From 56603c28d91690594423d073442ef9acec2aaa45 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 30 Jun 2026 14:01:17 +0100 Subject: [PATCH 01/95] perf/arm-cmn: Move DTM index data out of hw_perf_event The amount of data we need to store all the per-DTM counter and watchpoint allocations is already testing the limits of hw_perf_event, and future CMNs are only likely to keep growing larger, so move these arrays out to separate memory allocations. As part of that we can use an explicit union for allocating cycle counters to dtc_cycles events, which is arguably nicer anyway. Reviewed-by: Ilkka Koskinen Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/perf/arm-cmn.c | 91 ++++++++++++++++++++++++++++-------------- 1 file changed, 61 insertions(+), 30 deletions(-) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 6e5cc4086a9e..9392838408ff 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -598,17 +598,14 @@ static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) {} struct arm_cmn_hw_event { struct arm_cmn_node *dn; - u64 dtm_idx[DIV_ROUND_UP(CMN_MAX_NODES_PER_EVENT * 2, 64)]; + union { + unsigned long *dtm_idx; + int cc_idx; + }; + unsigned long *wp_idx; s8 dtc_idx[CMN_MAX_DTCS]; u8 num_dns; u8 dtm_offset; - - /* - * WP config registers are divided to UP and DOWN events. We need to - * keep to track only one of them. - */ - DECLARE_BITMAP(wp_idx, CMN_MAX_XPS); - bool wide_sel; enum cmn_filter_select filter_sel; }; @@ -626,25 +623,44 @@ static struct arm_cmn_hw_event *to_cmn_hw(struct perf_event *event) return (struct arm_cmn_hw_event *)&event->hw; } -static void arm_cmn_set_index(u64 x[], unsigned int pos, unsigned int val) +#define BPL2 (BITS_PER_LONG / 2) + +static void arm_cmn_set_dtm_idx(struct arm_cmn_hw_event *hw, unsigned int pos, unsigned int val) { - x[pos / 32] |= (u64)val << ((pos % 32) * 2); + hw->dtm_idx[pos / BPL2] |= (unsigned long)val << ((pos % BPL2) * 2); } -static unsigned int arm_cmn_get_index(u64 x[], unsigned int pos) +static unsigned int arm_cmn_get_dtm_idx(struct arm_cmn_hw_event *hw, unsigned int pos) { - return (x[pos / 32] >> ((pos % 32) * 2)) & 3; + return (hw->dtm_idx[pos / BPL2] >> ((pos % BPL2) * 2)) & 3; } -static void arm_cmn_set_wp_idx(unsigned long *wp_idx, unsigned int pos, bool val) +static unsigned long *arm_cmn_alloc_dtm_idx(void) +{ + return bitmap_zalloc(CMN_MAX_NODES_PER_EVENT * 2, GFP_KERNEL); +} + +static void arm_cmn_set_wp_idx(struct arm_cmn_hw_event *hw, unsigned int pos, bool val) { if (val) - set_bit(pos, wp_idx); + set_bit(pos, hw->wp_idx); } -static unsigned int arm_cmn_get_wp_idx(unsigned long *wp_idx, unsigned int pos) +static unsigned int arm_cmn_get_wp_idx(struct arm_cmn_hw_event *hw, unsigned int pos) { - return test_bit(pos, wp_idx); + return test_bit(pos, hw->wp_idx); +} + +static unsigned long *arm_cmn_alloc_wp_idx(void) +{ + return bitmap_zalloc(CMN_MAX_XPS, GFP_KERNEL); +} + +static void arm_cmn_clear_idx(struct arm_cmn_hw_event *hw) +{ + bitmap_zero(hw->dtm_idx, CMN_MAX_NODES_PER_EVENT * 2); + if (hw->wp_idx) + bitmap_zero(hw->wp_idx, CMN_MAX_XPS); } struct arm_cmn_event_attr { @@ -1377,7 +1393,7 @@ static int arm_cmn_get_assigned_wp_idx(struct perf_event *event, struct arm_cmn_hw_event *hw, unsigned int pos) { - return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw->wp_idx, pos); + return CMN_EVENT_EVENTID(event) + arm_cmn_get_wp_idx(hw, pos); } static void arm_cmn_claim_wp_idx(struct arm_cmn_dtm *dtm, @@ -1388,7 +1404,7 @@ static void arm_cmn_claim_wp_idx(struct arm_cmn_dtm *dtm, struct arm_cmn_hw_event *hw = to_cmn_hw(event); dtm->wp_event[wp_idx] = hw->dtc_idx[dtc]; - arm_cmn_set_wp_idx(hw->wp_idx, pos, wp_idx - CMN_EVENT_EVENTID(event)); + arm_cmn_set_wp_idx(hw, pos, wp_idx - CMN_EVENT_EVENTID(event)); } static u32 arm_cmn_wp_config(struct perf_event *event, int wp_idx) @@ -1459,7 +1475,7 @@ static u64 arm_cmn_read_dtm(struct arm_cmn *cmn, struct arm_cmn_hw_event *hw, dtm = &cmn->dtms[dn->dtm] + hw->dtm_offset; reg = readq_relaxed(dtm->base + offset); } - dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); + dtm_idx = arm_cmn_get_dtm_idx(hw, i); count += (u16)(reg >> (dtm_idx * 16)); } return count; @@ -1506,7 +1522,7 @@ static void arm_cmn_event_read(struct perf_event *event) unsigned long flags; if (CMN_EVENT_TYPE(event) == CMN_TYPE_DTC) { - delta = arm_cmn_read_cc(cmn->dtc + hw->dtc_idx[0]); + delta = arm_cmn_read_cc(cmn->dtc + hw->cc_idx); local64_add(delta, &event->count); return; } @@ -1573,7 +1589,7 @@ static void arm_cmn_event_start(struct perf_event *event, int flags) int i; if (type == CMN_TYPE_DTC) { - struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; + struct arm_cmn_dtc *dtc = cmn->dtc + hw->cc_idx; writel_relaxed(CMN_DT_DTC_CTL_DT_EN | CMN_DT_DTC_CTL_CG_DISABLE, dtc->base + CMN_DT_DTC_CTL); @@ -1591,7 +1607,7 @@ static void arm_cmn_event_start(struct perf_event *event, int flags) writeq_relaxed(mask, base + CMN_DTM_WPn_MASK(wp_idx)); } } else for_each_hw_dn(hw, dn, i) { - int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); + int dtm_idx = arm_cmn_get_dtm_idx(hw, i); arm_cmn_set_event_sel_lo(dn, dtm_idx, CMN_EVENT_EVENTID(event), hw->wide_sel); @@ -1607,7 +1623,7 @@ static void arm_cmn_event_stop(struct perf_event *event, int flags) int i; if (type == CMN_TYPE_DTC) { - struct arm_cmn_dtc *dtc = cmn->dtc + hw->dtc_idx[0]; + struct arm_cmn_dtc *dtc = cmn->dtc + hw->cc_idx; dtc->cc_active = false; writel_relaxed(CMN_DT_DTC_CTL_DT_EN, dtc->base + CMN_DT_DTC_CTL); @@ -1620,7 +1636,7 @@ static void arm_cmn_event_stop(struct perf_event *event, int flags) writeq_relaxed(~0ULL, base + CMN_DTM_WPn_VAL(wp_idx)); } } else for_each_hw_dn(hw, dn, i) { - int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); + int dtm_idx = arm_cmn_get_dtm_idx(hw, i); arm_cmn_set_event_sel_lo(dn, dtm_idx, 0, hw->wide_sel); } @@ -1764,6 +1780,14 @@ static enum cmn_filter_select arm_cmn_filter_sel(const struct arm_cmn *cmn, } +static void arm_cmn_event_destroy(struct perf_event *event) +{ + struct arm_cmn_hw_event *hw = to_cmn_hw(event); + + bitmap_free(hw->dtm_idx); + bitmap_free(hw->wp_idx); +} + static int arm_cmn_event_init(struct perf_event *event) { struct arm_cmn *cmn = to_cmn(event->pmu); @@ -1788,6 +1812,11 @@ static int arm_cmn_event_init(struct perf_event *event) if (type == CMN_TYPE_DTC) return arm_cmn_validate_group(cmn, event); + event->destroy = arm_cmn_event_destroy; + hw->dtm_idx = arm_cmn_alloc_dtm_idx(); + if (!hw->dtm_idx) + return -ENOMEM; + eventid = CMN_EVENT_EVENTID(event); /* For watchpoints we need the actual XP node here */ if (type == CMN_TYPE_WP) { @@ -1798,6 +1827,9 @@ static int arm_cmn_event_init(struct perf_event *event) /* ...but the DTM may depend on which port we're watching */ if (cmn->multi_dtm) hw->dtm_offset = CMN_EVENT_WP_DEV_SEL(event) / 2; + hw->wp_idx = arm_cmn_alloc_wp_idx(); + if (!hw->wp_idx) + return -ENOMEM; } else if (type == CMN_TYPE_XP && (cmn->part == PART_CMN700 || cmn->part == PART_CMN_S3)) { hw->wide_sel = true; @@ -1848,7 +1880,7 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, while (i--) { struct arm_cmn_dtm *dtm = &cmn->dtms[hw->dn[i].dtm] + hw->dtm_offset; - unsigned int dtm_idx = arm_cmn_get_index(hw->dtm_idx, i); + unsigned int dtm_idx = arm_cmn_get_dtm_idx(hw, i); if (type == CMN_TYPE_WP) { int wp_idx = arm_cmn_get_assigned_wp_idx(event, hw, i); @@ -1862,8 +1894,7 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); } - memset(hw->dtm_idx, 0, sizeof(hw->dtm_idx)); - memset(hw->wp_idx, 0, sizeof(hw->wp_idx)); + arm_cmn_clear_idx(hw); for_each_hw_dtc_idx(hw, j, idx) cmn->dtc[j].counters[idx] = NULL; @@ -1883,7 +1914,7 @@ static int arm_cmn_event_add(struct perf_event *event, int flags) return -ENOSPC; cmn->dtc[i].cycles = event; - hw->dtc_idx[0] = i; + hw->cc_idx = i; if (flags & PERF_EF_START) arm_cmn_event_start(event, 0); @@ -1948,7 +1979,7 @@ static int arm_cmn_event_add(struct perf_event *event, int flags) goto free_dtms; } - arm_cmn_set_index(hw->dtm_idx, i, dtm_idx); + arm_cmn_set_dtm_idx(hw, i, dtm_idx); dtm->input_sel[dtm_idx] = input_sel; shift = CMN__PMEVCNTn_GLOBAL_NUM_SHIFT(dtm_idx); @@ -1981,7 +2012,7 @@ static void arm_cmn_event_del(struct perf_event *event, int flags) arm_cmn_event_stop(event, PERF_EF_UPDATE); if (type == CMN_TYPE_DTC) - cmn->dtc[hw->dtc_idx[0]].cycles = NULL; + cmn->dtc[hw->cc_idx].cycles = NULL; else arm_cmn_event_clear(cmn, event, hw->num_dns); } From 9d0b1714e1af16622a2d0f7317ddaa9a4b27f353 Mon Sep 17 00:00:00 2001 From: Aviv Bakal Date: Tue, 30 Jun 2026 14:01:18 +0100 Subject: [PATCH 02/95] perf/arm-cmn: Add workarounds for CMN-S3 on Graviton5 Graviton5 uses a customised CMN-S3 implementation where certain discovery registers report zeroed fields. Add the following workarounds: - Introduce a dedicated ACPI HID to identify the Graviton5 CMN variant. - Derive the DTC domain from the XP node ID, since the unit info register reports it as zero. - Set the DTC logical ID from the XP's logical ID, since the node info register's logical ID field is also zeroed. Signed-off-by: Aviv Bakal Reviewed-by: Robin Murphy Reviewed-by: Ilkka Koskinen Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/perf/arm-cmn.c | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 9392838408ff..50402bc4a21d 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -31,7 +31,8 @@ #define CMN_CHILD_NODE_ADDR GENMASK(29, 0) #define CMN_CHILD_NODE_EXTERNAL BIT(31) -#define CMN_MAX_DIMENSION 12 +/* Some implementations use a mesh larger than the architectural max of 12 */ +#define CMN_MAX_DIMENSION 14 #define CMN_MAX_XPS (CMN_MAX_DIMENSION * CMN_MAX_DIMENSION) #define CMN_MAX_DTMS (CMN_MAX_XPS + (CMN_MAX_DIMENSION - 1) * 4) @@ -215,6 +216,8 @@ enum cmn_part { PART_CMN700 = 0x43c, PART_CI700 = 0x43a, PART_CMN_S3 = 0x43e, + /* Synthetic part number, overridden to PART_CMN_S3 during discovery */ + PART_GRAVITON5 = 0xa5, }; /* CMN-600 r0px shouldn't exist in silicon, thankfully */ @@ -2253,6 +2256,18 @@ static unsigned int arm_cmn_dtc_domain(struct arm_cmn *cmn, void __iomem *xp_reg return FIELD_GET(CMN_DTM_UNIT_INFO_DTC_DOMAIN, readl_relaxed(xp_region + offset)); } +static unsigned int arm_cmn_graviton5_dtc_domain(u16 xp_id) +{ + unsigned int x = (xp_id >> 7) & 0xf; + unsigned int y = (xp_id >> 3) & 0xf; + + /* + * The unit info register reads as zero; derive the DTC domain from + * the XP's mesh coordinates over the 10x14 mesh. + */ + return (x / 5) + (y / 7) * 2; +} + static void arm_cmn_init_node_info(struct arm_cmn *cmn, u32 offset, struct arm_cmn_node *node) { int level; @@ -2298,6 +2313,7 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) u64 reg; int i, j; size_t sz; + bool graviton5_workaround = false; arm_cmn_init_node_info(cmn, rgn_offset, &cfg); if (cfg.type != CMN_TYPE_CFG) @@ -2308,6 +2324,13 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) reg = readq_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_01); part = FIELD_GET(CMN_CFGM_PID0_PART_0, reg); part |= FIELD_GET(CMN_CFGM_PID1_PART_1, reg) << 8; + + /* Graviton5 has a customised CMN-S3 which needs some fixups */ + if (cmn->part == PART_GRAVITON5) { + cmn->part = PART_CMN_S3; + graviton5_workaround = true; + } + /* 600AE is close enough that it's not really worth more complexity */ if (part == PART_CMN600AE) part = PART_CMN600; @@ -2397,6 +2420,8 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) if (cmn->part == PART_CMN600) xp->dtc = -1; + else if (graviton5_workaround) + xp->dtc = arm_cmn_graviton5_dtc_domain(xp->id); else xp->dtc = arm_cmn_dtc_domain(cmn, xp_region); @@ -2475,6 +2500,10 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) switch (dn->type) { case CMN_TYPE_DTC: + if (graviton5_workaround) { + /* Node info logical ID is zeroed; use the XP's */ + dn->logid = xp->logid; + } cmn->num_dtcs++; dn++; break; @@ -2690,6 +2719,7 @@ static const struct acpi_device_id arm_cmn_acpi_match[] = { { "ARMHC650" }, { "ARMHC700" }, { "ARMHC003" }, + { "AMZN0070", PART_GRAVITON5 }, {} }; MODULE_DEVICE_TABLE(acpi, arm_cmn_acpi_match); From 49f2413f9b55dd975fb7d874c1fd35cf6a7335c6 Mon Sep 17 00:00:00 2001 From: Besar Wicaksono Date: Mon, 8 Jun 2026 23:41:35 +0000 Subject: [PATCH 03/95] perf/arm_pmu: Skip PMCCNTR_EL0 on NVIDIA Olympus The PMCCNTR_EL0 in NVIDIA Olympus CPU may increment while in WFI/WFE, which does not align with counting CPU_CYCLES on a programmable counter. Add a MIDR range entry and refuse PMCCNTR_EL0 for cycle events on affected parts so perf does not mix the two behaviors. Also keep PMCCNTR_EL0 unavailable to EL0 direct counter reads on affected CPUs. When userspace counter access is enabled, avoid setting PMUSERENR_EL0.CR for PMUs that must avoid PMCCNTR_EL0, while still allowing direct reads from programmable event counters. For 64-bit userspace CPU_CYCLES events on PMUs without native long event counters, reject the event if the only valid direct-read path would be PMCCNTR_EL0. Signed-off-by: Besar Wicaksono Signed-off-by: Will Deacon --- drivers/perf/arm_pmu.c | 7 +++- drivers/perf/arm_pmuv3.c | 64 +++++++++++++++++++++++++++++++----- include/linux/perf/arm_pmu.h | 2 +- 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c index 939bcbd433aa..aa1dac0b440f 100644 --- a/drivers/perf/arm_pmu.c +++ b/drivers/perf/arm_pmu.c @@ -931,8 +931,13 @@ int armpmu_register(struct arm_pmu *pmu) /* * By this stage we know our supported CPUs on either DT/ACPI platforms, * detect the SMT implementation. + * On SMT CPUs, the PMCCNTR_EL0 increments from the processor clock rather + * than the PE clock (ARM DDI0487 L.b D13.1.3) which means it'll continue + * counting on a WFI PE if one of its SMT sibling is not idle on a + * multi-threaded implementation. So don't use it on SMT cores. */ - pmu->has_smt = topology_core_has_smt(cpumask_first(&pmu->supported_cpus)); + pmu->avoid_pmccntr |= + topology_core_has_smt(cpumask_first(&pmu->supported_cpus)); if (!pmu->set_event_filter) pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE; diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 8014ff766cff..6d4d57342352 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -8,6 +8,7 @@ * This code is based heavily on the ARMv7 perf event code. */ +#include #include #include #include @@ -795,6 +796,7 @@ static void armv8pmu_disable_user_access(void) static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu) { int i; + u64 userenr = ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_UEN; struct pmu_hw_events *cpuc = this_cpu_ptr(cpu_pmu->hw_events); if (is_pmuv3p9(cpu_pmu->pmuver)) { @@ -817,7 +819,10 @@ static void armv8pmu_enable_user_access(struct arm_pmu *cpu_pmu) } } - update_pmuserenr(ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_UEN); + if (!cpu_pmu->avoid_pmccntr) + userenr |= ARMV8_PMU_USERENR_CR; + + update_pmuserenr(userenr); } static void armv8pmu_enable_event(struct perf_event *event) @@ -1002,13 +1007,7 @@ static bool armv8pmu_can_use_pmccntr(struct pmu_hw_events *cpuc, if (has_branch_stack(event)) return false; - /* - * The PMCCNTR_EL0 increments from the processor clock rather than - * the PE clock (ARM DDI0487 L.b D13.1.3) which means it'll continue - * counting on a WFI PE if one of its SMT sibling is not idle on a - * multi-threaded implementation. So don't use it on SMT cores. - */ - if (cpu_pmu->has_smt) + if (cpu_pmu->avoid_pmccntr) return false; return true; @@ -1250,7 +1249,8 @@ static int __armv8_pmuv3_map_event(struct perf_event *event, if (!(event->attach_state & PERF_ATTACH_TASK)) return -EINVAL; if (armv8pmu_event_is_64bit(event) && - (hw_event_id != ARMV8_PMUV3_PERFCTR_CPU_CYCLES) && + (hw_event_id != ARMV8_PMUV3_PERFCTR_CPU_CYCLES || + armpmu->avoid_pmccntr) && !armv8pmu_has_long_event(armpmu)) return -EOPNOTSUPP; @@ -1299,6 +1299,45 @@ static int armv8_vulcan_map_event(struct perf_event *event) &armv8_vulcan_perf_cache_map); } +#ifdef CONFIG_ARM64 +/* + * List of CPUs that should avoid using PMCCNTR_EL0. + */ +static struct midr_range armv8pmu_avoid_pmccntr_cpus[] = { + /* + * NVIDIA Olympus may expose different WFI/WFE behaviour between the + * PMCCNTR_EL0 and the CPU_CYCLES event on programmable counters. + * While the CPU is in WFI/WFE state, the PMCCNTR_EL0 may still increment + * but the programmable counter may not. This is an implementation specific + * behavior and not an erratum. Perf assumes those two paths are + * interchangeable, so avoid using PMCCNTR_EL0 for CPU_CYCLES event. + * + * From ARM DDI0487 D14.4: + * It is IMPLEMENTATION SPECIFIC whether CPU_CYCLES and PMCCNTR count + * when the PE is in WFI or WFE state, even if the clocks are not stopped. + * + * From ARM DDI0487 D24.5.2: + * All counters are subject to any changes in clock frequency, including + * clock stopping caused by the WFI and WFE instructions. + * This means that it is CONSTRAINED UNPREDICTABLE whether or not + * PMCCNTR_EL0 continues to increment when clocks are stopped by WFI and + * WFE instructions. + */ + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + {} +}; + +static bool armv8pmu_is_in_avoid_pmccntr_cpus(void) +{ + return is_midr_in_range_list(armv8pmu_avoid_pmccntr_cpus); +} +#else +static bool armv8pmu_is_in_avoid_pmccntr_cpus(void) +{ + return false; +} +#endif + struct armv8pmu_probe_info { struct arm_pmu *pmu; bool present; @@ -1348,6 +1387,13 @@ static void __armv8pmu_probe_pmu(void *info) else cpu_pmu->reg_pmmir = 0; + /* + * On some CPUs, PMCCNTR_EL0 does not match the behavior of CPU_CYCLES + * programmable counter, so avoid routing cycles through PMCCNTR_EL0 to + * prevent inconsistency in the results. + */ + cpu_pmu->avoid_pmccntr |= armv8pmu_is_in_avoid_pmccntr_cpus(); + brbe_probe(cpu_pmu); } diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 52b37f7bdbf9..02d2c7f45b52 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -119,7 +119,7 @@ struct arm_pmu { /* PMUv3 only */ int pmuver; - bool has_smt; + bool avoid_pmccntr; u64 reg_pmmir; u64 reg_brbidr; #define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40 From 10a47fb67340bca274276faf855d7d460afd9a7a Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 26 Jun 2026 15:17:28 +0100 Subject: [PATCH 04/95] tools: linux/types.h: Add 128-bit integer types for arm64 UAPI structures The arm64 UAPI exposes some 128-bit integer types to represent things such as fpsimd registers in the sigcontext. In preparation for defining these using the '__u128' typedef implemented by uapi/linux/types.h, copy that typedef over to the private linux/types.h header used by the tools directory. Cc: Arnd Bergmann Cc: Arnaldo Carvalho de Melo Cc: David Matlack Signed-off-by: Will Deacon --- tools/include/linux/types.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/include/linux/types.h b/tools/include/linux/types.h index d41f8a261bce..b6c473b7920d 100644 --- a/tools/include/linux/types.h +++ b/tools/include/linux/types.h @@ -23,6 +23,11 @@ typedef enum { __GFP_HIGH } gfp_t; +#ifdef __SIZEOF_INT128__ +typedef __signed__ __int128 __s128 __attribute__((aligned(16))); +typedef unsigned __int128 __u128 __attribute__((aligned(16))); +#endif + /* * We define u64 as uint64_t for every architecture * so that we can print it with "%"PRIx64 without getting warnings. From d4bf9e08d412289ae7c9072378bed238c0274c55 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 26 Jun 2026 15:17:29 +0100 Subject: [PATCH 05/95] arm64: uapi: Use __u128 instead of __uint128_t in UAPI headers The arm64 UAPI exposes '__uint128_t' types in the members of 'struct user_fpsimd_state', 'struct user_pac_address_keys' and in the signal frame via 'struct fpsimd_context'. Since the alignment of such a type appears to be non-portable (16 bytes on arm64, 8 bytes on s390), prefer the '__u128' typedef from uapi/linux/types.h, which makes the alignment explicit and allows the definitions to be reused by other host architectures. Cc: Arnd Bergmann Cc: Nick Desaulniers Cc: Steffen Eiden Cc: Andreas Grapentin Cc: Catalin Marinas Cc: Dave Martin Cc: Mark Rutland Cc: Marc Zyngier Acked-by: Mark Rutland Reviewed-by: Marc Zyngier Reviewed-by: Arnd Bergmann Signed-off-by: Will Deacon --- arch/arm64/include/uapi/asm/ptrace.h | 12 ++++++------ arch/arm64/include/uapi/asm/sigcontext.h | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h index 6fed93fb2536..15649a253a57 100644 --- a/arch/arm64/include/uapi/asm/ptrace.h +++ b/arch/arm64/include/uapi/asm/ptrace.h @@ -93,7 +93,7 @@ struct user_pt_regs { }; struct user_fpsimd_state { - __uint128_t vregs[32]; + __u128 vregs[32]; __u32 fpsr; __u32 fpcr; __u32 __reserved[2]; @@ -258,14 +258,14 @@ struct user_pac_mask { /* pointer authentication keys (NT_ARM_PACA_KEYS, NT_ARM_PACG_KEYS) */ struct user_pac_address_keys { - __uint128_t apiakey; - __uint128_t apibkey; - __uint128_t apdakey; - __uint128_t apdbkey; + __u128 apiakey; + __u128 apibkey; + __u128 apdakey; + __u128 apdbkey; }; struct user_pac_generic_keys { - __uint128_t apgakey; + __u128 apgakey; }; /* ZA state (NT_ARM_ZA) */ diff --git a/arch/arm64/include/uapi/asm/sigcontext.h b/arch/arm64/include/uapi/asm/sigcontext.h index e29bf3e2d0cc..d250ca7a1d46 100644 --- a/arch/arm64/include/uapi/asm/sigcontext.h +++ b/arch/arm64/include/uapi/asm/sigcontext.h @@ -78,7 +78,7 @@ struct fpsimd_context { struct _aarch64_ctx head; __u32 fpsr; __u32 fpcr; - __uint128_t vregs[32]; + __u128 vregs[32]; }; /* @@ -266,8 +266,8 @@ struct gcs_context { * - ---- ----------- * REGS the entire SVE context * - * ZREGS __uint128_t[SVE_NUM_ZREGS][vq] all Z-registers - * ZREG __uint128_t[vq] individual Z-register Zn + * ZREGS __u128[SVE_NUM_ZREGS][vq] all Z-registers + * ZREG __u128[vq] individual Z-register Zn * * PREGS uint16_t[SVE_NUM_PREGS][vq] all P-registers * PREG uint16_t[vq] individual P-register Pn From 62e11a7fde652026beb770077b1d9d4186a85b79 Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Mon, 29 Jun 2026 11:09:53 +0100 Subject: [PATCH 06/95] arm64: Clarify ARM64_WORKAROUND_REPEAT_TLBI semantics Will notes that the ARM64_WORKAROUND_REPEAT_TLBI name is potentially misleading, and that it would be nice to rename that and add some documentation. See: https://lore.kernel.org/linux-arm-kernel/ajKn_Pt50CmOUrsP@willie-the-truck/ To that end, I've renamed the Kconfig symbol and hwcap from: [CONFIG_]ARM64_WORKAROUND_REPEAT_TLBI ... to: [CONFIG_]ARM64_WORKAROUND_REPEAT_TLBI_SYNC ... and I've added some rationale alongside the Kconfig. As the Kconfig symbol isn't user selectable, the usual 'help' section won't appear in menuconfig, so I've added this as a comment. The rename was scripted with: git grep -l REPEAT_TLBI | while read F; do sed -i '{ s/WORKAROUND_REPEAT_TLBI\>/WORKAROUND_REPEAT_TLBI_SYNC/g }' $F; done Bikeshedding-wise, I considered a few names, including: * ARM64_WORKAROUND_REPEAT_TLBI_SYNC * ARM64_WORKAROUND_TLBI_REPEAT_SYNC * ARM64_WORKAROUND_BROADCAST_TLBI_REPEAT_SYNC ... and I settled on ARM64_WORKAROUND_REPEAT_TLBI_SYNC to try keep things simple, and to avoid unnecessary churn caused by moving definitions to retain alphabetical order. I'm happy to defer to Will and Catalin's preference. Signed-off-by: Mark Rutland Cc: Catalin Marinas Cc: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/Kconfig | 34 +++++++++++++++++++++++++------ arch/arm64/include/asm/cpucaps.h | 4 ++-- arch/arm64/include/asm/tlbflush.h | 2 +- arch/arm64/kernel/cpu_errata.c | 6 +++--- arch/arm64/tools/cpucaps | 2 +- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..757110421543 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -701,12 +701,34 @@ config ARM64_ERRATUM_1530923 If unsure, say Y. -config ARM64_WORKAROUND_REPEAT_TLBI +config ARM64_WORKAROUND_REPEAT_TLBI_SYNC bool + # This workaround is (only) suitable for TLB invalidation errata where + # all of the following conditions are true: + # + # - The effects of the errata are only a loss of ordering/completion + # for explicit memory accesses when the TLBI is completed with a DSB. + # The removal of TLB entries is not affected. + # + # Note that architecturally, S2-only invalidation does not remove + # combined S1+S2 entries, and does not complete accesses translated + # via those S1+S2 entries. Consequently, where this condition holds, + # the errata do not affect S2-only invalidation. + # + # - The errata only affect broadcast TLB invalidation operations (e.g. + # TLBI VMALLE1IS), and do not affect local TLB invalidation + # operations (e.g. TLBI VMALLE1). + # + # - After any number of affected TLBI operations are completed with a + # DSB, the errata can be mitigated by executing a single arbitrary + # broadcast TLBI (which targets an arbitrary translation regime), + # followed by a DSB. + # + # For more rationale, see commit a8f78680ee6bf795. config ARM64_ERRATUM_2441007 bool "Cortex-A55: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A55 erratum #2441007. @@ -722,7 +744,7 @@ config ARM64_ERRATUM_2441007 config ARM64_ERRATUM_1286807 bool "Cortex-A76: Modification of the translation table for a virtual address might lead to read-after-read ordering violation (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A76 erratum 1286807. @@ -944,7 +966,7 @@ config ARM64_ERRATUM_2224489 config ARM64_ERRATUM_2441009 bool "Cortex-A510: Completion of affected memory accesses might not be guaranteed by completion of a TLBI (rare)" - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for ARM Cortex-A510 erratum #2441009. @@ -1156,7 +1178,7 @@ config ARM64_ERRATUM_4193714 config ARM64_ERRATUM_4118414 bool "Various: Completion of affected memory accesses might not be guaranteed by completion of a TLBI" default y - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help This option adds a workaround for the following errata: @@ -1340,7 +1362,7 @@ config QCOM_FALKOR_ERRATUM_1003 config QCOM_FALKOR_ERRATUM_1009 bool "Falkor E1009: Prematurely complete a DSB after a TLBI" default y - select ARM64_WORKAROUND_REPEAT_TLBI + select ARM64_WORKAROUND_REPEAT_TLBI_SYNC help On Falkor v1, the CPU may prematurely complete a DSB following a TLBI xxIS invalidate maintenance operation. Repeat the TLBI operation diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h index 25c61cda901c..76350b38f0d7 100644 --- a/arch/arm64/include/asm/cpucaps.h +++ b/arch/arm64/include/asm/cpucaps.h @@ -60,8 +60,8 @@ cpucap_is_possible(const unsigned int cap) return IS_ENABLED(CONFIG_CAVIUM_ERRATUM_23154); case ARM64_WORKAROUND_DISABLE_CNP: return IS_ENABLED(CONFIG_ARM64_WORKAROUND_DISABLE_CNP); - case ARM64_WORKAROUND_REPEAT_TLBI: - return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI); + case ARM64_WORKAROUND_REPEAT_TLBI_SYNC: + return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC); case ARM64_WORKAROUND_SPECULATIVE_SSBS: return IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386); case ARM64_WORKAROUND_4193714: diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index e0e84332f51b..14a78ac0f800 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -236,7 +236,7 @@ static inline void __tlbi_level(tlbi_op op, u64 addr, u32 level) #define __repeat_tlbi_sync(op, arg...) \ do { \ - if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI)) \ + if (!alternative_has_cap_unlikely(ARM64_WORKAROUND_REPEAT_TLBI_SYNC)) \ break; \ __tlbi(op, ##arg); \ dsb(ish); \ diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 1995e1198648..685077d44ad1 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -309,7 +309,7 @@ static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *_ sysreg_clear_set_s(SYS_HACR_EL2, 0, BIT(56)); } -#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI +#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC static const struct arm64_cpu_capabilities arm64_repeat_tlbi_list[] = { #ifdef CONFIG_QCOM_FALKOR_ERRATUM_1009 { @@ -733,10 +733,10 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .match_list = qcom_erratum_1003_list, }, #endif -#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI +#ifdef CONFIG_ARM64_WORKAROUND_REPEAT_TLBI_SYNC { .desc = "Broken broadcast TLBI completion", - .capability = ARM64_WORKAROUND_REPEAT_TLBI, + .capability = ARM64_WORKAROUND_REPEAT_TLBI_SYNC, .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM, .matches = cpucap_multi_entry_cap_matches, .match_list = arm64_repeat_tlbi_list, diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9b85a84f6fd4..f8368e5d81a8 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -124,7 +124,7 @@ WORKAROUND_DISABLE_CNP WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF -WORKAROUND_REPEAT_TLBI +WORKAROUND_REPEAT_TLBI_SYNC WORKAROUND_SPECULATIVE_AT WORKAROUND_SPECULATIVE_SSBS WORKAROUND_SPECULATIVE_UNPRIV_LOAD From 24f55f511b9e1c19dc48d11bfe0dc60c86bdb376 Mon Sep 17 00:00:00 2001 From: Kohei Enju Date: Fri, 12 Jun 2026 20:09:21 +0900 Subject: [PATCH 07/95] virt: arm-cca-guest: use migrate_disable() for attestation token requests The RSI attestation token init and continue calls must be issued from the same CPU. arm_cca_report_new() currently snapshots the CPU number and uses smp_call_function_single() to issue those calls on that CPU. With CONFIG_DEBUG_PREEMPT=y, the smp_processor_id() call used for the snapshot triggers a debug splat [0] because it runs in preemptible context. The snapshot does not pin the task to that CPU; it is only used to choose the target CPU for smp_call_function_single(), which can fail if that CPU is no longer available. Use migrate_disable() and issue the token init and continue operations directly, without the smp_call_function_single() callbacks. This keeps the token request sequence on the same CPU while preserving a sleepable context for the GFP_KERNEL allocations needed after the init call. [0] BUG: using smp_processor_id() in preemptible [00000000] code: cca-workload-at/264 caller is debug_smp_processor_id+0x20/0x30 CPU: 0 UID: 0 PID: 264 Comm: cca-workload-at Not tainted 7.1.0-rc1-00044-g55542ab273f2 #80 PREEMPT(lazy) Hardware name: linux,dummy-virt (DT) Call trace: [...] check_preemption_disabled+0xd8/0xf8 debug_smp_processor_id+0x20/0x30 arm_cca_report_new+0x48/0x278 tsm_report_read+0x154/0x1f8 tsm_report_outblob_read+0x20/0x38 configfs_bin_read_iter+0x118/0x208 vfs_read+0x220/0x318 [...] Fixes: 7999edc484ca ("virt: arm-cca-guest: TSM_REPORT support for realms") Signed-off-by: Kohei Enju Reviewed-by: Suzuki K Poulose Tested-by: Suzuki K Poulose Reviewed-by: Gavin Shan Reviewed-by: Steven Price Signed-off-by: Will Deacon --- .../virt/coco/arm-cca-guest/arm-cca-guest.c | 97 +++++++------------ 1 file changed, 36 insertions(+), 61 deletions(-) diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c index 32cd038cb79b..dbbb2cc0e124 100644 --- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +++ b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c @@ -16,54 +16,38 @@ /** * struct arm_cca_token_info - a descriptor for the token buffer. - * @challenge: Pointer to the challenge data - * @challenge_size: Size of the challenge data * @granule: PA of the granule to which the token will be written * @offset: Offset within granule to start of buffer in bytes - * @result: result of rsi_attestation_token_continue operation */ struct arm_cca_token_info { - void *challenge; - unsigned long challenge_size; phys_addr_t granule; unsigned long offset; - unsigned long result; }; -static void arm_cca_attestation_init(void *param) -{ - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; - - info->result = rsi_attestation_token_init(info->challenge, - info->challenge_size); -} - /** * arm_cca_attestation_continue - Retrieve the attestation token data. * - * @param: pointer to the arm_cca_token_info + * @info: pointer to the arm_cca_token_info * * Attestation token generation is a long running operation and therefore * the token data may not be retrieved in a single call. Moreover, the * token retrieval operation must be requested on the same CPU on which the * attestation token generation was initialised. - * This helper function is therefore scheduled on the same CPU multiple + * This helper function must therefore be executed on the same CPU multiple * times until the entire token data is retrieved. */ -static void arm_cca_attestation_continue(void *param) +static unsigned long +arm_cca_attestation_continue(struct arm_cca_token_info *info) { + unsigned long ret; unsigned long len; unsigned long size; - struct arm_cca_token_info *info; - - info = (struct arm_cca_token_info *)param; size = RSI_GRANULE_SIZE - info->offset; - info->result = rsi_attestation_token_continue(info->granule, - info->offset, size, &len); + ret = rsi_attestation_token_continue(info->granule, info->offset, size, + &len); info->offset += len; + return ret; } /** @@ -74,8 +58,8 @@ static void arm_cca_attestation_continue(void *param) * * Initialise the attestation token generation using the challenge data * passed in the TSM descriptor. Allocate memory for the attestation token - * and schedule calls to retrieve the attestation token on the same CPU - * on which the attestation token generation was initialised. + * and retrieve the attestation token on the same CPU on which the + * attestation token generation was initialised. * * The challenge data must be at least 32 bytes and no more than 64 bytes. If * less than 64 bytes are provided it will be zero padded to 64 bytes. @@ -85,12 +69,11 @@ static void arm_cca_attestation_continue(void *param) * * %-EINVAL - A parameter was not valid. * * %-ENOMEM - Out of memory. * * %-EFAULT - Failed to get IPA for memory page(s). - * * A negative status code as returned by smp_call_function_single(). */ static int arm_cca_report_new(struct tsm_report *report, void *data) { - int ret; - int cpu; + int ret = 0; + unsigned long rsi_result; long max_size; unsigned long token_size = 0; struct arm_cca_token_info info; @@ -103,37 +86,33 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) /* * The attestation token 'init' and 'continue' calls must be - * performed on the same CPU. smp_call_function_single() is used - * instead of simply calling get_cpu() because of the need to - * allocate outblob based on the returned value from the 'init' - * call and that cannot be done in an atomic context. + * performed on the same CPU, so disable CPU migration around + * those operations. */ - cpu = smp_processor_id(); + migrate_disable(); - info.challenge = desc->inblob; - info.challenge_size = desc->inblob_len; - - ret = smp_call_function_single(cpu, arm_cca_attestation_init, - &info, true); - if (ret) - return ret; - max_size = info.result; - - if (max_size <= 0) - return -EINVAL; + max_size = rsi_attestation_token_init(desc->inblob, desc->inblob_len); + if (max_size <= 0) { + ret = -EINVAL; + goto exit_migrate_enable; + } /* Allocate outblob */ token = kvzalloc(max_size, GFP_KERNEL); - if (!token) - return -ENOMEM; + if (!token) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* * Since the outblob may not be physically contiguous, use a page * to bounce the buffer from RMM. */ buf = alloc_pages_exact(RSI_GRANULE_SIZE, GFP_KERNEL); - if (!buf) - return -ENOMEM; + if (!buf) { + ret = -ENOMEM; + goto exit_migrate_enable; + } /* Get the PA of the memory page(s) that were allocated */ info.granule = (unsigned long)virt_to_phys(buf); @@ -144,21 +123,15 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) info.offset = 0; do { /* - * Schedule a call to retrieve a sub-granule chunk - * of data per loop iteration. + * Retrieve a sub-granule chunk of data per loop + * iteration. */ - ret = smp_call_function_single(cpu, - arm_cca_attestation_continue, - (void *)&info, true); - if (ret != 0) { - token_size = 0; - goto exit_free_granule_page; - } - } while (info.result == RSI_INCOMPLETE && + rsi_result = arm_cca_attestation_continue(&info); + } while (rsi_result == RSI_INCOMPLETE && info.offset < RSI_GRANULE_SIZE); /* Break out in case of failure */ - if (info.result != RSI_SUCCESS && info.result != RSI_INCOMPLETE) { + if (rsi_result != RSI_SUCCESS && rsi_result != RSI_INCOMPLETE) { ret = -ENXIO; token_size = 0; goto exit_free_granule_page; @@ -173,12 +146,14 @@ static int arm_cca_report_new(struct tsm_report *report, void *data) break; memcpy(&token[token_size], buf, info.offset); token_size += info.offset; - } while (info.result == RSI_INCOMPLETE); + } while (rsi_result == RSI_INCOMPLETE); report->outblob = no_free_ptr(token); exit_free_granule_page: report->outblob_len = token_size; free_pages_exact(buf, RSI_GRANULE_SIZE); +exit_migrate_enable: + migrate_enable(); return ret; } From 442d366cc1aabc699015a2e10a41bba17fe416ad Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Wed, 3 Jun 2026 11:05:20 +0000 Subject: [PATCH 08/95] arm64/mm: Simplify SWIOTLB setup in arch_mm_preinit() At the moment, arch_mm_preinit() checks if the system has limited addressing or is running under CCA to enable SWIOTLB, only after to be forced to true anyway if it was false due to CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC being unconditionally true for arm64. Simplify this logic, by making it clear that SWIOTLB is always used but its size depends on the address layout of the system. Signed-off-by: Mostafa Saleh Reviewed-by: Catalin Marinas Reviewed-by: Aneesh Kumar K.V (Arm) Tested-by: Aneesh Kumar K.V (Arm) Signed-off-by: Will Deacon --- arch/arm64/mm/init.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 97987f850a33..9fb17043dd0d 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -336,25 +336,21 @@ void __init arch_setup_zero_pages(void) void __init arch_mm_preinit(void) { unsigned int flags = SWIOTLB_VERBOSE; - bool swiotlb = max_pfn > PFN_DOWN(arm64_dma_phys_limit); if (is_realm_world()) { - swiotlb = true; flags |= SWIOTLB_FORCE; - } - - if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) && !swiotlb) { + } else if (max_pfn <= PFN_DOWN(arm64_dma_phys_limit)) { /* * If no bouncing needed for ZONE_DMA, reduce the swiotlb * buffer for kmalloc() bouncing to 1MB per 1GB of RAM. */ unsigned long size = DIV_ROUND_UP(memblock_phys_mem_size(), 1024); + swiotlb_adjust_size(min(swiotlb_size_or_default(), size)); - swiotlb = true; } - swiotlb_init(swiotlb, flags); + swiotlb_init(true, flags); /* * Check boundaries twice: Some fundamental inconsistencies can be From e62decaf98e7c1385c4a22c61ba8bf94d24713a5 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Wed, 3 Jun 2026 11:05:22 +0000 Subject: [PATCH 09/95] arm64/coco: Add pKVM as a CC platform pKVM does support memory encryption, expose that to the rest of the kernel through cc_platform_has() At the moment, all devices inside the guest are emulated which requires its memory to be shared back to the host (decrypted), so set force_dma_unencrypted() to always return true. Although, typically pKVM guests rely on restricted-dma-pools to bounce traffic, with this change, it is possible to solely rely on the default SWIOTLB for that (assuming the appropriate size is set from the command line) Signed-off-by: Mostafa Saleh Reviewed-by: Catalin Marinas Tested-by: Aneesh Kumar K.V (Arm) Signed-off-by: Will Deacon --- arch/arm64/include/asm/hypervisor.h | 13 +++++++++++++ arch/arm64/include/asm/mem_encrypt.h | 3 ++- arch/arm64/kernel/rsi.c | 12 ------------ arch/arm64/mm/init.c | 15 ++++++++++++++- drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c | 3 +++ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/arch/arm64/include/asm/hypervisor.h b/arch/arm64/include/asm/hypervisor.h index a12fd897c877..8889a0ba1ec5 100644 --- a/arch/arm64/include/asm/hypervisor.h +++ b/arch/arm64/include/asm/hypervisor.h @@ -3,6 +3,9 @@ #define _ASM_ARM64_HYPERVISOR_H #include +#include + +DECLARE_STATIC_KEY_FALSE(pkvm_guest); void kvm_init_hyp_services(void); bool kvm_arm_hyp_service_available(u32 func_id); @@ -10,8 +13,18 @@ void kvm_arm_target_impl_cpu_init(void); #ifdef CONFIG_ARM_PKVM_GUEST void pkvm_init_hyp_services(void); + +static inline bool is_protected_kvm_guest(void) +{ + return static_branch_unlikely(&pkvm_guest); +} #else static inline void pkvm_init_hyp_services(void) { }; + +static inline bool is_protected_kvm_guest(void) +{ + return false; +} #endif static inline void kvm_arch_init_hyp_services(void) diff --git a/arch/arm64/include/asm/mem_encrypt.h b/arch/arm64/include/asm/mem_encrypt.h index 314b2b52025f..636f45b4d8af 100644 --- a/arch/arm64/include/asm/mem_encrypt.h +++ b/arch/arm64/include/asm/mem_encrypt.h @@ -2,6 +2,7 @@ #ifndef __ASM_MEM_ENCRYPT_H #define __ASM_MEM_ENCRYPT_H +#include #include struct device; @@ -20,7 +21,7 @@ int realm_register_memory_enc_ops(void); static inline bool force_dma_unencrypted(struct device *dev) { - return is_realm_world(); + return is_realm_world() || is_protected_kvm_guest(); } /* diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c index 92160f2e57ff..25ca75ce1a4d 100644 --- a/arch/arm64/kernel/rsi.c +++ b/arch/arm64/kernel/rsi.c @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -23,17 +22,6 @@ EXPORT_SYMBOL(prot_ns_shared); DEFINE_STATIC_KEY_FALSE_RO(rsi_present); EXPORT_SYMBOL(rsi_present); -bool cc_platform_has(enum cc_attr attr) -{ - switch (attr) { - case CC_ATTR_MEM_ENCRYPT: - return is_realm_world(); - default: - return false; - } -} -EXPORT_SYMBOL_GPL(cc_platform_has); - static bool rsi_version_matches(void) { unsigned long ver_lower, ver_higher; diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 9fb17043dd0d..3b9e1b8de6cb 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ #include #include +#include #include #include #include @@ -337,7 +339,7 @@ void __init arch_mm_preinit(void) { unsigned int flags = SWIOTLB_VERBOSE; - if (is_realm_world()) { + if (is_realm_world() || is_protected_kvm_guest()) { flags |= SWIOTLB_FORCE; } else if (max_pfn <= PFN_DOWN(arm64_dma_phys_limit)) { /* @@ -412,6 +414,17 @@ void dump_mem_limit(void) } } +bool cc_platform_has(enum cc_attr attr) +{ + switch (attr) { + case CC_ATTR_MEM_ENCRYPT: + return is_realm_world() || is_protected_kvm_guest(); + default: + return false; + } +} +EXPORT_SYMBOL_GPL(cc_platform_has); + #ifdef CONFIG_EXECMEM static u64 module_direct_base __ro_after_init = 0; static u64 module_plt_base __ro_after_init = 0; diff --git a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c index 4230b817a80b..6e966bad5ee1 100644 --- a/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c +++ b/drivers/virt/coco/pkvm-guest/arm-pkvm-guest.c @@ -17,6 +17,7 @@ #include static size_t pkvm_granule; +DEFINE_STATIC_KEY_FALSE_RO(pkvm_guest); static int arm_smccc_do_one_page(u32 func_id, phys_addr_t phys) { @@ -120,4 +121,6 @@ void pkvm_init_hyp_services(void) if (kvm_arm_hyp_service_available(ARM_SMCCC_KVM_FUNC_MMIO_GUARD)) arm64_ioremap_prot_hook_register(&mmio_guard_ioremap_hook); + + static_branch_enable(&pkvm_guest); } From 9aa7df52052399f7759b70f64c6ee561f901a28a Mon Sep 17 00:00:00 2001 From: Pengjie Zhang Date: Wed, 8 Jul 2026 16:28:17 +0800 Subject: [PATCH 10/95] ACPI: CPPC: add paired FFH feedback-counter read hook cppc_get_perf_ctrs() reads the delivered and reference performance counters one at a time. Allow architectures to provide both FFH feedback counters in one operation when that either narrows the sampling window or avoids extra cross-CPU reads. Add a small FFH-specific hook for that case and fall back to the existing per-register reads when unsupported. Tested-by: Sumit Gupta Reviewed-by: Sumit Gupta Tested-by: Vanshidhar Konda Reviewed-by: Vanshidhar Konda Signed-off-by: Pengjie Zhang Acked-by: Rafael J. Wysocki (Intel) Tested-by: Jeremy Linton Reviewed-by: Jeremy Linton Signed-off-by: Will Deacon --- drivers/acpi/cppc_acpi.c | 50 ++++++++++++++++++++++++++++++++++++---- include/acpi/cppc_acpi.h | 7 ++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c index 9f572f481241..b6356476a46e 100644 --- a/drivers/acpi/cppc_acpi.c +++ b/drivers/acpi/cppc_acpi.c @@ -1004,6 +1004,22 @@ int __weak cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) return -ENOTSUPP; } +/** + * cpc_read_ffh_fb_ctrs() - Read FFH feedback counters together + * @cpunum: Target CPU + * @reg1: first CPPC register information + * @val1: place holder for first return value + * @reg2: second CPPC register information + * @val2: place holder for second return value + * + * Return: 0 on success, error code otherwise + */ +int __weak cpc_read_ffh_fb_ctrs(int cpunum, struct cpc_reg *reg1, + u64 *val1, struct cpc_reg *reg2, u64 *val2) +{ + return -EOPNOTSUPP; +} + /** * cpc_write_ffh() - Write FFH register * @cpunum: CPU number to write @@ -1496,6 +1512,33 @@ bool cppc_perf_ctrs_in_pcc_cpu(unsigned int cpu) } EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc_cpu); +static int cppc_read_fb_ctrs(int cpunum, + struct cpc_register_resource *delivered_reg, + struct cpc_register_resource *reference_reg, + u64 *delivered, u64 *reference) +{ + int ret; + + /* + * For FFH feedback counters, try a paired read first to reduce + * sampling skew between delivered and reference counters. Fall + * back to the existing per-register reads if unsupported. + */ + if (CPC_IN_FFH(delivered_reg) && CPC_IN_FFH(reference_reg)) { + ret = cpc_read_ffh_fb_ctrs(cpunum, + &delivered_reg->cpc_entry.reg, delivered, + &reference_reg->cpc_entry.reg, reference); + if (ret != -EOPNOTSUPP) + return ret; + } + + ret = cpc_read(cpunum, delivered_reg, delivered); + if (ret) + return ret; + + return cpc_read(cpunum, reference_reg, reference); +} + /** * cppc_perf_ctrs_in_pcc - Check if any perf counters are in a PCC region. * @@ -1561,11 +1604,8 @@ int cppc_get_perf_ctrs(int cpunum, struct cppc_perf_fb_ctrs *perf_fb_ctrs) } } - ret = cpc_read(cpunum, delivered_reg, &delivered); - if (ret) - goto out_err; - - ret = cpc_read(cpunum, reference_reg, &reference); + ret = cppc_read_fb_ctrs(cpunum, delivered_reg, reference_reg, + &delivered, &reference); if (ret) goto out_err; diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h index 8693890a7275..5acebe62feac 100644 --- a/include/acpi/cppc_acpi.h +++ b/include/acpi/cppc_acpi.h @@ -176,6 +176,8 @@ extern int cppc_get_transition_latency(int cpu); extern bool cpc_ffh_supported(void); extern bool cpc_supported_by_cpu(void); extern int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val); +extern int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2); extern int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val); extern int cppc_get_epp_perf(int cpunum, u64 *epp_perf); extern int cppc_set_epp_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls, bool enable); @@ -250,6 +252,11 @@ static inline int cpc_read_ffh(int cpunum, struct cpc_reg *reg, u64 *val) { return -EOPNOTSUPP; } +static inline int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2) +{ + return -EOPNOTSUPP; +} static inline int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) { return -EOPNOTSUPP; From 42971d5329d9304eb115faebdb71d91650ada8dc Mon Sep 17 00:00:00 2001 From: Pengjie Zhang Date: Wed, 8 Jul 2026 16:28:18 +0800 Subject: [PATCH 11/95] arm64: topology: read CPPC FFH feedback counters in one operation arm64 implements CPPC FFH feedback-counter reads using AMU counters. Because those counters must be sampled on the target CPU, reading the delivered and reference counters separately widens the observation window between them. Implement the paired FFH feedback-counter read hook on arm64 and sample both AMU counters together before decoding the requested CPC register values. Also factor the FFH bitfield extraction logic into a helper and reuse it from the existing single-counter FFH read path. Tested-by: Sumit Gupta Reviewed-by: Sumit Gupta Tested-by: Vanshidhar Konda Reviewed-by: Vanshidhar Konda Signed-off-by: Pengjie Zhang Tested-by: Jeremy Linton Reviewed-by: Jeremy Linton Signed-off-by: Will Deacon --- arch/arm64/kernel/topology.c | 92 ++++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c index b32f13358fbb..d28438f8b83f 100644 --- a/arch/arm64/kernel/topology.c +++ b/arch/arm64/kernel/topology.c @@ -373,6 +373,16 @@ core_initcall(init_amu_fie); #ifdef CONFIG_ACPI_CPPC_LIB #include +struct amu_ffh_ctrs { + u64 corecnt; + u64 constcnt; +}; + +enum cpc_ffh_ctr_id { + CPC_FFH_CTR_CORE = 0x0, + CPC_FFH_CTR_CONST = 0x1, +}; + static void cpu_read_corecnt(void *val) { /* @@ -397,7 +407,7 @@ static void cpu_read_constcnt(void *val) } static inline -int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val) +int counters_read_on_cpu(int cpu, smp_call_func_t func, void *val) { /* * Abort call on counterless CPU. @@ -447,24 +457,90 @@ bool cpc_ffh_supported(void) return true; } +static void amu_read_core_const_ctrs(void *val) +{ + struct amu_ffh_ctrs *ctrs = val; + + /* + * cpu_read_constcnt() incurs slight latency due to the + * ARM64_WORKAROUND_2457168 check. Read it first to minimize + * the sampling skew between the const and core counters. + */ + cpu_read_constcnt(&ctrs->constcnt); + cpu_read_corecnt(&ctrs->corecnt); +} + +static u64 cpc_ffh_extract_bits(const struct cpc_reg *reg, u64 val) +{ + val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, + reg->bit_offset); + val >>= reg->bit_offset; + + return val; +} + +static void cpc_ffh_ctr_value(const struct cpc_reg *reg, + const struct amu_ffh_ctrs *ctrs, u64 *val) +{ + switch ((u64)reg->address) { + case CPC_FFH_CTR_CORE: + *val = ctrs->corecnt; + break; + case CPC_FFH_CTR_CONST: + *val = ctrs->constcnt; + break; + } + + *val = cpc_ffh_extract_bits(reg, *val); +} + +static bool is_amu_ctr_reg(const struct cpc_reg *reg) +{ + return reg->address == CPC_FFH_CTR_CORE || + reg->address == CPC_FFH_CTR_CONST; +} + +int cpc_read_ffh_fb_ctrs(int cpu, struct cpc_reg *reg1, u64 *val1, + struct cpc_reg *reg2, u64 *val2) +{ + struct amu_ffh_ctrs ctrs; + int ret; + + if (!is_amu_ctr_reg(reg1) || !is_amu_ctr_reg(reg2)) + return -EINVAL; + + ret = counters_read_on_cpu(cpu, amu_read_core_const_ctrs, &ctrs); + if (ret) { + /* + * If AMU is unsupported (-EOPNOTSUPP), translate the error + * to -ENODEV. This explicitly tells the generic CPPC layer + * to abort immediately and avoid falling back to pointless + * single-counter reads. + */ + return ret == -EOPNOTSUPP ? -ENODEV : ret; + } + + cpc_ffh_ctr_value(reg1, &ctrs, val1); + cpc_ffh_ctr_value(reg2, &ctrs, val2); + + return 0; +} + int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val) { int ret = -EOPNOTSUPP; switch ((u64)reg->address) { - case 0x0: + case CPC_FFH_CTR_CORE: ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val); break; - case 0x1: + case CPC_FFH_CTR_CONST: ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val); break; } - if (!ret) { - *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, - reg->bit_offset); - *val >>= reg->bit_offset; - } + if (!ret) + *val = cpc_ffh_extract_bits(reg, *val); return ret; } From ecc2e046869ea8caf24142ea349d0eba38e1b930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Uwe=20Kleine-K=C3=B6nig=20=28The=20Capable=20Hub=29?= Date: Wed, 17 Jun 2026 12:28:26 +0200 Subject: [PATCH 12/95] virt: arm-cca-guest: Drop unused assignment of platform_device_id driver data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver explicitly sets the .driver_data member of struct platform_device_id to zero without relying on that value. Drop this unused assignment. While touching this array use a named initializer for .name. Signed-off-by: Uwe Kleine-König (The Capable Hub) Signed-off-by: Will Deacon --- drivers/virt/coco/arm-cca-guest/arm-cca-guest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c index dbbb2cc0e124..0eeddd1ff05b 100644 --- a/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c +++ b/drivers/virt/coco/arm-cca-guest/arm-cca-guest.c @@ -198,7 +198,7 @@ module_exit(arm_cca_guest_exit); /* modalias, so userspace can autoload this module when RSI is available */ static const struct platform_device_id arm_cca_match[] __maybe_unused = { - { RSI_PDEV_NAME, 0}, + { .name = RSI_PDEV_NAME }, { } }; From 12aab25ca56ee9ab3051f7832402702d0c1953eb Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Wed, 15 Jul 2026 15:48:56 -0500 Subject: [PATCH 13/95] arm64: errata: work around NVIDIA Olympus device store/load ordering On systems with NVIDIA Olympus cores, a Device-nGnR* load can be observed by a peripheral before an older, non-overlapping Device-nGnR* store to the same peripheral. This breaks the program-order guarantee that software expects for Device-nGnR* accesses and can leave a peripheral in an incorrect state. The erratum can occur only when all of the following apply: - A PE executes a Device-nGnR* store followed by a younger Device-nGnR* load. - The store is not a store-release. - The accesses target the same peripheral and do not overlap in bytes. - There is at most one intervening Device-nGnR* store in program order, and there are no intervening Device-nGnR* loads. - There is no DSB or full DMB between the store and the load. - Specific microarchitectural and timing conditions occur. Insert a DMB OSH immediately before each raw MMIO load on affected CPUs. As a full barrier, DMB OSH orders the older Device store before the younger Device load and prevents the erroneous observation. Add the barrier directly to the __raw_read*() helpers, independently of the existing device-load-acquire alternative. On affected CPUs this adds one DMB OSH per raw MMIO load, including each load used by memcpy_fromio(). On unaffected CPUs the alternative remains a NOP. Co-developed-by: Vikram Sethi Signed-off-by: Vikram Sethi Signed-off-by: Shanker Donthineni Link: https://lore.kernel.org/all/akPQ8F3OgER621UP@willie-the-truck/ Reviewed-by: Vladimir Murzin Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 2 ++ arch/arm64/Kconfig | 22 +++++++++++++++++++++ arch/arm64/include/asm/io.h | 16 +++++++++++---- arch/arm64/kernel/cpu_errata.c | 8 ++++++++ arch/arm64/tools/cpucaps | 1 + 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 014aa1c215a1..02617007a57c 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -304,6 +304,8 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Carmel Core | N/A | NVIDIA_CARMEL_CNP_ERRATUM | +----------------+-----------------+-----------------+-----------------------------+ +| NVIDIA | Olympus core | T410-OLY-1027 | NVIDIA_OLYMPUS_1027_ERRATUM | ++----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | Olympus core | T410-OLY-1029 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ | NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A | diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 757110421543..3510b04468de 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1404,6 +1404,28 @@ config NVIDIA_CARMEL_CNP_ERRATUM If unsure, say Y. +config NVIDIA_OLYMPUS_1027_ERRATUM + bool "NVIDIA Olympus: device store/load ordering erratum" + default y + help + This option adds an alternative code sequence to work around an + NVIDIA Olympus core erratum where a Device-nGnR* store can be + observed by a peripheral after a younger Device-nGnR* load to the + same peripheral. This breaks the program order that drivers rely + on for MMIO and can leave a device in an incorrect state. + + The workaround inserts a DMB OSH immediately before raw MMIO loads. + The erratum cannot occur when a DMB that orders loads appears + between the store and load, preventing the younger load from being + observed before the older store. + + The alternatives framework patches in DMB OSH only when an affected + CPU is detected. Other CPUs execute a NOP in its place. Disabling + this option leaves the original MMIO read instruction stream + unchanged. + + If unsure, say Y. + config ROCKCHIP_ERRATUM_3568002 bool "Rockchip 3568002: GIC600 can not access physical addresses higher than 4GB" default y diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 21c8e400107c..49a7002661a9 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -54,7 +54,9 @@ static __always_inline void __raw_writeq(u64 val, volatile void __iomem *addr) static __always_inline u8 __raw_readb(const volatile void __iomem *addr) { u8 val; - asm volatile(ALTERNATIVE("ldrb %w0, [%1]", + asm volatile(ALTERNATIVE("nop", "dmb osh", + ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027) + ALTERNATIVE("ldrb %w0, [%1]", "ldarb %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); @@ -66,7 +68,9 @@ static __always_inline u16 __raw_readw(const volatile void __iomem *addr) { u16 val; - asm volatile(ALTERNATIVE("ldrh %w0, [%1]", + asm volatile(ALTERNATIVE("nop", "dmb osh", + ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027) + ALTERNATIVE("ldrh %w0, [%1]", "ldarh %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); @@ -77,7 +81,9 @@ static __always_inline u16 __raw_readw(const volatile void __iomem *addr) static __always_inline u32 __raw_readl(const volatile void __iomem *addr) { u32 val; - asm volatile(ALTERNATIVE("ldr %w0, [%1]", + asm volatile(ALTERNATIVE("nop", "dmb osh", + ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027) + ALTERNATIVE("ldr %w0, [%1]", "ldar %w0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); @@ -88,7 +94,9 @@ static __always_inline u32 __raw_readl(const volatile void __iomem *addr) static __always_inline u64 __raw_readq(const volatile void __iomem *addr) { u64 val; - asm volatile(ALTERNATIVE("ldr %0, [%1]", + asm volatile(ALTERNATIVE("nop", "dmb osh", + ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027) + ALTERNATIVE("ldr %0, [%1]", "ldar %0, [%1]", ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) : "=r" (val) : "r" (addr)); diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 685077d44ad1..b5ba9c455aef 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -850,6 +850,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = { ERRATA_MIDR_RANGE_LIST(cnp_erratum_cpus), }, #endif +#ifdef CONFIG_NVIDIA_OLYMPUS_1027_ERRATUM + { + /* NVIDIA Olympus core */ + .desc = "NVIDIA Olympus device store/load ordering erratum", + .capability = ARM64_WORKAROUND_NVIDIA_OLYMPUS_1027, + ERRATA_MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + }, +#endif #ifdef CONFIG_ARM64_WORKAROUND_TRBE_OVERWRITE_FILL_MODE { /* diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index f8368e5d81a8..e43c9095424b 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -121,6 +121,7 @@ WORKAROUND_CAVIUM_TX2_219_TVM WORKAROUND_CLEAN_CACHE WORKAROUND_DEVICE_LOAD_ACQUIRE WORKAROUND_DISABLE_CNP +WORKAROUND_NVIDIA_OLYMPUS_1027 WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF From 123b4fc0f8576ac29b965cfc362630f51fa0fe7e Mon Sep 17 00:00:00 2001 From: "Jose Fernandez (Anthropic)" Date: Tue, 9 Jun 2026 05:19:26 +0000 Subject: [PATCH 14/95] arm64: ftrace: prepare ftrace_modify_call() for use without CALL_OPS ftrace_modify_call() is guarded by CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS and calls ftrace_rec_set_ops(rec, arm64_rec_get_ops(rec)) directly, which only exists when CALL_OPS is enabled. Generic ftrace also needs ftrace_modify_call() when CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS is enabled, to retarget a callsite between two non-FTRACE_ADDR destinations, as happens when a direct trampoline is modified. The next patch allows DIRECT_CALLS without CALL_OPS, so widen the guard to cover both configurations and switch the body to the ftrace_rec_update_ops() wrapper, which already has a stub for the !CALL_OPS case. ftrace_make_call() already uses the same wrapper today. No functional change: with CALL_OPS enabled, ftrace_rec_update_ops() expands to the exact call this replaces. Assisted-by: Claude:unspecified Signed-off-by: Jose Fernandez (Anthropic) Acked-by: Xu Kuohai Tested-by: Nathan Chancellor Tested-by: Clayton Craft Reviewed-by: Puranjay Mohan Signed-off-by: Will Deacon --- arch/arm64/kernel/ftrace.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index 5a1554a44162..e1a3c0b3a051 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -409,7 +409,8 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) return ftrace_modify_code(pc, old, new, true); } -#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS +#if defined(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS) || \ + defined(CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS) int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, unsigned long addr) { @@ -417,7 +418,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr, u32 old, new; int ret; - ret = ftrace_rec_set_ops(rec, arm64_rec_get_ops(rec)); + ret = ftrace_rec_update_ops(rec); if (ret) return ret; From 9315e22b0c0a5be708798c03dc8f27549667e475 Mon Sep 17 00:00:00 2001 From: "Jose Fernandez (Anthropic)" Date: Tue, 9 Jun 2026 05:19:27 +0000 Subject: [PATCH 15/95] arm64: ftrace: allow DIRECT_CALLS without CALL_OPS Drop the CALL_OPS requirement from the HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS select. Configurations that keep CALL_OPS (!CFI clang builds, and GCC builds without CC_OPTIMIZE_FOR_SIZE) are unchanged. CALL_OPS-less configurations take the ftrace_caller ops-dispatch path for out-of-range direct calls, trading the per-callsite fast path for working BPF trampolines; in-range attachments still branch directly with no overhead. GCC -Os builds also gain DIRECT_CALLS as a side effect. That is intended: s390 and loongarch already ship DIRECT_CALLS without any per-callsite fast path. Assisted-by: Claude:unspecified Signed-off-by: Jose Fernandez (Anthropic) Acked-by: Xu Kuohai Tested-by: Nathan Chancellor Tested-by: Clayton Craft Reviewed-by: Puranjay Mohan Signed-off-by: Will Deacon --- arch/arm64/Kconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..0de419ed780f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -187,7 +187,7 @@ config ARM64 if (GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS || \ CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS) select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS \ - if DYNAMIC_FTRACE_WITH_ARGS && DYNAMIC_FTRACE_WITH_CALL_OPS + if DYNAMIC_FTRACE_WITH_ARGS select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS \ if (DYNAMIC_FTRACE_WITH_ARGS && !CFI && \ (CC_IS_CLANG || !CC_OPTIMIZE_FOR_SIZE)) From b6bd4d1a5b6c46be32bbe0d3447de59f8a5885e0 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Thu, 2 Jul 2026 20:11:16 +0100 Subject: [PATCH 16/95] arm64: Don't number registers in cpu-feature-registers.rst cpu-feature-regsters.rst documents the set of userspace visible ID registers. At present the section for each register is numbered, this has lead to the registers being documented in a haphazard order as new ones have been added to the end of the list to avoid renumbering. Remove the numbers so we can avoid this problem in future. Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- .../arch/arm64/cpu-feature-registers.rst | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Documentation/arch/arm64/cpu-feature-registers.rst b/Documentation/arch/arm64/cpu-feature-registers.rst index add66afc7b03..c6e5bc053c09 100644 --- a/Documentation/arch/arm64/cpu-feature-registers.rst +++ b/Documentation/arch/arm64/cpu-feature-registers.rst @@ -113,7 +113,7 @@ infrastructure: 4. List of registers with visible features ------------------------------------------- - 1) ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 + ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 +------------------------------+---------+---------+ | Name | bits | visible | @@ -146,7 +146,7 @@ infrastructure: +------------------------------+---------+---------+ - 2) ID_AA64PFR0_EL1 - Processor Feature Register 0 + ID_AA64PFR0_EL1 - Processor Feature Register 0 +------------------------------+---------+---------+ | Name | bits | visible | @@ -173,7 +173,7 @@ infrastructure: +------------------------------+---------+---------+ - 3) ID_AA64PFR1_EL1 - Processor Feature Register 1 + ID_AA64PFR1_EL1 - Processor Feature Register 1 +------------------------------+---------+---------+ | Name | bits | visible | @@ -188,7 +188,7 @@ infrastructure: +------------------------------+---------+---------+ - 4) MIDR_EL1 - Main ID Register + MIDR_EL1 - Main ID Register +------------------------------+---------+---------+ | Name | bits | visible | @@ -208,7 +208,7 @@ infrastructure: as available on the CPU where it is fetched and is not a system wide safe value. - 5) ID_AA64ISAR1_EL1 - Instruction set attribute register 1 + ID_AA64ISAR1_EL1 - Instruction set attribute register 1 +------------------------------+---------+---------+ | Name | bits | visible | @@ -240,7 +240,7 @@ infrastructure: | DPB | [3-0] | y | +------------------------------+---------+---------+ - 6) ID_AA64MMFR0_EL1 - Memory model feature register 0 + ID_AA64MMFR0_EL1 - Memory model feature register 0 +------------------------------+---------+---------+ | Name | bits | visible | @@ -248,7 +248,7 @@ infrastructure: | ECV | [63-60] | y | +------------------------------+---------+---------+ - 7) ID_AA64MMFR2_EL1 - Memory model feature register 2 + ID_AA64MMFR2_EL1 - Memory model feature register 2 +------------------------------+---------+---------+ | Name | bits | visible | @@ -256,7 +256,7 @@ infrastructure: | AT | [35-32] | y | +------------------------------+---------+---------+ - 8) ID_AA64ZFR0_EL1 - SVE feature ID register 0 + ID_AA64ZFR0_EL1 - SVE feature ID register 0 +------------------------------+---------+---------+ | Name | bits | visible | @@ -282,7 +282,7 @@ infrastructure: | SVEVer | [3-0] | y | +------------------------------+---------+---------+ - 8) ID_AA64MMFR1_EL1 - Memory model feature register 1 + ID_AA64MMFR1_EL1 - Memory model feature register 1 +------------------------------+---------+---------+ | Name | bits | visible | @@ -290,7 +290,7 @@ infrastructure: | AFP | [47-44] | y | +------------------------------+---------+---------+ - 9) ID_AA64ISAR2_EL1 - Instruction set attribute register 2 + ID_AA64ISAR2_EL1 - Instruction set attribute register 2 +------------------------------+---------+---------+ | Name | bits | visible | @@ -312,7 +312,7 @@ infrastructure: | WFXT | [3-0] | y | +------------------------------+---------+---------+ - 10) MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 + MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 +------------------------------+---------+---------+ | Name | bits | visible | @@ -320,7 +320,7 @@ infrastructure: | FPDP | [11-8] | y | +------------------------------+---------+---------+ - 11) MVFR1_EL1 - AArch32 Media and VFP Feature Register 1 + MVFR1_EL1 - AArch32 Media and VFP Feature Register 1 +------------------------------+---------+---------+ | Name | bits | visible | @@ -334,7 +334,7 @@ infrastructure: | SIMDLS | [11-8] | y | +------------------------------+---------+---------+ - 12) ID_ISAR5_EL1 - AArch32 Instruction Set Attribute Register 5 + ID_ISAR5_EL1 - AArch32 Instruction Set Attribute Register 5 +------------------------------+---------+---------+ | Name | bits | visible | From 4788adeca0e03cc34d0846a25f3d028ab75eadd1 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Thu, 2 Jul 2026 20:11:17 +0100 Subject: [PATCH 17/95] arm64: Document missing bitfields in cpu-feature-registers.rst We have been rather lax in updating the list of visible bitfields in the ID registers in cpu-feature-registers.rst, it is currently missing several of the registers and quite a few bitfields in existing registers. Bring it into sync with current -next. Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- .../arch/arm64/cpu-feature-registers.rst | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/Documentation/arch/arm64/cpu-feature-registers.rst b/Documentation/arch/arm64/cpu-feature-registers.rst index c6e5bc053c09..4b10980d4a40 100644 --- a/Documentation/arch/arm64/cpu-feature-registers.rst +++ b/Documentation/arch/arm64/cpu-feature-registers.rst @@ -113,6 +113,30 @@ infrastructure: 4. List of registers with visible features ------------------------------------------- + ID_AA64FPFR0_EL1 - Floating Point feature ID register 0 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | F8CVT | [31] | y | + +------------------------------+---------+---------+ + | F8FMA | [30] | y | + +------------------------------+---------+---------+ + | F8DP4 | [29] | y | + +------------------------------+---------+---------+ + | F8DP2 | [28] | y | + +------------------------------+---------+---------+ + | F8MM8 | [27] | y | + +------------------------------+---------+---------+ + | F8MM4 | [26] | y | + +------------------------------+---------+---------+ + | F16MM2 | [15] | y | + +------------------------------+---------+---------+ + | F8E4M3 | [1] | y | + +------------------------------+---------+---------+ + | F8E5M2 | [0] | y | + +------------------------------+---------+---------+ + ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 +------------------------------+---------+---------+ @@ -178,6 +202,8 @@ infrastructure: +------------------------------+---------+---------+ | Name | bits | visible | +------------------------------+---------+---------+ + | GCS | [47-44] | y | + +------------------------------+---------+---------+ | SME | [27-24] | y | +------------------------------+---------+---------+ | MTE | [11-8] | y | @@ -187,6 +213,17 @@ infrastructure: | BT | [3-0] | y | +------------------------------+---------+---------+ + ID_AA64PFR2_EL1 - Processor Feature Register 2 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | FPMR | [35-32] | y | + +------------------------------+---------+---------+ + | MTEFAR | [11-8] | y | + +------------------------------+---------+---------+ + | MTESTOREONLY | [7-4] | y | + +------------------------------+---------+---------+ MIDR_EL1 - Main ID Register @@ -213,6 +250,8 @@ infrastructure: +------------------------------+---------+---------+ | Name | bits | visible | +------------------------------+---------+---------+ + | LS64 | [63-60] | y | + +------------------------------+---------+---------+ | I8MM | [55-52] | y | +------------------------------+---------+---------+ | DGH | [51-48] | y | @@ -256,6 +295,68 @@ infrastructure: | AT | [35-32] | y | +------------------------------+---------+---------+ + ID_AA64MMFR3_EL1 - Memory model feature register 3 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | S1POE | [19-16] | y | + +------------------------------+---------+---------+ + + ID_AA64SMFR0_EL1 - SME feature ID register 0 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | FA64 | [63] | y | + +------------------------------+---------+---------+ + | LUT6 | [61] | y | + +------------------------------+---------+---------+ + | LUTv2 | [60] | y | + +------------------------------+---------+---------+ + | SMEver | [59-56] | y | + +------------------------------+---------+---------+ + | I16I64 | [55-52] | y | + +------------------------------+---------+---------+ + | F64F64 | [48] | y | + +------------------------------+---------+---------+ + | I16I32 | [47-44] | y | + +------------------------------+---------+---------+ + | B16B16 | [43] | y | + +------------------------------+---------+---------+ + | F16F16 | [42] | y | + +------------------------------+---------+---------+ + | F8F16 | [41] | y | + +------------------------------+---------+---------+ + | F8F32 | [40] | y | + +------------------------------+---------+---------+ + | I8I32 | [39-36] | y | + +------------------------------+---------+---------+ + | F16F32 | [35] | y | + +------------------------------+---------+---------+ + | B16F32 | [34] | y | + +------------------------------+---------+---------+ + | BI32I32 | [33] | y | + +------------------------------+---------+---------+ + | F32F32 | [32] | y | + +------------------------------+---------+---------+ + | SF8FMA | [30] | y | + +------------------------------+---------+---------+ + | SF8DP4 | [29] | y | + +------------------------------+---------+---------+ + | SF8DP2 | [28] | y | + +------------------------------+---------+---------+ + | SBitPerm | [25] | y | + +------------------------------+---------+---------+ + | AES | [24] | y | + +------------------------------+---------+---------+ + | SFEXPA | [23] | y | + +------------------------------+---------+---------+ + | STMOP | [16] | y | + +------------------------------+---------+---------+ + | SMOP4 | [0] | y | + +------------------------------+---------+---------+ + ID_AA64ZFR0_EL1 - SVE feature ID register 0 +------------------------------+---------+---------+ @@ -265,6 +366,8 @@ infrastructure: +------------------------------+---------+---------+ | F32MM | [55-52] | y | +------------------------------+---------+---------+ + | F16MM | [51-48] | y | + +------------------------------+---------+---------+ | I8MM | [47-44] | y | +------------------------------+---------+---------+ | SM4 | [43-40] | y | @@ -277,6 +380,8 @@ infrastructure: +------------------------------+---------+---------+ | BitPerm | [19-16] | y | +------------------------------+---------+---------+ + | EltPerm | [15-12] | y | + +------------------------------+---------+---------+ | AES | [7-4] | y | +------------------------------+---------+---------+ | SVEVer | [3-0] | y | @@ -295,6 +400,8 @@ infrastructure: +------------------------------+---------+---------+ | Name | bits | visible | +------------------------------+---------+---------+ + | LUT | [59-56] | y | + +------------------------------+---------+---------+ | CSSC | [55-52] | y | +------------------------------+---------+---------+ | RPRFM | [51-48] | y | @@ -312,6 +419,18 @@ infrastructure: | WFXT | [3-0] | y | +------------------------------+---------+---------+ + ID_AA64ISAR3_EL1 - Instruction set attribute register 3 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | FPRCVT | [31-28] | y | + +------------------------------+---------+---------+ + | LSFE | [19-16] | y | + +------------------------------+---------+---------+ + | FAMINMAX | [7-4] | y | + +------------------------------+---------+---------+ + MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 +------------------------------+---------+---------+ @@ -327,6 +446,10 @@ infrastructure: +------------------------------+---------+---------+ | SIMDFMAC | [31-28] | y | +------------------------------+---------+---------+ + | FPHP | [27-24] | y | + +------------------------------+---------+---------+ + | SIMDHP | [23-20] | y | + +------------------------------+---------+---------+ | SIMDSP | [19-16] | y | +------------------------------+---------+---------+ | SIMDInt | [15-12] | y | @@ -348,6 +471,29 @@ infrastructure: | AES | [7-4] | y | +------------------------------+---------+---------+ + ID_ISAR6_EL1 - AArch32 Instruction Set Attribute Register 6 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | I8MM | [27-24] | y | + +------------------------------+---------+---------+ + | BF16 | [23-20] | y | + +------------------------------+---------+---------+ + | SB | [15-12] | y | + +------------------------------+---------+---------+ + | FHM | [11-8] | y | + +------------------------------+---------+---------+ + | DP | [7-4] | y | + +------------------------------+---------+---------+ + + ID_PFR2_EL1 - AArch32 Processor Feature Register 2 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | SSBS | [7-4] | y | + +------------------------------+---------+---------+ Appendix I: Example ------------------- From 21a2ca869edd51373e3a953641ee97efee5c0557 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Thu, 2 Jul 2026 20:11:18 +0100 Subject: [PATCH 18/95] arm64: Sort registers in cpu-feature-registers.rst In order to make it a bit easier to work with sort the list of registers in cpu-feature-registers.rst lexically. There should be no content changes resulting from this patch. Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- .../arch/arm64/cpu-feature-registers.rst | 297 +++++++++--------- 1 file changed, 149 insertions(+), 148 deletions(-) diff --git a/Documentation/arch/arm64/cpu-feature-registers.rst b/Documentation/arch/arm64/cpu-feature-registers.rst index 4b10980d4a40..683bdd90c705 100644 --- a/Documentation/arch/arm64/cpu-feature-registers.rst +++ b/Documentation/arch/arm64/cpu-feature-registers.rst @@ -170,6 +170,108 @@ infrastructure: +------------------------------+---------+---------+ + ID_AA64ISAR1_EL1 - Instruction set attribute register 1 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | LS64 | [63-60] | y | + +------------------------------+---------+---------+ + | I8MM | [55-52] | y | + +------------------------------+---------+---------+ + | DGH | [51-48] | y | + +------------------------------+---------+---------+ + | BF16 | [47-44] | y | + +------------------------------+---------+---------+ + | SB | [39-36] | y | + +------------------------------+---------+---------+ + | FRINTTS | [35-32] | y | + +------------------------------+---------+---------+ + | GPI | [31-28] | y | + +------------------------------+---------+---------+ + | GPA | [27-24] | y | + +------------------------------+---------+---------+ + | LRCPC | [23-20] | y | + +------------------------------+---------+---------+ + | FCMA | [19-16] | y | + +------------------------------+---------+---------+ + | JSCVT | [15-12] | y | + +------------------------------+---------+---------+ + | API | [11-8] | y | + +------------------------------+---------+---------+ + | APA | [7-4] | y | + +------------------------------+---------+---------+ + | DPB | [3-0] | y | + +------------------------------+---------+---------+ + + ID_AA64ISAR2_EL1 - Instruction set attribute register 2 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | LUT | [59-56] | y | + +------------------------------+---------+---------+ + | CSSC | [55-52] | y | + +------------------------------+---------+---------+ + | RPRFM | [51-48] | y | + +------------------------------+---------+---------+ + | BC | [23-20] | y | + +------------------------------+---------+---------+ + | MOPS | [19-16] | y | + +------------------------------+---------+---------+ + | APA3 | [15-12] | y | + +------------------------------+---------+---------+ + | GPA3 | [11-8] | y | + +------------------------------+---------+---------+ + | RPRES | [7-4] | y | + +------------------------------+---------+---------+ + | WFXT | [3-0] | y | + +------------------------------+---------+---------+ + + ID_AA64ISAR3_EL1 - Instruction set attribute register 3 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | FPRCVT | [31-28] | y | + +------------------------------+---------+---------+ + | LSFE | [19-16] | y | + +------------------------------+---------+---------+ + | FAMINMAX | [7-4] | y | + +------------------------------+---------+---------+ + + ID_AA64MMFR0_EL1 - Memory model feature register 0 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | ECV | [63-60] | y | + +------------------------------+---------+---------+ + + ID_AA64MMFR1_EL1 - Memory model feature register 1 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | AFP | [47-44] | y | + +------------------------------+---------+---------+ + + ID_AA64MMFR2_EL1 - Memory model feature register 2 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | AT | [35-32] | y | + +------------------------------+---------+---------+ + + ID_AA64MMFR3_EL1 - Memory model feature register 3 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | S1POE | [19-16] | y | + +------------------------------+---------+---------+ + ID_AA64PFR0_EL1 - Processor Feature Register 0 +------------------------------+---------+---------+ @@ -225,84 +327,6 @@ infrastructure: | MTESTOREONLY | [7-4] | y | +------------------------------+---------+---------+ - MIDR_EL1 - Main ID Register - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | Implementer | [31-24] | y | - +------------------------------+---------+---------+ - | Variant | [23-20] | y | - +------------------------------+---------+---------+ - | Architecture | [19-16] | y | - +------------------------------+---------+---------+ - | PartNum | [15-4] | y | - +------------------------------+---------+---------+ - | Revision | [3-0] | y | - +------------------------------+---------+---------+ - - NOTE: The 'visible' fields of MIDR_EL1 will contain the value - as available on the CPU where it is fetched and is not a system - wide safe value. - - ID_AA64ISAR1_EL1 - Instruction set attribute register 1 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | LS64 | [63-60] | y | - +------------------------------+---------+---------+ - | I8MM | [55-52] | y | - +------------------------------+---------+---------+ - | DGH | [51-48] | y | - +------------------------------+---------+---------+ - | BF16 | [47-44] | y | - +------------------------------+---------+---------+ - | SB | [39-36] | y | - +------------------------------+---------+---------+ - | FRINTTS | [35-32] | y | - +------------------------------+---------+---------+ - | GPI | [31-28] | y | - +------------------------------+---------+---------+ - | GPA | [27-24] | y | - +------------------------------+---------+---------+ - | LRCPC | [23-20] | y | - +------------------------------+---------+---------+ - | FCMA | [19-16] | y | - +------------------------------+---------+---------+ - | JSCVT | [15-12] | y | - +------------------------------+---------+---------+ - | API | [11-8] | y | - +------------------------------+---------+---------+ - | APA | [7-4] | y | - +------------------------------+---------+---------+ - | DPB | [3-0] | y | - +------------------------------+---------+---------+ - - ID_AA64MMFR0_EL1 - Memory model feature register 0 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | ECV | [63-60] | y | - +------------------------------+---------+---------+ - - ID_AA64MMFR2_EL1 - Memory model feature register 2 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | AT | [35-32] | y | - +------------------------------+---------+---------+ - - ID_AA64MMFR3_EL1 - Memory model feature register 3 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | S1POE | [19-16] | y | - +------------------------------+---------+---------+ - ID_AA64SMFR0_EL1 - SME feature ID register 0 +------------------------------+---------+---------+ @@ -387,76 +411,6 @@ infrastructure: | SVEVer | [3-0] | y | +------------------------------+---------+---------+ - ID_AA64MMFR1_EL1 - Memory model feature register 1 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | AFP | [47-44] | y | - +------------------------------+---------+---------+ - - ID_AA64ISAR2_EL1 - Instruction set attribute register 2 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | LUT | [59-56] | y | - +------------------------------+---------+---------+ - | CSSC | [55-52] | y | - +------------------------------+---------+---------+ - | RPRFM | [51-48] | y | - +------------------------------+---------+---------+ - | BC | [23-20] | y | - +------------------------------+---------+---------+ - | MOPS | [19-16] | y | - +------------------------------+---------+---------+ - | APA3 | [15-12] | y | - +------------------------------+---------+---------+ - | GPA3 | [11-8] | y | - +------------------------------+---------+---------+ - | RPRES | [7-4] | y | - +------------------------------+---------+---------+ - | WFXT | [3-0] | y | - +------------------------------+---------+---------+ - - ID_AA64ISAR3_EL1 - Instruction set attribute register 3 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | FPRCVT | [31-28] | y | - +------------------------------+---------+---------+ - | LSFE | [19-16] | y | - +------------------------------+---------+---------+ - | FAMINMAX | [7-4] | y | - +------------------------------+---------+---------+ - - MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | FPDP | [11-8] | y | - +------------------------------+---------+---------+ - - MVFR1_EL1 - AArch32 Media and VFP Feature Register 1 - - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | SIMDFMAC | [31-28] | y | - +------------------------------+---------+---------+ - | FPHP | [27-24] | y | - +------------------------------+---------+---------+ - | SIMDHP | [23-20] | y | - +------------------------------+---------+---------+ - | SIMDSP | [19-16] | y | - +------------------------------+---------+---------+ - | SIMDInt | [15-12] | y | - +------------------------------+---------+---------+ - | SIMDLS | [11-8] | y | - +------------------------------+---------+---------+ - ID_ISAR5_EL1 - AArch32 Instruction Set Attribute Register 5 +------------------------------+---------+---------+ @@ -495,6 +449,53 @@ infrastructure: | SSBS | [7-4] | y | +------------------------------+---------+---------+ + MIDR_EL1 - Main ID Register + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | Implementer | [31-24] | y | + +------------------------------+---------+---------+ + | Variant | [23-20] | y | + +------------------------------+---------+---------+ + | Architecture | [19-16] | y | + +------------------------------+---------+---------+ + | PartNum | [15-4] | y | + +------------------------------+---------+---------+ + | Revision | [3-0] | y | + +------------------------------+---------+---------+ + + NOTE: The 'visible' fields of MIDR_EL1 will contain the value + as available on the CPU where it is fetched and is not a system + wide safe value. + + MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | FPDP | [11-8] | y | + +------------------------------+---------+---------+ + + MVFR1_EL1 - AArch32 Media and VFP Feature Register 1 + + +------------------------------+---------+---------+ + | Name | bits | visible | + +------------------------------+---------+---------+ + | SIMDFMAC | [31-28] | y | + +------------------------------+---------+---------+ + | FPHP | [27-24] | y | + +------------------------------+---------+---------+ + | SIMDHP | [23-20] | y | + +------------------------------+---------+---------+ + | SIMDSP | [19-16] | y | + +------------------------------+---------+---------+ + | SIMDInt | [15-12] | y | + +------------------------------+---------+---------+ + | SIMDLS | [11-8] | y | + +------------------------------+---------+---------+ + + Appendix I: Example ------------------- From bc7f7ddc64df4b821d7dce97db58c5bf84e0efd1 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Thu, 2 Jul 2026 20:11:19 +0100 Subject: [PATCH 19/95] arm64: Remove hidden bitfields from cpu-feature-registers.rst We currently have a visibility column in the tables for the registers in cpu-feature-registers.rst but this is always "y" for every register other than ID_AA64PFR0_EL1. Given that the documentation of the full set of bitfields is readily available in the architecture documentation it is redundant for us to explicitly document things we don't advertise, and the kernel documentation will inevitably lag the architecture. Just remove the visibility column and hidden bitfields. Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- .../arch/arm64/cpu-feature-registers.rst | 618 +++++++++--------- 1 file changed, 303 insertions(+), 315 deletions(-) diff --git a/Documentation/arch/arm64/cpu-feature-registers.rst b/Documentation/arch/arm64/cpu-feature-registers.rst index 683bdd90c705..fc63ccb666bf 100644 --- a/Documentation/arch/arm64/cpu-feature-registers.rst +++ b/Documentation/arch/arm64/cpu-feature-registers.rst @@ -115,355 +115,343 @@ infrastructure: ID_AA64FPFR0_EL1 - Floating Point feature ID register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | F8CVT | [31] | y | - +------------------------------+---------+---------+ - | F8FMA | [30] | y | - +------------------------------+---------+---------+ - | F8DP4 | [29] | y | - +------------------------------+---------+---------+ - | F8DP2 | [28] | y | - +------------------------------+---------+---------+ - | F8MM8 | [27] | y | - +------------------------------+---------+---------+ - | F8MM4 | [26] | y | - +------------------------------+---------+---------+ - | F16MM2 | [15] | y | - +------------------------------+---------+---------+ - | F8E4M3 | [1] | y | - +------------------------------+---------+---------+ - | F8E5M2 | [0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | F8CVT | [31] | + +------------------------------+---------+ + | F8FMA | [30] | + +------------------------------+---------+ + | F8DP4 | [29] | + +------------------------------+---------+ + | F8DP2 | [28] | + +------------------------------+---------+ + | F8MM8 | [27] | + +------------------------------+---------+ + | F8MM4 | [26] | + +------------------------------+---------+ + | F16MM2 | [15] | + +------------------------------+---------+ + | F8E4M3 | [1] | + +------------------------------+---------+ + | F8E5M2 | [0] | + +------------------------------+---------+ ID_AA64ISAR0_EL1 - Instruction Set Attribute Register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | RNDR | [63-60] | y | - +------------------------------+---------+---------+ - | TS | [55-52] | y | - +------------------------------+---------+---------+ - | FHM | [51-48] | y | - +------------------------------+---------+---------+ - | DP | [47-44] | y | - +------------------------------+---------+---------+ - | SM4 | [43-40] | y | - +------------------------------+---------+---------+ - | SM3 | [39-36] | y | - +------------------------------+---------+---------+ - | SHA3 | [35-32] | y | - +------------------------------+---------+---------+ - | RDM | [31-28] | y | - +------------------------------+---------+---------+ - | ATOMICS | [23-20] | y | - +------------------------------+---------+---------+ - | CRC32 | [19-16] | y | - +------------------------------+---------+---------+ - | SHA2 | [15-12] | y | - +------------------------------+---------+---------+ - | SHA1 | [11-8] | y | - +------------------------------+---------+---------+ - | AES | [7-4] | y | + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | RNDR | [63-60] | + +------------------------------+---------+ + | TS | [55-52] | + +------------------------------+---------+ + | FHM | [51-48] | + +------------------------------+---------+ + | DP | [47-44] | + +------------------------------+---------+ + | SM4 | [43-40] | + +------------------------------+---------+ + | SM3 | [39-36] | + +------------------------------+---------+ + | SHA3 | [35-32] | + +------------------------------+---------+ + | RDM | [31-28] | + +------------------------------+---------+ + | ATOMICS | [23-20] | + +------------------------------+---------+ + | CRC32 | [19-16] | + +------------------------------+---------+ + | SHA2 | [15-12] | + +------------------------------+---------+ + | SHA1 | [11-8] | + +------------------------------+---------+ + | AES | [7-4] | +------------------------------+---------+---------+ ID_AA64ISAR1_EL1 - Instruction set attribute register 1 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | LS64 | [63-60] | y | - +------------------------------+---------+---------+ - | I8MM | [55-52] | y | - +------------------------------+---------+---------+ - | DGH | [51-48] | y | - +------------------------------+---------+---------+ - | BF16 | [47-44] | y | - +------------------------------+---------+---------+ - | SB | [39-36] | y | - +------------------------------+---------+---------+ - | FRINTTS | [35-32] | y | - +------------------------------+---------+---------+ - | GPI | [31-28] | y | - +------------------------------+---------+---------+ - | GPA | [27-24] | y | - +------------------------------+---------+---------+ - | LRCPC | [23-20] | y | - +------------------------------+---------+---------+ - | FCMA | [19-16] | y | - +------------------------------+---------+---------+ - | JSCVT | [15-12] | y | - +------------------------------+---------+---------+ - | API | [11-8] | y | - +------------------------------+---------+---------+ - | APA | [7-4] | y | - +------------------------------+---------+---------+ - | DPB | [3-0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | LS64 | [63-60] | + +------------------------------+---------+ + | I8MM | [55-52] | + +------------------------------+---------+ + | DGH | [51-48] | + +------------------------------+---------+ + | BF16 | [47-44] | + +------------------------------+---------+ + | SB | [39-36] | + +------------------------------+---------+ + | FRINTTS | [35-32] | + +------------------------------+---------+ + | GPI | [31-28] | + +------------------------------+---------+ + | GPA | [27-24] | + +------------------------------+---------+ + | LRCPC | [23-20] | + +------------------------------+---------+ + | FCMA | [19-16] | + +------------------------------+---------+ + | JSCVT | [15-12] | + +------------------------------+---------+ + | API | [11-8] | + +------------------------------+---------+ + | APA | [7-4] | + +------------------------------+---------+ + | DPB | [3-0] | + +------------------------------+---------+ ID_AA64ISAR2_EL1 - Instruction set attribute register 2 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | LUT | [59-56] | y | - +------------------------------+---------+---------+ - | CSSC | [55-52] | y | - +------------------------------+---------+---------+ - | RPRFM | [51-48] | y | - +------------------------------+---------+---------+ - | BC | [23-20] | y | - +------------------------------+---------+---------+ - | MOPS | [19-16] | y | - +------------------------------+---------+---------+ - | APA3 | [15-12] | y | - +------------------------------+---------+---------+ - | GPA3 | [11-8] | y | - +------------------------------+---------+---------+ - | RPRES | [7-4] | y | - +------------------------------+---------+---------+ - | WFXT | [3-0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | LUT | [59-56] | + +------------------------------+---------+ + | CSSC | [55-52] | + +------------------------------+---------+ + | RPRFM | [51-48] | + +------------------------------+---------+ + | BC | [23-20] | + +------------------------------+---------+ + | MOPS | [19-16] | + +------------------------------+---------+ + | APA3 | [15-12] | + +------------------------------+---------+ + | GPA3 | [11-8] | + +------------------------------+---------+ + | RPRES | [7-4] | + +------------------------------+---------+ + | WFXT | [3-0] | + +------------------------------+---------+ ID_AA64ISAR3_EL1 - Instruction set attribute register 3 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | FPRCVT | [31-28] | y | - +------------------------------+---------+---------+ - | LSFE | [19-16] | y | - +------------------------------+---------+---------+ - | FAMINMAX | [7-4] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | FPRCVT | [31-28] | + +------------------------------+---------+ + | LSFE | [19-16] | + +------------------------------+---------+ + | FAMINMAX | [7-4] | + +------------------------------+---------+ ID_AA64MMFR0_EL1 - Memory model feature register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | ECV | [63-60] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | ECV | [63-60] | + +------------------------------+---------+ ID_AA64MMFR1_EL1 - Memory model feature register 1 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | AFP | [47-44] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | AFP | [47-44] | + +------------------------------+---------+ ID_AA64MMFR2_EL1 - Memory model feature register 2 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | AT | [35-32] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | AT | [35-32] | + +------------------------------+---------+ ID_AA64MMFR3_EL1 - Memory model feature register 3 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | S1POE | [19-16] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | S1POE | [19-16] | + +------------------------------+---------+ ID_AA64PFR0_EL1 - Processor Feature Register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | DIT | [51-48] | y | - +------------------------------+---------+---------+ - | MPAM | [43-40] | n | - +------------------------------+---------+---------+ - | SVE | [35-32] | y | - +------------------------------+---------+---------+ - | GIC | [27-24] | n | - +------------------------------+---------+---------+ - | AdvSIMD | [23-20] | y | - +------------------------------+---------+---------+ - | FP | [19-16] | y | - +------------------------------+---------+---------+ - | EL3 | [15-12] | n | - +------------------------------+---------+---------+ - | EL2 | [11-8] | n | - +------------------------------+---------+---------+ - | EL1 | [7-4] | n | - +------------------------------+---------+---------+ - | EL0 | [3-0] | n | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | DIT | [51-48] | + +------------------------------+---------+ + | SVE | [35-32] | + +------------------------------+---------+ + | AdvSIMD | [23-20] | + +------------------------------+---------+ + | FP | [19-16] | + +------------------------------+---------+ ID_AA64PFR1_EL1 - Processor Feature Register 1 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | GCS | [47-44] | y | - +------------------------------+---------+---------+ - | SME | [27-24] | y | - +------------------------------+---------+---------+ - | MTE | [11-8] | y | - +------------------------------+---------+---------+ - | SSBS | [7-4] | y | - +------------------------------+---------+---------+ - | BT | [3-0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | GCS | [47-44] | + +------------------------------+---------+ + | SME | [27-24] | + +------------------------------+---------+ + | MTE | [11-8] | + +------------------------------+---------+ + | SSBS | [7-4] | + +------------------------------+---------+ + | BT | [3-0] | + +------------------------------+---------+ ID_AA64PFR2_EL1 - Processor Feature Register 2 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | FPMR | [35-32] | y | - +------------------------------+---------+---------+ - | MTEFAR | [11-8] | y | - +------------------------------+---------+---------+ - | MTESTOREONLY | [7-4] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | FPMR | [35-32] | + +------------------------------+---------+ + | MTEFAR | [11-8] | + +------------------------------+---------+ + | MTESTOREONLY | [7-4] | + +------------------------------+---------+ ID_AA64SMFR0_EL1 - SME feature ID register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | FA64 | [63] | y | - +------------------------------+---------+---------+ - | LUT6 | [61] | y | - +------------------------------+---------+---------+ - | LUTv2 | [60] | y | - +------------------------------+---------+---------+ - | SMEver | [59-56] | y | - +------------------------------+---------+---------+ - | I16I64 | [55-52] | y | - +------------------------------+---------+---------+ - | F64F64 | [48] | y | - +------------------------------+---------+---------+ - | I16I32 | [47-44] | y | - +------------------------------+---------+---------+ - | B16B16 | [43] | y | - +------------------------------+---------+---------+ - | F16F16 | [42] | y | - +------------------------------+---------+---------+ - | F8F16 | [41] | y | - +------------------------------+---------+---------+ - | F8F32 | [40] | y | - +------------------------------+---------+---------+ - | I8I32 | [39-36] | y | - +------------------------------+---------+---------+ - | F16F32 | [35] | y | - +------------------------------+---------+---------+ - | B16F32 | [34] | y | - +------------------------------+---------+---------+ - | BI32I32 | [33] | y | - +------------------------------+---------+---------+ - | F32F32 | [32] | y | - +------------------------------+---------+---------+ - | SF8FMA | [30] | y | - +------------------------------+---------+---------+ - | SF8DP4 | [29] | y | - +------------------------------+---------+---------+ - | SF8DP2 | [28] | y | - +------------------------------+---------+---------+ - | SBitPerm | [25] | y | - +------------------------------+---------+---------+ - | AES | [24] | y | - +------------------------------+---------+---------+ - | SFEXPA | [23] | y | - +------------------------------+---------+---------+ - | STMOP | [16] | y | - +------------------------------+---------+---------+ - | SMOP4 | [0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | FA64 | [63] | + +------------------------------+---------+ + | LUT6 | [61] | + +------------------------------+---------+ + | LUTv2 | [60] | + +------------------------------+---------+ + | SMEver | [59-56] | + +------------------------------+---------+ + | I16I64 | [55-52] | + +------------------------------+---------+ + | F64F64 | [48] | + +------------------------------+---------+ + | I16I32 | [47-44] | + +------------------------------+---------+ + | B16B16 | [43] | + +------------------------------+---------+ + | F16F16 | [42] | + +------------------------------+---------+ + | F8F16 | [41] | + +------------------------------+---------+ + | F8F32 | [40] | + +------------------------------+---------+ + | I8I32 | [39-36] | + +------------------------------+---------+ + | F16F32 | [35] | + +------------------------------+---------+ + | B16F32 | [34] | + +------------------------------+---------+ + | BI32I32 | [33] | + +------------------------------+---------+ + | F32F32 | [32] | + +------------------------------+---------+ + | SF8FMA | [30] | + +------------------------------+---------+ + | SF8DP4 | [29] | + +------------------------------+---------+ + | SF8DP2 | [28] | + +------------------------------+---------+ + | SBitPerm | [25] | + +------------------------------+---------+ + | AES | [24] | + +------------------------------+---------+ + | SFEXPA | [23] | + +------------------------------+---------+ + | STMOP | [16] | + +------------------------------+---------+ + | SMOP4 | [0] | + +------------------------------+---------+ ID_AA64ZFR0_EL1 - SVE feature ID register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | F64MM | [59-56] | y | - +------------------------------+---------+---------+ - | F32MM | [55-52] | y | - +------------------------------+---------+---------+ - | F16MM | [51-48] | y | - +------------------------------+---------+---------+ - | I8MM | [47-44] | y | - +------------------------------+---------+---------+ - | SM4 | [43-40] | y | - +------------------------------+---------+---------+ - | SHA3 | [35-32] | y | - +------------------------------+---------+---------+ - | B16B16 | [27-24] | y | - +------------------------------+---------+---------+ - | BF16 | [23-20] | y | - +------------------------------+---------+---------+ - | BitPerm | [19-16] | y | - +------------------------------+---------+---------+ - | EltPerm | [15-12] | y | - +------------------------------+---------+---------+ - | AES | [7-4] | y | - +------------------------------+---------+---------+ - | SVEVer | [3-0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | F64MM | [59-56] | + +------------------------------+---------+ + | F32MM | [55-52] | + +------------------------------+---------+ + | F16MM | [51-48] | + +------------------------------+---------+ + | I8MM | [47-44] | + +------------------------------+---------+ + | SM4 | [43-40] | + +------------------------------+---------+ + | SHA3 | [35-32] | + +------------------------------+---------+ + | B16B16 | [27-24] | + +------------------------------+---------+ + | BF16 | [23-20] | + +------------------------------+---------+ + | BitPerm | [19-16] | + +------------------------------+---------+ + | EltPerm | [15-12] | + +------------------------------+---------+ + | AES | [7-4] | + +------------------------------+---------+ + | SVEVer | [3-0] | + +------------------------------+---------+ ID_ISAR5_EL1 - AArch32 Instruction Set Attribute Register 5 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | CRC32 | [19-16] | y | - +------------------------------+---------+---------+ - | SHA2 | [15-12] | y | - +------------------------------+---------+---------+ - | SHA1 | [11-8] | y | - +------------------------------+---------+---------+ - | AES | [7-4] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | CRC32 | [19-16] | + +------------------------------+---------+ + | SHA2 | [15-12] | + +------------------------------+---------+ + | SHA1 | [11-8] | + +------------------------------+---------+ + | AES | [7-4] | + +------------------------------+---------+ ID_ISAR6_EL1 - AArch32 Instruction Set Attribute Register 6 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | I8MM | [27-24] | y | - +------------------------------+---------+---------+ - | BF16 | [23-20] | y | - +------------------------------+---------+---------+ - | SB | [15-12] | y | - +------------------------------+---------+---------+ - | FHM | [11-8] | y | - +------------------------------+---------+---------+ - | DP | [7-4] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | I8MM | [27-24] | + +------------------------------+---------+ + | BF16 | [23-20] | + +------------------------------+---------+ + | SB | [15-12] | + +------------------------------+---------+ + | FHM | [11-8] | + +------------------------------+---------+ + | DP | [7-4] | + +------------------------------+---------+ ID_PFR2_EL1 - AArch32 Processor Feature Register 2 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | SSBS | [7-4] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | SSBS | [7-4] | + +------------------------------+---------+ MIDR_EL1 - Main ID Register - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | Implementer | [31-24] | y | - +------------------------------+---------+---------+ - | Variant | [23-20] | y | - +------------------------------+---------+---------+ - | Architecture | [19-16] | y | - +------------------------------+---------+---------+ - | PartNum | [15-4] | y | - +------------------------------+---------+---------+ - | Revision | [3-0] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | Implementer | [31-24] | + +------------------------------+---------+ + | Variant | [23-20] | + +------------------------------+---------+ + | Architecture | [19-16] | + +------------------------------+---------+ + | PartNum | [15-4] | + +------------------------------+---------+ + | Revision | [3-0] | + +------------------------------+---------+ NOTE: The 'visible' fields of MIDR_EL1 will contain the value as available on the CPU where it is fetched and is not a system @@ -471,29 +459,29 @@ infrastructure: MVFR0_EL1 - AArch32 Media and VFP Feature Register 0 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | FPDP | [11-8] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | FPDP | [11-8] | + +------------------------------+---------+ MVFR1_EL1 - AArch32 Media and VFP Feature Register 1 - +------------------------------+---------+---------+ - | Name | bits | visible | - +------------------------------+---------+---------+ - | SIMDFMAC | [31-28] | y | - +------------------------------+---------+---------+ - | FPHP | [27-24] | y | - +------------------------------+---------+---------+ - | SIMDHP | [23-20] | y | - +------------------------------+---------+---------+ - | SIMDSP | [19-16] | y | - +------------------------------+---------+---------+ - | SIMDInt | [15-12] | y | - +------------------------------+---------+---------+ - | SIMDLS | [11-8] | y | - +------------------------------+---------+---------+ + +------------------------------+---------+ + | Name | bits | + +------------------------------+---------+ + | SIMDFMAC | [31-28] | + +------------------------------+---------+ + | FPHP | [27-24] | + +------------------------------+---------+ + | SIMDHP | [23-20] | + +------------------------------+---------+ + | SIMDSP | [19-16] | + +------------------------------+---------+ + | SIMDInt | [15-12] | + +------------------------------+---------+ + | SIMDLS | [11-8] | + +------------------------------+---------+ Appendix I: Example From 221049874b6a78c7d87bc826581b0695cd338e2b Mon Sep 17 00:00:00 2001 From: Kohei Enju Date: Wed, 15 Jul 2026 21:28:50 +0900 Subject: [PATCH 20/95] arm64: RSI: fix field-spanning write warning in attestation token init The challenge is passed in registers a1 through a8. However, copying to ®s.a1 makes FORTIFY treat the destination as the single a1 field, resulting in a field-spanning write warning. [1] Overlay the SMCCC register structure with an RSI-specific argument layout and copy the challenge into an explicit 64-byte array. This keeps the existing a1-a8 argument encoding while giving the copy a correctly sized destination object. [1] memcpy: detected field-spanning write (size 64) of single field "®s.a1" at ./arch/arm64/include/asm/rsi_cmds.h:119 (size 8) WARNING: ./arch/arm64/include/asm/rsi_cmds.h:119 at rsi_attestation_token_init+0xdc/0xf8 [arm_cca_guest], CPU#0: cat/3314 Fixes: b880a80011f5 ("arm64: rsi: Add RSI definitions") Signed-off-by: Kohei Enju Signed-off-by: Will Deacon --- arch/arm64/include/asm/rsi_cmds.h | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/rsi_cmds.h b/arch/arm64/include/asm/rsi_cmds.h index 2c8763876dfb..c1fab41f671e 100644 --- a/arch/arm64/include/asm/rsi_cmds.h +++ b/arch/arm64/include/asm/rsi_cmds.h @@ -88,6 +88,14 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, return res.a0; } +#define RSI_ATTEST_CHALLENGE_MIN_SIZE 32 +#define RSI_ATTEST_CHALLENGE_MAX_SIZE 64 + +struct rsi_attestation_token_init_args { + unsigned long fid; + u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE]; +}; + /** * rsi_attestation_token_init - Initialise the operation to retrieve an * attestation token. @@ -109,18 +117,21 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, static inline long rsi_attestation_token_init(const u8 *challenge, unsigned long size) { - struct arm_smccc_1_2_regs regs = { 0 }; + union { + struct arm_smccc_1_2_regs regs; + struct rsi_attestation_token_init_args init; + } args = { 0 }; - /* The challenge must be at least 32bytes and at most 64bytes */ - if (!challenge || size < 32 || size > 64) + if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE || + size > RSI_ATTEST_CHALLENGE_MAX_SIZE) return -EINVAL; - regs.a0 = SMC_RSI_ATTESTATION_TOKEN_INIT; - memcpy(®s.a1, challenge, size); - arm_smccc_1_2_smc(®s, ®s); + args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT; + memcpy(args.init.challenge, challenge, size); + arm_smccc_1_2_smc(&args.regs, &args.regs); - if (regs.a0 == RSI_SUCCESS) - return regs.a1; + if (args.regs.a0 == RSI_SUCCESS) + return args.regs.a1; return -EINVAL; } From 25ef34b541eb216c09cb304778915fec845ef7ec Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 23 Jul 2026 14:03:04 +0100 Subject: [PATCH 21/95] arm64: futex: Consolidate 'old == new' check in __lsui_cmpxchg32() The LSUI futex implementation relies on a cmpxchg() loop to implement FUTEX_OP_XOR, as the architecture doesn't provide unprivileged *EOR atomics. Since the unprivileged 'CAST' instructions used to implement the cmpxchg() can only operate on 64-bit memory locations, the __lsui_cmpxchg32() helper function performs a song and dance to marshall the 32-bit futex value into the correct part of a 64-bit register and fill the remaining bytes with the neighbouring data. A consequence of this structure is that the 'CAST' failure/success condition ends up being split into two separate 32-bit checks across __lsui_cmpxchg32() and its caller. This is a little fiddly to read and introduces some additional local variables which can be avoided if the check is done in one place. Tweak __lsui_cmpxchg32() so that it performs the full 64-bit check on the value returned from the 'CAST' instruction and returns success to its caller only in the case that the cmpxchg() operation has succeeded. With that in place, simplify the outer loop in __lsui_futex_atomic_eor() to pass 'oldval' by reference and return unless the cmpxchg() operation returns -EAGAIN. __lsui_futex_cmpxchg() then swallows the -EAGAIN if the futex word has changed. Cc: Catalin Marinas Cc: Yeoreum Yun Reviewed-by: Yeoreum Yun Signed-off-by: Will Deacon --- arch/arm64/include/asm/futex.h | 61 ++++++++++++---------------------- 1 file changed, 22 insertions(+), 39 deletions(-) diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index d1d2ff9d323a..79c6d86c38a9 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -151,42 +151,31 @@ __lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval) } static __always_inline int -__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +__lsui_cmpxchg32(u32 __user *uaddr, u32 *oldval, u32 newval) { u64 __user *uaddr64; bool futex_pos, other_pos; - u32 other, orig_other; union { u32 futex[2]; u64 raw; - } oval64, orig64, nval64; + } orig64, oval64, nval64; uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64)); futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64)); other_pos = !futex_pos; - oval64.futex[futex_pos] = oldval; - if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos)) + orig64.futex[futex_pos] = *oldval; + if (get_user(orig64.futex[other_pos], (u32 __user *)uaddr64 + other_pos)) return -EFAULT; - orig64.raw = oval64.raw; - + nval64 = oval64 = orig64; nval64.futex[futex_pos] = newval; - nval64.futex[other_pos] = oval64.futex[other_pos]; if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw)) return -EFAULT; - oldval = oval64.futex[futex_pos]; - other = oval64.futex[other_pos]; - orig_other = orig64.futex[other_pos]; - - if (other != orig_other) - return -EAGAIN; - - *oval = oldval; - - return 0; + *oldval = oval64.futex[futex_pos]; + return oval64.raw == orig64.raw ? 0 : -EAGAIN; } static __always_inline int @@ -202,7 +191,7 @@ __lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval) static __always_inline int __lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) { - u32 oldval, newval, val; + u32 oldval, newval; int ret, i; if (get_user(oldval, uaddr)) @@ -214,33 +203,27 @@ __lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) for (i = 0; i < FUTEX_MAX_LOOPS; i++) { newval = oldval ^ oparg; - ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val); - switch (ret) { - case -EFAULT: - return ret; - case -EAGAIN: - continue; - } - - if (val == oldval) { - *oval = val; - return 0; - } - - oldval = val; + ret = __lsui_cmpxchg32(uaddr, &oldval, newval); + if (ret != -EAGAIN) + break; } - return -EAGAIN; + *oval = oldval; + return ret; } static __always_inline int __lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) { - /* - * Callers of futex_atomic_cmpxchg_inatomic() already retry on - * -EAGAIN, no need for another loop of max retries. - */ - return __lsui_cmpxchg32(uaddr, oldval, newval, oval); + u32 curval = oldval; + int ret; + + ret = __lsui_cmpxchg32(uaddr, &curval, newval); + if (ret == -EAGAIN && curval != oldval) + ret = 0; + + *oval = curval; + return ret; } #endif /* CONFIG_ARM64_LSUI */ From d50a908ffde534596107072c4897cf17c5ccd9e5 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sun, 26 Jul 2026 17:10:02 -0700 Subject: [PATCH 22/95] arm64: fix cpu-feature-registers Malformed table The "visible" column was dropped so that last line's extra pieces should also be dropped to prevent a docs build error: Documentation/arch/arm64/cpu-feature-registers.rst:170: ERROR: Malformed table. Right border not aligned or missing. +------------------------------+---------+ | Name | bits | +------------------------------+---------+ | RNDR | [63-60] | +------------------------------+---------+ | TS | [55-52] | +------------------------------+---------+ ... +------------------------------+---------+ | AES | [7-4] | +------------------------------+---------+---------+ [docutils] Fixes: bc7f7ddc64df ("arm64: Remove hidden bitfields from cpu-feature-registers.rst") Signed-off-by: Randy Dunlap Signed-off-by: Will Deacon --- Documentation/arch/arm64/cpu-feature-registers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/arch/arm64/cpu-feature-registers.rst b/Documentation/arch/arm64/cpu-feature-registers.rst index fc63ccb666bf..f603afe323d6 100644 --- a/Documentation/arch/arm64/cpu-feature-registers.rst +++ b/Documentation/arch/arm64/cpu-feature-registers.rst @@ -167,7 +167,7 @@ infrastructure: | SHA1 | [11-8] | +------------------------------+---------+ | AES | [7-4] | - +------------------------------+---------+---------+ + +------------------------------+---------+ ID_AA64ISAR1_EL1 - Instruction set attribute register 1 From 2f6fc0612607c95489c960aaefc8cb5578cdab8c Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Sun, 26 Jul 2026 20:22:53 +0200 Subject: [PATCH 23/95] arm64: proton-pack: Restore the nospectre_bhb command-line option Commit 7f1635737823 ("arm64: proton-pack: Fix hard lockup due to print in scheduler context") moved the "mitigation disabled" printks into spectre_print_disabled_mitigations(). For spectre-v2 and spectre-v4 only the pr_info_once() calls were removed, but for spectre-bhb the whole branch went with the print: - } else if (cpu_mitigations_off() || __nospectre_bhb) { - pr_info_once("spectre-bhb mitigation disabled ...\n"); spectre_bhb_enable_mitigation() therefore no longer tests __nospectre_bhb or cpu_mitigations_off() and the mitigation is enabled regardless of the command line. The parameter is still parsed and its flag is still checked by spectre_print_disabled_mitigations(), so the kernel prints "spectre-bhb mitigation disabled by command-line option" while /sys/devices/system/cpu/vulnerabilities/spectre_v2 reports "Mitigation: CSV2, BHB" and the vectors are switched to EL1_VECTOR_BHB_LOOP. The only remaining escape is the SPECTRE_VULNERABLE arm at the top of the chain, which a CSV2 core never reaches, so from Cortex-A76 and Neoverse N1 onwards both nospectre_bhb and mitigations=off are ignored. Both are documented in Documentation/admin-guide/kernel-parameters.txt. The identical mistake was made on the neighbouring compile-time-option branch immediately before this regression and fixed shortly afterwards; this command-line branch was missed. build_bhb_mitigation() in arch/arm64/net/bpf_jit_comp.c still tests both flags, so nospectre_bhb currently keeps the exception-vector loop while dropping the cBPF epilogue mitigation. Restore the check, folded into a spectre_bhb_mitigations_off() helper alongside its spectre_v2/v4 counterparts, and use it for the boot-time print in spectre_print_disabled_mitigations() as well. The print itself already lives there and does not need restoring. Tested under QEMU with -cpu neoverse-n1 (CSV2, no ECBHB, no CLRBHB). Before, spectre_v2 read "Mitigation: CSV2, BHB" with and without the option; after, nospectre_bhb and mitigations=off both give "Mitigation: CSV2, but not BHB" and a boot without either is unchanged. Fixes: 7f1635737823 ("arm64: proton-pack: Fix hard lockup due to print in scheduler context") Assisted-by: Claude:claude-opus-5 Cc: stable@vger.kernel.org Signed-off-by: Karl Mehltretter Signed-off-by: Will Deacon --- arch/arm64/kernel/proton-pack.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c index 7bb6553fec08..3bcf86154d95 100644 --- a/arch/arm64/kernel/proton-pack.c +++ b/arch/arm64/kernel/proton-pack.c @@ -1023,6 +1023,11 @@ static int __init parse_spectre_bhb_param(char *str) } early_param("nospectre_bhb", parse_spectre_bhb_param); +static bool spectre_bhb_mitigations_off(void) +{ + return __nospectre_bhb || cpu_mitigations_off(); +} + void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) { bp_hardening_cb_t cpu_cb; @@ -1036,6 +1041,8 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry) /* No point mitigating Spectre-BHB alone. */ } else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) { /* Do nothing */ + } else if (spectre_bhb_mitigations_off()) { + /* Mitigation disabled on the command line */ } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) { state = SPECTRE_MITIGATED; set_bit(BHB_HW, &system_bhb_mitigations); @@ -1201,6 +1208,6 @@ void spectre_print_disabled_mitigations(void) if (spectre_v4_mitigations_off()) pr_info("spectre-v4 %s", spectre_disabled_suffix); - if (__nospectre_bhb || cpu_mitigations_off()) + if (spectre_bhb_mitigations_off()) pr_info("spectre-bhb %s", spectre_disabled_suffix); } From fcc5eaea2d234162dfb8258372dd897bc2a1b862 Mon Sep 17 00:00:00 2001 From: Leo Yan Date: Thu, 2 Jul 2026 18:05:21 +0100 Subject: [PATCH 24/95] perf: arm_spe: Make wakeup range check overflow safe The current code checks whether the wakeup point is in the current writable range by comparing it with handle->head + handle->size. The perf AUX head is a monotonically increasing index, so that addition can overflow when head is close to ULONG_MAX. In that case, a wakeup point which is still inside the free space range can be missed. Use unsigned subtraction to compare the distance from head to wakeup against the handle->size. This can dismiss the issue when addition overflow. This is unlikely to happen in practice, but the change makes the watermark check logically correct. Fixes: d5d9696b0380 ("drivers/perf: Add support for ARMv8.2 Statistical Profiling Extension") Signed-off-by: Leo Yan Signed-off-by: Will Deacon --- drivers/perf/arm_spe_pmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index dbd0da111639..b64cf2313a20 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -577,7 +577,7 @@ static u64 __arm_spe_pmu_next_off(struct perf_output_handle *handle) * the page boundary following it. Keep the tail boundary if * that's lower. */ - if (handle->wakeup < (handle->head + handle->size) && head <= wakeup) + if ((handle->wakeup - handle->head) < handle->size && head <= wakeup) limit = min(limit, round_up(wakeup, PAGE_SIZE)); if (limit > head) From 6e0aa591af9948c433d239e8b3f7203a014e2513 Mon Sep 17 00:00:00 2001 From: Geetha sowjanya Date: Mon, 1 Jun 2026 13:03:16 +0530 Subject: [PATCH 25/95] dt-bindings: perf: marvell: Add CN20K DDR PMU binding Marvell CN20K SoCs integrate a DDR Performance Monitoring Unit (PMU) associated with the DDR controller. The block provides hardware counters to monitor DDR traffic and performance events and is accessed via a dedicated MMIO region. The CN20K DDR PMU is functionally equivalent to the CN10K DDR PMU, with minor register offset differences. Signed-off-by: Geetha sowjanya Reviewed-by: Krzysztof Kozlowski Signed-off-by: Will Deacon --- .../devicetree/bindings/perf/marvell-cn10k-ddr.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/perf/marvell-cn10k-ddr.yaml b/Documentation/devicetree/bindings/perf/marvell-cn10k-ddr.yaml index a18dd0a8c43a..f2f0d6b61eac 100644 --- a/Documentation/devicetree/bindings/perf/marvell-cn10k-ddr.yaml +++ b/Documentation/devicetree/bindings/perf/marvell-cn10k-ddr.yaml @@ -4,16 +4,22 @@ $id: http://devicetree.org/schemas/perf/marvell-cn10k-ddr.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Marvell CN10K DDR performance monitor +title: Marvell CN10K / CN20K DDR performance monitor + +description: + Performance Monitoring Unit (PMU) for the DDR controller on Marvell + CN10K and CN20K SoCs. The block is accessed via a dedicated MMIO region. maintainers: - Bharat Bhushan + - Geetha sowjanya properties: compatible: items: - enum: - marvell,cn10k-ddr-pmu + - marvell,cn20k-ddr-pmu reg: maxItems: 1 From 9b551a2446b2efaa25d6034e88f2f31c7df18e4d Mon Sep 17 00:00:00 2001 From: Geetha sowjanya Date: Mon, 1 Jun 2026 13:03:17 +0530 Subject: [PATCH 26/95] perf: marvell: Add CN20K DDR PMU support The CN20K DRAM Subsystem exposes eight programmable performance counters and two fixed counters for DDR read and write traffic. Software selects events for the programmable counters from traffic at the DDR PHY interface, the CHI interconnect, or inside the DDR controller. Add CN20K register offsets, event maps, and sysfs attributes; match the device via OF (marvell,cn20k-ddr-pmu) and ACPI (MRVL000B). Represent the SoC variant in platform data with bit flags so CN20K can reuse the CN10K PMU code path where appropriate. Signed-off-by: Geetha sowjanya Signed-off-by: Will Deacon --- drivers/perf/marvell_cn10k_ddr_pmu.c | 228 +++++++++++++++++++++++++-- 1 file changed, 213 insertions(+), 15 deletions(-) diff --git a/drivers/perf/marvell_cn10k_ddr_pmu.c b/drivers/perf/marvell_cn10k_ddr_pmu.c index 72ac17efd846..6f638dfe829b 100644 --- a/drivers/perf/marvell_cn10k_ddr_pmu.c +++ b/drivers/perf/marvell_cn10k_ddr_pmu.c @@ -13,31 +13,43 @@ #include #include #include +#include + +/* SoC variant flags for struct ddr_pmu_platform_data (mutually exclusive in pdata) */ +#define IS_CN10K BIT(0) +#define IS_ODY BIT(1) +#define IS_CN20K BIT(2) /* Performance Counters Operating Mode Control Registers */ #define CN10K_DDRC_PERF_CNT_OP_MODE_CTRL 0x8020 #define ODY_DDRC_PERF_CNT_OP_MODE_CTRL 0x20020 +#define CN20K_DDRC_PERF_CNT_OP_MODE_CTRL 0x20000 #define OP_MODE_CTRL_VAL_MANUAL 0x1 /* Performance Counters Start Operation Control Registers */ #define CN10K_DDRC_PERF_CNT_START_OP_CTRL 0x8028 #define ODY_DDRC_PERF_CNT_START_OP_CTRL 0x200A0 +#define CN20K_DDRC_PERF_CNT_START_OP_CTRL 0x20080 #define START_OP_CTRL_VAL_START 0x1ULL #define START_OP_CTRL_VAL_ACTIVE 0x2 /* Performance Counters End Operation Control Registers */ #define CN10K_DDRC_PERF_CNT_END_OP_CTRL 0x8030 #define ODY_DDRC_PERF_CNT_END_OP_CTRL 0x200E0 +#define CN20K_DDRC_PERF_CNT_END_OP_CTRL 0x200C0 #define END_OP_CTRL_VAL_END 0x1ULL /* Performance Counters End Status Registers */ #define CN10K_DDRC_PERF_CNT_END_STATUS 0x8038 #define ODY_DDRC_PERF_CNT_END_STATUS 0x20120 +#define CN20K_DDRC_PERF_CNT_END_STATUS 0x20100 #define END_STATUS_VAL_END_TIMER_MODE_END 0x1 /* Performance Counters Configuration Registers */ #define CN10K_DDRC_PERF_CFG_BASE 0x8040 #define ODY_DDRC_PERF_CFG_BASE 0x20160 +#define CN20K_DDRC_PERF_CFG_BASE 0x20140 +#define CN20K_DDRC_PERF_CFG1_BASE 0x20180 /* 8 Generic event counter + 2 fixed event counters */ #define DDRC_PERF_NUM_GEN_COUNTERS 8 @@ -61,6 +73,24 @@ * DO NOT change these event-id numbers, they are used to * program event bitmap in h/w. */ + +/* CN20K specific events */ +#define EVENT_PERF_OP_IS_RD16 61 +#define EVENT_PERF_OP_IS_RD32 60 +#define EVENT_PERF_OP_IS_WR16 59 +#define EVENT_PERF_OP_IS_WR32 58 +#define EVENT_OP_IS_ENTER_DSM 44 +#define EVENT_OP_IS_RFM 43 + + +#define EVENT_CN20K_OP_IS_ZQLATCH 62 +#define EVENT_CN20K_OP_IS_ZQSTART 63 +#define EVENT_CN20K_OP_IS_TCR_MRR 50 +#define EVENT_CN20K_OP_IS_DQSOSC_MRR 49 +#define EVENT_CN20K_OP_IS_DQSOSC_MPC 48 +#define EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_WR 47 +#define EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_RD 46 + #define EVENT_DFI_CMD_IS_RETRY 61 #define EVENT_RD_UC_ECC_ERROR 60 #define EVENT_RD_CRC_ERROR 59 @@ -87,6 +117,9 @@ #define EVENT_OP_IS_SPEC_REF 41 #define EVENT_OP_IS_CRIT_REF 40 #define EVENT_OP_IS_REFRESH 39 +#define EVENT_OP_IS_CAS_WCK_SUS 38 +#define EVENT_OP_IS_CAS_WS_OFF 37 +#define EVENT_OP_IS_CAS_WS 36 #define EVENT_OP_IS_ENTER_MPSM 35 #define EVENT_OP_IS_ENTER_POWERDOWN 31 #define EVENT_OP_IS_ENTER_SELFREF 27 @@ -183,8 +216,8 @@ struct ddr_pmu_platform_data { u64 cnt_freerun_clr; u64 cnt_value_wr_op; u64 cnt_value_rd_op; - bool is_cn10k; - bool is_ody; + u64 cfg1_base; + unsigned int silicon_flags; /* IS_CN10K, IS_ODY, or IS_CN20K */ }; static ssize_t cn10k_ddr_pmu_event_show(struct device *dev, @@ -336,6 +369,80 @@ static struct attribute *odyssey_ddr_perf_events_attrs[] = { NULL }; +static struct attribute *cn20k_ddr_perf_events_attrs[] = { + /* Programmable */ + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_wr_data_access, + EVENT_DFI_WR_DATA_CYCLES), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_rd_data_access, + EVENT_DFI_RD_DATA_CYCLES), + CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access, + EVENT_HPR_XACT_WHEN_CRITICAL), + CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access, + EVENT_LPR_XACT_WHEN_CRITICAL), + CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access, + EVENT_WR_XACT_WHEN_CRITICAL), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, + EVENT_OP_IS_RD_OR_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, + EVENT_OP_IS_RD_ACTIVATE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, + EVENT_PRECHARGE_FOR_RDWR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other, + EVENT_PRECHARGE_FOR_OTHER), + CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF), + CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, + EVENT_OP_IS_ENTER_POWERDOWN), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_ws, EVENT_OP_IS_CAS_WS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_ws_off, EVENT_OP_IS_CAS_WS_OFF), + CN10K_DDR_PMU_EVENT_ATTR(ddr_cas_wck_sus, EVENT_OP_IS_CAS_WCK_SUS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH), + CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF), + CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF), + CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE), + CN10K_DDR_PMU_EVENT_ATTR(ddr_rfm, EVENT_OP_IS_RFM), + CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_dsm, EVENT_OP_IS_ENTER_DSM), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cycles, EVENT_DFI_CYCLES), + CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd, + EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_RD), + CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr, + EVENT_CN20K_VISIBLE_WIN_LIMIT_REACHED_WR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_CN20K_OP_IS_DQSOSC_MPC), + CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_CN20K_OP_IS_DQSOSC_MRR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_CN20K_OP_IS_TCR_MRR), + CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_CN20K_OP_IS_ZQSTART), + CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_CN20K_OP_IS_ZQLATCH), + CN10K_DDR_PMU_EVENT_ATTR(ddr_read16, EVENT_PERF_OP_IS_RD16), + CN10K_DDR_PMU_EVENT_ATTR(ddr_read32, EVENT_PERF_OP_IS_RD32), + CN10K_DDR_PMU_EVENT_ATTR(ddr_write16, EVENT_PERF_OP_IS_WR16), + CN10K_DDR_PMU_EVENT_ATTR(ddr_write32, EVENT_PERF_OP_IS_WR32), + /* Free run event counters */ + CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS), + CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES), + NULL +}; + +static struct attribute_group cn20k_ddr_perf_events_attr_group = { + .name = "events", + .attrs = cn20k_ddr_perf_events_attrs, +}; + static struct attribute_group odyssey_ddr_perf_events_attr_group = { .name = "events", .attrs = odyssey_ddr_perf_events_attrs, @@ -393,6 +500,13 @@ static const struct attribute_group *odyssey_attr_groups[] = { NULL }; +static const struct attribute_group *cn20k_attr_groups[] = { + &cn20k_ddr_perf_events_attr_group, + &cn10k_ddr_perf_format_attr_group, + &cn10k_ddr_perf_cpumask_attr_group, + NULL +}; + /* Default poll timeout is 100 sec, which is very sufficient for * 48 bit counter incremented max at 5.6 GT/s, which may take many * hours to overflow. @@ -411,14 +525,38 @@ static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap, int err = 0; switch (eventid) { + case EVENT_CN20K_OP_IS_ZQLATCH ... EVENT_CN20K_OP_IS_ZQSTART: + if (ddr_pmu->p_data->silicon_flags & IS_CN20K) { + *event_bitmap = (1ULL << (eventid - 42)); + break; + } + err = -EINVAL; + break; case EVENT_DFI_PARITY_POISON ...EVENT_DFI_CMD_IS_RETRY: - if (!ddr_pmu->p_data->is_ody) { + /* + * 58..61: CN20K perf width events share numeric IDs with Odyssey + * DFI events; same 1ULL << (eventid - 1) bitmap on both paths. + */ + if (eventid >= EVENT_PERF_OP_IS_WR32 && + eventid <= EVENT_PERF_OP_IS_RD16) { + if (ddr_pmu->p_data->silicon_flags & IS_CN20K) { + *event_bitmap = (1ULL << (eventid - 1)); + break; + } + if (!(ddr_pmu->p_data->silicon_flags & IS_ODY)) { + err = -EINVAL; + break; + } + *event_bitmap = (1ULL << (eventid - 1)); + break; + } + if (!(ddr_pmu->p_data->silicon_flags & IS_ODY)) { err = -EINVAL; break; } fallthrough; case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD: - case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH: + case EVENT_OP_IS_CAS_WS ... EVENT_OP_IS_ZQLATCH: *event_bitmap = (1ULL << (eventid - 1)); break; case EVENT_OP_IS_ENTER_SELFREF: @@ -524,9 +662,9 @@ static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu, int counter, bool enable) { const struct ddr_pmu_platform_data *p_data = pmu->p_data; + unsigned int silicon_flags = pmu->p_data->silicon_flags; u64 ctrl_reg = pmu->p_data->cnt_op_mode_ctrl; const struct ddr_pmu_ops *ops = pmu->ops; - bool is_ody = pmu->p_data->is_ody; u32 reg; u64 val; @@ -546,7 +684,7 @@ static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu, writeq_relaxed(val, pmu->base + reg); - if (is_ody) { + if ((silicon_flags & IS_ODY) || (silicon_flags & IS_CN20K)) { if (enable) { /* * Setup the PMU counter to work in @@ -621,6 +759,7 @@ static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) { struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); const struct ddr_pmu_platform_data *p_data = pmu->p_data; + unsigned int silicon_flags = pmu->p_data->silicon_flags; const struct ddr_pmu_ops *ops = pmu->ops; struct hw_perf_event *hwc = &event->hw; u8 config = event->attr.config; @@ -642,10 +781,27 @@ static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) if (counter < DDRC_PERF_NUM_GEN_COUNTERS) { /* Generic counters, configure event id */ reg_offset = DDRC_PERF_CFG(p_data->cfg_base, counter); + ret = ddr_perf_get_event_bitmap(config, &val, pmu); if (ret) - return ret; + goto err_free_counter; + if (silicon_flags & IS_CN20K) { + if (config == EVENT_CN20K_OP_IS_ZQSTART || + config == EVENT_CN20K_OP_IS_ZQLATCH) { + /* ZQ lives in CFG1; clear stale event mask in CFG0 */ + writeq_relaxed(0, pmu->base + + DDRC_PERF_CFG(p_data->cfg_base, + counter)); + reg_offset = DDRC_PERF_CFG(p_data->cfg1_base, + counter); + } else { + /* Clear CFG1 so a prior ZQ select cannot linger */ + writeq_relaxed(0, pmu->base + + DDRC_PERF_CFG(p_data->cfg1_base, + counter)); + } + } writeq_relaxed(val, pmu->base + reg_offset); } else { /* fixed event counter, clear counter value */ @@ -661,6 +817,14 @@ static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) cn10k_ddr_perf_event_start(event, flags); return 0; + +err_free_counter: + if (pmu->active_events == 1) + hrtimer_cancel(&pmu->hrtimer); + pmu->active_events--; + cn10k_ddr_perf_free_counter(pmu, counter); + hwc->idx = -1; + return ret; } static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags) @@ -952,7 +1116,25 @@ static const struct ddr_pmu_platform_data cn10k_ddr_pmu_pdata = { .cnt_freerun_clr = 0, .cnt_value_wr_op = CN10K_DDRC_PERF_CNT_VALUE_WR_OP, .cnt_value_rd_op = CN10K_DDRC_PERF_CNT_VALUE_RD_OP, - .is_cn10k = TRUE, + .silicon_flags = IS_CN10K, +}; + +static const struct ddr_pmu_platform_data cn20k_ddr_pmu_pdata = { + .counter_overflow_val = 0, + .counter_max_val = GENMASK_ULL(63, 0), + .cnt_base = ODY_DDRC_PERF_CNT_VALUE_BASE, + .cfg_base = CN20K_DDRC_PERF_CFG_BASE, + .cfg1_base = CN20K_DDRC_PERF_CFG1_BASE, + .cnt_op_mode_ctrl = CN20K_DDRC_PERF_CNT_OP_MODE_CTRL, + .cnt_start_op_ctrl = CN20K_DDRC_PERF_CNT_START_OP_CTRL, + .cnt_end_op_ctrl = CN20K_DDRC_PERF_CNT_END_OP_CTRL, + .cnt_end_status = CN20K_DDRC_PERF_CNT_END_STATUS, + .cnt_freerun_en = 0, + .cnt_freerun_ctrl = ODY_DDRC_PERF_CNT_FREERUN_CTRL, + .cnt_freerun_clr = ODY_DDRC_PERF_CNT_FREERUN_CLR, + .cnt_value_wr_op = ODY_DDRC_PERF_CNT_VALUE_WR_OP, + .cnt_value_rd_op = ODY_DDRC_PERF_CNT_VALUE_RD_OP, + .silicon_flags = IS_CN20K, }; #endif @@ -979,7 +1161,7 @@ static const struct ddr_pmu_platform_data odyssey_ddr_pmu_pdata = { .cnt_freerun_clr = ODY_DDRC_PERF_CNT_FREERUN_CLR, .cnt_value_wr_op = ODY_DDRC_PERF_CNT_VALUE_WR_OP, .cnt_value_rd_op = ODY_DDRC_PERF_CNT_VALUE_RD_OP, - .is_ody = TRUE, + .silicon_flags = IS_ODY, }; #endif @@ -989,8 +1171,7 @@ static int cn10k_ddr_perf_probe(struct platform_device *pdev) struct cn10k_ddr_pmu *ddr_pmu; struct resource *res; void __iomem *base; - bool is_cn10k; - bool is_ody; + unsigned int silicon_flags; char *name; int ret; @@ -1014,10 +1195,9 @@ static int cn10k_ddr_perf_probe(struct platform_device *pdev) ddr_pmu->base = base; ddr_pmu->p_data = dev_data; - is_cn10k = ddr_pmu->p_data->is_cn10k; - is_ody = ddr_pmu->p_data->is_ody; + silicon_flags = ddr_pmu->p_data->silicon_flags; - if (is_cn10k) { + if (silicon_flags & IS_CN10K) { ddr_pmu->ops = &ddr_pmu_ops; /* Setup the PMU counter to work in manual mode */ writeq_relaxed(OP_MODE_CTRL_VAL_MANUAL, ddr_pmu->base + @@ -1039,7 +1219,7 @@ static int cn10k_ddr_perf_probe(struct platform_device *pdev) }; } - if (is_ody) { + if (silicon_flags & IS_ODY) { ddr_pmu->ops = &ddr_pmu_ody_ops; ddr_pmu->pmu = (struct pmu) { @@ -1056,6 +1236,22 @@ static int cn10k_ddr_perf_probe(struct platform_device *pdev) }; } + if (silicon_flags & IS_CN20K) { + ddr_pmu->ops = &ddr_pmu_ody_ops; + + ddr_pmu->pmu = (struct pmu) { + .module = THIS_MODULE, + .capabilities = PERF_PMU_CAP_NO_EXCLUDE, + .task_ctx_nr = perf_invalid_context, + .attr_groups = cn20k_attr_groups, + .event_init = cn10k_ddr_perf_event_init, + .add = cn10k_ddr_perf_event_add, + .del = cn10k_ddr_perf_event_del, + .start = cn10k_ddr_perf_event_start, + .stop = cn10k_ddr_perf_event_stop, + .read = cn10k_ddr_perf_event_update, + }; + } /* Choose this cpu to collect perf data */ ddr_pmu->cpu = raw_smp_processor_id(); @@ -1098,6 +1294,7 @@ static void cn10k_ddr_perf_remove(struct platform_device *pdev) #ifdef CONFIG_OF static const struct of_device_id cn10k_ddr_pmu_of_match[] = { { .compatible = "marvell,cn10k-ddr-pmu", .data = &cn10k_ddr_pmu_pdata }, + { .compatible = "marvell,cn20k-ddr-pmu", .data = &cn20k_ddr_pmu_pdata }, { }, }; MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match); @@ -1107,6 +1304,7 @@ MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match); static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = { {"MRVL000A", (kernel_ulong_t)&cn10k_ddr_pmu_pdata }, {"MRVL000C", (kernel_ulong_t)&odyssey_ddr_pmu_pdata}, + {"MRVL000B", (kernel_ulong_t)&cn20k_ddr_pmu_pdata}, {}, }; MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match); From 7dbcb0268d1e2a1d3badb0c6e77ee70e550b1787 Mon Sep 17 00:00:00 2001 From: Geetha sowjanya Date: Mon, 1 Jun 2026 13:03:18 +0530 Subject: [PATCH 27/95] perf: marvell: Cancel CN10K DDR PMU hrtimer on device remove cn10k_ddr_perf_remove() did not cancel the poll hrtimer before returning. If the device was unbound while perf events were still active the timer callback could run post-free. To fix the issue by adding hrtimer_cancel() in remove(). Signed-off-by: Geetha sowjanya Signed-off-by: Will Deacon --- drivers/perf/marvell_cn10k_ddr_pmu.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/perf/marvell_cn10k_ddr_pmu.c b/drivers/perf/marvell_cn10k_ddr_pmu.c index 6f638dfe829b..9c82bf4ee2c4 100644 --- a/drivers/perf/marvell_cn10k_ddr_pmu.c +++ b/drivers/perf/marvell_cn10k_ddr_pmu.c @@ -1284,6 +1284,12 @@ static void cn10k_ddr_perf_remove(struct platform_device *pdev) { struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev); + /* + * Cancel the poll timer before further teardown so the handler + * cannot run after this function returns. + */ + hrtimer_cancel(&ddr_pmu->hrtimer); + cpuhp_state_remove_instance_nocalls( CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, &ddr_pmu->node); From cf65b4d765b5f09db46baaa2dadf10fd53708a64 Mon Sep 17 00:00:00 2001 From: Yicong Yang Date: Wed, 8 Jul 2026 16:45:03 +0800 Subject: [PATCH 28/95] perf/dwc_pcie: Add support for Picoheart vendor devices Add PCI_VENDOR_ID_PICOHEART in pci_ids.h. Update the DWC PCIe vendor table with Picoheart PCIe Vendor ID to enable the PCIe PMU support. Acked-by: Bjorn Helgaas Reviewed-by: Shuai Xue Signed-off-by: Yicong Yang Signed-off-by: Will Deacon --- include/linux/pci_ids.h | 2 ++ include/linux/pcie-dwc.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 1c9d40e09107..2c17239aacea 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2640,6 +2640,8 @@ #define PCI_VENDOR_ID_SUNIX 0x1fd4 #define PCI_DEVICE_ID_SUNIX_1999 0x1999 +#define PCI_VENDOR_ID_PICOHEART 0x20fa + #define PCI_VENDOR_ID_HINT 0x3388 #define PCI_DEVICE_ID_HINT_VXPROII_IDE 0x8013 diff --git a/include/linux/pcie-dwc.h b/include/linux/pcie-dwc.h index 8ff778e7aec0..d3cd701a58e3 100644 --- a/include/linux/pcie-dwc.h +++ b/include/linux/pcie-dwc.h @@ -26,6 +26,8 @@ static const struct dwc_pcie_vsec_id dwc_pcie_rasdes_vsec_ids[] = { .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_AMPERE, .vsec_id = 0x02, .vsec_rev = 0x4 }, + { .vendor_id = PCI_VENDOR_ID_PICOHEART, + .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_QCOM, .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_ROCKCHIP, From 18ec1f543ae1466fdd437803c64ab7c5e43cc36c Mon Sep 17 00:00:00 2001 From: Yufan Dou Date: Wed, 8 Jul 2026 16:45:04 +0800 Subject: [PATCH 29/95] perf/dwc_pcie: Support narrowed time-based counter for long time monitoring The DWC PCIe Time-Based Analysis Data Register (the counter for time-based events) is architected as 64-bit, but some hardware implementations do not implement the full width. On these implementations the counter stops after reaching its implemented width. This will limit the usage for short time monitoring only. The counter will only cover ~15s for monitoring RX TLP payloads on our platform. Add an optional hrtimer that fires every 2 seconds. It'll take the role as the counter overflow interrupt to read-update-reset the counter and event counts to break the limits of the narrow counters. It'll only apply on timer-based counter. The 2 seconds update period is the half of the maximum counting period (4s) of the time-based counter under period counting mode of the hardware. Because fully-implemented 64-bit counters do not need this workaround, enable this hrtimer on the platforms known to have narrowed counter. Before this patch, when counting fio for 10m the counts is incorrect: root@localhost:/tmp# perf stat -e dwc_rootport_20000/rx_pcie_tlp_data_payload/ -- fio --runtime=10m fio_job.config [...] Run status group 0 (all jobs): READ: bw=5594MiB/s (5865MB/s), 5594MiB/s-5594MiB/s (5865MB/s-5865MB/s), io=3278GiB (3519GB), run=600010-600010msec [...] Performance counter stats for 'system wide': 137,438,953,456 dwc_rootport_20000/rx_pcie_tlp_data_payload/ After this patch the counts is as expected: root@localhost:/tmp# perf stat -e dwc_rootport_20000/rx_pcie_tlp_data_payload/ -- fio --runtime=10m fio_job.config [...] Run status group 0 (all jobs): READ: bw=5632MiB/s (5905MB/s), 5632MiB/s-5632MiB/s (5905MB/s-5905MB/s), io=3300GiB (3543GB), run=600013-600013msec [...] Performance counter stats for 'system wide': 3,543,850,268,576 dwc_rootport_20000/rx_pcie_tlp_data_payload/ Signed-off-by: Yufan Dou Signed-off-by: Yicong Yang Signed-off-by: Will Deacon --- drivers/perf/dwc_pcie_pmu.c | 71 ++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/drivers/perf/dwc_pcie_pmu.c b/drivers/perf/dwc_pcie_pmu.c index 5385401fa9cf..f5f3e5b83f5b 100644 --- a/drivers/perf/dwc_pcie_pmu.c +++ b/drivers/perf/dwc_pcie_pmu.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -83,6 +84,7 @@ enum dwc_pcie_event_type { #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) +#define DWC_PCIE_PMU_TIMER_PERIOD_NS (2 * NSEC_PER_SEC) struct dwc_pcie_pmu { struct pmu pmu; @@ -93,6 +95,8 @@ struct dwc_pcie_pmu { /* Groups #6 and #7 */ DECLARE_BITMAP(lane_events, 2 * DWC_PCIE_LANE_MAX_EVENTS_PER_GROUP); struct perf_event *time_based_event; + bool timer_enable; + struct hrtimer hrtimer; struct hlist_node cpuhp_node; int on_cpu; @@ -354,6 +358,26 @@ static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) return val; } +static void dwc_pcie_pmu_reset_time_based_counter(struct perf_event *event) +{ + struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); + struct hw_perf_event *hwc = &event->hw; + u64 prev; + + dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); + + /* + * The hardware counter is reset to zero when disabled. Synchronize + * prev_count so that the next event_update() computes the correct + * delta against the new counter baseline. + */ + do { + prev = local64_read(&hwc->prev_count); + } while (local64_cmpxchg(&hwc->prev_count, prev, 0) != prev); + + dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); +} + static void dwc_pcie_pmu_event_update(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; @@ -429,6 +453,26 @@ static int dwc_pcie_pmu_validate_group(struct perf_event *event) return 0; } +static enum hrtimer_restart dwc_pcie_pmu_hrtimer_callback(struct hrtimer *hrtimer) +{ + struct dwc_pcie_pmu *pcie_pmu = container_of(hrtimer, struct dwc_pcie_pmu, hrtimer); + struct perf_event *event = pcie_pmu->time_based_event; + struct hw_perf_event *hwc; + + if (!event) + return HRTIMER_NORESTART; + + hwc = &event->hw; + if (hwc->state & PERF_HES_STOPPED) + return HRTIMER_NORESTART; + + dwc_pcie_pmu_event_update(event); + dwc_pcie_pmu_reset_time_based_counter(event); + hrtimer_forward_now(hrtimer, ns_to_ktime(DWC_PCIE_PMU_TIMER_PERIOD_NS)); + + return HRTIMER_RESTART; +} + static int dwc_pcie_pmu_event_init(struct perf_event *event) { struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); @@ -478,10 +522,15 @@ static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) hwc->state = 0; local64_set(&hwc->prev_count, 0); - if (type == DWC_PCIE_LANE_EVENT) + if (type == DWC_PCIE_LANE_EVENT) { dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, true); - else if (type == DWC_PCIE_TIME_BASE_EVENT) + } else if (type == DWC_PCIE_TIME_BASE_EVENT) { dwc_pcie_pmu_time_based_event_enable(pcie_pmu, true); + if (pcie_pmu->timer_enable) + hrtimer_start(&pcie_pmu->hrtimer, + ns_to_ktime(DWC_PCIE_PMU_TIMER_PERIOD_NS), + HRTIMER_MODE_REL_PINNED_HARD); + } } static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) @@ -495,11 +544,15 @@ static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) dwc_pcie_pmu_event_update(event); - if (type == DWC_PCIE_LANE_EVENT) + if (type == DWC_PCIE_LANE_EVENT) { dwc_pcie_pmu_lane_event_enable(pcie_pmu, event, false); - else if (type == DWC_PCIE_TIME_BASE_EVENT) + } else if (type == DWC_PCIE_TIME_BASE_EVENT) { dwc_pcie_pmu_time_based_event_enable(pcie_pmu, false); + if (pcie_pmu->timer_enable) + hrtimer_cancel(&pcie_pmu->hrtimer); + } + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } @@ -726,6 +779,16 @@ static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) pcie_pmu->ras_des_offset = vsec; pcie_pmu->nr_lanes = pcie_get_width_cap(pdev); pcie_pmu->on_cpu = -1; + hrtimer_setup(&pcie_pmu->hrtimer, dwc_pcie_pmu_hrtimer_callback, + CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED_HARD); + + /* + * Use timer for updating time-based counts on platforms known + * to have narrowed counter. + */ + if (pdev->vendor == PCI_VENDOR_ID_PICOHEART) + pcie_pmu->timer_enable = true; + pcie_pmu->pmu = (struct pmu){ .name = name, .parent = &plat_dev->dev, From 233b5554e07b3902be9b0dd25bbdcc718824b022 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 23 Jul 2026 14:03:21 +0100 Subject: [PATCH 30/95] arm64: mm: Treat all devices as dma-coherent when CLIDR_EL1.LoC == 0 On systems where CLIDR_EL1.LoC == 0, no cache maintenance is required when cleaning or invalidating to the Point of Coherency and therefore all DMA agents can be treated as coherent. Extend arch_setup_dma_ops() to take CLIDR_EL1.LoC into account when setting the DMA ops for a device, emitting a warning message if the firmware advertises a non-coherent device on a fully coherent system. Cc: Steffen Eiden Cc: Andreas Grapentin Cc: Mark Rutland Cc: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/mm/dma-mapping.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c index 994b7b36e2b9..7f6f2f40dba2 100644 --- a/arch/arm64/mm/dma-mapping.c +++ b/arch/arm64/mm/dma-mapping.c @@ -42,6 +42,11 @@ void arch_setup_dma_ops(struct device *dev, bool coherent) { int cls = cache_line_size_of_cpu(); + if (!coherent && !CLIDR_LOC(read_sysreg(clidr_el1))) { + dev_warn(dev, "CLIDR_EL1.LoC == 0, treating as coherent\n"); + coherent = true; + } + WARN_TAINT(!coherent && cls > ARCH_DMA_MINALIGN, TAINT_CPU_OUT_OF_SPEC, "%s %s: ARCH_DMA_MINALIGN smaller than CTR_EL0.CWG (%d < %d)", From 6c74dcb1e1b73b381950f8991fa7493bdefbf3c9 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 30 Jul 2026 14:27:57 +0100 Subject: [PATCH 31/95] arm64: cpucaps: Remove stale comment about keeping capabilities sorted There is no functional requirement to keep the entries in arm64's tools/cpucaps file sorted alphabetically and, in fact, they have fallen out of order over time. Given that only the arm64 tree touches this file and the perceived benefit of sorting was to help with conflicts, just remove the stale comment and accept that we're collectively not very good at the alphabet. In the limited cases where a specific ordering is important, we enforce it through build-time assertions (e.g. in can_use_gic_priorities()). Cc: Catalin Marinas Acked-by: Mark Rutland Reviewed-by: Fuad Tabba Reported-by: Marc Zyngier Acked-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm64/tools/cpucaps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9b85a84f6fd4..3fcac789b0a4 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 # -# Internal CPU capabilities constants, keep this list sorted +# Internal CPU capabilities constants ALWAYS_BOOT ALWAYS_SYSTEM From 5686f2bed02d98d03e3f38f4b5569804510fba31 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Mon, 29 Jun 2026 16:07:15 +0100 Subject: [PATCH 32/95] firmware: arm_sdei: add sdei_is_present() invoke_sdei_fn() returns -EIO when no SDEI conduit was probed, and the core warns ("Failed to create event ...") on any registration that hits that. An optional consumer that registers an event from an unconditional initcall would therefore make every boot on a non-SDEI system emit that warning for what is simply absent firmware. Expose whether SDEI firmware is present so such a consumer can skip registration -- and the warning -- when there is nothing to talk to. Signed-off-by: Kiryl Shutsemau (Meta) Reviewed-by: Douglas Anderson Tested-by: Yin Fengwei Signed-off-by: Will Deacon --- drivers/firmware/arm_sdei.c | 10 ++++++++++ include/linux/arm_sdei.h | 3 +++ 2 files changed, 13 insertions(+) diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c index f39ed7ba3a38..c161cf263547 100644 --- a/drivers/firmware/arm_sdei.c +++ b/drivers/firmware/arm_sdei.c @@ -339,6 +339,16 @@ static void _ipi_unmask_cpu(void *ignored) sdei_unmask_local_cpu(); } +/* + * Was SDEI firmware probed and is it usable? Lets optional consumers skip + * registering an event -- and the warning a failed registration emits -- on + * systems with no SDEI. + */ +bool sdei_is_present(void) +{ + return sdei_firmware_call; +} + static void _ipi_private_reset(void *ignored) { int err; diff --git a/include/linux/arm_sdei.h b/include/linux/arm_sdei.h index f652a5028b59..b07113eeeff7 100644 --- a/include/linux/arm_sdei.h +++ b/include/linux/arm_sdei.h @@ -37,6 +37,9 @@ int sdei_event_unregister(u32 event_num); int sdei_event_enable(u32 event_num); int sdei_event_disable(u32 event_num); +/* Was SDEI firmware probed and usable? */ +bool sdei_is_present(void); + /* GHES register/unregister helpers */ int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb, sdei_event_callback *critical_cb); From 9141eda9eec76b538bb64e89185034ec61d341ca Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Mon, 29 Jun 2026 16:07:16 +0100 Subject: [PATCH 33/95] firmware: arm_sdei: add SDEI_EVENT_SIGNAL support Add sdei_event_signal(), a thin wrapper over the SDEI_EVENT_SIGNAL call (DEN0054) that makes the software-signalled event (event 0) pending on a target PE -- delivered NMI-like even when that PE has interrupts masked. It takes no locks, so it is safe to call from NMI / crash context. Signed-off-by: Kiryl Shutsemau (Meta) Reviewed-by: Douglas Anderson Tested-by: Yin Fengwei Signed-off-by: Will Deacon --- drivers/firmware/arm_sdei.c | 12 ++++++++++++ include/linux/arm_sdei.h | 6 ++++++ include/uapi/linux/arm_sdei.h | 1 + 3 files changed, 19 insertions(+) diff --git a/drivers/firmware/arm_sdei.c b/drivers/firmware/arm_sdei.c index c161cf263547..e8dd2f0f3919 100644 --- a/drivers/firmware/arm_sdei.c +++ b/drivers/firmware/arm_sdei.c @@ -339,6 +339,18 @@ static void _ipi_unmask_cpu(void *ignored) sdei_unmask_local_cpu(); } +/* + * Signal the software-signalled event (event 0) to @mpidr. Does nothing + * but the SMC -- no locks, no event lookup -- so it is safe from NMI / + * crash context (e.g. the cross-CPU NMI service). + */ +int sdei_event_signal(u32 event_num, u64 mpidr) +{ + return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_SIGNAL, event_num, + mpidr, 0, 0, 0, NULL); +} +NOKPROBE_SYMBOL(sdei_event_signal); + /* * Was SDEI firmware probed and is it usable? Lets optional consumers skip * registering an event -- and the warning a failed registration emits -- on diff --git a/include/linux/arm_sdei.h b/include/linux/arm_sdei.h index b07113eeeff7..b9dc21c241be 100644 --- a/include/linux/arm_sdei.h +++ b/include/linux/arm_sdei.h @@ -37,6 +37,12 @@ int sdei_event_unregister(u32 event_num); int sdei_event_enable(u32 event_num); int sdei_event_disable(u32 event_num); +/* + * Signal the software-signalled event (event 0) to another PE, NMI-like. + * @mpidr is the target's MPIDR affinity. + */ +int sdei_event_signal(u32 event_num, u64 mpidr); + /* Was SDEI firmware probed and usable? */ bool sdei_is_present(void); diff --git a/include/uapi/linux/arm_sdei.h b/include/uapi/linux/arm_sdei.h index af0630ba5437..22eb61612673 100644 --- a/include/uapi/linux/arm_sdei.h +++ b/include/uapi/linux/arm_sdei.h @@ -22,6 +22,7 @@ #define SDEI_1_0_FN_SDEI_PE_UNMASK SDEI_1_0_FN(0x0C) #define SDEI_1_0_FN_SDEI_INTERRUPT_BIND SDEI_1_0_FN(0x0D) #define SDEI_1_0_FN_SDEI_INTERRUPT_RELEASE SDEI_1_0_FN(0x0E) +#define SDEI_1_0_FN_SDEI_EVENT_SIGNAL SDEI_1_0_FN(0x0F) #define SDEI_1_0_FN_SDEI_PRIVATE_RESET SDEI_1_0_FN(0x11) #define SDEI_1_0_FN_SDEI_SHARED_RESET SDEI_1_0_FN(0x12) From 0c2c56598cc11b4e422b739ccd0f77bcd1f216ee Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Mon, 29 Jun 2026 16:07:17 +0100 Subject: [PATCH 34/95] drivers/firmware: add SDEI cross-CPU NMI service for arm64 Deliver an NMI-like event to an interrupt-masked arm64 CPU via the standard SDEI software-signalled event (event 0), without the pseudo-NMI hot-path cost: register a handler for event 0 and poke a target with sdei_event_signal(0, mpidr). First user is arch_trigger_cpumask_backtrace() (sysrq-l, RCU stalls, hung-task/soft-lockup dumps), which otherwise rides an IPI that can't reach a masked CPU. Falls back to the IPI path when SDEI is absent; no watchdog backend yet, so the stock detector is untouched. Signed-off-by: Kiryl Shutsemau (Meta) Reviewed-by: Douglas Anderson Tested-by: Yin Fengwei Signed-off-by: Will Deacon --- MAINTAINERS | 2 +- arch/arm64/include/asm/nmi.h | 24 +++++ arch/arm64/kernel/smp.c | 11 +++ drivers/firmware/Kconfig | 19 ++++ drivers/firmware/Makefile | 1 + drivers/firmware/arm_sdei_nmi.c | 159 ++++++++++++++++++++++++++++++++ 6 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 arch/arm64/include/asm/nmi.h create mode 100644 drivers/firmware/arm_sdei_nmi.c diff --git a/MAINTAINERS b/MAINTAINERS index 806bd2d80d15..151d356f2940 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -25093,7 +25093,7 @@ M: James Morse L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained F: Documentation/devicetree/bindings/arm/firmware/sdei.txt -F: drivers/firmware/arm_sdei.c +F: drivers/firmware/arm_sdei* F: include/linux/arm_sdei.h F: include/uapi/linux/arm_sdei.h diff --git a/arch/arm64/include/asm/nmi.h b/arch/arm64/include/asm/nmi.h new file mode 100644 index 000000000000..9366be419d18 --- /dev/null +++ b/arch/arm64/include/asm/nmi.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_NMI_H +#define __ASM_NMI_H + +#include + +/* + * Cross-CPU NMI provider hooks, consulted by the arm64 arch code before + * its regular-IRQ / pseudo-NMI IPI paths. The SDEI provider in + * drivers/firmware/arm_sdei_nmi.c implements them when active; a future + * FEAT_NMI provider could slot in here too. The stubs let callers stay + * unconditional when ARM_SDEI_NMI is off. + */ +#ifdef CONFIG_ARM_SDEI_NMI +bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu); +#else +static inline bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, + int exclude_cpu) +{ + return false; +} +#endif + +#endif /* __ASM_NMI_H */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index cdcdd160e5b6..88ecc0910db9 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -932,6 +933,16 @@ static void arm64_backtrace_ipi(cpumask_t *mask) void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { + /* + * Prefer the SDEI cross-CPU NMI provider when active: firmware + * dispatches the event out of EL3 and reaches CPUs that have + * interrupts locally masked, without the per-IRQ-mask cost that + * pseudo-NMI pays for the same reach. The plain IPI path below + * can't reach such a CPU unless pseudo-NMI is enabled. + */ + if (sdei_nmi_trigger_cpumask_backtrace(mask, exclude_cpu)) + return; + /* * NOTE: though nmi_trigger_cpumask_backtrace() has "nmi_" in the name, * nothing about it truly needs to be implemented using an NMI, it's diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 12dc70254842..07ab3090d219 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -36,6 +36,25 @@ config ARM_SDE_INTERFACE standard for registering callbacks from the platform firmware into the OS. This is typically used to implement RAS notifications. +config ARM_SDEI_NMI + bool "SDEI-based cross-CPU NMI service (arm64)" + depends on ARM_SDE_INTERFACE + help + Provides SDEI-based cross-CPU NMI delivery for hooks that need + to reach interrupt-masked CPUs on silicon that lacks FEAT_NMI: + + - arch_trigger_cpumask_backtrace() (sysrq-l, RCU stalls, + hardlockup_all_cpu_backtrace, soft-lockup secondary dumps, + hung-task auxiliary dumps) + + The driver registers a handler for the SDEI software-signalled + event (event 0) and reaches a target CPU by signalling it with + SDEI_EVENT_SIGNAL. Firmware delivers the event out of EL3 + regardless of the target's PSTATE.DAIF -- forced delivery into a + CPU wedged with interrupts locally masked. + + If unsure, say N. + config EDD tristate "BIOS Enhanced Disk Drive calls determine boot disk" depends on X86 diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index 4ddec2820c96..be46f1e1dc77 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -4,6 +4,7 @@ # obj-$(CONFIG_ARM_SCPI_PROTOCOL) += arm_scpi.o obj-$(CONFIG_ARM_SDE_INTERFACE) += arm_sdei.o +obj-$(CONFIG_ARM_SDEI_NMI) += arm_sdei_nmi.o obj-$(CONFIG_DMI) += dmi_scan.o obj-$(CONFIG_DMI_SYSFS) += dmi-sysfs.o obj-$(CONFIG_EDD) += edd.o diff --git a/drivers/firmware/arm_sdei_nmi.c b/drivers/firmware/arm_sdei_nmi.c new file mode 100644 index 000000000000..4047fcda0bc3 --- /dev/null +++ b/drivers/firmware/arm_sdei_nmi.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arm64 SDEI-based cross-CPU NMI service. + * + * Delivering an "NMI-shaped" event to an EL1 context that has locally + * masked interrupts, on silicon without FEAT_NMI, can be done two ways: + * + * - pseudo-NMI: mask "interrupts" via the GIC priority register + * (ICC_PMR_EL1) instead of PSTATE.DAIF, leaving a high-priority band + * deliverable. Functionally this works -- but it reimplements every + * local_irq_disable()/enable() and exception entry/exit as a PMR + * write plus synchronisation, a cost paid on that hot path forever, + * whether or not an NMI is ever delivered. + * + * - SDEI: leave interrupt masking as the cheap PSTATE.DAIF operation + * and have the firmware bounce an EL3-routed Group-0 SGI back to + * NS-EL1 as an event callback. The cost is a firmware round-trip, + * but only at the rare moment delivery is actually needed. + * + * This driver takes the second path: it keeps the IRQ-mask hot path + * free and pays only when it fires, which is what makes cross-CPU NMI + * affordable on hardware where the pseudo-NMI tax isn't, until FEAT_NMI + * makes NMI masking cheap in the architecture itself. + * + * Capabilities provided: + * + * - sdei_nmi_trigger_cpumask_backtrace() — override for arm64's + * arch_trigger_cpumask_backtrace(), so sysrq-l, RCU stall dumps, + * hardlockup_all_cpu_backtrace, soft-lockup/hung-task secondary + * dumps all reach interrupt-masked CPUs. + * + * Delivery uses the standard SDEI software-signalled event (event 0) and + * SDEI_EVENT_SIGNAL. We register a handler for event 0, enable it, and + * poke a target CPU with sdei_event_signal(0, mpidr): firmware makes + * event 0 pending on that PE and dispatches the handler NMI-like, + * regardless of the target's DAIF. + * Availability is simply whether event 0 registers and enables -- if SDEI + * and its software-signalled event are present we use it, otherwise the + * driver stays inert. + */ + +#define pr_fmt(fmt) "sdei_nmi: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static bool sdei_nmi_available; + +#define SDEI_NMI_EVENT 0 + +static int sdei_nmi_handler(u32 event, struct pt_regs *regs, void *arg) +{ + /* + * nmi_cpu_backtrace() no-ops unless this CPU's bit is set in the + * global backtrace mask (driven by nmi_trigger_cpumask_backtrace()), + * so a fire that reaches a CPU not being backtraced is harmless. + */ + nmi_cpu_backtrace(regs); + return SDEI_EV_HANDLED; +} +NOKPROBE_SYMBOL(sdei_nmi_handler); + +static void sdei_nmi_fire(unsigned int target_cpu) +{ + int err = sdei_event_signal(SDEI_NMI_EVENT, cpu_logical_map(target_cpu)); + + if (err) + pr_warn("SDEI_EVENT_SIGNAL to CPU %u failed: %d\n", + target_cpu, err); +} + +/* + * Raise callback for nmi_trigger_cpumask_backtrace(): signal event 0 + * at every CPU still pending in @mask. The framework excludes the local + * CPU from @mask before calling us. + */ +static void sdei_nmi_raise_backtrace(cpumask_t *mask) +{ + unsigned int cpu; + + /* + * Publish backtrace_mask (set by nmi_trigger_cpumask_backtrace()) + * before signalling. As in the stop path, the SMC is not a memory + * store, so dsb(ishst) is needed for the target to observe the mask. + */ + dsb(ishst); + + for_each_cpu(cpu, mask) + sdei_nmi_fire(cpu); +} + +/* + * Override hook for arch_trigger_cpumask_backtrace() (see + * arch/arm64/kernel/smp.c). Returns true when SDEI handled the request, + * which is the case whenever SDEI is active; on a false return the arch + * falls back to its regular-IRQ (or pseudo-NMI, if enabled) IPI. + * + * On a kernel built without paying the pseudo-NMI hot-path cost (the + * usual case for this driver's target), the IPI can't reach a CPU that + * has interrupts masked -- so the backtrace of the one CPU you care + * about comes back empty. SDEI is dispatched out of EL3 and lands + * regardless of the target's DAIF, without taxing the IRQ-mask path. + */ +bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +{ + if (!sdei_nmi_available) + return false; + + nmi_trigger_cpumask_backtrace(mask, exclude_cpu, + sdei_nmi_raise_backtrace); + return true; +} + +/* + * device_initcall (after arch_initcall(sdei_init), so the SDEI subsystem + * is up): probe the firmware, register the event, and turn on the + * cross-CPU service. If the probe fails the driver stays inert and the + * override hooks decline, leaving the arch's own paths in place. + */ +static int __init sdei_nmi_init(void) +{ + int err; + + if (!sdei_is_present()) + return 0; + + err = sdei_event_register(SDEI_NMI_EVENT, sdei_nmi_handler, NULL); + if (err) { + pr_err("sdei_event_register(%u) failed: %d\n", + SDEI_NMI_EVENT, err); + return 0; + } + + err = sdei_event_enable(SDEI_NMI_EVENT); + if (err) { + pr_err("sdei_event_enable(%u) failed: %d\n", + SDEI_NMI_EVENT, err); + sdei_event_unregister(SDEI_NMI_EVENT); + return 0; + } + + sdei_nmi_available = true; + pr_info("using SDEI cross-CPU NMI (SDEI_EVENT_SIGNAL, event %u)\n", + SDEI_NMI_EVENT); + + return 0; +} +device_initcall(sdei_nmi_init); From d639f762629212db129ba76bde5526da902654a4 Mon Sep 17 00:00:00 2001 From: "Kiryl Shutsemau (Meta)" Date: Mon, 29 Jun 2026 16:07:18 +0100 Subject: [PATCH 35/95] arm64: escalate smp_send_stop() to an SDEI NMI as a last resort A CPU wedged with interrupts masked ignores the stop IPI, and without pseudo-NMI there is no NMI IPI to escalate to: a reboot proceeds with the CPU still running, and a kdump misses its registers. Add a third rung to smp_send_stop(): once the IPI (and pseudo-NMI IPI, if enabled) rungs have run, signal SDEI event 0 at whatever stayed online. Firmware delivers it regardless of the target's DAIF, so it reaches a CPU a plain IPI cannot; the target acks by going offline, which the caller already polls for. Fold the stop bookkeeping into one arm64_nmi_cpu_stop(regs, die_on_crash), shared by the stop IPI handlers, panic_smp_self_stop() and the SDEI handler, replacing the near-duplicate local_cpu_stop() and ipi_cpu_crash_stop(). @die_on_crash is the only difference: the IPI handlers pass true and PSCI CPU_OFF the CPU on a crash stop so a capture kernel can reclaim it; the SDEI handler and self-stop pass false and park. The SDEI park is required, not conservative -- its handler runs inside an SDEI event that is never completed (completing it resumes the wedged context), and a CPU_OFF from that unfinished-event context wedges EL3 on some firmware (left as a follow-up). The dump is unaffected; only re-onlining the CPU in an SMP capture kernel is lost. Suggested-by: Douglas Anderson Signed-off-by: Kiryl Shutsemau (Meta) Reviewed-by: Douglas Anderson Tested-by: Yin Fengwei Signed-off-by: Will Deacon --- arch/arm64/include/asm/nmi.h | 24 +++++++ arch/arm64/kernel/smp.c | 113 +++++++++++++++++++++----------- drivers/firmware/Kconfig | 2 + drivers/firmware/arm_sdei_nmi.c | 87 ++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 38 deletions(-) diff --git a/arch/arm64/include/asm/nmi.h b/arch/arm64/include/asm/nmi.h index 9366be419d18..2e8974ff8d63 100644 --- a/arch/arm64/include/asm/nmi.h +++ b/arch/arm64/include/asm/nmi.h @@ -4,21 +4,45 @@ #include +struct pt_regs; + /* * Cross-CPU NMI provider hooks, consulted by the arm64 arch code before * its regular-IRQ / pseudo-NMI IPI paths. The SDEI provider in * drivers/firmware/arm_sdei_nmi.c implements them when active; a future * FEAT_NMI provider could slot in here too. The stubs let callers stay * unconditional when ARM_SDEI_NMI is off. + * + * sdei_nmi_active() lets a caller test for the service before committing + * to (and waiting on) the SDEI stop rung; sdei_nmi_stop_cpus() then signals + * the targets, which ack by going offline. */ #ifdef CONFIG_ARM_SDEI_NMI bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu); +bool sdei_nmi_active(void); +void sdei_nmi_stop_cpus(const cpumask_t *mask); #else static inline bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) { return false; } + +static inline bool sdei_nmi_active(void) +{ + return false; +} + +static inline void sdei_nmi_stop_cpus(const cpumask_t *mask) { } #endif +/* + * The common "stop this CPU" entry every arm64 stop path funnels through: + * the regular/pseudo-NMI stop IPI handlers, panic_smp_self_stop(), and the + * SDEI cross-CPU NMI handler. @die_on_crash powers the CPU off on the kdump + * crash path (IPI handlers) instead of parking it (SDEI / self-stop). + * Defined in arch/arm64/kernel/smp.c. + */ +void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash); + #endif /* __ASM_NMI_H */ diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 88ecc0910db9..fe09fc5838b0 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -867,14 +868,62 @@ void arch_irq_work_raise(void) } #endif -static void __noreturn local_cpu_stop(unsigned int cpu) +/** + * arm64_nmi_cpu_stop() - stop the local CPU after it is told to stop. + * @regs: register state to record in the vmcore on a crash stop, or NULL for + * panic_smp_self_stop(), which has no interrupted context to save. + * @die_on_crash: on the kdump crash path, power the CPU off via PSCI CPU_OFF + * (so a capture kernel can reclaim it) rather than parking it. + * + * The single point every arm64 stop path funnels through, keeping the + * bookkeeping (mask interrupts, save the crash context, mark offline, mask + * SDEI, optionally power off) in one place: + * + * - the regular IPI_CPU_STOP and pseudo-NMI IPI_CPU_STOP_NMI handlers; + * - panic_smp_self_stop(), a CPU parking itself on a parallel panic(); + * - the SDEI cross-CPU NMI handler (drivers/firmware/arm_sdei_nmi.c), + * which reaches CPUs the stop IPIs could not. + * + * The IPI stop handlers pass @die_on_crash true. The SDEI handler and + * panic_smp_self_stop() pass false and only park. For SDEI that is required, + * not just conservative: it runs inside an SDEI event that is deliberately + * never completed (completing it has firmware resume the wedged context), and + * a CPU_OFF from that not-yet-completed context wedges EL3 on some firmware -- + * a documented follow-up. Parking also matches this path's own fallback when + * CPU_OFF is unavailable. + */ +void __noreturn arm64_nmi_cpu_stop(struct pt_regs *regs, bool die_on_crash) { + unsigned int cpu = smp_processor_id(); + bool crash = IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop; + + /* + * Use local_daif_mask() instead of local_irq_disable() to make sure + * that pseudo-NMIs are disabled. The "stop" code starts with an IRQ + * and falls back to NMI (which might be pseudo). If the IRQ finally + * goes through right as we're timing out then the NMI could interrupt + * us. It's better to prevent the NMI and let the IRQ finish since the + * pt_regs will be better. + */ + local_daif_mask(); + +#ifdef CONFIG_KEXEC_CORE + if (crash && regs) + crash_save_cpu(regs, cpu); +#endif + + /* the ack a stop requester (e.g. smp_send_stop()) polls for */ set_cpu_online(cpu, false); - local_daif_mask(); sdei_mask_local_cpu(); + + if (crash && die_on_crash) + __cpu_try_die(cpu); + + /* just in case */ cpu_park_loop(); } +NOKPROBE_SYMBOL(arm64_nmi_cpu_stop); /* * We need to implement panic_smp_self_stop() for parallel panic() calls, so @@ -883,36 +932,7 @@ static void __noreturn local_cpu_stop(unsigned int cpu) */ void __noreturn panic_smp_self_stop(void) { - local_cpu_stop(smp_processor_id()); -} - -static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) -{ -#ifdef CONFIG_KEXEC_CORE - /* - * Use local_daif_mask() instead of local_irq_disable() to make sure - * that pseudo-NMIs are disabled. The "crash stop" code starts with - * an IRQ and falls back to NMI (which might be pseudo). If the IRQ - * finally goes through right as we're timing out then the NMI could - * interrupt us. It's better to prevent the NMI and let the IRQ - * finish since the pt_regs will be better. - */ - local_daif_mask(); - - crash_save_cpu(regs, cpu); - - set_cpu_online(cpu, false); - - sdei_mask_local_cpu(); - - if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) - __cpu_try_die(cpu); - - /* just in case */ - cpu_park_loop(); -#else - BUG(); -#endif + arm64_nmi_cpu_stop(NULL, false); } static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr) @@ -989,12 +1009,7 @@ static void do_handle_IPI(int ipinr) case IPI_CPU_STOP: case IPI_CPU_STOP_NMI: - if (IS_ENABLED(CONFIG_KEXEC_CORE) && crash_stop) { - ipi_cpu_crash_stop(cpu, get_irq_regs()); - unreachable(); - } else { - local_cpu_stop(cpu); - } + arm64_nmi_cpu_stop(get_irq_regs(), true); break; #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST @@ -1268,6 +1283,28 @@ void smp_send_stop(void) udelay(1); } + /* + * If CPUs are *still* online, try the SDEI cross-CPU NMI. Firmware + * delivers it regardless of the target's DAIF state, so it reaches + * a CPU spinning with interrupts masked, which neither rung above + * could (without pseudo-NMI there is no NMI rung at all). Allow + * 100ms: a firmware round-trip per CPU, with headroom. + */ + if (num_other_online_cpus() && sdei_nmi_active()) { + /* re-snapshot after the rungs above took CPUs offline */ + smp_rmb(); + cpumask_copy(&mask, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), &mask); + + pr_info("SMP: retry stop with SDEI NMI for CPUs %*pbl\n", + cpumask_pr_args(&mask)); + + sdei_nmi_stop_cpus(&mask); + timeout = USEC_PER_MSEC * 100; + while (num_other_online_cpus() && timeout--) + udelay(1); + } + if (num_other_online_cpus()) { smp_rmb(); cpumask_copy(&mask, cpu_online_mask); diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index 07ab3090d219..dbaca140a1b0 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -46,6 +46,8 @@ config ARM_SDEI_NMI - arch_trigger_cpumask_backtrace() (sysrq-l, RCU stalls, hardlockup_all_cpu_backtrace, soft-lockup secondary dumps, hung-task auxiliary dumps) + - smp_send_stop() escalation (reboot/halt and the + panic / kdump crash stop) The driver registers a handler for the SDEI software-signalled event (event 0) and reaches a target CPU by signalling it with diff --git a/drivers/firmware/arm_sdei_nmi.c b/drivers/firmware/arm_sdei_nmi.c index 4047fcda0bc3..038eb79e28ac 100644 --- a/drivers/firmware/arm_sdei_nmi.c +++ b/drivers/firmware/arm_sdei_nmi.c @@ -29,6 +29,11 @@ * hardlockup_all_cpu_backtrace, soft-lockup/hung-task secondary * dumps all reach interrupt-masked CPUs. * + * - sdei_nmi_stop_cpus() — the last rung of smp_send_stop()'s + * escalation (reboot/halt and the panic/kdump crash stop alike), + * reaching CPUs that ignored the stop IPIs; on the kdump path the + * wedged context is captured into the vmcore before the CPU parks. + * * Delivery uses the standard SDEI software-signalled event (event 0) and * SDEI_EVENT_SIGNAL. We register a handler for event 0, enable it, and * poke a target CPU with sdei_event_signal(0, mpidr): firmware makes @@ -59,8 +64,57 @@ static bool sdei_nmi_available; #define SDEI_NMI_EVENT 0 +/* + * Backtrace and stop both ride SDEI event 0. That is not a chosen economy: + * event 0 is the only architecturally software-signalled event -- the sole + * event SDEI_EVENT_SIGNAL can target at an arbitrary PE. Every other event + * number is a firmware/platform interrupt-bound event, not something the + * kernel can raise cross-CPU, so a dedicated "stop" event would need + * firmware to define and bind it -- exactly the firmware dependency this + * driver sets out to avoid. + * + * Sharing one event means the handler must tell a stop apart from a + * backtrace. A stop is terminal and system-wide -- sdei_nmi_stop_cpus() is + * only reached from smp_send_stop() (reboot/halt/panic/kdump), which never + * returns -- so once a stop is requested, every later event-0 fire is a + * stop too. A single write-once flag therefore carries as much as a + * per-CPU mask would: sdei_nmi_stop_cpus() sets it before signalling, and + * the handler reads a set flag as "stop this CPU" and a clear flag as + * "backtrace" (handled by nmi_cpu_backtrace(), which self-gates on the + * framework's backtrace mask). A backtrace fire that races in after a stop + * has begun just stops that CPU instead -- harmless, it is going down. + */ +static bool sdei_nmi_stopping; + static int sdei_nmi_handler(u32 event, struct pt_regs *regs, void *arg) { + /* + * No smp_rmb() pairing sdei_nmi_stop_cpus()'s dsb(ishst): the flag is + * the only shared value, and this handler runs only because firmware + * delivered the event -- a round-trip past that store -- so the read + * cannot be stale and there is no second load for a barrier to order. + */ + if (READ_ONCE(sdei_nmi_stopping)) { + /* + * Never returns, and deliberately never completes the SDEI + * event: SDEI_EVENT_COMPLETE has firmware restore the + * interrupted context, which would land the CPU back in + * the wedged loop (or in do_idle, which BUGs at + * cpuhp_report_idle_dead once it sees itself offline). + * Returning a modified pt_regs doesn't help -- + * arch/arm64/kernel/sdei.c::do_sdei_event only honours a PC + * override via its IRQ-state heuristic and otherwise hands + * EL3 its own saved-context slot back. + * + * Trade-off: EL3 retains ~one saved-context slot per parked + * CPU until the next hardware reset (~hundreds of bytes per + * CPU). Recoverability is unchanged versus an IPI-stopped + * CPU: neither comes back without a reset. + */ + arm64_nmi_cpu_stop(regs, false); + /* unreachable */ + } + /* * nmi_cpu_backtrace() no-ops unless this CPU's bit is set in the * global backtrace mask (driven by nmi_trigger_cpumask_backtrace()), @@ -122,6 +176,39 @@ bool sdei_nmi_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) return true; } +bool sdei_nmi_active(void) +{ + return sdei_nmi_available; +} + +/* + * Last rung of the stop escalation in smp_send_stop() (see + * arch/arm64/kernel/smp.c). The caller runs the regular stop IPI (and + * the pseudo-NMI stop IPI, where available) first; @mask holds whatever + * stayed online through those -- typically CPUs wedged with interrupts + * masked, unreachable by an IPI. Mark the stop in progress and signal + * event 0 at each target; a target acks by marking itself offline, which + * the caller polls for. The caller has already confirmed sdei_nmi_active(). + */ +void sdei_nmi_stop_cpus(const cpumask_t *mask) +{ + unsigned int cpu; + + WRITE_ONCE(sdei_nmi_stopping, true); + + /* + * Publish the flag before signalling. The signal goes out via an SMC + * to firmware, not a memory store, so smp_wmb() ordering is not + * enough: use dsb(ishst) to make the store globally visible before the + * SMC executes, as gic_ipi_send_mask() does for its SGI. The SDEI spec + * does not require the dispatch to order the caller's prior stores. + */ + dsb(ishst); + + for_each_cpu(cpu, mask) + sdei_nmi_fire(cpu); +} + /* * device_initcall (after arch_initcall(sdei_init), so the SDEI subsystem * is up): probe the firmware, register the event, and turn on the From 4c9c81a0860415284e9d260f998fbd755d3a7469 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Fri, 31 Jul 2026 13:26:46 +0100 Subject: [PATCH 36/95] arm64: smp: Fix IPI teardown for GICv5 flow Sashiko reported that during CPU offlining, __cpu_disable() is executed by the stopper thread via take_cpu_down() with local interrupts disabled. __cpu_disable() calls ipi_teardown(), which invokes ipi_lpi_disable(). For the GICv5 flow, this eventually calls the sleepable disable_irq(). This can be reproduced easily with CONFIG_DEBUG_ATOMIC_SLEEP=y by offlining a CPU: BUG: sleeping function called from invalid context at kernel/irq/manage.c:702 in_atomic(): 1, irqs_disabled(): 1, non_block: 0, pid: 20, name: migration/1 preempt_count: 1, expected: 0 no locks held by migration/1/20. irq event stamp: 186 hardirqs last enabled at (185): [] _raw_spin_unlock_irq+0x38/0x68 hardirqs last disabled at (186): [] multi_cpu_stop+0xc8/0x190 softirqs last enabled at (80): [] handle_softirqs+0x410/0x468 softirqs last disabled at (75): [] __do_softirq+0x1c/0x28 Fix this by using disable_irq_nosync() instead, which is safe in this atomic context. Fixes: ba1004f861d1 ("arm64: smp: Support non-SGIs for IPIs") Signed-off-by: Vladimir Murzin Signed-off-by: Will Deacon --- arch/arm64/kernel/smp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index cdcdd160e5b6..3ab90aa24efb 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -1086,7 +1086,7 @@ static void ipi_teardown(int cpu) disable_percpu_irq(ipi_irq_base + i); } } else { - disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i))); + disable_irq_nosync(irq_desc_get_irq(get_ipi_desc(cpu, i))); } } } From 436d111d918ae106d7545b1fad6fe42294447603 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 31 Jul 2026 18:52:38 +0100 Subject: [PATCH 37/95] arm_mpam: resctrl: Pick classes for use as MBM counters resctrl has two types of bandwidth counters, NUMA-local and global. MPAM can only count globally; either using MSC at the L3 cache or in the memory controllers. When global and local equate to the same thing continue just to call it global. Pick the corresponding MPAM classes to back the MBM counters. As resctrl requires all monitors to be at the L3 cache, we can only use the counters at the memory controllers when they have the same topology as the L3 cache and the traffic they see if the same. In particular, for the bandwidth counters at the memory controllers to be exposed to resctrl it is required there is a single L3 cache and a single NUMA node as otherwise cross NUMA traffic will be counted at the wrong instance. Signed-off-by: James Morse Signed-off-by: Ben Horgan Tested-by: Shaopeng Tan Tested-by: Zeng Heng Tested-by: Fenghua Yu Tested-by: Gavin Shan Reviewed-by: Shaopeng Tan Reviewed-by: Jonathan Cameron Reviewed-by: Fenghua Yu Reviewed-by: Gavin Shan Signed-off-by: Will Deacon --- drivers/resctrl/mpam_resctrl.c | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 226ff6f532fa..0fce9703b869 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -606,6 +606,16 @@ static bool cache_has_usable_csu(struct mpam_class *class) return true; } +static bool class_has_usable_mbwu(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) + return false; + + return true; +} + /* * Calculate the worst-case percentage change from each implemented step * in the control. @@ -983,6 +993,24 @@ static void mpam_resctrl_pick_counters(void) break; } } + + if (class_has_usable_mbwu(class) && + topology_matches_l3(class) && + traffic_matches_l3(class)) { + pr_debug("class %u has usable MBWU, and matches L3 topology and traffic\n", + class->level); + + /* + * An MSC measures bandwidth for a path determined by + * its location in hardware. We can't distinguish + * traffic by destination so we don't know if it's + * staying on the same NUMA node. Hence, we can't + * calculate mbm_local except when we only have one L3 + * and it's equivalent to mbm_total and so always use + * mbm_total. + */ + counter_update_class(QOS_L3_MBM_TOTAL_EVENT_ID, class); + } } } From 779cfd65316806b55a8c8fa50579e0d025310bc0 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 31 Jul 2026 18:52:39 +0100 Subject: [PATCH 38/95] arm_mpam: resctrl: Pre-allocate assignable monitors MPAM is able to emulate ABMC, i.e. mbm_event mode, by making memory bandwidth monitors assignable. Rather than supporting the 'default' mbm_assign_mode always use 'mbm_event' mode even if there are sufficient memory bandwidth monitors. The per monitor event configuration is only provided by resctrl when in 'mbm_event' mode and so only allowing 'mbm_event' mode will make it easier to support per-monitor event configuration for MPAM. For the moment, the only event supported is mbm_total_event with no bandwidth type configuration. The 'mbm_assign_mode' file will still show 'default' when there is no support for memory bandwidth monitoring. The monitors need to be allocated from the driver, and mapped to whichever control/monitor group resctrl wants to use them with. Add a second array to hold the monitor values indexed by resctrl's cntr_id. When CDP is in use, two monitors are needed so the available number of counters halves. Platforms with one monitor will have zero monitors when CDP is in use. Co-developed-by: James Morse Signed-off-by: James Morse Signed-off-by: Ben Horgan Tested-by: Shaopeng Tan Tested-by: Fenghua Yu Tested-by: Gavin Shan Reviewed-by: Shaopeng Tan Reviewed-by: Fenghua Yu Reviewed-by: Gavin Shan Signed-off-by: Will Deacon --- drivers/resctrl/mpam_internal.h | 6 +- drivers/resctrl/mpam_resctrl.c | 143 +++++++++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 4 deletions(-) diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h index 04d1a59f02af..def0e3a65c23 100644 --- a/drivers/resctrl/mpam_internal.h +++ b/drivers/resctrl/mpam_internal.h @@ -409,7 +409,11 @@ struct mpam_resctrl_res { struct mpam_resctrl_mon { struct mpam_class *class; - /* per-class data that resctrl needs will live here */ + /* Array of allocated MBWU monitors, indexed by (closid, rmid). */ + int *mbwu_idx_to_mon; + + /* Array of assigned MBWU monitors, indexed by resctrl's cntr_id. */ + int *assigned_counters; }; static inline int mpam_alloc_csu_mon(struct mpam_class *class) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 0fce9703b869..0698a235d15c 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -140,7 +140,7 @@ int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) { - return false; + return (r == &mpam_resctrl_controls[RDT_RESOURCE_L3].resctrl_res); } int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable) @@ -185,6 +185,26 @@ static void resctrl_reset_task_closids(void) read_unlock(&tasklist_lock); } +static void mpam_resctrl_monitor_sync_abmc_vals(struct rdt_resource *l3) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[QOS_L3_MBM_TOTAL_EVENT_ID]; + + if (!mon->class) + return; + + if (!mon->assigned_counters) + return; + + l3->mon.num_mbm_cntrs = mon->class->props.num_mbwu_mon; + if (cdp_enabled) + l3->mon.num_mbm_cntrs /= 2; + + /* + * Continue as normal even if enabling cdp causes there to be + * zero counters. This avoids giving resctrl mixed messages. + */ +} + int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) { u32 partid_i = RESCTRL_RESERVED_CLOSID, partid_d = RESCTRL_RESERVED_CLOSID; @@ -244,6 +264,7 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level rid, bool enable) WRITE_ONCE(arm64_mpam_global_default, mpam_get_regval(current)); resctrl_reset_task_closids(); + mpam_resctrl_monitor_sync_abmc_vals(l3); for_each_possible_cpu(cpu) mpam_set_cpu_defaults(cpu, partid_d, partid_i, 0, 0); @@ -613,6 +634,9 @@ static bool class_has_usable_mbwu(struct mpam_class *class) if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) return false; + if (!cprops->num_mbwu_mon) + return false; + return true; } @@ -935,6 +959,50 @@ static void mpam_resctrl_pick_mba(void) } } +static void __free_mbwu_mon(struct mpam_class *class, int *array, + u16 num_mbwu_mon) +{ + for (int i = 0; i < num_mbwu_mon; i++) { + if (array[i] < 0) + continue; + + mpam_free_mbwu_mon(class, array[i]); + array[i] = -1; + } +} + +static int __alloc_mbwu_mon(struct mpam_class *class, int *array, + u16 num_mbwu_mon) +{ + for (int i = 0; i < num_mbwu_mon; i++) { + int mbwu_mon = mpam_alloc_mbwu_mon(class); + + if (mbwu_mon < 0) { + __free_mbwu_mon(class, array, num_mbwu_mon); + return mbwu_mon; + } + array[i] = mbwu_mon; + } + + return 0; +} + +static int *__alloc_mbwu_array(struct mpam_class *class, u16 num_mbwu_mon) +{ + int err; + + int *array __free(kvfree) = kvmalloc_objs(*array, num_mbwu_mon); + if (!array) + return ERR_PTR(-ENOMEM); + + memset(array, -1, num_mbwu_mon * sizeof(*array)); + + err = __alloc_mbwu_mon(class, array, num_mbwu_mon); + if (err) + return ERR_PTR(err); + return_ptr(array); +} + static void counter_update_class(enum resctrl_event_id evt_id, struct mpam_class *class) { @@ -1091,6 +1159,43 @@ static int mpam_resctrl_pick_domain_id(int cpu, struct mpam_component *comp) return comp->comp_id; } +/* + * This must run after all event counters have been picked so that any free + * running counters have already been allocated. + */ +static int mpam_resctrl_monitor_init_abmc(struct mpam_resctrl_mon *mon) +{ + struct mpam_resctrl_res *res = &mpam_resctrl_controls[RDT_RESOURCE_L3]; + size_t num_rmid = resctrl_arch_system_num_rmid_idx(); + struct rdt_resource *l3 = &res->resctrl_res; + struct mpam_class *class = mon->class; + u16 num_mbwu_mon; + int *cntrs; + + int *rmid_array __free(kvfree) = kvmalloc_objs(*rmid_array, num_rmid); + if (!rmid_array) { + pr_debug("Failed to allocate RMID array\n"); + return -ENOMEM; + } + memset(rmid_array, -1, num_rmid * sizeof(*rmid_array)); + + num_mbwu_mon = class->props.num_mbwu_mon; + cntrs = __alloc_mbwu_array(mon->class, num_mbwu_mon); + if (IS_ERR(cntrs)) + return PTR_ERR(cntrs); + mon->assigned_counters = cntrs; + mon->mbwu_idx_to_mon = no_free_ptr(rmid_array); + + l3->mon.mbm_cntr_assignable = true; + l3->mon.mbm_assign_on_mkdir = true; + l3->mon.mbm_cntr_configurable = false; + l3->mon.mbm_cntr_assign_fixed = true; + + mpam_resctrl_monitor_sync_abmc_vals(l3); + + return 0; +} + static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon, enum resctrl_event_id type) { @@ -1135,8 +1240,21 @@ static int mpam_resctrl_monitor_init(struct mpam_resctrl_mon *mon, */ l3->mon.num_rmid = resctrl_arch_system_num_rmid_idx(); - if (resctrl_enable_mon_event(type, false, 0, NULL)) - l3->mon_capable = true; + if (type == QOS_L3_MBM_TOTAL_EVENT_ID) { + int err; + + err = mpam_resctrl_monitor_init_abmc(mon); + if (err) + return err; + + static_assert(MAX_EVT_CONFIG_BITS == 0x7f); + l3->mon.mbm_cfg_mask = MAX_EVT_CONFIG_BITS; + } + + if (!resctrl_enable_mon_event(type, false, 0, NULL)) + return -EINVAL; + + l3->mon_capable = true; return 0; } @@ -1699,6 +1817,23 @@ void mpam_resctrl_exit(void) resctrl_exit(); } +static void mpam_resctrl_teardown_mon(struct mpam_resctrl_mon *mon, struct mpam_class *class) +{ + u32 num_mbwu_mon = class->props.num_mbwu_mon; + + if (!mon->mbwu_idx_to_mon) + return; + + if (mon->assigned_counters) { + __free_mbwu_mon(class, mon->assigned_counters, num_mbwu_mon); + kvfree(mon->assigned_counters); + mon->assigned_counters = NULL; + } + + kvfree(mon->mbwu_idx_to_mon); + mon->mbwu_idx_to_mon = NULL; +} + /* * The driver is detaching an MSC from this class, if resctrl was using it, * pull on resctrl_exit(). @@ -1721,6 +1856,8 @@ void mpam_resctrl_teardown_class(struct mpam_class *class) for_each_mpam_resctrl_mon(mon, eventid) { if (mon->class == class) { mon->class = NULL; + + mpam_resctrl_teardown_mon(mon, class); break; } } From 6e0e538a75ceb95583b91d05abfe315bdbc5c967 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 31 Jul 2026 18:52:40 +0100 Subject: [PATCH 39/95] arm_mpam: resctrl: Add resctrl_arch_config_cntr() for ABMC use ABMC, mbm_event mode, has a helper resctrl_arch_config_cntr() for changing the mapping between 'cntr_id' and a CLOSID/RMID pair. Add the helper. For MPAM this is done by updating the mon->mbwu_idx_to_mon[] array, and as usual CDP means it needs doing in three different ways. Signed-off-by: James Morse Signed-off-by: Ben Horgan Tested-by: Shaopeng Tan Tested-by: Fenghua Yu Tested-by: Gavin Shan Reviewed-by: Jonathan Cameron Reviewed-by: Shaopeng Tan Reviewed-by: Fenghua Yu Reviewed-by: Gavin Shan Signed-off-by: Will Deacon --- drivers/resctrl/mpam_resctrl.c | 48 +++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index 0698a235d15c..ab322865c51b 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -125,12 +125,6 @@ void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d { } -void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - enum resctrl_event_id evtid, u32 rmid, u32 closid, - u32 cntr_id, bool assign) -{ -} - int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, u32 unused, u32 rmid, int cntr_id, enum resctrl_event_id eventid, u64 *val) @@ -1082,6 +1076,48 @@ static void mpam_resctrl_pick_counters(void) } } +static void __config_cntr(struct mpam_resctrl_mon *mon, u32 cntr_id, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid, + bool assign) +{ + /* Same CDP index remap as closid; maps cntr_id to assigned_counters[]. */ + u32 mbwu_idx, mon_idx = resctrl_get_config_index(cntr_id, cdp_type); + + closid = resctrl_get_config_index(closid, cdp_type); + mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + if (assign) + mon->mbwu_idx_to_mon[mbwu_idx] = mon->assigned_counters[mon_idx]; + else + mon->mbwu_idx_to_mon[mbwu_idx] = -1; +} + +void resctrl_arch_config_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + enum resctrl_event_id evtid, u32 rmid, u32 closid, + u32 cntr_id, bool assign) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[evtid]; + + if (evtid != QOS_L3_MBM_TOTAL_EVENT_ID) { + pr_debug("unexpected event id\n"); + return; + } + + if (!mon->mbwu_idx_to_mon || !mon->assigned_counters) { + pr_debug("monitor arrays not allocated\n"); + return; + } + + if (cdp_enabled) { + __config_cntr(mon, cntr_id, CDP_CODE, closid, rmid, assign); + __config_cntr(mon, cntr_id, CDP_DATA, closid, rmid, assign); + } else { + __config_cntr(mon, cntr_id, CDP_NONE, closid, rmid, assign); + } + + resctrl_arch_reset_cntr(r, d, closid, rmid, cntr_id, QOS_L3_MBM_TOTAL_EVENT_ID); +} + static int mpam_resctrl_control_init(struct mpam_resctrl_res *res) { struct mpam_class *class = res->class; From a5ff6ef2d73e87ff001c19c8d877d6dc8c249b67 Mon Sep 17 00:00:00 2001 From: James Morse Date: Fri, 31 Jul 2026 18:52:41 +0100 Subject: [PATCH 40/95] arm_mpam: resctrl: Add resctrl_arch_cntr_read() & resctrl_arch_reset_cntr() When used in 'mbm_event' mode, ABMC emulation, resctrl uses arch hooks to read and reset the memory bandwidth utilization (MBWU) counters. Add these. Signed-off-by: James Morse Signed-off-by: Ben Horgan Tested-by: Shaopeng Tan Tested-by: Fenghua Yu Tested-by: Gavin Shan Reviewed-by: Jonathan Cameron Reviewed-by: Shaopeng Tan Reviewed-by: Fenghua Yu Reviewed-by: Gavin Shan Signed-off-by: Will Deacon --- drivers/resctrl/mpam_resctrl.c | 99 +++++++++++++++++++++++++++++----- 1 file changed, 86 insertions(+), 13 deletions(-) diff --git a/drivers/resctrl/mpam_resctrl.c b/drivers/resctrl/mpam_resctrl.c index ab322865c51b..9d223057953a 100644 --- a/drivers/resctrl/mpam_resctrl.c +++ b/drivers/resctrl/mpam_resctrl.c @@ -119,19 +119,6 @@ void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_l3_mon_domain *d { } -void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - u32 closid, u32 rmid, int cntr_id, - enum resctrl_event_id eventid) -{ -} - -int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, - u32 unused, u32 rmid, int cntr_id, - enum resctrl_event_id eventid, u64 *val) -{ - return -EOPNOTSUPP; -} - bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r) { return (r == &mpam_resctrl_controls[RDT_RESOURCE_L3].resctrl_res); @@ -469,6 +456,14 @@ static int __read_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_c /* Shift closid to account for CDP */ closid = resctrl_get_config_index(closid, cdp_type); + if (mon_idx == USE_PRE_ALLOCATED) { + int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + mon_idx = mon->mbwu_idx_to_mon[mbwu_idx]; + if (mon_idx == -1) + return -ENOENT; + } + if (irqs_disabled()) { /* Check if we can access this domain without an IPI */ return -EIO; @@ -541,6 +536,84 @@ int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain_hdr *hdr, closid, rmid, val); } +/* MBWU counters when in ABMC mode */ +int resctrl_arch_cntr_read(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int mon_idx, + enum resctrl_event_id eventid, u64 *val) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + + if (!mpam_is_enabled()) + return -EINVAL; + + if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class) + return -EINVAL; + + l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom); + mon_comp = l3_dom->mon_comp[eventid]; + + return read_mon_cdp_safe(mon, mon_comp, mpam_feat_msmon_mbwu, + USE_PRE_ALLOCATED, closid, rmid, val); +} + +static void __reset_mon(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + int mon_idx, + enum resctrl_conf_type cdp_type, u32 closid, u32 rmid) +{ + struct mon_cfg cfg = { }; + + if (!mpam_is_enabled()) + return; + + /* Shift closid to account for CDP */ + closid = resctrl_get_config_index(closid, cdp_type); + + if (mon_idx == USE_PRE_ALLOCATED) { + int mbwu_idx = resctrl_arch_rmid_idx_encode(closid, rmid); + + mon_idx = mon->mbwu_idx_to_mon[mbwu_idx]; + } + + if (mon_idx == -1) + return; + cfg.mon = mon_idx; + mpam_msmon_reset_mbwu(mon_comp, &cfg); +} + +static void reset_mon_cdp_safe(struct mpam_resctrl_mon *mon, struct mpam_component *mon_comp, + int mon_idx, u32 closid, u32 rmid) +{ + if (cdp_enabled) { + __reset_mon(mon, mon_comp, mon_idx, CDP_CODE, closid, rmid); + __reset_mon(mon, mon_comp, mon_idx, CDP_DATA, closid, rmid); + } else { + __reset_mon(mon, mon_comp, mon_idx, CDP_NONE, closid, rmid); + } +} + +/* Reset an assigned counter */ +void resctrl_arch_reset_cntr(struct rdt_resource *r, struct rdt_l3_mon_domain *d, + u32 closid, u32 rmid, int cntr_id, + enum resctrl_event_id eventid) +{ + struct mpam_resctrl_mon *mon = &mpam_resctrl_counters[eventid]; + struct mpam_resctrl_dom *l3_dom; + struct mpam_component *mon_comp; + + if (!mpam_is_enabled()) + return; + + if (eventid == QOS_L3_OCCUP_EVENT_ID || !mon->class) + return; + + l3_dom = container_of(d, struct mpam_resctrl_dom, resctrl_mon_dom); + mon_comp = l3_dom->mon_comp[eventid]; + + reset_mon_cdp_safe(mon, mon_comp, USE_PRE_ALLOCATED, closid, rmid); +} + /* * The rmid realloc threshold should be for the smallest cache exposed to * resctrl. From 6b4ee75215c9dbcf6a0fd8cf7af2449906e2932c Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Fri, 31 Jul 2026 18:52:42 +0100 Subject: [PATCH 41/95] arm64: mpam: Add memory bandwidth usage (MBWU) documentation Memory bandwidth monitoring make uses of MBWU monitors and is now exposed to the user via resctrl. Add some documentation so the user knows what to expect. Co-developed-by: James Morse Signed-off-by: James Morse Signed-off-by: Ben Horgan Reviewed-by: Shaopeng Tan Reviewed-by: Fenghua Yu Reviewed-by: Gavin Shan Signed-off-by: Will Deacon --- Documentation/arch/arm64/mpam.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Documentation/arch/arm64/mpam.rst b/Documentation/arch/arm64/mpam.rst index 570f51a8d4eb..67fe515ed501 100644 --- a/Documentation/arch/arm64/mpam.rst +++ b/Documentation/arch/arm64/mpam.rst @@ -65,6 +65,28 @@ The supported features are: there is at least one CSU monitor on each MSC that makes up the L3 group. Exposing CSU counters from other caches or devices is not supported. +* Memory Bandwidth Usage (MBWU) on or after the L3 cache. resctrl uses the + L3 cache-id to identify where the memory bandwidth is measured. For this + reason the platform must have an L3 cache with cache-id's supplied by + firmware. (The platform doesn't need to support MPAM.) + + Memory bandwidth monitoring makes use of MBWU monitors in each MSC that + makes up the L3 group. If the memory bandwidth monitoring is on the memory + rather than the L3 then there must be a single global L3 as otherwise it + is unknown which L3 the traffic came from. + + To expose 'mbm_total_bytes', the topology of the group of MSC chosen must + match the topology of the L3 cache so that the cache-id's can be + repainted. For example: Platforms with Memory bandwidth monitors on + CPU-less NUMA nodes cannot expose 'mbm_total_bytes' as these nodes do not + have a corresponding L3 cache. 'mbm_local_bytes' is not exposed as MPAM + cannot distinguish local traffic from global traffic. + + All these restrictions based on L3 cache are due to resctrl, currently, only + supporting monitoring at the L3 scope. It is expected that going forward more + MBWU monitors can be exposed to the user after support for more monitoring + scopes is added to resctrl. + Reporting Bugs ============== If you are not seeing the counters or controls you expect please share the From 2c43aced9be353f8428678a75495ca9d631dc927 Mon Sep 17 00:00:00 2001 From: Shanker Donthineni Date: Fri, 31 Jul 2026 18:52:43 +0100 Subject: [PATCH 42/95] arm_mpam: Apply T241-MPAM-6 to 63-bit counters T241-MPAM-6 causes all MBWU counter formats to count 64-byte requests instead of bytes. Commit dc48eb1ff27c excluded the 63-bit MSMON_MBWU_LWD format while scaling the shorter counters. Systems selecting the preferred 63-bit counter consequently report bandwidth values that are 64 times too small. Apply the scale to both the sampled value and overflow correction for the 63-bit format. Unsigned arithmetic retains modulo-u64 behavior when the scaled counter range exceeds u64. Fixes: dc48eb1ff27c ("arm_mpam: Add workaround for T241-MPAM-6") Link: https://lore.kernel.org/lkml/20240816131432.993859-1-sdonthineni@nvidia.com/ Signed-off-by: Shanker Donthineni Reviewed-by: Fenghua Yu Tested-by: Fenghua Yu Reviewed-by: Ben Horgan Signed-off-by: Ben Horgan Signed-off-by: Will Deacon --- drivers/resctrl/mpam_devices.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index b69f99488111..53942b331269 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -1196,8 +1196,7 @@ static u64 mpam_msmon_overflow_val(enum mpam_device_features type, { u64 overflow_val = __mpam_msmon_overflow_val(type); - if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && - type != mpam_feat_msmon_mbwu_63counter) + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) overflow_val *= 64; return overflow_val; @@ -1293,8 +1292,7 @@ static void __ris_msmon_read(void *arg) now = FIELD_GET(MSMON___VALUE, now); } - if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc) && - m->type != mpam_feat_msmon_mbwu_63counter) + if (mpam_has_quirk(T241_MBW_COUNTER_SCALE_64, msc)) now *= 64; if (nrdy) From c3f83d021162571bcd87b062ec9e587828d2b7f3 Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Wed, 29 Jul 2026 02:42:54 +0200 Subject: [PATCH 43/95] arm64/fpsimd: ptrace: Fix inactive SVE and SSVE regsets sve_init_header_from_task() takes header as a pointer, so for the inactive mode header->size = sizeof(header); stores 8 rather than sizeof(struct user_sve_header), which is 16. Userspace sees an impossible size smaller than the header it describes. The inactive-mode check in sve_get_common() compares header.size against sizeof(header) as well, but there header is a struct, so the check can never fire. Reads of NT_ARM_SVE and NT_ARM_SSVE for the inactive mode therefore still return the other mode's FPSIMD data, exactly the situation the check was added to prevent. Fix the size, and make the check return the remaining membuf space instead of 0, which regset_get() would interpret as the entire (zero-filled) buffer having been populated. Fixes: b93e685ecff7 ("arm64/fpsimd: ptrace: Do not present register data for inactive mode") Assisted-by: Claude:claude-opus-5 Signed-off-by: Karl Mehltretter Signed-off-by: Will Deacon --- arch/arm64/kernel/ptrace.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 4d08598e2891..2a72c61a8af9 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -801,7 +801,7 @@ static void sve_init_header_from_task(struct user_sve_header *header, if (active) header->size = SVE_PT_SIZE(vq, header->flags); else - header->size = sizeof(header); + header->size = sizeof(*header); header->max_size = SVE_PT_SIZE(sve_vq_from_vl(header->max_vl), SVE_PT_REGS_SVE); } @@ -837,7 +837,7 @@ static int sve_get_common(struct task_struct *target, * from the other mode to userspace. */ if (header.size == sizeof(header)) - return 0; + return to.left; switch ((header.flags & SVE_PT_REGS_MASK)) { case SVE_PT_REGS_FPSIMD: From bd290e7fc245f9f85607f305fe8213c6c47a416c Mon Sep 17 00:00:00 2001 From: Karl Mehltretter Date: Wed, 29 Jul 2026 02:42:55 +0200 Subject: [PATCH 44/95] kselftest/arm64: fp-ptrace: Fix checks for inactive SVE and SSVE regsets The checks on the header size reported for the inactive regset of the NT_ARM_SVE/NT_ARM_SSVE pair compare it against sizeof(sve), but sve is a struct user_sve_header *, so this is 8 rather than the intended 16. The kernel carried the identical typo when filling in the header, so kernel and test agreed on the wrong value and the test passed. Compare against sizeof(*sve), stop after the header checks for an inactive regset since it has no payload to compare, and prefill the buffer with a sentinel to verify that reading an inactive regset leaves everything after the header untouched. This also covers the getter's return value, which determines how many bytes ptrace copies back to userspace. Fixes: 864f3ddcd715 ("kselftest/arm64: fp-ptrace: Adjust to new inactive mode behaviour") Assisted-by: Claude:claude-opus-5 Signed-off-by: Karl Mehltretter Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/fp/fp-ptrace.c | 47 +++++++++++++++++--- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 22c584b78be5..b435837c8c0e 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -65,6 +65,9 @@ /* VL 128..2048 in powers of 2 */ #define MAX_NUM_VLS 5 +/* Sentinel for detecting buffer bytes the kernel did not write */ +#define REGSET_SENTINEL 0xa5 + /* * FPMR bits we can set without doing feature checks to see if values * are valid. @@ -181,6 +184,20 @@ static bool compare_buffer(const char *name, void *out, return false; } +static bool buffer_is_filled(const void *buffer, size_t size, + unsigned char value) +{ + const unsigned char *bytes = buffer; + size_t i; + + for (i = 0; i < size; i++) { + if (bytes[i] != value) + return false; + } + + return true; +} + struct test_config { int sve_vl_in; int sve_vl_expected; @@ -401,6 +418,7 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -409,14 +427,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sve_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SVE: %s (%d)\n", @@ -440,10 +460,16 @@ static bool check_ptrace_values_sve(pid_t child, struct test_config *config) } if (svcr_in & SVCR_SM) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SVE reports data with PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SVE wrote beyond its header with PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SVE header size: %d != %lu\n", @@ -485,6 +511,7 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) struct user_sve_header *sve; struct user_fpsimd_state *fpsimd; struct iovec iov; + size_t buf_size; int ret, vq; bool pass = true; @@ -493,14 +520,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) vq = __sve_vq_from_vl(config->sme_vl_in); - iov.iov_len = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); - iov.iov_base = malloc(iov.iov_len); + buf_size = SVE_PT_SVE_OFFSET + SVE_PT_SVE_SIZE(vq, SVE_PT_REGS_SVE); + iov.iov_len = buf_size; + iov.iov_base = malloc(buf_size); if (!iov.iov_base) { ksft_print_msg("OOM allocating %lu byte SSVE buffer\n", iov.iov_len); return false; } + memset(iov.iov_base, REGSET_SENTINEL, buf_size); ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_SSVE, &iov); if (ret != 0) { ksft_print_msg("Failed to read initial SSVE: %s (%d)\n", @@ -523,10 +552,16 @@ static bool check_ptrace_values_ssve(pid_t child, struct test_config *config) } if (!(svcr_in & SVCR_SM)) { - if (sve->size != sizeof(sve)) { + if (sve->size != sizeof(*sve)) { ksft_print_msg("NT_ARM_SSVE reports data without PSTATE.SM\n"); pass = false; } + if (!buffer_is_filled(iov.iov_base + sizeof(*sve), + buf_size - sizeof(*sve), REGSET_SENTINEL)) { + ksft_print_msg("NT_ARM_SSVE wrote beyond its header without PSTATE.SM\n"); + pass = false; + } + goto out; } else { if (sve->size != SVE_PT_SIZE(vq, sve->flags)) { ksft_print_msg("Mismatch in SSVE header size: %d != %lu\n", From 2fcbc4adf99790564b83381b2f239294185603d5 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Tue, 28 Jul 2026 10:11:21 +0800 Subject: [PATCH 45/95] kselftest/arm64: Add seccomp ptrace x0 bypass test As Kees suggested, add a test that verifies that seccomp observes the correct first argument after a ptracer modifies x0 at a syscall-enter-stop on arm64. The first syscall argument and the return value share register x0. The original value is saved in orig_x0 on entry and used by syscall_get_arguments(), but ptrace changes to x0 were not automatically reflected there. This test checks the kernel re-syncs orig_x0 after a ptrace stop so that seccomp sees the modified argument. A seccomp filter allows write(2,...) and kills the task for any other fd. The tracer changes fd from 2 to 1 at entry. If orig_x0 remains stale, the child exits normally (bypass, test fails). If orig_x0 is correctly updated, the child is killed by SIGSYS (test passes). Before the fix: ./seccomp_ptrace_x0_bypass TAP version 13 1..1 not ok 1 seccomp_ptrace_x0_bypass # Totals: pass:0 fail:1 xfail:0 xpass:0 skip:0 error:0 After the fix: # ./seccomp_ptrace_x0_bypass TAP version 13 1..1 [ 19.475951] audit: type=1326 audit(1784254846.284:2): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=227 comm="seccomp_ptrace_" exe="/mnt/seccomp0 [ 19.477852] audit: type=1701 audit(1784254846.284:3): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=227 comm="seccomp_ptrace_" exe="/mnt/seccomp1 ok 1 seccomp_ptrace_x0_bypass # Totals: pass:1 fail:0 xfail:0 xpass:0 skip:0 error:0 Cc: Kees Cook Cc: Will Deacon Cc: Catalin Marinas Cc: Mark Rutland Link: https://lore.kernel.org/all/20260716120640.6590-1-will@kernel.org/ Link: https://lore.kernel.org/all/202607152004.DEA95D63@keescook/ Suggested-by: Kees Cook Signed-off-by: Jinjie Ruan Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/abi/.gitignore | 1 + tools/testing/selftests/arm64/abi/Makefile | 2 +- .../arm64/abi/seccomp_ptrace_x0_bypass.c | 195 ++++++++++++++++++ 3 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c diff --git a/tools/testing/selftests/arm64/abi/.gitignore b/tools/testing/selftests/arm64/abi/.gitignore index 44f8b80f37e3..39129a9907c7 100644 --- a/tools/testing/selftests/arm64/abi/.gitignore +++ b/tools/testing/selftests/arm64/abi/.gitignore @@ -1,4 +1,5 @@ hwcap ptrace +seccomp_ptrace_x0_bypass syscall-abi tpidr2 diff --git a/tools/testing/selftests/arm64/abi/Makefile b/tools/testing/selftests/arm64/abi/Makefile index 483488f8c2ad..5a16db379bd4 100644 --- a/tools/testing/selftests/arm64/abi/Makefile +++ b/tools/testing/selftests/arm64/abi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright (C) 2021 ARM Limited -TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2 +TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2 seccomp_ptrace_x0_bypass include ../../lib.mk diff --git a/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c b/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c new file mode 100644 index 000000000000..a00621a8949c --- /dev/null +++ b/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test that seccomp, tracepoints and audit observe the correct syscall + * arguments after a ptracer has modified them at syscall-enter-stop. + * + * On arm64, both the first argument and the return value of a syscall + * are passed in register x0. The original x0 is saved in + * pt_regs::orig_x0 during syscall entry and returned as the first + * argument by syscall_get_arguments(). Because ptrace modifications + * to x0 are not automatically reflected in orig_x0, seccomp, tracepoints + * and audit may see a stale value unless orig_x0 is explicitly + * re-synchronised after a ptrace stop. + * + * This test sets up a seccomp filter that allows write(2, ...) but kills + * the task for any other fd. A ptracer changes the fd argument from 2 + * to 1 at the syscall-enter stop. If the orig_x0 re-sync works, seccomp + * sees the modified argument (fd=1) and kills the child with SIGSYS + * (test passes). If orig_x0 is not re-synced, seccomp sees the original + * fd=2, the write succeeds and the child exits normally (test fails, + * vulnerability present). + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" + +#ifndef __NR_write +#define __NR_write 64 +#endif + +#define EXPECTED_TESTS 1 + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define ARG0_OFFSET (offsetof(struct seccomp_data, args)) +#else +#define ARG0_OFFSET (offsetof(struct seccomp_data, args) + 4) +#endif + +static int do_child(void) +{ + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL)) + ksft_exit_fail_perror("PTRACE_TRACEME"); + + if (raise(SIGSTOP)) + ksft_exit_fail_perror("raise(SIGSTOP)"); + + /* + * Seccomp filter: + * If syscall is not write -> ALLOW + * If syscall is write: + * - If args[0] (fd) == 2 -> ALLOW + * - Otherwise -> KILL + */ + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), /* nr */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_write, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, ARG0_OFFSET), /* args[0] */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 2, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog prog = { + .len = ARRAY_SIZE(filter), + .filter = filter, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + ksft_exit_fail_perror("prctl NO_NEW_PRIVS"); + + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) + ksft_exit_fail_perror("prctl SECCOMP"); + + /* + * Invoke write(2, ...) while the tracer will change the first + * argument (fd) from 2 to 1 at syscall entry. + */ + syscall(__NR_write, 2, NULL, 0); + _exit(0); +} + +static int do_parent(pid_t child) +{ + bool bypass = false; + int status; + + /* Wait for the initial SIGSTOP */ + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("waitpid failed"); + + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGSTOP) + ksft_exit_fail_msg("unexpected stop status"); + + if (ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD | PTRACE_O_EXITKILL)) + ksft_exit_fail_perror("PTRACE_SETOPTIONS"); + + if (ptrace(PTRACE_SYSCALL, child, 0, 0)) + ksft_exit_fail_perror("PTRACE_SYSCALL"); + + while (1) { + int sig; + + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("waitpid lost child"); + + if (WIFEXITED(status)) { + /* Child exited normally – bypass succeeded */ + bypass = true; + break; + } + + if (WIFSIGNALED(status)) { + sig = WTERMSIG(status); + if (sig == SIGSYS) + break; + ksft_exit_fail_msg("child died unexpectedly from signal %d (%s)", + sig, strsignal(sig)); + } + + if (!WIFSTOPPED(status)) + ksft_exit_fail_msg("unexpected wait status"); + + sig = WSTOPSIG(status); + + if (sig == (SIGTRAP | 0x80)) { + struct user_regs_struct regs; + struct iovec iov = { + .iov_base = ®s, + .iov_len = sizeof(regs), + }; + + if (ptrace(PTRACE_GETREGSET, child, NT_PRSTATUS, &iov)) + ksft_exit_fail_perror("PTRACE_GETREGSET"); + + unsigned long syscall_nr = regs.regs[8]; + unsigned long x0 = regs.regs[0]; + + /* Modify fd from 2 to 1 at write entry */ + if (syscall_nr == __NR_write && x0 == 2) { + regs.regs[0] = 1; + if (ptrace(PTRACE_SETREGSET, child, NT_PRSTATUS, &iov)) + ksft_exit_fail_perror("PTRACE_SETREGSET"); + } + + if (ptrace(PTRACE_SYSCALL, child, 0, 0)) + ksft_exit_fail_perror("PTRACE_SYSCALL"); + } else { + /* Forward other signals */ + if (ptrace(PTRACE_SYSCALL, child, 0, sig)) + ksft_exit_fail_perror("PTRACE_SYSCALL"); + } + } + + /* bypass == true means vulnerability exists -> test fails */ + return bypass ? EXIT_FAILURE : EXIT_SUCCESS; +} + +int main(void) +{ + pid_t child; + + ksft_print_header(); + ksft_set_plan(EXPECTED_TESTS); + + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork failed: %s", strerror(errno)); + + if (!child) + return do_child(); + + /* + * do_parent() returns EXIT_SUCCESS if the child was killed by + * SIGSYS (i.e. seccomp correctly saw the modified argument), + * and EXIT_FAILURE if the child exited normally (bypass). + */ + int result = do_parent(child); + + ksft_test_result(result == EXIT_SUCCESS, "seccomp_ptrace_x0_bypass\n"); + + ksft_print_cnts(); + return result; +} From 21e37da12071da1e2d2b995219c3f394dd358300 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Tue, 28 Jul 2026 10:11:22 +0800 Subject: [PATCH 46/95] kselftest/arm64: Add testcase for SECCOMP_RET_TRACE orig_x0 bypass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a selftest that verifies the kernel re-evaluates a seccomp filter with the correct (ptrace-modified) first argument after a SECCOMP_RET_TRACE stop. On arm64, syscall_get_arguments() reads the first argument from orig_x0, which may be stale if the tracer modified regs->regs[0] but orig_x0 was not synced. This can cause the filter to see an old argument and incorrectly allow a syscall that it should have rejected. The child installs a filter that: - TRACEs write() when fd == 2 - returns ERRNO(EPERM) when fd == 1 The parent catches the SECCOMP event, changes x0 (fd) from 2 to 1, and resumes the child. If the seccomp re-evaluation sees the stale orig_x0 (fd=2) the filter returns TRACE again and the kernel (with recheck_after_trace=true) allows the syscall to proceed – write succeeds and the child exits 0. If the seccomp re-evaluation sees the new value (fd=1) the filter returns ERRNO(EPERM), write fails and the child exits non-zero. The test passes only when the write fails (child exit != 0). Before the fix: # ./seccomp_ret_trace_x0_bypass TAP version 13 1..1 not ok 1 write succeeded, orig_x0 bypass likely # Totals: pass:0 fail:1 xfail:0 xpass:0 skip:0 error:0 After the fix: # ./seccomp_ret_trace_x0_bypass TAP version 13 1..1 ok 1 seccomp correctly denied modified syscall # Totals: pass:1 fail:0 xfail:0 xpass:0 skip:0 error:0 Cc: Kees Cook Cc: Will Deacon Cc: Catalin Marinas Cc: Mark Rutland Link: https://lore.kernel.org/all/20260717182758.17111-1-will@kernel.org/ Link: https://lore.kernel.org/all/20260716120640.6590-1-will@kernel.org/ Link: https://lore.kernel.org/all/202607152004.DEA95D63@keescook/ Suggested-by: Kees Cook Signed-off-by: Jinjie Ruan Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/abi/.gitignore | 1 + tools/testing/selftests/arm64/abi/Makefile | 2 +- .../arm64/abi/seccomp_ret_trace_x0_bypass.c | 204 ++++++++++++++++++ 3 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c diff --git a/tools/testing/selftests/arm64/abi/.gitignore b/tools/testing/selftests/arm64/abi/.gitignore index 39129a9907c7..491a80db9dff 100644 --- a/tools/testing/selftests/arm64/abi/.gitignore +++ b/tools/testing/selftests/arm64/abi/.gitignore @@ -1,5 +1,6 @@ hwcap ptrace seccomp_ptrace_x0_bypass +seccomp_ret_trace_x0_bypass syscall-abi tpidr2 diff --git a/tools/testing/selftests/arm64/abi/Makefile b/tools/testing/selftests/arm64/abi/Makefile index 5a16db379bd4..a01d3806eba8 100644 --- a/tools/testing/selftests/arm64/abi/Makefile +++ b/tools/testing/selftests/arm64/abi/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright (C) 2021 ARM Limited -TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2 seccomp_ptrace_x0_bypass +TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2 seccomp_ptrace_x0_bypass seccomp_ret_trace_x0_bypass include ../../lib.mk diff --git a/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c b/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c new file mode 100644 index 000000000000..3a69f53f4f88 --- /dev/null +++ b/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test for SECCOMP_RET_TRACE argument modification bypass + * via stale orig_x0 during filter re-evaluation. + * + * On arm64, syscall_get_arguments() reads the first argument from + * regs->orig_x0. When a seccomp filter returns SECCOMP_RET_TRACE, + * ptrace may modify regs->regs[0] while orig_x0 remains unchanged. + * The kernel then re-evaluates the filter; if it sees the stale + * orig_x0, it may incorrectly allow a syscall that the tracer intended + * to block. + * + * This test installs a filter that: + * - TRACEs write() when fd == 2 + * - returns ERRNO(EPERM) when fd == 1 + * - allows all other syscalls + * + * The child calls write(2, ...). The parent catches the SECCOMP stop, + * changes x0 (fd) from 2 to 1, and resumes the child. + * + * If re-evaluation sees the old fd=2 (stale orig_x0), the filter + * returns TRACE again; because recheck_after_trace is true, the kernel + * allows the syscall to proceed. write(1, ...) succeeds, child exits 0. + * -> test FAIL (bypass detected). + * + * If re-evaluation sees the new fd=1 (synced orig_x0), the filter + * returns ERRNO(EPERM), write fails, child exits 1. + * -> test PASS (no bypass). + * + * No special privileges required beyond CAP_SYS_PTRACE. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kselftest.h" + +#ifndef __NR_write +#define __NR_write 64 +#endif + +#define PTRACE_EVENT_MASK(status) ((status) >> 16) + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define ARG0_OFFSET (offsetof(struct seccomp_data, args)) +#else +#define ARG0_OFFSET (offsetof(struct seccomp_data, args) + 4) +#endif + +static int do_child(void) +{ + long ret; + + if (ptrace(PTRACE_TRACEME, 0, NULL, NULL)) + _exit(2); + + raise(SIGSTOP); /* synchronize with parent */ + + /* + * Filter: + * if syscall == write: + * if fd == 2 -> TRACE + * if fd == 1 -> ERRNO(EPERM) + * else -> ALLOW + * else -> ALLOW + */ + struct sock_filter filter[] = { + /* Load syscall number */ + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + /* If not write, allow */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_write, 0, 5), + /* Load first argument (fd) */ + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, ARG0_OFFSET), + /* fd == 2 ? */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 2, 0, 1), + /* Yes: TRACE */ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_TRACE), + /* fd == 1 ? */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 1, 0, 1), + /* Yes: ERRNO(EPERM) */ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | (EPERM & SECCOMP_RET_DATA)), + /* Other fd: ALLOW */ + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + + struct sock_fprog prog = { + .len = ARRAY_SIZE(filter), + .filter = filter, + }; + + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) + _exit(3); + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &prog)) + _exit(4); + + /* + * write(2, ...) triggers TRACE, parent changes fd to 1. + * If re-eval sees fd=1 -> ERRNO -> write fails, ret = -EPERM. + * If re-eval sees fd=2 -> TRACE again -> allowed -> write succeeds. + */ + ret = syscall(__NR_write, 2, "", 0); + _exit(ret == 0 ? 0 : 1); +} + +int main(void) +{ + struct user_pt_regs regs; + struct iovec iov = { .iov_base = ®s, .iov_len = sizeof(regs) }; + pid_t child; + int status; + + ksft_print_header(); + ksft_set_plan(1); + + child = fork(); + if (child < 0) + ksft_exit_fail_msg("fork failed: %s", strerror(errno)); + + if (!child) + return do_child(); + + /* 1. Wait for initial SIGSTOP */ + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("waitpid SIGSTOP"); + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGSTOP) + ksft_exit_fail_msg("unexpected initial stop"); + + /* 2. Enable SECCOMP ptrace events */ + if (ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESECCOMP)) + ksft_exit_fail_msg("PTRACE_SETOPTIONS"); + + /* 3. Continue child to hit SECCOMP stop */ + if (ptrace(PTRACE_CONT, child, 0, 0)) + ksft_exit_fail_msg("PTRACE_CONT"); + + /* 4. Wait for SECCOMP stop */ + while (1) { + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("waitpid SECCOMP"); + if (WIFEXITED(status)) { + ksft_test_result_fail("child exited before SECCOMP stop\n"); + goto out; + } + if (WIFSIGNALED(status)) { + ksft_test_result_fail("child killed unexpectedly\n"); + goto out; + } + if (WIFSTOPPED(status) && + WSTOPSIG(status) == SIGTRAP && + PTRACE_EVENT_MASK(status) == PTRACE_EVENT_SECCOMP) + break; + ptrace(PTRACE_CONT, child, 0, WSTOPSIG(status)); + } + + /* 5. Modify x0 (fd) from 2 to 1 */ + if (ptrace(PTRACE_GETREGSET, child, NT_PRSTATUS, &iov)) + ksft_exit_fail_perror("GETREGSET"); + if (regs.regs[8] != __NR_write || regs.regs[0] != 2) { + ksft_test_result_fail("unexpected regs: syscall=%llu, x0=%llu\n", + regs.regs[8], regs.regs[0]); + goto out; + } + regs.regs[0] = 1; + if (ptrace(PTRACE_SETREGSET, child, NT_PRSTATUS, &iov)) + ksft_exit_fail_perror("SETREGSET"); + + /* 6. Resume child */ + if (ptrace(PTRACE_CONT, child, 0, 0)) + ksft_exit_fail_perror("PTRACE_CONT"); + + /* 7. Reap child – must exit normally */ + if (waitpid(child, &status, 0) != child) + ksft_exit_fail_msg("final waitpid"); + + if (!WIFEXITED(status)) { + ksft_test_result_fail("child did not exit normally\n"); + goto out; + } + + if (WEXITSTATUS(status) != 0) + ksft_test_result_pass("seccomp correctly denied modified syscall\n"); + else + ksft_test_result_fail("write succeeded, orig_x0 bypass likely\n"); + +out: + if (child > 0) { + kill(child, SIGKILL); + waitpid(child, NULL, 0); + } + ksft_print_cnts(); + return ksft_get_fail_cnt() ? EXIT_FAILURE : EXIT_SUCCESS; +} From 88b839ce497ccb1ff92f7ae742c78dd2937ba572 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 30 Jul 2026 14:26:50 +0100 Subject: [PATCH 47/95] arm64: ptrace: Keep 'orig_x0' in-sync with x0 on syscall entry Commit e057b9477232 ("arm64: syscall: Ensure saved x0 is kept in-sync with tracer updates") attempted to resolve a long-standing issue with syscall entry tracing, where a tracer is able to manipulate the first syscall argument without being subjected to seccomp or audit checking. Unfortunately, that fix was incomplete [1], as it failed to update 'orig_x0' between a tracer updating x0 during a seccomp ptrace exit (SECCOMP_RET_TRACE) and the seccomp filter being re-evaluated. Rather than add hooks to the core seccomp code, instead move the synchronisation code into the ptrace GPR and syscall setting code so that 'orig_x0' is kept up to date with x0 whenever we're stopped on the syscall entry path. Cc: Kees Cook Cc: Jinjie Ruan Cc: Mark Rutland Link: https://sashiko.dev/#/patchset/20260716120640.6590-1-will@kernel.org [1] Reported-by: Yiqi Sun Link: https://lore.kernel.org/all/20260529065444.1336608-1-sunyiqixm@gmail.com/ Fixes: e057b9477232 ("arm64: syscall: Ensure saved x0 is kept in-sync with tracer updates") Fixes: a5cd110cb836 ("arm64/ptrace: run seccomp after ptrace") Tested-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/kernel/ptrace.c | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index 2a72c61a8af9..4955166723b8 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -560,6 +560,42 @@ static int gpr_get(struct task_struct *target, return membuf_write(&to, uregs, sizeof(*uregs)); } +static void update_syscall_orig_x0_after_ptrace(struct task_struct *target) +{ + struct pt_regs *regs = task_pt_regs(target); + struct kernel_siginfo *info = target->last_siginfo; + + /* + * Skip the update for NO_SYSCALL (set either by the user or the + * tracer), as regs[0] holds the return value (see the comment in + * el0_svc_common()) and can be unwound using syscall_rollback(). + */ + if (regs->syscallno == NO_SYSCALL) + return; + + /* We should only be called when target is in a ptrace stop */ + if (WARN_ON_ONCE(!info)) + return; + + /* + * For compat tasks, orig_r0 is provided directly through GPR index + * 17. + */ + if (is_compat_thread(task_thread_info(target))) + return; + + /* + * Don't update orig_x0 for a syscall-exit-stop, as x0 now contains the + * return value of the system call. + */ + if ((info->si_code & ~0x80) == SIGTRAP && + target->ptrace_message == PTRACE_EVENTMSG_SYSCALL_EXIT) { + return; + } + + regs->orig_x0 = regs->regs[0]; +} + static int gpr_set(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void __user *ubuf) @@ -575,6 +611,14 @@ static int gpr_set(struct task_struct *target, const struct user_regset *regset, return -EINVAL; task_pt_regs(target)->user_regs = newregs; + + /* + * Keep orig_x0 authoritative so that seccomp (via + * syscall_get_arguments()), audit and the restart path all see the same + * first argument the syscall is dispatched with, even if it has been + * updated by a tracer. + */ + update_syscall_orig_x0_after_ptrace(target); return 0; } @@ -753,6 +797,12 @@ static int system_call_set(struct task_struct *target, return ret; task_pt_regs(target)->syscallno = syscallno; + + /* + * Re-sync orig_x0 in case the syscall number has been changed + * from NO_SYSCALL. + */ + update_syscall_orig_x0_after_ptrace(target); return ret; } From 5a87e8c7f3702eba8a1ed17431a807e01da3ae51 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 30 Jul 2026 14:26:51 +0100 Subject: [PATCH 48/95] arm64: syscall: Pass 'orig_x0' as first argument to native system call syscall_get_arguments() returns 'regs->orig_x0' for the first system call argument so as to avoid aliasing with the syscall return value in 'regs->regs[0]' on the return path, however the actual syscall invocation passes 'regs->regs[0]' as the first parameter. Although the two registers should be kept in sync during syscall entry for native tasks, pass 'regs->orig_x0' as the first syscall parameter for consistency with the syscall argument APIs. Compat tasks continue to use 'regs->regs[0]' for compatibility with the behaviour of the 32-bit kernel. Suggested-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/include/asm/syscall_wrapper.h | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/syscall_wrapper.h b/arch/arm64/include/asm/syscall_wrapper.h index abb57bc54305..395152ef5372 100644 --- a/arch/arm64/include/asm/syscall_wrapper.h +++ b/arch/arm64/include/asm/syscall_wrapper.h @@ -10,13 +10,13 @@ #include -#define SC_ARM64_REGS_TO_ARGS(x, ...) \ +#ifdef CONFIG_COMPAT + +#define COMPAT_SC_ARM64_REGS_TO_ARGS(x, ...) \ __MAP(x,__SC_ARGS \ ,,regs->regs[0],,regs->regs[1],,regs->regs[2] \ ,,regs->regs[3],,regs->regs[4],,regs->regs[5]) -#ifdef CONFIG_COMPAT - #define COMPAT_SYSCALL_DEFINEx(x, name, ...) \ asmlinkage long __arm64_compat_sys##name(const struct pt_regs *regs); \ ALLOW_ERROR_INJECTION(__arm64_compat_sys##name, ERRNO); \ @@ -24,7 +24,7 @@ static inline long __do_compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \ asmlinkage long __arm64_compat_sys##name(const struct pt_regs *regs) \ { \ - return __se_compat_sys##name(SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \ + return __se_compat_sys##name(COMPAT_SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \ } \ static long __se_compat_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ { \ @@ -46,6 +46,11 @@ #endif /* CONFIG_COMPAT */ +#define SC_ARM64_REGS_TO_ARGS(x, ...) \ + __MAP(x,__SC_ARGS \ + ,,regs->orig_x0,,regs->regs[1],,regs->regs[2] \ + ,,regs->regs[3],,regs->regs[4],,regs->regs[5]) + #define __SYSCALL_DEFINEx(x, name, ...) \ asmlinkage long __arm64_sys##name(const struct pt_regs *regs); \ ALLOW_ERROR_INJECTION(__arm64_sys##name, ERRNO); \ From 2b989c411ab98fa76b7bb3b87ba7a2a4c3b5e946 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Fri, 31 Jul 2026 21:50:52 +0100 Subject: [PATCH 49/95] kselftest/arm64: Don't write to P0 in irritator on SME only systems Commit 3e360ef0c0a1f ("kselftest/arm64: Corrupt P0 in the irritator when testing SSVE") added corruption of P0 to the sve-test case in order to ensure that the predicate registers were covered as part of the corruption. On SME only systems this results in an illegal instruction since signal handlers are run out of streaming mode and the predicate registers do not exist out of streaming mode without SVE. Switch to entering and exiting streaming mode in the irritator, this will reset all relevant registers to 0 if they somehow weren't already by the signal entry. Fixes: 3e360ef0c0a1f ("kselftest/arm64: Corrupt P0 in the irritator when testing SSVE") Reported-by: Mark Rutland Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/fp/sve-test.S | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/arm64/fp/sve-test.S b/tools/testing/selftests/arm64/fp/sve-test.S index 80e072f221cd..7ef7835389e7 100644 --- a/tools/testing/selftests/arm64/fp/sve-test.S +++ b/tools/testing/selftests/arm64/fp/sve-test.S @@ -298,15 +298,20 @@ function irritator_handler add x0, x0, #1 str x0, [x2, #ucontext_regs + 8 * 23] +#ifndef SSVE // Corrupt some random Z-regs movi v0.8b, #1 movi v9.16b, #2 movi v31.8b, #3 // And P0 ptrue p0.d -#ifndef SSVE // And FFR wrffr p15.b +#else + // Enter and exit streaming mode, will reset all of the V, Z, P + // and FFR registers that the system has. + smstart_sm + smstop #endif ret From ea434e8fd3a539e9c53285b10d3c7e539e228591 Mon Sep 17 00:00:00 2001 From: Davidlohr Bueso Date: Wed, 15 Jul 2026 12:14:52 -0700 Subject: [PATCH 50/95] perf/cxlpmu: Fix 64-bit write to 32-bit HDM filter register The HDM decoder filter configuration register is 32 bits wide, but the driver programs it with a 64-bit writeq(). The filter value never exceeds 32 bits, so the upper half of the write is always zero and lands in the adjacent Filter ID 1 (Channel/Rank/Bank) configuration register at offset+4. Fixes: 5d7107c72796 ("perf: CXL Performance Monitoring Unit driver") Signed-off-by: Davidlohr Bueso Reviewed-by: Richard Cheng Reviewed-by: Dave Jiang Signed-off-by: Will Deacon --- drivers/perf/cxl_pmu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 68a54d97d2a8..39b46550a510 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -635,7 +635,7 @@ static void cxl_pmu_event_start(struct perf_event *event, int flags) cfg = cxl_pmu_config2_get_hdm_decoder(event); else cfg = GENMASK(31, 0); /* No filtering if 0xFFFF_FFFF */ - writeq(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); + writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); } cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); From 22713779214890613f412674051d9d4d2b07c8af Mon Sep 17 00:00:00 2001 From: Harshal Thakkar Date: Wed, 15 Jul 2026 12:14:53 -0700 Subject: [PATCH 51/95] perf/cxlpmu: Add missing CXL 4.0 events Add support for CXL 4.0 events that are exposed by the CPMU hardware but not present in the driver. Such events are defined in Table 13-5 of the spec. Signed-off-by: Harshal Thakkar [davidlohr: add missing throttle and queue occupancy events] Signed-off-by: Davidlohr Bueso Reviewed-by: Richard Cheng Reviewed-by: Dave Jiang Signed-off-by: Will Deacon --- drivers/perf/cxl_pmu.c | 50 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 39b46550a510..1fc83858f653 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -77,6 +77,10 @@ #define CXL_PMU_GID_S2M_NDR 0x0024 #define CXL_PMU_GID_S2M_DRS 0x0025 #define CXL_PMU_GID_DDR 0x8000 +#define CXL_PMU_GID_QUEUE_OCC 0x8001 +#define CXL_PMU_GID_QUEUE_RESID 0x8002 +#define CXL_PMU_GID_RETRY_EVENTS 0x8003 +#define CXL_PMU_GID_THROTTLE 0x8004 static int cxl_pmu_cpuhp_state_num; @@ -385,13 +389,23 @@ static struct attribute *cxl_pmu_event_attrs[] = { CXL_PMU_EVENT_CXL_ATTR(m2s_req_memwrfwd, CXL_PMU_GID_M2S_REQ, BIT(4)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrdtee, CXL_PMU_GID_M2S_REQ, BIT(5)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_memrddatatee, CXL_PMU_GID_M2S_REQ, BIT(6)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvtee, CXL_PMU_GID_M2S_REQ, BIT(7)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrd, CXL_PMU_GID_M2S_REQ, BIT(8)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvnt, CXL_PMU_GID_M2S_REQ, BIT(9)), CXL_PMU_EVENT_CXL_ATTR(m2s_req_memcleanevict, CXL_PMU_GID_M2S_REQ, BIT(10)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_meminvptee, CXL_PMU_GID_M2S_REQ, BIT(11)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_memspecrdtee, CXL_PMU_GID_M2S_REQ, BIT(12)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_teupdate, CXL_PMU_GID_M2S_REQ, BIT(13)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevcttee, CXL_PMU_GID_M2S_REQ, BIT(14)), + CXL_PMU_EVENT_CXL_ATTR(m2s_req_memclnevctu, CXL_PMU_GID_M2S_REQ, BIT(15)), /* CXL rev 3.0 Table 3-35 M2S RwD Memory Opcodes */ CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwr, CXL_PMU_GID_M2S_RWD, BIT(1)), CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptl, CXL_PMU_GID_M2S_RWD, BIT(2)), CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_biconflict, CXL_PMU_GID_M2S_RWD, BIT(4)), + CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfill, CXL_PMU_GID_M2S_RWD, BIT(5)), + CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrtee, CXL_PMU_GID_M2S_RWD, BIT(9)), + CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memwrptltee, CXL_PMU_GID_M2S_RWD, BIT(10)), + CXL_PMU_EVENT_CXL_ATTR(m2s_rwd_memrdfilltee, CXL_PMU_GID_M2S_RWD, BIT(13)), /* CXL rev 3.0 Table 3-38 M2S BIRsp Memory Opcodes */ CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_i, CXL_PMU_GID_M2S_BIRSP, BIT(0)), CXL_PMU_EVENT_CXL_ATTR(m2s_birsp_s, CXL_PMU_GID_M2S_BIRSP, BIT(1)), @@ -406,15 +420,25 @@ static struct attribute *cxl_pmu_event_attrs[] = { CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblk, CXL_PMU_GID_S2M_BISNP, BIT(4)), CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datblk, CXL_PMU_GID_S2M_BISNP, BIT(5)), CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblk, CXL_PMU_GID_S2M_BISNP, BIT(6)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curtee, CXL_PMU_GID_S2M_BISNP, BIT(8)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datatee, CXL_PMU_GID_S2M_BISNP, BIT(9)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invtee, CXL_PMU_GID_S2M_BISNP, BIT(10)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_curblktee, CXL_PMU_GID_S2M_BISNP, BIT(12)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_datablktee, CXL_PMU_GID_S2M_BISNP, BIT(13)), + CXL_PMU_EVENT_CXL_ATTR(s2m_bisnp_invblktee, CXL_PMU_GID_S2M_BISNP, BIT(14)), /* CXL rev 3.1 Table 3-50 S2M NDR Opcodes */ CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmp, CXL_PMU_GID_S2M_NDR, BIT(0)), CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmps, CXL_PMU_GID_S2M_NDR, BIT(1)), CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpe, CXL_PMU_GID_S2M_NDR, BIT(2)), CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmpm, CXL_PMU_GID_S2M_NDR, BIT(3)), CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_biconflictack, CXL_PMU_GID_S2M_NDR, BIT(4)), + CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee, CXL_PMU_GID_S2M_NDR, BIT(5)), + CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_s, CXL_PMU_GID_S2M_NDR, BIT(6)), + CXL_PMU_EVENT_CXL_ATTR(s2m_ndr_cmptee_e, CXL_PMU_GID_S2M_NDR, BIT(7)), /* CXL rev 3.0 Table 3-46 S2M DRS opcodes */ CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdata, CXL_PMU_GID_S2M_DRS, BIT(0)), CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatanxm, CXL_PMU_GID_S2M_DRS, BIT(1)), + CXL_PMU_EVENT_CXL_ATTR(s2m_drs_memdatatee, CXL_PMU_GID_S2M_DRS, BIT(2)), /* CXL rev 3.0 Table 13-5 directly lists these */ CXL_PMU_EVENT_CXL_ATTR(ddr_act, CXL_PMU_GID_DDR, BIT(0)), CXL_PMU_EVENT_CXL_ATTR(ddr_pre, CXL_PMU_GID_DDR, BIT(1)), @@ -423,6 +447,32 @@ static struct attribute *cxl_pmu_event_attrs[] = { CXL_PMU_EVENT_CXL_ATTR(ddr_refresh, CXL_PMU_GID_DDR, BIT(4)), CXL_PMU_EVENT_CXL_ATTR(ddr_selfrefreshent, CXL_PMU_GID_DDR, BIT(5)), CXL_PMU_EVENT_CXL_ATTR(ddr_rfm, CXL_PMU_GID_DDR, BIT(6)), + /* CXL 4.0 Table 13-5 DDR add-on events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(ddr_cas_rd_ap, CXL_PMU_GID_DDR, BIT(7)), + CXL_PMU_EVENT_CXL_ATTR(ddr_cas_wr_ap, CXL_PMU_GID_DDR, BIT(8)), + CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_all_banks, CXL_PMU_GID_DDR, BIT(9)), + CXL_PMU_EVENT_CXL_ATTR(ddr_refresh_same_bank, CXL_PMU_GID_DDR, BIT(10)), + CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_entry, CXL_PMU_GID_DDR, BIT(11)), + CXL_PMU_EVENT_CXL_ATTR(ddr_pwrdn_exit, CXL_PMU_GID_DDR, BIT(12)), + CXL_PMU_EVENT_CXL_ATTR(ddr_rd_wr_ddr_bus_switching, CXL_PMU_GID_DDR, BIT(13)), + CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_rd_req, CXL_PMU_GID_DDR, BIT(14)), + CXL_PMU_EVENT_CXL_ATTR(ddr_incoming_wr_req, CXL_PMU_GID_DDR, BIT(15)), + /* CXL 4.0 Table 13-5 QUEUE OCCUPANCY events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(rd_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(0)), + CXL_PMU_EVENT_CXL_ATTR(wr_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(1)), + CXL_PMU_EVENT_CXL_ATTR(rd_wr_merged_queue_occ, CXL_PMU_GID_QUEUE_OCC, BIT(2)), + CXL_PMU_EVENT_CXL_ATTR(pwrdn_event, CXL_PMU_GID_QUEUE_OCC, BIT(3)), + /* CXL 4.0 Table 13-5 QUEUE RESIDENCY events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(mc_rd_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(0)), + CXL_PMU_EVENT_CXL_ATTR(mc_wr_resid_cnt, CXL_PMU_GID_QUEUE_RESID, BIT(1)), + /* CXL 4.0 Table 13-5 RETRY events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_rd_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(0)), + CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_wr_crc, CXL_PMU_GID_RETRY_EVENTS, BIT(1)), + CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ca_parity, CXL_PMU_GID_RETRY_EVENTS, BIT(2)), + CXL_PMU_EVENT_CXL_ATTR(retry_event_trig_by_ecc, CXL_PMU_GID_RETRY_EVENTS, BIT(3)), + /* CXL 4.0 Table 13-5 THROTTLE events opcodes */ + CXL_PMU_EVENT_CXL_ATTR(thermal_throttle_event, CXL_PMU_GID_THROTTLE, BIT(0)), + CXL_PMU_EVENT_CXL_ATTR(power_throttle_event, CXL_PMU_GID_THROTTLE, BIT(1)), NULL }; From 0263e0b788d5d91fa1539787a003c67456c88718 Mon Sep 17 00:00:00 2001 From: Harshal Thakkar Date: Wed, 15 Jul 2026 12:14:54 -0700 Subject: [PATCH 52/95] perf/cxlpmu: Support Channel/Rank/Bank filter Implement CRB filtering per CXL 4.0 8.2.7.2.2, and extend the current filtering support beyond HDM. CRB filtering is only permitted for the DDR Interface, Queue Occupancy, Queue Residency and Retry event groups (CXL 4.0 Table 13-5), and only when counting a single event (a single mask bit). Because these group IDs are scoped by the CXL vendor ID, events from other vendors are also rejected. For example, to count DDR activates on channel 2 only: perf stat -a -e cxl_pmu_mem0.0/ddr_act,crb_filter_en=1,crb=0x02FFFFFF/ Placing the 32-bit CRB value at config2:32-63 leaves the existing HDM value at config2:0-15 untouched and avoids needing a new config3. Signed-off-by: Harshal Thakkar [davidlohr: multiple fixes] Signed-off-by: Davidlohr Bueso Reviewed-by: Richard Cheng Reviewed-by: Dave Jiang Signed-off-by: Will Deacon --- drivers/perf/cxl_pmu.c | 67 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/drivers/perf/cxl_pmu.c b/drivers/perf/cxl_pmu.c index 1fc83858f653..bb32cd1084e9 100644 --- a/drivers/perf/cxl_pmu.c +++ b/drivers/perf/cxl_pmu.c @@ -110,6 +110,7 @@ struct cxl_pmu_info { int on_cpu; struct hlist_node node; bool filter_hdm; + bool filter_crb; int irq; }; @@ -146,6 +147,8 @@ static int cxl_pmu_parse_caps(struct device *dev, struct cxl_pmu_info *info) info->num_event_capabilities = FIELD_GET(CXL_PMU_CAP_NUM_EVN_CAP_REG_SUP_MSK, val) + 1; info->filter_hdm = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & CXL_PMU_FILTER_HDM; + info->filter_crb = FIELD_GET(CXL_PMU_CAP_FILTERS_SUP_MSK, val) & + CXL_PMU_FILTER_CHAN_RANK_BANK; if (FIELD_GET(CXL_PMU_CAP_INT, val)) info->irq = FIELD_GET(CXL_PMU_CAP_MSI_N_MSK, val); else @@ -229,6 +232,8 @@ enum { cxl_pmu_edge_attr, cxl_pmu_hdm_filter_en_attr, cxl_pmu_hdm_attr, + cxl_pmu_crb_filter_en_attr, + cxl_pmu_crb_attr, }; static struct attribute *cxl_pmu_format_attr[] = { @@ -240,6 +245,8 @@ static struct attribute *cxl_pmu_format_attr[] = { [cxl_pmu_edge_attr] = CXL_PMU_FORMAT_ATTR(edge, "config1:17"), [cxl_pmu_hdm_filter_en_attr] = CXL_PMU_FORMAT_ATTR(hdm_filter_en, "config1:18"), [cxl_pmu_hdm_attr] = CXL_PMU_FORMAT_ATTR(hdm, "config2:0-15"), + [cxl_pmu_crb_filter_en_attr] = CXL_PMU_FORMAT_ATTR(crb_filter_en, "config1:19"), + [cxl_pmu_crb_attr] = CXL_PMU_FORMAT_ATTR(crb, "config2:32-63"), NULL }; @@ -250,7 +257,9 @@ static struct attribute *cxl_pmu_format_attr[] = { #define CXL_PMU_ATTR_CONFIG1_INVERT_MSK BIT(16) #define CXL_PMU_ATTR_CONFIG1_EDGE_MSK BIT(17) #define CXL_PMU_ATTR_CONFIG1_FILTER_EN_MSK BIT(18) +#define CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK BIT(19) #define CXL_PMU_ATTR_CONFIG2_HDM_MSK GENMASK(15, 0) +#define CXL_PMU_ATTR_CONFIG2_CRB_MSK GENMASK_ULL(63, 32) static umode_t cxl_pmu_format_is_visible(struct kobject *kobj, struct attribute *attr, int a) @@ -267,6 +276,11 @@ static umode_t cxl_pmu_format_is_visible(struct kobject *kobj, attr == cxl_pmu_format_attr[cxl_pmu_hdm_attr])) return 0; + if (!info->filter_crb && + (attr == cxl_pmu_format_attr[cxl_pmu_crb_filter_en_attr] || + attr == cxl_pmu_format_attr[cxl_pmu_crb_attr])) + return 0; + return attr->mode; } @@ -323,6 +337,17 @@ static u16 cxl_pmu_config2_get_hdm_decoder(struct perf_event *event) return FIELD_GET(CXL_PMU_ATTR_CONFIG2_HDM_MSK, event->attr.config2); } +static u16 cxl_pmu_config1_crb_filter_en(struct perf_event *event) +{ + return FIELD_GET(CXL_PMU_ATTR_CONFIG1_CRB_FILTER_EN_MSK, + event->attr.config1); +} + +static u32 cxl_pmu_config2_get_crb(struct perf_event *event) +{ + return FIELD_GET(CXL_PMU_ATTR_CONFIG2_CRB_MSK, event->attr.config2); +} + static ssize_t cxl_pmu_event_sysfs_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -621,6 +646,36 @@ static int cxl_pmu_event_init(struct perf_event *event) return -EOPNOTSUPP; /* TODO: Validation of any filter */ + if (cxl_pmu_config1_crb_filter_en(event)) { + if (!info->filter_crb) + return -EINVAL; + /* event group IDs are scoped by the CXL vendor ID */ + if (cxl_pmu_config_get_vid(event) != PCI_VENDOR_ID_CXL) + return -EINVAL; + + /* + * CRB filtering (Filter ID 1) is only valid for the DDR + * Interface, Queue Occupancy, Queue Residency and Retry + * event groups (CXL 4.0 Table 13-5). + */ + switch (cxl_pmu_config_get_gid(event)) { + case CXL_PMU_GID_DDR: + case CXL_PMU_GID_QUEUE_OCC: + case CXL_PMU_GID_QUEUE_RESID: + case CXL_PMU_GID_RETRY_EVENTS: + break; + default: + return -EINVAL; + } + + /* + * Filtering while counting multiple events is + * undefined behavior. + */ + if (hweight32(cxl_pmu_config_get_mask(event)) > 1) + return -EINVAL; + } + /* * Verify that it is possible to count what was requested. Either must * be a fixed counter that is a precise match or a configurable counter @@ -677,8 +732,8 @@ static void cxl_pmu_event_start(struct perf_event *event, int flags) hwc->state = 0; /* - * Currently only hdm filter control is implemented, this code will - * want generalizing when more filters are added. + * Filter ID=0: HDM decoder filter + * Filter ID=1: Channel/Rank/Bank (CRB) filter */ if (info->filter_hdm) { if (cxl_pmu_config1_hdm_filter_en(event)) @@ -688,6 +743,14 @@ static void cxl_pmu_event_start(struct perf_event *event, int flags) writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 0)); } + if (info->filter_crb) { + if (cxl_pmu_config1_crb_filter_en(event)) + cfg = cxl_pmu_config2_get_crb(event); + else + cfg = GENMASK(31, 0); /* no filtering if 0xFFFF_FFFF */ + writel(cfg, base + CXL_PMU_FILTER_CFG_REG(hwc->idx, 1)); + } + cfg = readq(base + CXL_PMU_COUNTER_CFG_REG(hwc->idx)); cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_INT_ON_OVRFLW, 1); cfg |= FIELD_PREP(CXL_PMU_COUNTER_CFG_FREEZE_ON_OVRFLW, 1); From 04aa909b38bfe9a7e9b052f2c66b98d33f6f8a64 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 16 Jul 2026 15:56:34 +0100 Subject: [PATCH 53/95] ACPI/APMT: Use stable device ID The APMT node format includes a unique identifier, so we can use this as the platform device ID to give userspace stable and identifiable device names, rather than auto numbering dependent on how the table is parsed. Cc: Lorenzo Pieralisi Cc: Hanjun Guo Cc: Sudeep Holla Reviewed-by: Ilkka Koskinen Signed-off-by: Robin Murphy Reviewed-by: Hanjun Guo Signed-off-by: Will Deacon --- drivers/acpi/arm64/apmt.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/acpi/arm64/apmt.c b/drivers/acpi/arm64/apmt.c index bb010f6164e5..91fcdd289e63 100644 --- a/drivers/acpi/arm64/apmt.c +++ b/drivers/acpi/arm64/apmt.c @@ -76,10 +76,12 @@ static int __init apmt_add_platform_device(struct acpi_apmt_node *node, struct fwnode_handle *fwnode) { struct platform_device *pdev; - int ret, count; + int ret, count, uid = node->id & INT_MAX; struct resource res[DEV_MAX_RESOURCE_COUNT]; - pdev = platform_device_alloc(DEV_NAME, PLATFORM_DEVID_AUTO); + if (uid != node->id) + pr_warn("Unexpectedly large UID 0x%x, truncated to 0x%x\n", node->id, uid); + pdev = platform_device_alloc(DEV_NAME, uid); if (!pdev) return -ENOMEM; From 745a724f9c81abd3c37c2d07daa855bd87e88120 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 16 Jul 2026 15:56:35 +0100 Subject: [PATCH 54/95] perf/arm_cspmu: Improve APMT-based PMU naming On ACPI systems, it has not actually been possible for userspace to reliably tell which PMU corresponds to which APMT entry for types other than "ACPI device" - the evidence trail only leads from the arbitrarily-numbered PMU device to its arbitrarily-numbered parent platform device that has no distinguishing features either. While we've now improved the platform device creation to associate the actual APMT unique ID, we may as well also tweak the PMU devices to substitute the arbitrary number with a different arbitrary number that might be more directly meaningful based on the APMT definitions. We don't have an equivalent for Devicetree, but in that case the platform devices are at least identifiable via their sysfs-visible of_node. Signed-off-by: Robin Murphy Reviewed-by: Ilkka Koskinen [will: Remove unneeded semicolon reported by coccinelle] Signed-off-by: Will Deacon --- drivers/perf/arm_cspmu/arm_cspmu.c | 33 +++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index 80fb314d5135..f4f071c79263 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -250,38 +250,43 @@ static const char *arm_cspmu_get_name(const struct arm_cspmu *cspmu) struct device *dev; struct acpi_apmt_node *apmt_node; u8 pmu_type; - char *name; char acpi_hid_string[ACPI_ID_LEN] = { 0 }; - static atomic_t pmu_idx[ACPI_APMT_NODE_TYPE_COUNT] = { 0 }; + static atomic_t pmu_idx; + u32 id; dev = cspmu->dev; apmt_node = arm_cspmu_apmt_node(dev); if (!apmt_node) return devm_kasprintf(dev, GFP_KERNEL, PMUNAME "_%u", - atomic_fetch_inc(&pmu_idx[0])); + atomic_fetch_inc(&pmu_idx)); pmu_type = apmt_node->type; - - if (pmu_type >= ACPI_APMT_NODE_TYPE_COUNT) { + switch (pmu_type) { + default: dev_err(dev, "unsupported PMU type-%u\n", pmu_type); return NULL; - } - - if (pmu_type == ACPI_APMT_NODE_TYPE_ACPI) { + case ACPI_APMT_NODE_TYPE_ACPI: memcpy(acpi_hid_string, &apmt_node->inst_primary, sizeof(apmt_node->inst_primary)); - name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME, + return devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME, arm_cspmu_type_str[pmu_type], acpi_hid_string, apmt_node->inst_secondary); - } else { - name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%d", PMUNAME, - arm_cspmu_type_str[pmu_type], - atomic_fetch_inc(&pmu_idx[pmu_type])); + case ACPI_APMT_NODE_TYPE_MC: + id = apmt_node->id; + break; + case ACPI_APMT_NODE_TYPE_SMMU: + case ACPI_APMT_NODE_TYPE_PCIE_ROOT: + id = apmt_node->inst_primary; + break; + case ACPI_APMT_NODE_TYPE_CACHE: + id = apmt_node->inst_secondary; + break; } - return name; + return devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%u", PMUNAME, + arm_cspmu_type_str[pmu_type], id); } static ssize_t arm_cspmu_cpumask_show(struct device *dev, From 60b234db6bf127ad0ab831c0fddb511e9bf63fe5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 16 Jul 2026 15:56:36 +0100 Subject: [PATCH 55/95] perf/arm_cspmu: Improve sub-module error reporting When waiting for a sub-module to register, we return a bare -EPROBE_DEFER that ends up showing the end user: platform arm-cs-arch-pmu.1: deferred probe pending (no reason) wherein it's not necessarily clear that they might need to take some action to ensure the appropriate module is available to load. Let's use dev_err_probe() here so we can show exactly what we're waiting for. Similarly, in the case where something's gone horribly wrong with an already-registered module, we can use dev_WARN() to standardise the device/driver attribution rather than just open-coding "arm_cspmu". Reviewed-by: Ilkka Koskinen Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/perf/arm_cspmu/arm_cspmu.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index f4f071c79263..d641119c2df4 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -437,13 +437,15 @@ static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu) if (ret) module_put(match->module); } else { - WARN(1, "arm_cspmu failed to get module: %s\n", + dev_WARN(cspmu->dev, "Failed to get module: %s\n", match->module_name); ret = -EINVAL; } } else { request_module_nowait(match->module_name); - ret = -EPROBE_DEFER; + ret = dev_err_probe(cspmu->dev, -EPROBE_DEFER, + "Waiting for module %s to load\n", + match->module_name); } mutex_unlock(&arm_cspmu_lock); From 36e200f0199192c48e563bd365fe6e88e4cb0862 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 16 Jul 2026 15:56:37 +0100 Subject: [PATCH 56/95] perf/arm_cspmu: Make IRQ more optional If we have 64-bit counters, we can reasonably assume we'll never have to handle an overflow before the end of the universe (since we're a system PMU with no sampling). Thus even if firmware does specify an IRQ, we can still continue in the event of being unable to request it. This can help systems where IRQs cannot be claimed exclusively, or are broken in other ways. Reviewed-by: Ilkka Koskinen Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/perf/arm_cspmu/arm_cspmu.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index d641119c2df4..1074225ff450 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -1256,8 +1256,11 @@ static int arm_cspmu_device_probe(struct platform_device *pdev) return ret; ret = arm_cspmu_request_irq(cspmu); - if (ret) - return ret; + if (ret) { + if (counter_size(cspmu) < 64) + return ret; + dev_info(cspmu->dev, "Continuing without IRQ\n"); + } ret = arm_cspmu_get_cpus(cspmu); if (ret) From e445816a488a66ea75f228f3b9b451aeff953250 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 25 Jul 2026 17:26:27 +0100 Subject: [PATCH 57/95] perf: arm_pmu_acpi: Get rid of the edge-triggered interrupt oddity The ACPI spec bizarrely indicates that the PMU interrupt can be edge-triggered, which contradicts the very basics of the PMU architecture (SW is required to clear the interrupt condition for the level to drop). Remove the code parsing this flag and always flag the interrupt as level triggered, no matter what firmware says. Signed-off-by: Marc Zyngier Reviewed-by: Zenghui Yu Signed-off-by: Will Deacon --- drivers/perf/arm_pmu_acpi.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/perf/arm_pmu_acpi.c b/drivers/perf/arm_pmu_acpi.c index e80f76d95e68..ca6bf8f86cfe 100644 --- a/drivers/perf/arm_pmu_acpi.c +++ b/drivers/perf/arm_pmu_acpi.c @@ -22,7 +22,7 @@ static DEFINE_PER_CPU(int, pmu_irqs); static int arm_pmu_acpi_register_irq(int cpu) { struct acpi_madt_generic_interrupt *gicc; - int gsi, trigger; + int gsi; gicc = acpi_cpu_get_madt_gicc(cpu); @@ -38,11 +38,6 @@ static int arm_pmu_acpi_register_irq(int cpu) if (!gsi) return 0; - if (gicc->flags & ACPI_MADT_PERFORMANCE_IRQ_MODE) - trigger = ACPI_EDGE_SENSITIVE; - else - trigger = ACPI_LEVEL_SENSITIVE; - /* * Helpfully, the MADT GICC doesn't have a polarity flag for the * "performance interrupt". Luckily, on compliant GICs the polarity is @@ -53,8 +48,12 @@ static int arm_pmu_acpi_register_irq(int cpu) * may not match the real polarity, but that should not matter. * * Other interrupt controllers are not supported with ACPI. + * + * The spec also indicates that the PMU interrupt can be edge + * triggered, which doesn't make any sense (SW needs to clear the + * interrupt condition for the level to drop). Ignore the silly flag. */ - return acpi_register_gsi(NULL, gsi, trigger, ACPI_ACTIVE_HIGH); + return acpi_register_gsi(NULL, gsi, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_HIGH); } static void arm_pmu_acpi_unregister_irq(int cpu) From a2d0c34f6dc0b00f677cbe2c27d19b55bfa0c541 Mon Sep 17 00:00:00 2001 From: Pan Chuang Date: Fri, 17 Jul 2026 18:31:17 +0800 Subject: [PATCH 58/95] perf: Remove redundant dev_err()/dev_err_probe() Since commit 55b48e23f5c4 ("genirq/devres: Add error handling in devm_request_*_irq()"), devm_request_irq() automatically logs detailed error messages on failure. Remove the now-redundant driver-specific dev_err() and dev_err_probe() calls. Signed-off-by: Pan Chuang Reviewed-by: Shuai Xue Acked-by: Frank Li Reviewed-by: Xu Yang Signed-off-by: Will Deacon --- drivers/perf/alibaba_uncore_drw_pmu.c | 5 +---- drivers/perf/fsl_imx8_ddr_perf.c | 4 +--- drivers/perf/fsl_imx9_ddr_perf.c | 4 +--- drivers/perf/fujitsu_uncore_pmu.c | 2 +- drivers/perf/qcom_l2_pmu.c | 5 +---- drivers/perf/qcom_l3_pmu.c | 5 +---- drivers/perf/starfive_starlink_pmu.c | 2 +- drivers/perf/xgene_pmu.c | 4 +--- 8 files changed, 8 insertions(+), 23 deletions(-) diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c index ac49d3b2dad6..de77da83df2e 100644 --- a/drivers/perf/alibaba_uncore_drw_pmu.c +++ b/drivers/perf/alibaba_uncore_drw_pmu.c @@ -449,11 +449,8 @@ static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device */ ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr, IRQF_SHARED, dev_name(&pdev->dev), irq); - if (ret < 0) { - dev_err(&pdev->dev, - "Fail to request IRQ:%d ret:%d\n", irq_num, ret); + if (ret < 0) goto out_free; - } ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu)); if (ret) diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c index bcdf5575d71c..6d1e99abf110 100644 --- a/drivers/perf/fsl_imx8_ddr_perf.c +++ b/drivers/perf/fsl_imx8_ddr_perf.c @@ -858,10 +858,8 @@ static int ddr_perf_probe(struct platform_device *pdev) IRQF_NOBALANCING | IRQF_NO_THREAD, DDR_CPUHP_CB_NAME, pmu); - if (ret < 0) { - dev_err(&pdev->dev, "Request irq failed: %d", ret); + if (ret < 0) goto ddr_perf_err; - } pmu->irq = irq; ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)); diff --git a/drivers/perf/fsl_imx9_ddr_perf.c b/drivers/perf/fsl_imx9_ddr_perf.c index 7050b48c0467..9dbc3187ef52 100644 --- a/drivers/perf/fsl_imx9_ddr_perf.c +++ b/drivers/perf/fsl_imx9_ddr_perf.c @@ -830,10 +830,8 @@ static int ddr_perf_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, irq, ddr_perf_irq_handler, IRQF_NOBALANCING | IRQF_NO_THREAD, DDR_CPUHP_CB_NAME, pmu); - if (ret < 0) { - dev_err(&pdev->dev, "Request irq failed: %d", ret); + if (ret < 0) goto ddr_perf_err; - } pmu->irq = irq; ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)); diff --git a/drivers/perf/fujitsu_uncore_pmu.c b/drivers/perf/fujitsu_uncore_pmu.c index aeeb68c66e1e..8c87d91e64b5 100644 --- a/drivers/perf/fujitsu_uncore_pmu.c +++ b/drivers/perf/fujitsu_uncore_pmu.c @@ -526,7 +526,7 @@ static int fujitsu_uncore_pmu_probe(struct platform_device *pdev) IRQF_NOBALANCING | IRQF_NO_THREAD, name, uncorepmu); if (ret) - return dev_err_probe(dev, ret, "Failed to request IRQ:%d\n", irq); + return ret; ret = irq_set_affinity(irq, cpumask_of(uncorepmu->cpu)); if (ret) diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c index ea8c85729937..57a28ddb9b0e 100644 --- a/drivers/perf/qcom_l2_pmu.c +++ b/drivers/perf/qcom_l2_pmu.c @@ -869,11 +869,8 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN, "l2-cache-pmu", cluster); - if (err) { - dev_err(&pdev->dev, - "Unable to request IRQ%d for L2 PMU counters\n", irq); + if (err) return err; - } dev_info(&pdev->dev, "Registered L2 cache PMU cluster %lld\n", fw_cluster_id); diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c index 66e6cabd6fff..5897ec5560fc 100644 --- a/drivers/perf/qcom_l3_pmu.c +++ b/drivers/perf/qcom_l3_pmu.c @@ -767,11 +767,8 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev) ret = devm_request_irq(&pdev->dev, ret, qcom_l3_cache__handle_irq, 0, name, l3pmu); - if (ret) { - dev_err(&pdev->dev, "Request for IRQ failed for slice @%pa\n", - &memrc->start); + if (ret) return ret; - } /* Add this instance to the list used by the offline callback */ ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_QCOM_L3_ONLINE, &l3pmu->node); diff --git a/drivers/perf/starfive_starlink_pmu.c b/drivers/perf/starfive_starlink_pmu.c index b1c7dc4869bd..57b73575c43f 100644 --- a/drivers/perf/starfive_starlink_pmu.c +++ b/drivers/perf/starfive_starlink_pmu.c @@ -435,7 +435,7 @@ static int starlink_setup_irqs(struct starlink_pmu *starlink_pmu, ret = devm_request_irq(&pdev->dev, irq, starlink_pmu_handle_irq, 0, STARLINK_PMU_PDEV_NAME, starlink_pmu); if (ret) - return dev_err_probe(&pdev->dev, ret, "Failed to request IRQ\n"); + return ret; starlink_pmu->irq = irq; diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c index 33b5497bdc06..b79409c750e6 100644 --- a/drivers/perf/xgene_pmu.c +++ b/drivers/perf/xgene_pmu.c @@ -1876,10 +1876,8 @@ static int xgene_pmu_probe(struct platform_device *pdev) rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr, IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(&pdev->dev), xgene_pmu); - if (rc) { - dev_err(&pdev->dev, "Could not request IRQ %d\n", irq); + if (rc) return rc; - } xgene_pmu->irq = irq; From 3bee2a4d210b73b25af5cac70a466c6e962d6527 Mon Sep 17 00:00:00 2001 From: Pan Chuang Date: Fri, 17 Jul 2026 18:31:18 +0800 Subject: [PATCH 59/95] perf: arm_cspmu: Remove redundant dev_err() Since commit 55b48e23f5c4 ("genirq/devres: Add error handling in devm_request_*_irq()"), devm_request_irq() automatically logs detailed error messages on failure. Remove the now-redundant driver-specific dev_err() calls. Signed-off-by: Pan Chuang Reviewed-by: Ilkka Koskinen Signed-off-by: Will Deacon --- drivers/perf/arm_cspmu/arm_cspmu.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index 1074225ff450..a83810c554eb 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -1077,10 +1077,8 @@ static int arm_cspmu_request_irq(struct arm_cspmu *cspmu) ret = devm_request_irq(dev, irq, arm_cspmu_handle_irq, IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(dev), cspmu); - if (ret) { - dev_err(dev, "Could not request IRQ %d\n", irq); + if (ret) return ret; - } cspmu->irq = irq; From 5f254ed7149e5fc2d02d1fcea48baf22c82a3ff8 Mon Sep 17 00:00:00 2001 From: Pan Chuang Date: Fri, 17 Jul 2026 18:31:19 +0800 Subject: [PATCH 60/95] drivers/perf: hisi: Remove redundant dev_err()/dev_err_probe() Since commit 55b48e23f5c4 ("genirq/devres: Add error handling in devm_request_*_irq()"), devm_request_irq() automatically logs detailed error messages on failure. Remove the now-redundant driver-specific dev_err() and dev_err_probe() calls. Signed-off-by: Pan Chuang Acked-by: Yushan Wang Signed-off-by: Will Deacon --- drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c | 3 +-- drivers/perf/hisilicon/hisi_uncore_pmu.c | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c index f963e4f9e552..56a88fb0d3c2 100644 --- a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c @@ -604,8 +604,7 @@ static int hisi_l3c_pmu_init_ext(struct hisi_pmu *l3c_pmu, struct platform_devic IRQF_NOBALANCING | IRQF_NO_THREAD, irqname, l3c_pmu); if (ret < 0) - return dev_err_probe(&pdev->dev, ret, - "Fail to request EXT IRQ: %d.\n", irq); + return ret; hisi_l3c_pmu->ext_irq[i] = irq; } diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pmu.c index de71dcf11653..f6f5b4470efe 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_pmu.c @@ -192,11 +192,8 @@ int hisi_uncore_pmu_init_irq(struct hisi_pmu *hisi_pmu, ret = devm_request_irq(&pdev->dev, irq, hisi_uncore_pmu_isr, IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(&pdev->dev), hisi_pmu); - if (ret < 0) { - dev_err(&pdev->dev, - "Fail to request IRQ: %d ret: %d.\n", irq, ret); + if (ret < 0) return ret; - } hisi_pmu->irq = irq; From 71dae04644e19bddda9e06d8e32849e0618f9cc9 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 3 Aug 2026 17:42:30 +0100 Subject: [PATCH 61/95] MAINTAINERS: arm64: Add Mark Rutland as an official Reviewer Mark has been part of the arm64 furniture for as long as the code has existed in mainline and is consistently one of the most active and knowledgeable reviewers that we have. Make it official. Acked-by: Ard Biesheuvel Acked-by: Mark Rutland Acked-by: Catalin Marinas Acked-by: Marc Zyngier Signed-off-by: Will Deacon --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 806bd2d80d15..08328eb1064d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3913,6 +3913,7 @@ F: drivers/platform/arm64/ ARM64 PORT (AARCH64 ARCHITECTURE) M: Catalin Marinas M: Will Deacon +R: Mark Rutland L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git From 807b9ecead03712f0cec2525e90fe75a8bb4e3ee Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 31 Jul 2026 16:22:58 +0100 Subject: [PATCH 62/95] perf/arm_cspmu: Support 64-bit programmers' model The 64-bit Programmers' model extension, now named FEAT_CSPMU_EXT64, makes all the non-counter registers 64-bit as well, of which we most need to care PMEVFILTR, PMEVTYPER, and PMCR since it changes location. Our event config fields are ready for this internally, but we need a few more tweaks to propagate 64-bit values properly from end to end. Since 64-bit events leave no room for our existing "cycles" encoding, and we don't really expect to see implementations of it anyway, we deliberately omit generic support for the architectural fixed cycle counter in this case to save some bother. At worst, vendor modules can still implement their own encoding if they really want to. Signed-off-by: Robin Murphy Reviewed-by: Ilkka Koskinen Signed-off-by: Will Deacon --- drivers/perf/arm_cspmu/arm_cspmu.c | 121 +++++++++++++++++++++----- drivers/perf/arm_cspmu/arm_cspmu.h | 25 +++--- drivers/perf/arm_cspmu/nvidia_cspmu.c | 2 +- 3 files changed, 114 insertions(+), 34 deletions(-) diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index a83810c554eb..60005f89892d 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -78,13 +78,13 @@ static struct acpi_apmt_node *arm_cspmu_apmt_node(struct device *dev) } /* - * In CoreSight PMU architecture, all of the MMIO registers are 32-bit except - * counter register. The counter register can be implemented as 32-bit or 64-bit - * register depending on the value of PMCFGR.SIZE field. For 64-bit access, - * single-copy 64-bit atomic support is implementation defined. APMT node flag - * is used to identify if the PMU supports 64-bit single copy atomic. If 64-bit - * single copy atomic is not supported, the driver treats the register as a pair - * of 32-bit register. + * With FEAT_CSPMU_EXT32, all of the MMIO registers are 32-bit except the + * counter registers, which are either 32-bit or 64-bit depending on the value + * of PMCFGR.SIZE. It is implementation-defined whether single-copy-atomic + * 64-bit accesses are supported, so we rely on a firmware flag to identify + * that, and otherwise treat a 64-bit counter as a non-atomic pair of 32-bit + * registers. With FEAT_CSPMU_EXT64, everything is 64-bit, but we may still + * have to deal with atomicity being broken. */ /* @@ -173,13 +173,30 @@ arm_cspmu_event_attr_is_visible(struct kobject *kobj, eattr = container_of(attr, typeof(*eattr), attr.attr); /* Hide cycle event if not supported */ - if (!supports_cycle_counter(cspmu) && + if ((cspmu->has_ext64 || !supports_cycle_counter(cspmu)) && eattr->id == ARM_CSPMU_EVT_CYCLES_DEFAULT) return 0; return attr->mode; } +ssize_t arm_cspmu_default_format_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct perf_pmu_events_attr *fmt = container_of(attr, typeof(*fmt), attr); + struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev)); + u64 field = cspmu->has_ext64 ? U64_MAX : U32_MAX; + DECLARE_BITMAP(bits, 64) = { BITMAP_FROM_U64(field) }; + + if (!fmt->id) { + set_bit(32, bits); /* For 32-bit "cycles" event */ + return sysfs_emit(buf, "config:%*pbl\n", 64, bits); + } + + return sysfs_emit(buf, "config%lld:%*pbl\n", fmt->id, 64, bits); +} +EXPORT_SYMBOL_GPL(arm_cspmu_default_format_show); + static struct attribute *arm_cspmu_format_attrs[] = { ARM_CSPMU_FORMAT_EVENT_ATTR, ARM_CSPMU_FORMAT_FILTER_ATTR, @@ -198,9 +215,9 @@ arm_cspmu_get_format_attrs(const struct arm_cspmu *cspmu) return attrs; } -static u32 arm_cspmu_event_type(const struct perf_event *event) +static u64 arm_cspmu_event_type(const struct perf_event *event) { - return event->attr.config & ARM_CSPMU_EVENT_MASK; + return event->attr.config; } static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event) @@ -208,6 +225,16 @@ static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event) return (event->attr.config == ARM_CSPMU_EVT_CYCLES_DEFAULT); } +static u64 arm_cspmu_filter(const struct perf_event *event) +{ + return event->attr.config1; +} + +static u64 arm_cspmu_filter2(const struct perf_event *event) +{ + return event->attr.config2; +} + static ssize_t arm_cspmu_identifier_show(struct device *dev, struct device_attribute *attr, char *page) @@ -417,6 +444,17 @@ static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu) DEFAULT_IMPL_OP(event_attr_is_visible), }; + /* + * With 64-bit events, since our default "cycles" encoding won't work, + * and the architecture recommends against implementing it anyway, we + * choose to effectively ignore FEAT_CSPMU_CCNTR, unless a vendor + * module really wants to provide its own encoding and ops. + */ + if (cspmu->has_ext64) { + cspmu->impl.ops.is_cycle_counter_event = NULL; + cspmu->impl.ops.set_cc_filter = NULL; + } + /* Firmware may override implementer/product ID from PMIIDR */ if (apmt_node && apmt_node->impl_id) cspmu->impl.pmiidr = apmt_node->impl_id; @@ -518,19 +556,24 @@ static int arm_cspmu_alloc_attr_groups(struct arm_cspmu *cspmu) return 0; } +static inline int arm_cspmu_pmcr(struct arm_cspmu *cspmu) +{ + return cspmu->has_ext64 ? PMCR_64 : PMCR; +} + static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu) { - writel(PMCR_C | PMCR_P, cspmu->base0 + PMCR); + writel(PMCR_C | PMCR_P, cspmu->base0 + arm_cspmu_pmcr(cspmu)); } static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu) { - writel(PMCR_E, cspmu->base0 + PMCR); + writel(PMCR_E, cspmu->base0 + arm_cspmu_pmcr(cspmu)); } static inline void arm_cspmu_stop_counters(struct arm_cspmu *cspmu) { - writel(0, cspmu->base0 + PMCR); + writel(0, cspmu->base0 + arm_cspmu_pmcr(cspmu)); } static void arm_cspmu_enable(struct pmu *pmu) @@ -561,7 +604,8 @@ static int arm_cspmu_get_event_idx(struct arm_cspmu_hw_events *hw_events, struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu); if (supports_cycle_counter(cspmu)) { - if (cspmu->impl.ops.is_cycle_counter_event(event)) { + if (cspmu->impl.ops.is_cycle_counter_event && + cspmu->impl.ops.is_cycle_counter_event(event)) { /* Search for available cycle counter. */ if (test_and_set_bit(cspmu->cycle_counter_logical_idx, hw_events->used_ctrs)) @@ -804,26 +848,33 @@ static void arm_cspmu_event_update(struct perf_event *event) static inline void arm_cspmu_set_event(struct arm_cspmu *cspmu, struct hw_perf_event *hwc) { - u32 offset = PMEVTYPER + (4 * hwc->idx); - - writel(hwc->config, cspmu->base0 + offset); + if (cspmu->has_ext64) + writeq(hwc->config, cspmu->base0 + PMEVTYPER + (8 * hwc->idx)); + else + writel(hwc->config, cspmu->base0 + PMEVTYPER + (4 * hwc->idx)); } static void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu, const struct perf_event *event) { - u32 filter = event->attr.config1 & ARM_CSPMU_FILTER_MASK; - u32 filter2 = event->attr.config2 & ARM_CSPMU_FILTER_MASK; - u32 offset = 4 * event->hw.idx; + u64 filter = arm_cspmu_filter(event); + u64 filter2 = arm_cspmu_filter2(event); + int n = event->hw.idx; - writel(filter, cspmu->base0 + PMEVFILTR + offset); - writel(filter2, cspmu->base0 + PMEVFILT2R + offset); + if (cspmu->has_ext64) { + writeq(filter, cspmu->base0 + PMEVFILTR + (8 * n)); + writeq(filter2, cspmu->base0 + PMEVFILT2R + (8 * n)); + } else { + writel(filter, cspmu->base0 + PMEVFILTR + (4 * n)); + writel(filter2, cspmu->base0 + PMEVFILT2R + (4 * n)); + } } +/* Note we deliberately don't expect 64-bit filters here; see init_impl_ops */ static void arm_cspmu_set_cc_filter(struct arm_cspmu *cspmu, const struct perf_event *event) { - u32 filter = event->attr.config1 & ARM_CSPMU_FILTER_MASK; + u32 filter = arm_cspmu_filter(event); writel(filter, cspmu->base0 + PMCCFILTR); } @@ -976,6 +1027,30 @@ static int arm_cspmu_init_mmio(struct arm_cspmu *cspmu) } } + /* + * We can infer FEAT_CSPMU_EXT64 from PMCNTEN, or hope that anything + * that failed to get that right has at least implemented the optional + * PMDEVARCH correctly... + * + * Note that architecturally, has_ext64 *should* imply has_atomic_dword, + * but enough implementations have ignored that already that we'll just + * have to still rely on the firmware flag. + */ + writel(~0U, cspmu->base0 + PMCNTENCLR); + writel(~0U, cspmu->base0 + PMCNTEN); + if (readl(cspmu->base0 + PMCNTENCLR)) { + cspmu->has_ext64 = true; + writel(0, cspmu->base0 + PMCNTEN); + } else { + u32 reg = readl(cspmu->base0 + PMDEVARCH); + + if (reg & ARM_CSPMU_PMDEVARCH_PRESENT) { + reg &= ARM_CSPMU_PMDEVARCH_ARCHPART; + if (reg == 0xaf4 || reg == 0xaf5) + cspmu->has_ext64 = true; + } + } + cspmu->pmcfgr = readl(cspmu->base0 + PMCFGR); cspmu->num_logical_ctrs = FIELD_GET(PMCFGR_N, cspmu->pmcfgr) + 1; diff --git a/drivers/perf/arm_cspmu/arm_cspmu.h b/drivers/perf/arm_cspmu/arm_cspmu.h index 3fc5c8d77266..c4058d602477 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.h +++ b/drivers/perf/arm_cspmu/arm_cspmu.h @@ -35,21 +35,15 @@ PMU_EVENT_ATTR_ID(_name, arm_cspmu_sysfs_event_show, _config) -/* Default event id mask */ -#define ARM_CSPMU_EVENT_MASK GENMASK_ULL(63, 0) - -/* Default filter value mask */ -#define ARM_CSPMU_FILTER_MASK GENMASK_ULL(63, 0) - /* Default event format */ #define ARM_CSPMU_FORMAT_EVENT_ATTR \ - ARM_CSPMU_FORMAT_ATTR(event, "config:0-32") + PMU_EVENT_ATTR_ID(event, arm_cspmu_default_format_show, 0) /* Default filter format */ #define ARM_CSPMU_FORMAT_FILTER_ATTR \ - ARM_CSPMU_FORMAT_ATTR(filter, "config1:0-31") + PMU_EVENT_ATTR_ID(filter, arm_cspmu_default_format_show, 1) #define ARM_CSPMU_FORMAT_FILTER2_ATTR \ - ARM_CSPMU_FORMAT_ATTR(filter2, "config2:0-31") + PMU_EVENT_ATTR_ID(filter2, arm_cspmu_default_format_show, 2) /* * This is the default event number for cycle count, if supported, since the @@ -78,6 +72,7 @@ #define PMEVFILT2R 0x800 #define PMEVFILTR 0xA00 #define PMCNTENSET 0xC00 +#define PMCNTEN 0xC10 #define PMCNTENCLR 0xC20 #define PMINTENSET 0xC40 #define PMINTENCLR 0xC60 @@ -87,6 +82,8 @@ #define PMCFGR 0xE00 #define PMCR 0xE04 #define PMIIDR 0xE08 +#define PMCR_64 0xE10 +#define PMDEVARCH 0xFBC #define PMPIDR0 0xFE0 #define PMPIDR1 0xFE4 #define PMPIDR2 0xFE8 @@ -154,6 +151,10 @@ #define ARM_CSPMU_IMPL_ID_NVIDIA 0x36B #define ARM_CSPMU_IMPL_ID_AMPERE 0xA16 +/* PMDEVARCH */ +#define ARM_CSPMU_PMDEVARCH_PRESENT BIT(20) +#define ARM_CSPMU_PMDEVARCH_ARCHPART GENMASK(11, 0) + struct arm_cspmu; /* This tracks the events assigned to each counter in the PMU. */ @@ -183,7 +184,7 @@ struct arm_cspmu_impl_ops { /* Check if the event corresponds to cycle count event */ bool (*is_cycle_counter_event)(const struct perf_event *event); /* Decode event type/id from configs */ - u32 (*event_type)(const struct perf_event *event); + u64 (*event_type)(const struct perf_event *event); /* Set/reset event filters */ void (*set_cc_filter)(struct arm_cspmu *cspmu, const struct perf_event *event); @@ -234,6 +235,7 @@ struct arm_cspmu { int irq; bool has_atomic_dword; + bool has_ext64; u32 pmcfgr; u32 num_logical_ctrs; u32 num_set_clr_reg; @@ -250,6 +252,9 @@ ssize_t arm_cspmu_sysfs_event_show(struct device *dev, struct device_attribute *attr, char *buf); +ssize_t arm_cspmu_default_format_show(struct device *dev, + struct device_attribute *attr, char *buf); + /* Register vendor backend. */ int arm_cspmu_impl_register(const struct arm_cspmu_impl_match *impl_match); diff --git a/drivers/perf/arm_cspmu/nvidia_cspmu.c b/drivers/perf/arm_cspmu/nvidia_cspmu.c index bac83e424d6d..a4c1ab886709 100644 --- a/drivers/perf/arm_cspmu/nvidia_cspmu.c +++ b/drivers/perf/arm_cspmu/nvidia_cspmu.c @@ -762,7 +762,7 @@ static void pcie_tgt_pmu_reset_ev_filter(struct arm_cspmu *cspmu, pcie_tgt_pmu_config_addr_filter(cspmu, false, base, mask, idx); } -static u32 pcie_tgt_pmu_event_type(const struct perf_event *event) +static u64 pcie_tgt_pmu_event_type(const struct perf_event *event) { return event->attr.config & NV_PCIE_TGT_EV_TYPE_MASK; } From 3b56ebafecc43367b811361459c09c9ef9bfc167 Mon Sep 17 00:00:00 2001 From: Yeoreum Yun Date: Wed, 22 Jul 2026 16:30:28 +0100 Subject: [PATCH 63/95] arm64: pgtable: convert pte_present() from macro to static inline pte_present() is used as the basis for both pmd_present() and pud_present(). It is currently implemented as a macro composed of pte_val() and pte_present_invalid(). When pte_present() or its higher-level variants are used directly with ptep_get() or pXdp_get(), for example: pte_present(ptep_get(pte)); pmd_present(pmdp_get(pmd)); pud_present(pudp_get(pud)); the macro expansion causes the compiler to evaluate the argument twice, resulting in redundant loads. For example, pte_present() expands to: !pte_val(READ_ONCE(*pte) || pte_present_invalid(READ_ONCE(*pte)) A typical example is pud_free_pmd_page(), where the expansion of pmd_present() generates: ... /* pmd_present() (x20 = pmdp) */ 1b88: f9400288 ldr x8, [x20] // read pmdp. 1b8c: f9000fa8 str x8, [x29, #0x18] 1b90: 3707fec8 tbnz w8, #0x0, 0x1b68 1b94: f9400288 ldr x8, [x20] // redundant read of pmdp. 1b98: 8a170109 and x9, x8, x23 1b9c: f9000fa8 str x8, [x29, #0x18] 1ba0: f120013f cmp x9, #0x800 1ba4: 54fffe20 b.eq 0x1b68 1ba8: 17fffff4 b 0x1b78 ... Convert pte_present() to static inline function so that prevent the generation of redundant code and move pte_valid() and pte_present_invalid() further up so the inline function can use them. After this change, the generated code becomes: ... /* pmd_present() (x20 = pmdp) */ 1a30: f9400288 ldr x8, [x20] 1a34: 8a170109 and x9, x8, x23 1a38: f9000fa8 str x8, [x29, #0x18] 1a3c: f120013f cmp x9, #0x800 1a40: 54fffe80 b.eq 0x1a10 1a44: 3607fee8 tbz w8, #0x0, 0x1a20 1a48: 17fffff2 b 0x1a10 ... This eliminates the redundant load and also reduces code size at call sites using this pattern. For example, pud_free_pmd_page() shrinks from 7,500 bytes to 7,148 bytes, a reduction of approximately 4.7%. Signed-off-by: Yeoreum Yun Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgtable.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 27689c62bd25..25001694ae8e 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -140,10 +140,17 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) #define pte_none(pte) (!pte_val(pte)) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) +#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) +#define pte_present_invalid(pte) \ + ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) + /* * The following only work if pte_present(). Undefined behaviour otherwise. */ -#define pte_present(pte) (pte_valid(pte) || pte_present_invalid(pte)) +static __always_inline bool pte_present(pte_t pte) +{ + return pte_valid(pte) || pte_present_invalid(pte); +} #define pte_young(pte) (!!(pte_val(pte) & PTE_AF)) #define pte_special(pte) (!!(pte_val(pte) & PTE_SPECIAL)) #define pte_write(pte) (!!(pte_val(pte) & PTE_WRITE)) @@ -168,9 +175,6 @@ static inline pteval_t __phys_to_pte_val(phys_addr_t phys) #define pte_sw_dirty(pte) (!!(pte_val(pte) & PTE_DIRTY)) #define pte_dirty(pte) (pte_sw_dirty(pte) || pte_hw_dirty(pte)) -#define pte_valid(pte) (!!(pte_val(pte) & PTE_VALID)) -#define pte_present_invalid(pte) \ - ((pte_val(pte) & (PTE_VALID | PTE_PRESENT_INVALID)) == PTE_PRESENT_INVALID) /* * Execute-only user mappings do not have the PTE_USER bit set. All valid * kernel mappings have the PTE_UXN bit set. From 482145e0327f46f5051dfa2da4254867039928b2 Mon Sep 17 00:00:00 2001 From: Linu Cherian Date: Thu, 23 Jul 2026 10:10:28 +0530 Subject: [PATCH 64/95] arm64: cputype: Add Cortex-A520AE definitions Add cputype definitions for Cortex-A520AE. The definition can be found in Cortex-A520AE TRM, https://developer.arm.com/documentation/107726/0001/ as part of MIDR_EL1 bit descriptions. This is going to be used in the bbml3 support list. Reviewed-by: Gavin Shan Reviewed-by: Anshuman Khandual Signed-off-by: Linu Cherian Signed-off-by: Will Deacon --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 1b9f0cda1336..e41fae46426b 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -82,6 +82,7 @@ #define ARM_CPU_PART_CORTEX_X1 0xD44 #define ARM_CPU_PART_CORTEX_A510 0xD46 #define ARM_CPU_PART_CORTEX_A520 0xD80 +#define ARM_CPU_PART_CORTEX_A520AE 0xD88 #define ARM_CPU_PART_CORTEX_A710 0xD47 #define ARM_CPU_PART_CORTEX_A715 0xD4D #define ARM_CPU_PART_CORTEX_X2 0xD48 @@ -176,6 +177,7 @@ #define MIDR_CORTEX_X1 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X1) #define MIDR_CORTEX_A510 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A510) #define MIDR_CORTEX_A520 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A520) +#define MIDR_CORTEX_A520AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A520AE) #define MIDR_CORTEX_A710 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A710) #define MIDR_CORTEX_A715 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A715) #define MIDR_CORTEX_X2 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_X2) From 357230cb5276b0d224327ae2bb372d13f2243320 Mon Sep 17 00:00:00 2001 From: Linu Cherian Date: Thu, 23 Jul 2026 10:10:29 +0530 Subject: [PATCH 65/95] arm64: cputype: Add C1-Nano definitions Add cputype definitions for C1-Nano. The definition can be found in C1-Nano TRM, https://developer.arm.com/documentation/107753/0002 as part of MIDR_EL1 bit descriptions. This is going to be used in the bbml3 support list. Reviewed-by: Gavin Shan Reviewed-by: Anshuman Khandual Signed-off-by: Linu Cherian Signed-off-by: Will Deacon --- arch/arm64/include/asm/cputype.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index e41fae46426b..1fa29616e586 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -100,6 +100,7 @@ #define ARM_CPU_PART_CORTEX_A720AE 0xD89 #define ARM_CPU_PART_C1_ULTRA 0xD8C #define ARM_CPU_PART_NEOVERSE_N3 0xD8E +#define ARM_CPU_PART_C1_NANO 0xD8A #define ARM_CPU_PART_C1_PRO 0xD8B #define ARM_CPU_PART_C1_PREMIUM 0xD90 @@ -195,6 +196,7 @@ #define MIDR_CORTEX_A720AE MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A720AE) #define MIDR_C1_ULTRA MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_ULTRA) #define MIDR_NEOVERSE_N3 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_NEOVERSE_N3) +#define MIDR_C1_NANO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_NANO) #define MIDR_C1_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PRO) #define MIDR_C1_PREMIUM MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_C1_PREMIUM) #define MIDR_THUNDERX MIDR_CPU_MODEL(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX) From a7ac60414475fdd811575e45ffc45e1f59e404b3 Mon Sep 17 00:00:00 2001 From: Linu Cherian Date: Thu, 23 Jul 2026 10:10:30 +0530 Subject: [PATCH 66/95] arm64: cpufeature: Extend bbml2_noabort support list Add below cpus to the midr list, which supports BBML2_NOABORT. Cortex A520(AE) Cortex A715 Cortex A720(AE) Cortex A725 Neoverse N3 C1-Nano C1-Pro C1-Ultra C1-Premium C1-Ultra and C1-Premium both suffer from erratum 3683289, where Break-Before-Make must be followed to avoid a livelock. For both CPUs, the erratum is fixed from r1p1. Hence we do not enable BBML2_NOABORT for CPU revisions <= r1p0. The relevant SDENs are: * C1-Ultra: https://developer.arm.com/documentation/111077/9-00/ * C1-Premium: https://developer.arm.com/documentation/111078/9-00/ Reviewed-by: Gavin Shan Reviewed-by: Anshuman Khandual Signed-off-by: Linu Cherian Signed-off-by: Will Deacon --- Documentation/arch/arm64/silicon-errata.rst | 4 ++++ arch/arm64/kernel/cpufeature.c | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst index 014aa1c215a1..57c778446936 100644 --- a/Documentation/arch/arm64/silicon-errata.rst +++ b/Documentation/arch/arm64/silicon-errata.rst @@ -242,10 +242,14 @@ stable kernels. +----------------+-----------------+-----------------+-----------------------------+ | ARM | Neoverse-V3AE | #4193784 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Premium | #3683289 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Premium | #4193780 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Pro | #4193714 | ARM64_ERRATUM_4193714 | +----------------+-----------------+-----------------+-----------------------------+ +| ARM | C1-Ultra | #3683289 | N/A | ++----------------+-----------------+-----------------+-----------------------------+ | ARM | C1-Ultra | #4193780 | ARM64_ERRATUM_4118414 | +----------------+-----------------+-----------------+-----------------------------+ | ARM | MMU-500 | #562869, | ARM_SMMU_MMU_500_CPRE_ERRATA| diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 9a22df0c5120..98d2cfc2dc90 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2152,6 +2152,16 @@ bool cpu_supports_bbml2_noabort(void) MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), MIDR_ALL_VERSIONS(MIDR_AMPERE1), MIDR_ALL_VERSIONS(MIDR_AMPERE1A), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A520AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A715), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A725), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N3), + MIDR_ALL_VERSIONS(MIDR_C1_NANO), + MIDR_ALL_VERSIONS(MIDR_C1_PRO), + /* Erratum 3683289 fixed in r1p1 */ + MIDR_RANGE(MIDR_C1_ULTRA, 1, 1, 0xf, 0xf), + MIDR_RANGE(MIDR_C1_PREMIUM, 1, 1, 0xf, 0xf), {} }; From e4fa246243083391a65a721dbbc340766668d829 Mon Sep 17 00:00:00 2001 From: Linu Cherian Date: Thu, 23 Jul 2026 10:10:31 +0530 Subject: [PATCH 67/95] arm64: sysreg: Add BBM_3 Add BBM_3 definition for ID_AA64MMFR2_EL1 register. Reviewed-by: Gavin Shan Reviewed-by: Anshuman Khandual Signed-off-by: Linu Cherian Signed-off-by: Will Deacon --- arch/arm64/tools/sysreg | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg index 7cb61aca3797..e070ada06196 100644 --- a/arch/arm64/tools/sysreg +++ b/arch/arm64/tools/sysreg @@ -2259,6 +2259,7 @@ UnsignedEnum 55:52 BBM 0b0000 0 0b0001 1 0b0010 2 + 0b0011 3 EndEnum UnsignedEnum 51:48 TTL 0b0000 NI From 94104e3cfa8036ac2457aa0c2ca1351e16d77159 Mon Sep 17 00:00:00 2001 From: Linu Cherian Date: Thu, 23 Jul 2026 10:10:32 +0530 Subject: [PATCH 68/95] arm64: cpufeature: Rename BBML2_NOABORT as BBML3 - As bbml2_noabort is functionally equivalent to bbml3, rename cpu/system_supports_bbml2_noabort to cpu/system_supports_bbml3. The ARM64 capability name is also renamed accordingly. - As BBML2_NOABORT or the equivalent BBML3 is the kernel requirement for setting up linear map with block/contpte mappings and not BBML2, replace all bbml2 references with bbml3. FEAT_BBML3, is introduced as part of 2025 Architecture Extensions. https://developer.arm.com/documentation/109697/2026_03/2025-Architecture-Extensions No functional changes are introduced with this patch. Reviewed-by: Gavin Shan Reviewed-by: Anshuman Khandual Signed-off-by: Linu Cherian Signed-off-by: Will Deacon --- arch/arm64/include/asm/cpufeature.h | 6 ++-- arch/arm64/kernel/cpufeature.c | 30 +++++----------- arch/arm64/mm/contpte.c | 21 +++++------ arch/arm64/mm/mmu.c | 54 ++++++++++++++--------------- arch/arm64/mm/proc.S | 4 +-- arch/arm64/tools/cpucaps | 2 +- 6 files changed, 50 insertions(+), 67 deletions(-) diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index a57870fa96db..d90040fb9de6 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -878,11 +878,11 @@ static inline bool system_supports_pmuv3(void) return cpus_have_final_cap(ARM64_HAS_PMUV3); } -bool cpu_supports_bbml2_noabort(void); +bool cpu_supports_bbml3(void); -static inline bool system_supports_bbml2_noabort(void) +static inline bool system_supports_bbml3(void) { - return alternative_has_cap_unlikely(ARM64_HAS_BBML2_NOABORT); + return alternative_has_cap_unlikely(ARM64_HAS_BBML3); } int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt); diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 98d2cfc2dc90..896bafdb00b1 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2131,21 +2131,10 @@ static bool hvhe_possible(const struct arm64_cpu_capabilities *entry, return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE); } -bool cpu_supports_bbml2_noabort(void) +bool cpu_supports_bbml3(void) { - /* - * We want to allow usage of BBML2 in as wide a range of kernel contexts - * as possible. This list is therefore an allow-list of known-good - * implementations that both support BBML2 and additionally, fulfill the - * extra constraint of never generating TLB conflict aborts when using - * the relaxed BBML2 semantics (such aborts make use of BBML2 in certain - * kernel contexts difficult to prove safe against recursive aborts). - * - * Note that implementations can only be considered "known-good" if their - * implementors attest to the fact that the implementation never raises - * TLB conflict aborts for BBML2 mapping granularity changes. - */ - static const struct midr_range supports_bbml2_noabort_list[] = { + /* CPUs that support BBML3 but dont advertise through ID_AA64MMFR2_EL1 */ + static const struct midr_range supports_bbml3_list[] = { MIDR_REV_RANGE(MIDR_CORTEX_X4, 0, 3, 0xf), MIDR_REV_RANGE(MIDR_NEOVERSE_V3, 0, 2, 0xf), MIDR_REV_RANGE(MIDR_NEOVERSE_V3AE, 0, 2, 0xf), @@ -2165,8 +2154,7 @@ bool cpu_supports_bbml2_noabort(void) {} }; - /* Does our cpu guarantee to never raise TLB conflict aborts? */ - if (!is_midr_in_range_list(supports_bbml2_noabort_list)) + if (!is_midr_in_range_list(supports_bbml3_list)) return false; /* @@ -2177,9 +2165,9 @@ bool cpu_supports_bbml2_noabort(void) return true; } -static bool has_bbml2_noabort(const struct arm64_cpu_capabilities *caps, int scope) +static bool has_bbml3(const struct arm64_cpu_capabilities *caps, int scope) { - return cpu_supports_bbml2_noabort(); + return cpu_supports_bbml3(); } static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) @@ -3072,10 +3060,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = { ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP) }, { - .desc = "BBM Level 2 without TLB conflict abort", - .capability = ARM64_HAS_BBML2_NOABORT, + .desc = "BBM Level 3", + .capability = ARM64_HAS_BBML3, .type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE, - .matches = has_bbml2_noabort, + .matches = has_bbml3, }, { .desc = "52-bit Virtual Addressing for KVM (LPA2)", diff --git a/arch/arm64/mm/contpte.c b/arch/arm64/mm/contpte.c index 2de12656b4d8..0acab179fc1a 100644 --- a/arch/arm64/mm/contpte.c +++ b/arch/arm64/mm/contpte.c @@ -89,7 +89,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, } /* - * On eliding the __tlb_flush_range() under BBML2+noabort: + * On eliding the __tlb_flush_range() under BBML3: * * NOTE: Instead of using N=16 as the contiguous block length, we use * N=4 for clarity. @@ -135,7 +135,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * contiguous TLB entry, which is a micro-optimisation opportunity, * but does not affect correctness. * - * In the BBML2 case, the change is avoiding the intermediate tlbi+dsb. + * In the BBML3 case, the change is avoiding the intermediate tlbi+dsb. * This means a few things, but notably other PEs will still "see" any * stale cached TLB entries. This could lead to a "contiguous bit * misprogramming" issue until the final tlbi+dsb of the changed page, @@ -158,21 +158,16 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * are present, and a write is made to this address, do we fault or * is the write permitted (via amalgamation)? * - * The relevant Arm ARM DDI 0487L.a requirements are RNGLXZ and RJQQTC, - * and together state that when BBML1 or BBML2 are implemented, either - * a TLB conflict abort is raised (which we expressly forbid), or will - * "produce an OA, access permissions, and memory attributes that are - * consistent with any of the programmed translation table values". - * - * That is to say, will either raise a TLB conflict, or produce one of - * the cached TLB entries, but never amalgamate. + * With BBML3 implemented, no TLB conflict abort is raised and the OA, + * access permissions and memory attributes produced is one of the cached + * TLB entries, but never amalgamate. * * Thus, as the page tables are only considered "consistent" after * the final tlbi+dsb (which evicts both the single stale (RW,n) TLB * entry as well as the new contiguous (RO,c) TLB entry), omitting the * initial tlbi+dsb is correct. * - * It is also important to note that at the end of the BBML2 folding + * It is also important to note that at the end of the BBML3 folding * case, we are still left with potentially all N TLB entries still * cached (the N-1 non-contiguous ptes, and the single contiguous * block). However, over time, natural TLB pressure will cause the @@ -214,7 +209,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * * |____| <--- tlbi + dsb * - * For BBML2, we again remove the intermediate tlbi+dsb. Here, there + * For BBML3, we again remove the intermediate tlbi+dsb. Here, there * are no issues, as the final tlbi+dsb covering the changed page is * guaranteed to remove the original large contiguous (RW,c) TLB entry, * as well as the intermediate (RW,n) TLB entry; the next access will @@ -224,7 +219,7 @@ static void contpte_convert(struct mm_struct *mm, unsigned long addr, * regardless. */ - if (!system_supports_bbml2_noabort()) + if (!system_supports_bbml3()) __flush_tlb_range(&vma, start_addr, addr, PAGE_SIZE, 3, TLBF_NOWALKCACHE); diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index a25d8beacc83..fad71d6bff40 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -779,18 +779,18 @@ out: static inline bool force_pte_mapping(void) { - const bool bbml2 = system_capabilities_finalized() ? - system_supports_bbml2_noabort() : cpu_supports_bbml2_noabort(); + const bool bbml3 = system_capabilities_finalized() ? + system_supports_bbml3() : cpu_supports_bbml3(); if (debug_pagealloc_enabled()) return true; - if (bbml2) + if (bbml3) return false; return rodata_full || arm64_kfence_can_set_direct_map() || is_realm_world(); } static DEFINE_MUTEX(pgtable_split_lock); -static bool linear_map_requires_bbml2; +static bool linear_map_requires_bbml3; int split_kernel_leaf_mapping(unsigned long start, unsigned long end) { @@ -803,15 +803,15 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end) * always pte-mapped), we must not go any further because taking the * mutex below may sleep. Do not call force_pte_mapping() here because * it could return a confusing result if called from a secondary cpu - * prior to finalizing caps. Instead, linear_map_requires_bbml2 gives us + * prior to finalizing caps. Instead, linear_map_requires_bbml3 gives us * what we need. */ - if (!linear_map_requires_bbml2 || is_kfence_address((void *)start)) + if (!linear_map_requires_bbml3 || is_kfence_address((void *)start)) return 0; - if (!system_supports_bbml2_noabort()) { + if (!system_supports_bbml3()) { /* - * !BBML2_NOABORT systems should not be trying to change + * BBML3 systems should not be trying to change * permissions on anything that is not pte-mapped in the first * place. Just return early and let the permission change code * raise a warning if not already pte-mapped. @@ -828,8 +828,8 @@ int split_kernel_leaf_mapping(unsigned long start, unsigned long end) /* * Boot-time: Started secondary cpus but don't know if they - * support BBML2_NOABORT yet. Can't allow splitting in this - * window in case they don't. + * support BBML3 yet. Can't allow splitting in this window + * in case they don't. */ if (WARN_ON(num_online_cpus() > 1)) return -EBUSY; @@ -934,11 +934,11 @@ static int range_split_to_ptes(unsigned long start, unsigned long end, gfp_t gfp return ret; } -u32 idmap_kpti_bbml2_flag; +u32 idmap_kpti_bbml3_flag; -static void __init init_idmap_kpti_bbml2_flag(void) +static void __init init_idmap_kpti_bbml3_flag(void) { - WRITE_ONCE(idmap_kpti_bbml2_flag, 1); + WRITE_ONCE(idmap_kpti_bbml3_flag, 1); /* Must be visible to other CPUs before stop_machine() is called. */ smp_mb(); } @@ -947,7 +947,7 @@ static int __init linear_map_split_to_ptes(void *__unused) { /* * Repainting the linear map must be done by CPU0 (the boot CPU) because - * that's the only CPU that we know supports BBML2. The other CPUs will + * that's the only CPU that we know supports BBML3. The other CPUs will * be held in a waiting area with the idmap active. */ if (!smp_processor_id()) { @@ -960,7 +960,7 @@ static int __init linear_map_split_to_ptes(void *__unused) /* * Wait for all secondary CPUs to be put into the waiting area. */ - smp_cond_load_acquire(&idmap_kpti_bbml2_flag, VAL == num_online_cpus()); + smp_cond_load_acquire(&idmap_kpti_bbml3_flag, VAL == num_online_cpus()); /* * Walk all of the linear map [lstart, lend), except the kernel @@ -979,7 +979,7 @@ static int __init linear_map_split_to_ptes(void *__unused) * Relies on dsb in flush_tlb_kernel_range() to avoid reordering * before any page table split operations. */ - WRITE_ONCE(idmap_kpti_bbml2_flag, 0); + WRITE_ONCE(idmap_kpti_bbml3_flag, 0); } else { typedef void (wait_split_fn)(void); extern wait_split_fn wait_linear_map_split_to_ptes; @@ -988,7 +988,7 @@ static int __init linear_map_split_to_ptes(void *__unused) wait_fn = (void *)__pa_symbol(wait_linear_map_split_to_ptes); /* - * At least one secondary CPU doesn't support BBML2 so cannot + * At least one secondary CPU doesn't support BBML3 so cannot * tolerate the size of the live mappings changing. So have the * secondary CPUs wait for the boot CPU to make the changes * with the idmap active and init_mm inactive. @@ -1003,8 +1003,8 @@ static int __init linear_map_split_to_ptes(void *__unused) void __init linear_map_maybe_split_to_ptes(void) { - if (linear_map_requires_bbml2 && !system_supports_bbml2_noabort()) { - init_idmap_kpti_bbml2_flag(); + if (linear_map_requires_bbml3 && !system_supports_bbml3()) { + init_idmap_kpti_bbml3_flag(); stop_machine(linear_map_split_to_ptes, NULL, cpu_online_mask); } } @@ -1127,7 +1127,7 @@ bool arch_kfence_init_pool(void) mutex_unlock(&pgtable_split_lock); /* - * Since the system supports bbml2_noabort, tlb invalidation is not + * Since the system supports bbml3, tlb invalidation is not * required here; the pgtable mappings have been split to pte but larger * entries may safely linger in the TLB. */ @@ -1166,7 +1166,7 @@ static void __init map_mem(void) arm64_kfence_map_pool(); - linear_map_requires_bbml2 = !force_pte_mapping() && can_set_direct_map(); + linear_map_requires_bbml3 = !force_pte_mapping() && can_set_direct_map(); if (force_pte_mapping()) flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS; @@ -1333,7 +1333,7 @@ void __init kpti_install_ng_mappings(void) if (arm64_use_ng_mappings) return; - init_idmap_kpti_bbml2_flag(); + init_idmap_kpti_bbml3_flag(); stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask); } @@ -1394,7 +1394,7 @@ void __pi_map_range(phys_addr_t *pte, u64 start, u64 end, phys_addr_t pa, u64 va_offset); static u8 idmap_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init, - kpti_bbml2_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init; + kpti_bbml3_ptes[IDMAP_LEVELS - 1][PAGE_SIZE] __aligned(PAGE_SIZE) __ro_after_init; static void __init create_idmap(void) { @@ -1406,17 +1406,17 @@ static void __init create_idmap(void) IDMAP_ROOT_LEVEL, (pte_t *)idmap_pg_dir, false, __phys_to_virt(ptep) - ptep); - if (linear_map_requires_bbml2 || + if (linear_map_requires_bbml3 || (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0) && !arm64_use_ng_mappings)) { - phys_addr_t pa = __pa_symbol(&idmap_kpti_bbml2_flag); + phys_addr_t pa = __pa_symbol(&idmap_kpti_bbml3_flag); /* * The KPTI G-to-nG conversion code needs a read-write mapping * of its synchronization flag in the ID map. This is also used * when splitting the linear map to ptes if a secondary CPU - * doesn't support bbml2. + * doesn't support bbml3. */ - ptep = __pa_symbol(kpti_bbml2_ptes); + ptep = __pa_symbol(kpti_bbml3_ptes); __pi_map_range(&ptep, pa, pa + sizeof(u32), pa, PAGE_KERNEL, IDMAP_ROOT_LEVEL, (pte_t *)idmap_pg_dir, false, __phys_to_virt(ptep) - ptep); diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be37..f4e4e71a0ea8 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -287,7 +287,7 @@ SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings) mov x5, x3 // preserve temp_pte arg mrs swapper_ttb, ttbr1_el1 - adr_l flag_ptr, idmap_kpti_bbml2_flag + adr_l flag_ptr, idmap_kpti_bbml3_flag cbnz cpu, __idmap_kpti_secondary @@ -445,7 +445,7 @@ SYM_TYPED_FUNC_START(wait_linear_map_split_to_ptes) flag_ptr .req x4 mrs swapper_ttb, ttbr1_el1 - adr_l flag_ptr, idmap_kpti_bbml2_flag + adr_l flag_ptr, idmap_kpti_bbml3_flag __idmap_cpu_set_reserved_ttbr1 x16, x17 scondary_cpu_wait: diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 3fcac789b0a4..0b94837ac238 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -14,6 +14,7 @@ HAS_ADDRESS_AUTH_ARCH_QARMA5 HAS_ADDRESS_AUTH_IMP_DEF HAS_AMU_EXTN HAS_ARMv8_4_TTL +HAS_BBML3 HAS_CACHE_DIC HAS_CACHE_IDC HAS_CNP @@ -51,7 +52,6 @@ HAS_LS64_V HAS_LSUI HAS_MOPS HAS_NESTED_VIRT -HAS_BBML2_NOABORT HAS_PAN HAS_PMUV3 HAS_S1PIE From 879aca5119cd7487bac64089127927aaae233a63 Mon Sep 17 00:00:00 2001 From: Linu Cherian Date: Thu, 23 Jul 2026 10:10:33 +0530 Subject: [PATCH 69/95] arm64: cpufeature: Detect BBML3 based on ID_AA64MMFR2_EL1.BBM Add ID_AA64MMFR2_EL1.BBM based BBML3 feature detection in cpu_supports_bbml3() so that cpus with the feature would not have to be added into MIDR based supports_bbml3_list. Reviewed-by: Gavin Shan Reviewed-by: Anshuman Khandual Signed-off-by: Linu Cherian [will: Tidy up cpu_supports_bbml3()] Signed-off-by: Will Deacon --- arch/arm64/kernel/cpufeature.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index 896bafdb00b1..4025d51d2d93 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -2153,16 +2153,12 @@ bool cpu_supports_bbml3(void) MIDR_RANGE(MIDR_C1_PREMIUM, 1, 1, 0xf, 0xf), {} }; + u64 mmfr2 = __read_sysreg_by_encoding(SYS_ID_AA64MMFR2_EL1); - if (!is_midr_in_range_list(supports_bbml3_list)) - return false; + if (SYS_FIELD_GET(ID_AA64MMFR2_EL1, BBM, mmfr2) >= ID_AA64MMFR2_EL1_BBM_3) + return true; - /* - * We currently ignore the ID_AA64MMFR2_EL1 register, and only care - * about whether the MIDR check passes. - */ - - return true; + return is_midr_in_range_list(supports_bbml3_list); } static bool has_bbml3(const struct arm64_cpu_capabilities *caps, int scope) From b7f741717d1e7bcdca2d5eaf3babbb5026cdb3a4 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 27 Jul 2026 17:34:09 +0100 Subject: [PATCH 70/95] arm64: ptrace: Remove INIT_PSTATE_EL2 Last user of INIT_PSTATE_EL2 has gone with ae4b7e38e9a9 ("arm64: Allow sticky E2H when entering EL1"), so remove it. Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/include/asm/ptrace.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/arm64/include/asm/ptrace.h b/arch/arm64/include/asm/ptrace.h index 39582511ad72..f7dc5fb9427d 100644 --- a/arch/arm64/include/asm/ptrace.h +++ b/arch/arm64/include/asm/ptrace.h @@ -18,8 +18,6 @@ #define INIT_PSTATE_EL1 \ (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL1h) -#define INIT_PSTATE_EL2 \ - (PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT | PSR_MODE_EL2h) #include From 7cf2d6efb86fc98c4f5f78e438e2e32d88d76ff7 Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:10 +0100 Subject: [PATCH 71/95] arm64: debug: don't mask DAIF for mdscr_write() Masking DAIF around the write to MDSCR_EL1 doesn't do anything: we can write to sysregs with interrupts unmasked, and writing to PSTATE is not a context synchronization event so it does not synchronize it. This is done in the context of a general interrupt handling cleanup, so it does not address the missing context synchronization for the MDSCR_EL1 write, staying consistent with the current state. This should be addressed in a future patch. Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Signed-off-by: Will Deacon --- arch/arm64/kernel/debug-monitors.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 29307642f4c9..e271fbac5f82 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -40,10 +40,7 @@ u8 debug_monitors_arch(void) */ static void mdscr_write(u64 mdscr) { - unsigned long flags; - flags = local_daif_save(); write_sysreg(mdscr, mdscr_el1); - local_daif_restore(flags); } NOKPROBE_SYMBOL(mdscr_write); From 684bde100117931f4c51c644a95f42f2dab041bc Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:11 +0100 Subject: [PATCH 72/95] arm64: hibernate: mask DAIF before restoring hibernated kernel The arm64 hibernate code manages the exception masking in an unsound way, leading to potential crashes and/or warnings during resume. When a hibernation image is saved in `swsusp_arch_suspend()`, all DAIF exceptions are masked (by virtue of `local_daif_save()`), and the suspended image is saved assuming that all DAIF exceptions will remain masked when the image is restored. When a hibernation image is resumed by `swsusp_arch_resume()`, only interrupts are masked (by virtue of `local_irq_disable()` in `resume_target_kernel()`). When pseudo-NMI is enabled the DAIF.IF bits will be clear, and regardless of pseudo-NMI the DAIF.DA bits will be clear. This means that there are two problems: (1) It is possible to take Debug, SError, or pseudo-NMI exceptions during the resume process. This is unsafe, as during the resume process both the old ane new kernels will tranisently be in an inconsistent state, and swsusp_arch_suspend_exit() won't retain an executable mapping of any exception vectors. Any exception taken here will be fatal and silent. (2) When re-entering the resumed kernel, some DAIF bits will be clear unexpectedly. This permits Debug, SError, or pseudo-NMI exceptions to be taken for a short period while the resumed kernel is not yet in a consistent state. This is detected by CONFIG_ARM64_DEBUG_PRIORITY_MASKING. Avoid these issues by masking all DAIF exceptions during resume. Fixes: 82869ac57b5d ("arm64: kernel: Add support for hibernate/suspend-to-disk") Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/kernel/hibernate.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 9717568518ba..1eb1c1074c5b 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -465,9 +465,21 @@ int __nocfi swsusp_arch_resume(void) if (el2_reset_needed()) __hyp_set_vectors(el2_vectors); + /* + * It is necessary to mask all DAIF exceptions here as: + * + * - The copy of swsusp_arch_suspend_exit() in the hibernation + * text cannot handle taking any exceptions. + * + * - The suspended kernel masked all DAIF exceptions in + * swsusp_arch_resume(), and expects to be re-entered in the + * same state : with all DAIF exceptions masked. + */ + local_daif_save(); hibernate_exit(virt_to_phys(tmp_pg_dir), resume_hdr.ttbr1_el1, resume_hdr.reenter_kernel, restore_pblist, resume_hdr.__hyp_stub_vectors, virt_to_phys(zero_page)); + unreachable(); return 0; } From 541549827889d0380fd73f8aacb5de6ef7a5a1ac Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 27 Jul 2026 17:34:12 +0100 Subject: [PATCH 73/95] arm64: hibernate: Restore DAIF state on error Sashiko AI has reported that if swsusp_mte_save_tags() for some reason fails we return from swsusp_arch_suspend() with DAIF being masked - that is not what we'd expect. Restore the saved DAIF state before returning from the error path. Fixes: ee11f332af96 ("arm64: mte: Save tags when hibernating") Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/kernel/hibernate.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index 1eb1c1074c5b..7bf117427777 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -348,8 +348,10 @@ int swsusp_arch_suspend(void) crash_prepare_suspend(); ret = swsusp_mte_save_tags(); - if (ret) + if (ret) { + local_daif_restore(flags); return ret; + } sleep_cpu = smp_processor_id(); ret = swsusp_save(); From 630ca6c58fcfeaddd96dd46e7b11826013977955 Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:13 +0100 Subject: [PATCH 74/95] arm64: suspend: rely on daif helpers to handle PMR Commit 77345ef70445 ("arm64: suspend: Use cpuidle context helpers in cpu_suspend()") added cpuidle helpers to handle PMR manipulation and restoration to ensure that the CPU receives interrupts when suspended and pseudo-NMIs are enabled. However, those helpers are called in between a pair of `local_daif_save()` and `local_daif_restore()`, which already configure the PMR as expected. Effectively, `arm_cpuidle_save_irq_context()` is a no-op here, even when using pseudo-NMIs, and `arm_cpuidle_restore_irq_context()` would not restore proper interrupt masking configuration early enough if there were unexpected changes during suspend or resume. (This can be observed with Trusted Firmware A (TF-A) at EL3 handling suspend through PSCI. Even though it should not be the case, TF-A can reset `ICC_PMR_EL1` during CPU_SUSPEND, thus resuming the kernel with an inconsistent priority mask value on hardware implementing more than the minimum number of priority levels, such as Morello.) Thus : remove the cpuidle context helpers as they do not do anything, but keep the comment mentioning the need for interrupts to reach the CPU if we are using pseudo-NMIs. Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/kernel/suspend.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index eaaff94329cd..c41724a40b75 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -99,7 +99,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int ret = 0; unsigned long flags; struct sleep_stack_data state; - struct arm_cpuidle_irq_context context; /* * Some portions of CPU state (e.g. PSTATE.{PAN,DIT}) are initialized @@ -121,6 +120,9 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) * Strictly speaking the trace_hardirqs_off() here is superfluous, * hardirqs should be firmly off by now. This really ought to use * something like raw_local_daif_save(). + * + * This also unmasks interrupts in PMR in order to reliably + * resume if we're using pseudo-NMIs. */ flags = local_daif_save(); @@ -131,12 +133,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) */ pause_graph_tracing(); - /* - * Switch to using DAIF.IF instead of PMR in order to reliably - * resume if we're using pseudo-NMIs. - */ - arm_cpuidle_save_irq_context(&context); - ct_cpuidle_enter(); if (__cpu_suspend_enter(&state)) { @@ -159,8 +155,6 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) __cpu_suspend_exit(); } - arm_cpuidle_restore_irq_context(&context); - unpause_graph_tracing(); /* From d3ebdc1772c0714faca14eec789ff082f6e00535 Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 27 Jul 2026 17:34:14 +0100 Subject: [PATCH 75/95] arm64: suspend: Initialize PMR on resume When we resume from cpu_suspend() context tracking, specially, ct_idle_exit() performs IRQ save/restore sequence. It doesn't cause any functional issues since we have masked all exceptions prior suspend and have not restored them. However, in case of pseudo-NMI PMR can be set by firmware to arbitrary value, thus IRQ save/restore routines manipulates this arbitrary value. Again, it doesn't cause any issues since PMR variant of IRQ save helper carries a __pmr_irqs_disabled_flags() guard. Going forward __pmr_irqs_disabled_flags() guard will be gone and we will call __pmr_local_irq_disable() unconditionally - that would cause warning in case CONFIG_ARM64_DEBUG_PRIORITY_MASKING is set. Initialize PMR to a value known to Linux on resume until the normal exception restore path restores the saved DAIF and PMR state. Signed-off-by: Vladimir Murzin Signed-off-by: Will Deacon --- arch/arm64/mm/proc.S | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 22866b49be37..06c8bc9a85d6 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -169,6 +169,13 @@ alternative_if ARM64_HAS_RAS_EXTN msr_s SYS_DISR_EL1, xzr alternative_else_nop_endif +#ifdef CONFIG_ARM64_PSEUDO_NMI +alternative_if ARM64_HAS_GIC_PRIO_MASKING + mov x1, #GIC_PRIO_IRQON + msr_s SYS_ICC_PMR_EL1, x1 +alternative_else_nop_endif +#endif + ptrauth_keys_install_kernel_nosync x14, x1, x2, x3 isb ret From 0d774e0517f30b9684e936c2559fb50681a45329 Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:15 +0100 Subject: [PATCH 76/95] arm64: entry: mask DAIF before returning from C EL1 handlers Most EL1 exceptions already call local_daif_mask() before returning, with the exception of debug exception handlers which do not change DAIF, and the IRQ/FIQ/Error handlers. However, DAIF get masked in kernel_exit() in all cases when returning from EL1 C handlers anyway. Move this masking from assembly to C by calling local_daif_mask() before irqentry_nmi_exit(). Unlike the raw DAIF masking helper, local_daif_mask() invokes trace_hardirqs_off(), so it must execute while RCU is still watching. Remove the disable_daif assembly macro, as this was its only use. Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Signed-off-by: Will Deacon --- arch/arm64/include/asm/assembler.h | 4 ---- arch/arm64/kernel/entry-common.c | 2 ++ arch/arm64/kernel/entry.S | 4 ---- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index effae53e9739..0b58b550e8dc 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -34,10 +34,6 @@ wx\n .req w\n .endr - .macro disable_daif - msr daifset, #0xf - .endm - /* * Save/restore interrupts. */ diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index ceb4eb11232a..2be42d7f4eaa 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -495,6 +495,7 @@ static __always_inline void __el1_pnmi(struct pt_regs *regs, state = irqentry_nmi_enter(regs); do_interrupt_handler(regs, handler); + local_daif_mask(); irqentry_nmi_exit(regs, state); } @@ -540,6 +541,7 @@ asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) local_daif_restore(DAIF_ERRCTX); state = irqentry_nmi_enter(regs); do_serror(regs, esr); + local_daif_mask(); irqentry_nmi_exit(regs, state); } diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index e0db14e9c843..f63049ac32dc 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -333,10 +333,6 @@ alternative_else_nop_endif .endm .macro kernel_exit, el - .if \el != 0 - disable_daif - .endif - #ifdef CONFIG_ARM64_PSEUDO_NMI alternative_if_not ARM64_HAS_GIC_PRIO_MASKING b .Lskip_pmr_restore\@ From d97afae6f16a4f8ac7d50af0070816270b5380c1 Mon Sep 17 00:00:00 2001 From: Yureka Lilian Date: Tue, 4 Aug 2026 19:23:15 +0200 Subject: [PATCH 77/95] arch: arm64: add early_param idle= Overriding the idle mechanism might be useful for debugging and performance testing. Add a cmdline parameter for it, similar to the existing idle= parameter already present for the x86 and ppc architectures. It is also useful on platforms where the WFI instruction misbehaves, such as Apple Silicon SoCs. Generally, a misbehaving instruction should be treated as an erratum and patched using the alternatives framework. However, in the Apple Silicon case we need more flexibility because it is difficult to detect whether the erratum applies. For example, Linux VMs inside macOS have the same MIDR and may even seem like they're running in EL2 in the case of NV, but should continue using WFI (it's trapped and handled correctly by the hypervisor there). Thus, we prefer to let the m1n1 bootloader add the idle=nop parameter[1]. Link[1]: https://lore.kernel.org/all/99b69262-e54b-424e-baa2-96ef7013b87a@kernel.org/ Suggested-by: Will Deacon Signed-off-by: Yureka Lilian Signed-off-by: Will Deacon --- .../admin-guide/kernel-parameters.txt | 23 +++++++++++++ arch/arm64/kernel/idle.c | 32 +++++++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b5493a7f8f22..7e9cc49b90a9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2236,6 +2236,29 @@ Kernel parameters idle=nomwait: Disable mwait for CPU C-states + [ARM64,EARLY] + Format: idle=wfi, idle=yield, idle=nop + + idle=wfi: Use the WFI (Wait For Interrupt) hint + instruction in the idle loop. This is the default and + allows the CPU to enter a low-power state until an + interrupt arrives. + + idle=yield: Use the YIELD hint instruction instead of + WFI. CPUs supporting simultaneous multi-threading (SMT), + can continue executing another thread when the current + thread reaches the idle loop. This will make the CPUs + eat more power, but may be useful to get slightly better + performance in some applications, since the CPUs will + not enter a low-power state. + + idle=nop: Do not execute any idle instruction in the + idle loop. This is useful on platforms where WFI + misbehaves, leading to system instability or loss of CPU + state. This will make the CPUs eat more power, but may + give slightly better performance in some applications, + since the CPUs will not enter a low-power state. + idxd.sva= [HW] Format: Allow force disabling of Shared Virtual Memory (SVA) diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c index 05cfb347ec26..42c5543f9589 100644 --- a/arch/arm64/kernel/idle.c +++ b/arch/arm64/kernel/idle.c @@ -11,6 +11,29 @@ #include #include +enum { + ARM64_IDLE_WFI, + ARM64_IDLE_YIELD, + ARM64_IDLE_NOP, +} idle = ARM64_IDLE_WFI; + +static int __init setup_idle(char *arg) +{ + if (!arg) + return -1; + else if (!strcmp(arg, "wfi")) + idle = ARM64_IDLE_WFI; + else if (!strcmp(arg, "yield")) + idle = ARM64_IDLE_YIELD; + else if (!strcmp(arg, "nop")) + idle = ARM64_IDLE_NOP; + else + return -1; + + return 0; +} +early_param("idle", setup_idle); + /* * cpu_do_idle() * @@ -26,8 +49,13 @@ void __cpuidle cpu_do_idle(void) arm_cpuidle_save_irq_context(&context); - dsb(sy); - wfi(); + if (likely(idle == ARM64_IDLE_WFI)) { + dsb(sy); + wfi(); + } else if (idle == ARM64_IDLE_YIELD) { + dsb(sy); + asm volatile("yield" ::: "memory"); + } arm_cpuidle_restore_irq_context(&context); } From eb78ef9cf77a156b4bdebc4417427ef99aab3125 Mon Sep 17 00:00:00 2001 From: Jinjie Ruan Date: Thu, 6 Aug 2026 19:15:46 +0800 Subject: [PATCH 78/95] kselftest/arm64: Fix abi test compilation errors The arm64 ABI selftests fail to compile due to missing include paths for kernel headers, causing errors like incomplete type struct sock_filter and implicit BPF macro declarations. Add $(KHDR_INCLUDES) and -I$(top_srcdir)/tools/include to CFLAGS to resolve the header search path. Also remove the hardcoded __NR_write macro and include to obtain the correct syscall number. Fixes: 21e37da12071 ("kselftest/arm64: Add testcase for SECCOMP_RET_TRACE orig_x0 bypass") Fixes: 2fcbc4adf997 ("kselftest/arm64: Add seccomp ptrace x0 bypass test") Reported-by: kernel test robot Closes: https://lore.kernel.org/r/202608021842.jp6IBrFi-lkp@intel.com/ Suggested-by: Mark Brown Reviewed-by: Mark Brown Tested-by: Mark Brown Signed-off-by: Jinjie Ruan Signed-off-by: Will Deacon --- tools/testing/selftests/arm64/abi/Makefile | 2 ++ tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c | 5 +---- .../selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c | 5 +---- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tools/testing/selftests/arm64/abi/Makefile b/tools/testing/selftests/arm64/abi/Makefile index a01d3806eba8..e91d4cdf17ad 100644 --- a/tools/testing/selftests/arm64/abi/Makefile +++ b/tools/testing/selftests/arm64/abi/Makefile @@ -1,6 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright (C) 2021 ARM Limited +CFLAGS += $(KHDR_INCLUDES) -I$(top_srcdir)/tools/include + TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2 seccomp_ptrace_x0_bypass seccomp_ret_trace_x0_bypass include ../../lib.mk diff --git a/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c b/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c index a00621a8949c..4ee8e5aaad6f 100644 --- a/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c +++ b/tools/testing/selftests/arm64/abi/seccomp_ptrace_x0_bypass.c @@ -34,13 +34,10 @@ #include #include #include +#include #include "kselftest.h" -#ifndef __NR_write -#define __NR_write 64 -#endif - #define EXPECTED_TESTS 1 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ diff --git a/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c b/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c index 3a69f53f4f88..a23081763328 100644 --- a/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c +++ b/tools/testing/selftests/arm64/abi/seccomp_ret_trace_x0_bypass.c @@ -44,13 +44,10 @@ #include #include #include +#include #include "kselftest.h" -#ifndef __NR_write -#define __NR_write 64 -#endif - #define PTRACE_EVENT_MASK(status) ((status) >> 16) #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ From 7a1f400ff5e57f41e23c8145a54ea084039b023c Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Wed, 5 Aug 2026 22:32:23 +0100 Subject: [PATCH 79/95] tools: Ensure tools copy of linux/filter.h exports the UAPI Normally when there is an include/foo.h and an include/uapi/foo.h the non-UAPI copy includes the UAPI copy. This is the case for the in kernel copy of linux/filter.h but not for the copy in tools/ which results in build breaks for the newly added arm64 seccomp_ptrace_x0_bypass selftest. Add an explicit include of the uapi to fix the test and avoid future surprises. Fixes: 2fcbc4adf997 ("kselftest/arm64: Add seccomp ptrace x0 bypass test") Fixes: f143c11bb7b9 ("tools: bpf: Use local copy of headers including uapi/linux/filter.h") Signed-off-by: Mark Brown Signed-off-by: Will Deacon --- tools/include/linux/filter.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/include/linux/filter.h b/tools/include/linux/filter.h index bcc6df79301a..4ead4e72097c 100644 --- a/tools/include/linux/filter.h +++ b/tools/include/linux/filter.h @@ -6,6 +6,7 @@ #define __TOOLS_LINUX_FILTER_H #include +#include /* ArgX, context and stack frame pointer register positions. Note, * Arg1, Arg2, Arg3, etc are used as argument mappings of function From 7ace06a01efaba1ec585630aa4c98ac39b38452a Mon Sep 17 00:00:00 2001 From: liulhong617 Date: Wed, 13 May 2026 09:02:55 +0800 Subject: [PATCH 80/95] arm64: mm: fix accidental linear mapping of no-map reserved memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When reserved-memory regions with the "no-map" property are not page-aligned, the kernel may accidentally map them into the linear mapping, contradicting the no-map semantics. The root cause is a mismatch between /proc/iomem's address boundaries and the actual page table mapping boundaries: 1. /proc/iomem derives its ranges from memblock via memblock_region_reserved_base_pfn/memblock_region_reserved_end_pfn, which perform PFN rounding so the displayed boundaries are page-aligned. This gives the impression that the no-map region occupies whole pages. 2. However, memblock_mark_nomap() splits memblock.memory regions at exact byte boundaries (memblock_isolate_range preserves raw DT base/size with no alignment). When for_each_mem_range iterates the non-NOMAP regions adjacent to a no-map region, it returns start/end values that are NOT page-aligned — they are the precise byte boundaries from the memblock split. 3. These sub-page-aligned values are passed to __create_pgd_mapping_locked(), which does: phys &= PAGE_MASK; addr = virt & PAGE_MASK; end = PAGE_ALIGN(virt + size); The downward rounding of phys via PAGE_MASK extends the mapped range backward into the adjacent no-map region, effectively including no-map memory in the linear mapping. For example, with 64K pages, reserved_region@A2000000 (base=0xA2000000, size=0x8000, no-map) causes for_each_mem_range to return start=0xA2008000 for the next mappable region. After phys &= PAGE_MASK, the actual mapping starts at 0xA2000000 — the entire no-map region is incorrectly mapped. Fix this by rounding the mappable range inward to PAGE_SIZE boundaries before passing it to __map_memblock: start is rounded UP and end is rounded DOWN. This ensures the mapped area never overlaps with adjacent no-map regions. The cost is at most one page of unmapped gap at each boundary, which is preferable to violating no-map semantics. Signed-off-by: liulhong617 Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index a25d8beacc83..f51566316ba4 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -1190,6 +1190,20 @@ static void __init map_mem(void) /* map all the memory banks */ for_each_mem_range(i, &start, &end) { + /* + * for_each_mem_range may return sub-page-aligned boundaries + * after memblock_mark_nomap() splits regions at byte precision. + * __create_pgd_mapping_locked aligns phys down to PAGE_MASK, + * which could accidentally map no-map memory on the boundary. + * Round the mappable range inward: start UP, end DOWN, so + * that the mapped area never overlaps with adjacent no-map + * regions. The cost is at most one page of unmapped gap at + * each boundary. + */ + start = PAGE_ALIGN(start); + end = end & PAGE_MASK; + if (start >= end) + continue; /* * The linear map must allow allocation tags reading/writing * if MTE is present. Otherwise, it has the same attributes as From 55c671f965b1e823972162a8dde10b12a5dde1a4 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 28 Jul 2026 16:59:21 +0100 Subject: [PATCH 81/95] perf/arm-cmn: Rename filter variables for clarity CMN has already grown many more event-specific filters than the original Occupancy ID, but since they are all independent of each other we've just overloaded them onto the same name. Before we add yet more, and they begin to overlap, rename all our "occupid" variables to "filter" so that things can be a bit clearer and more consistent (but leaving the format attribute itself, to avoid UAPI concerns). Reviewed-by: Ilkka Koskinen Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/perf/arm-cmn.c | 53 ++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 50402bc4a21d..2a8a67da72c3 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -163,13 +163,13 @@ /* Event attributes */ #define CMN_CONFIG_TYPE GENMASK_ULL(15, 0) #define CMN_CONFIG_EVENTID GENMASK_ULL(26, 16) -#define CMN_CONFIG_OCCUPID GENMASK_ULL(30, 27) +#define CMN_CONFIG_FILTER GENMASK_ULL(30, 27) #define CMN_CONFIG_BYNODEID BIT_ULL(31) #define CMN_CONFIG_NODEID GENMASK_ULL(47, 32) #define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config) #define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config) -#define CMN_EVENT_OCCUPID(event) FIELD_GET(CMN_CONFIG_OCCUPID, (event)->attr.config) +#define CMN_EVENT_FILTER(event) FIELD_GET(CMN_CONFIG_FILTER, (event)->attr.config) #define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config) #define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config) @@ -301,7 +301,7 @@ struct arm_cmn_node { struct { u8 val : 4; u8 count : 4; - } occupid[SEL_MAX]; + } filter[SEL_MAX]; union { u8 event[4]; __le32 event_sel; @@ -672,7 +672,7 @@ struct arm_cmn_event_attr { enum cmn_node_type type; enum cmn_filter_select fsel; u16 eventid; - u8 occupid; + u8 filter; }; struct arm_cmn_format_attr { @@ -681,13 +681,13 @@ struct arm_cmn_format_attr { int config; }; -#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _occupid, _fsel)\ +#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _filter, _fsel)\ (&((struct arm_cmn_event_attr[]) {{ \ .attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL), \ .model = _model, \ .type = _type, \ .eventid = _eventid, \ - .occupid = _occupid, \ + .filter = _filter, \ .fsel = _fsel, \ }})[0].attr.attr) #define CMN_EVENT_ATTR(_model, _name, _type, _eventid) \ @@ -709,8 +709,8 @@ static ssize_t arm_cmn_event_show(struct device *dev, eattr->type, eattr->eventid); if (eattr->fsel > SEL_NONE) - return sysfs_emit(buf, "type=0x%x,eventid=0x%x,occupid=0x%x\n", - eattr->type, eattr->eventid, eattr->occupid); + return sysfs_emit(buf, "type=0x%x,eventid=0x%x,filter=0x%x\n", + eattr->type, eattr->eventid, eattr->filter); return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type, eattr->eventid); @@ -1320,7 +1320,7 @@ static ssize_t arm_cmn_format_show(struct device *dev, static struct attribute *arm_cmn_format_attrs[] = { CMN_FORMAT_ATTR(type, CMN_CONFIG_TYPE), CMN_FORMAT_ATTR(eventid, CMN_CONFIG_EVENTID), - CMN_FORMAT_ATTR(occupid, CMN_CONFIG_OCCUPID), + CMN_FORMAT_ATTR(filter, CMN_CONFIG_FILTER), CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID), CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID), @@ -1333,6 +1333,9 @@ static struct attribute *arm_cmn_format_attrs[] = { _CMN_FORMAT_ATTR(wp_val, 1, CMN_CONFIG1_WP_VAL), _CMN_FORMAT_ATTR(wp_mask, 2, CMN_CONFIG2_WP_MASK), + /* Old name for UAPI compatibility */ + CMN_FORMAT_ATTR(occupid, CMN_CONFIG_FILTER), + NULL }; @@ -1544,30 +1547,30 @@ static void arm_cmn_event_read(struct perf_event *event) } static int arm_cmn_set_event_sel_hi(struct arm_cmn_node *dn, - enum cmn_filter_select fsel, u8 occupid) + enum cmn_filter_select fsel, u8 filter) { u64 reg; if (fsel == SEL_NONE) return 0; - if (!dn->occupid[fsel].count) { - dn->occupid[fsel].val = occupid; + if (!dn->filter[fsel].count) { + dn->filter[fsel].val = filter; reg = FIELD_PREP(CMN__PMU_CBUSY_SNTHROTTLE_SEL, - dn->occupid[SEL_CBUSY_SNTHROTTLE_SEL].val) | + dn->filter[SEL_CBUSY_SNTHROTTLE_SEL].val) | FIELD_PREP(CMN__PMU_SN_HOME_SEL, - dn->occupid[SEL_SN_HOME_SEL].val) | + dn->filter[SEL_SN_HOME_SEL].val) | FIELD_PREP(CMN__PMU_HBT_LBT_SEL, - dn->occupid[SEL_HBT_LBT_SEL].val) | + dn->filter[SEL_HBT_LBT_SEL].val) | FIELD_PREP(CMN__PMU_CLASS_OCCUP_ID, - dn->occupid[SEL_CLASS_OCCUP_ID].val) | + dn->filter[SEL_CLASS_OCCUP_ID].val) | FIELD_PREP(CMN__PMU_OCCUP1_ID, - dn->occupid[SEL_OCCUP1ID].val); + dn->filter[SEL_OCCUP1ID].val); writel_relaxed(reg >> 32, dn->pmu_base + CMN_PMU_EVENT_SEL + 4); - } else if (dn->occupid[fsel].val != occupid) { + } else if (dn->filter[fsel].val != filter) { return -EBUSY; } - dn->occupid[fsel].count++; + dn->filter[fsel].count++; return 0; } @@ -1649,7 +1652,7 @@ static void arm_cmn_event_stop(struct perf_event *event, int flags) struct arm_cmn_val { u8 dtm_count[CMN_MAX_DTMS]; - u8 occupid[CMN_MAX_DTMS][SEL_MAX]; + u8 filter[CMN_MAX_DTMS][SEL_MAX]; u8 wp[CMN_MAX_DTMS][4]; u8 wp_combine[CMN_MAX_DTMS][2]; int dtc_count[CMN_MAX_DTCS]; @@ -1694,7 +1697,7 @@ static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val, val->dtm_count[dtm]++; if (sel > SEL_NONE) - val->occupid[dtm][sel] = CMN_EVENT_OCCUPID(event) + 1; + val->filter[dtm][sel] = CMN_EVENT_FILTER(event) + 1; if (type != CMN_TYPE_WP) continue; @@ -1745,8 +1748,8 @@ static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event) if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) goto done; - if (sel > SEL_NONE && val->occupid[dtm][sel] && - val->occupid[dtm][sel] != CMN_EVENT_OCCUPID(event) + 1) + if (sel > SEL_NONE && val->filter[dtm][sel] && + val->filter[dtm][sel] != CMN_EVENT_FILTER(event) + 1) goto done; if (type != CMN_TYPE_WP) @@ -1892,7 +1895,7 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, } if (hw->filter_sel > SEL_NONE) - hw->dn[i].occupid[hw->filter_sel].count--; + hw->dn[i].filter[hw->filter_sel].count--; dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); @@ -1978,7 +1981,7 @@ static int arm_cmn_event_add(struct perf_event *event, int flags) input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx + (nid.port << 4) + (nid.dev << 2); - if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_OCCUPID(event))) + if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_FILTER(event))) goto free_dtms; } From 4da12d5c8bef4ea92fc1f7088b1803f6e1de8118 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 28 Jul 2026 16:59:22 +0100 Subject: [PATCH 82/95] perf/arm-cmn: Refactor event filter programming We're soon going to need to cope with events having multiple filters, plus the filter fields themselves moving around, wherein any more inline if/else logic will struggle to scale. Add a more general abstraction for the node-specific filter controls, and rejig the pmu_event_sel filter programming around it in a more extensible manner. Reviewed-by: Ilkka Koskinen Signed-off-by: Robin Murphy Signed-off-by: Will Deacon --- drivers/perf/arm-cmn.c | 139 ++++++++++++++++++++++++++++++----------- 1 file changed, 101 insertions(+), 38 deletions(-) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 2a8a67da72c3..14c267d2e2f9 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -278,8 +278,8 @@ enum cmn_node_type { }; enum cmn_filter_select { - SEL_NONE = -1, - SEL_OCCUP1ID, + SEL_NONE, + SEL_OCCUP1_ID, SEL_CLASS_OCCUP_ID, SEL_CBUSY_SNTHROTTLE_SEL, SEL_HBT_LBT_SEL, @@ -599,6 +599,57 @@ static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) static void arm_cmn_debugfs_init(struct arm_cmn *cmn, int id) {} #endif +enum cmn_filter_type { + FILT_NONE, + FILT_OCCUP1_ID, + FILT_HNF_700, + FILT_HNS, +}; +#define CMN_FILTER(_sel) [SEL_##_sel] = CMN__PMU_##_sel + +static const u64 arm_cmn_filters[][SEL_MAX] = { + [FILT_NONE] = {}, + /* DVM etc. */ + [FILT_OCCUP1_ID] = { + CMN_FILTER(OCCUP1_ID) + }, + /* Newer HN-F */ + [FILT_HNF_700] = { + CMN_FILTER(OCCUP1_ID), + CMN_FILTER(CLASS_OCCUP_ID), + CMN_FILTER(CBUSY_SNTHROTTLE_SEL) + }, + /* HN-S */ + [FILT_HNS] = { + CMN_FILTER(OCCUP1_ID), + CMN_FILTER(CLASS_OCCUP_ID), + CMN_FILTER(CBUSY_SNTHROTTLE_SEL), + CMN_FILTER(HBT_LBT_SEL), + CMN_FILTER(SN_HOME_SEL) + }, +}; + +static enum cmn_filter_type arm_cmn_filter(enum cmn_node_type node, + enum cmn_model model) +{ + switch (node) { + default: + return FILT_NONE; + case CMN_TYPE_DVM: + case CMN_TYPE_CXRA: + case CMN_TYPE_CXHA: + case CMN_TYPE_CCRA: + case CMN_TYPE_CCHA: + return FILT_OCCUP1_ID; + case CMN_TYPE_HNF: + if (model < CMN700) + return FILT_OCCUP1_ID; + return FILT_HNF_700; + case CMN_TYPE_HNS: + return FILT_HNS; + }; +} + struct arm_cmn_hw_event { struct arm_cmn_node *dn; union { @@ -708,7 +759,7 @@ static ssize_t arm_cmn_event_show(struct device *dev, "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n", eattr->type, eattr->eventid); - if (eattr->fsel > SEL_NONE) + if (eattr->fsel) return sysfs_emit(buf, "type=0x%x,eventid=0x%x,filter=0x%x\n", eattr->type, eattr->eventid, eattr->filter); @@ -832,16 +883,16 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, #define CMN_EVENT_DVM(_model, _name, _event) \ _CMN_EVENT_DVM(_model, _name, _event, 0, SEL_NONE) #define CMN_EVENT_DVM_OCC(_model, _name, _event) \ - _CMN_EVENT_DVM(_model, _name##_all, _event, 0, SEL_OCCUP1ID), \ - _CMN_EVENT_DVM(_model, _name##_dvmop, _event, 1, SEL_OCCUP1ID), \ - _CMN_EVENT_DVM(_model, _name##_dvmsync, _event, 2, SEL_OCCUP1ID) + _CMN_EVENT_DVM(_model, _name##_all, _event, 0, SEL_OCCUP1_ID), \ + _CMN_EVENT_DVM(_model, _name##_dvmop, _event, 1, SEL_OCCUP1_ID), \ + _CMN_EVENT_DVM(_model, _name##_dvmsync, _event, 2, SEL_OCCUP1_ID) #define CMN_EVENT_HN_OCC(_model, _name, _type, _event) \ - _CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 1, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 2, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(_model, _name##_atomic, _type, _event, 3, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(_model, _name##_stash, _type, _event, 4, SEL_OCCUP1ID) + _CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_OCCUP1_ID), \ + _CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 1, SEL_OCCUP1_ID), \ + _CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 2, SEL_OCCUP1_ID), \ + _CMN_EVENT_ATTR(_model, _name##_atomic, _type, _event, 3, SEL_OCCUP1_ID), \ + _CMN_EVENT_ATTR(_model, _name##_stash, _type, _event, 4, SEL_OCCUP1_ID) #define CMN_EVENT_HN_CLS(_model, _name, _type, _event) \ _CMN_EVENT_ATTR(_model, _name##_class0, _type, _event, 0, SEL_CLASS_OCCUP_ID), \ _CMN_EVENT_ATTR(_model, _name##_class1, _type, _event, 1, SEL_CLASS_OCCUP_ID), \ @@ -865,9 +916,9 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, #define CMN_EVENT_HNS_OCC(_name, _event) \ CMN_EVENT_HN_OCC(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_rxsnp, CMN_TYPE_HNS, _event, 5, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 6, SEL_OCCUP1ID), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 7, SEL_OCCUP1ID) + _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_rxsnp, CMN_TYPE_HNS, _event, 5, SEL_OCCUP1_ID), \ + _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 6, SEL_OCCUP1_ID), \ + _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 7, SEL_OCCUP1_ID) #define CMN_EVENT_HNS_CLS( _name, _event) \ CMN_EVENT_HN_CLS(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) #define CMN_EVENT_HNS_SNT(_name, _event) \ @@ -1547,27 +1598,20 @@ static void arm_cmn_event_read(struct perf_event *event) } static int arm_cmn_set_event_sel_hi(struct arm_cmn_node *dn, - enum cmn_filter_select fsel, u8 filter) + enum cmn_filter_select fsel, u8 val) { - u64 reg; - - if (fsel == SEL_NONE) - return 0; - if (!dn->filter[fsel].count) { - dn->filter[fsel].val = filter; - reg = FIELD_PREP(CMN__PMU_CBUSY_SNTHROTTLE_SEL, - dn->filter[SEL_CBUSY_SNTHROTTLE_SEL].val) | - FIELD_PREP(CMN__PMU_SN_HOME_SEL, - dn->filter[SEL_SN_HOME_SEL].val) | - FIELD_PREP(CMN__PMU_HBT_LBT_SEL, - dn->filter[SEL_HBT_LBT_SEL].val) | - FIELD_PREP(CMN__PMU_CLASS_OCCUP_ID, - dn->filter[SEL_CLASS_OCCUP_ID].val) | - FIELD_PREP(CMN__PMU_OCCUP1_ID, - dn->filter[SEL_OCCUP1ID].val); + const u64 *filter = arm_cmn_filters[dn->filter[SEL_NONE].val]; + u64 reg = 0; + + dn->filter[fsel].val = val; + for (int i = SEL_OCCUP1_ID; i < SEL_MAX; i++) { + if (filter[i]) + reg |= field_prep(filter[i], dn->filter[i].val); + } + writel_relaxed(reg >> 32, dn->pmu_base + CMN_PMU_EVENT_SEL + 4); - } else if (dn->filter[fsel].val != filter) { + } else if (dn->filter[fsel].val != val) { return -EBUSY; } dn->filter[fsel].count++; @@ -1696,7 +1740,7 @@ static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val, val->dtm_count[dtm]++; - if (sel > SEL_NONE) + if (sel) val->filter[dtm][sel] = CMN_EVENT_FILTER(event) + 1; if (type != CMN_TYPE_WP) @@ -1748,7 +1792,7 @@ static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event) if (val->dtm_count[dtm] == CMN_DTM_NUM_COUNTERS) goto done; - if (sel > SEL_NONE && val->filter[dtm][sel] && + if (sel && val->filter[dtm][sel] && val->filter[dtm][sel] != CMN_EVENT_FILTER(event) + 1) goto done; @@ -1894,7 +1938,7 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, dtm->wp_event[wp_idx] = -1; } - if (hw->filter_sel > SEL_NONE) + if (hw->filter_sel) hw->dn[i].filter[hw->filter_sel].count--; dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); @@ -1906,6 +1950,17 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, cmn->dtc[j].counters[idx] = NULL; } +static int arm_cmn_set_event_filter(struct arm_cmn_node *dn, struct perf_event *event) +{ + enum cmn_filter_select fsel = to_cmn_hw(event)->filter_sel; + int ret = 0; + + if (fsel) + ret = arm_cmn_set_event_sel_hi(dn, fsel, CMN_EVENT_FILTER(event)); + + return ret; +} + static int arm_cmn_event_add(struct perf_event *event, int flags) { struct arm_cmn *cmn = to_cmn(event->pmu); @@ -1981,7 +2036,7 @@ static int arm_cmn_event_add(struct perf_event *event, int flags) input_sel = CMN__PMEVCNT0_INPUT_SEL_DEV + dtm_idx + (nid.port << 4) + (nid.dev << 2); - if (arm_cmn_set_event_sel_hi(dn, hw->filter_sel, CMN_EVENT_FILTER(event))) + if (arm_cmn_set_event_filter(dn, event)) goto free_dtms; } @@ -2311,6 +2366,7 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) void __iomem *cfg_region, __iomem *xp_region; struct arm_cmn_node cfg, *dn; struct arm_cmn_dtm *dtm; + enum cmn_model model; enum cmn_part part; u16 child_count, child_poff; u64 reg; @@ -2342,12 +2398,14 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) "Firmware binding mismatch: expected part number 0x%x, found 0x%x\n", cmn->part, part); cmn->part = part; - if (!arm_cmn_model(cmn)) - dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part); reg = readl_relaxed(cfg_region + CMN_CFGM_PERIPH_ID_23); cmn->rev = FIELD_GET(CMN_CFGM_PID2_REVISION, reg); + model = arm_cmn_model(cmn); + if (!model) + dev_warn(cmn->dev, "Unknown part number: 0x%x\n", part); + /* * With the device isolation feature, if firmware has neglected to enable * an XP port then we risk locking up if we try to access anything behind @@ -2500,6 +2558,11 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) dev_err(cmn->dev, "Node ID invalid for supported CMN versions: %d\n", dn->logid); return -ENODEV; } + /* + * We can utilise the "wasted" filter array slot to store + * the index for referencing the filter encodings later. + */ + dn->filter[SEL_NONE].val = arm_cmn_filter(dn->type, model); switch (dn->type) { case CMN_TYPE_DTC: From f1b844fdc0d66767b8438ff2f2bb31ce2701c8c3 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 28 Jul 2026 16:59:23 +0100 Subject: [PATCH 83/95] perf/arm-cmn: Refactor event filter data The ABI hole I have dug myself into requires the driver to know which event encodings are associated with which particular filter control. Since we will soon have a notion of multiple filters per event, refactor the event data to encapsulate filters in an explicit structure, which can then more easily scale as an array in future. Signed-off-by: Robin Murphy Reviewed-by: Ilkka Koskinen Signed-off-by: Will Deacon --- drivers/perf/arm-cmn.c | 95 ++++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 14c267d2e2f9..74bc2aae4a06 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -717,13 +717,17 @@ static void arm_cmn_clear_idx(struct arm_cmn_hw_event *hw) bitmap_zero(hw->wp_idx, CMN_MAX_XPS); } +struct arm_cmn_filter_attr { + enum cmn_filter_select sel; + u8 val; +}; + struct arm_cmn_event_attr { struct device_attribute attr; enum cmn_model model; enum cmn_node_type type; - enum cmn_filter_select fsel; u16 eventid; - u8 filter; + struct arm_cmn_filter_attr filter[1]; }; struct arm_cmn_format_attr { @@ -732,24 +736,25 @@ struct arm_cmn_format_attr { int config; }; -#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _filter, _fsel)\ +#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _fa, _fb, ...) \ (&((struct arm_cmn_event_attr[]) {{ \ .attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL), \ .model = _model, \ .type = _type, \ .eventid = _eventid, \ - .filter = _filter, \ - .fsel = _fsel, \ + .filter = {{_fa, _fb}}, \ }})[0].attr.attr) -#define CMN_EVENT_ATTR(_model, _name, _type, _eventid) \ - _CMN_EVENT_ATTR(_model, _name, _type, _eventid, 0, SEL_NONE) +#define CMN_EVENT_ATTR(_model, _name, _type, _eventid, _filter...) \ + _CMN_EVENT_ATTR(_model, _name, _type, _eventid, ##_filter, 0, 0) static ssize_t arm_cmn_event_show(struct device *dev, struct device_attribute *attr, char *buf) { struct arm_cmn_event_attr *eattr; + struct arm_cmn_filter_attr *filter; eattr = container_of(attr, typeof(*eattr), attr); + filter = eattr->filter; if (eattr->type == CMN_TYPE_DTC) return sysfs_emit(buf, "type=0x%x\n", eattr->type); @@ -759,9 +764,9 @@ static ssize_t arm_cmn_event_show(struct device *dev, "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n", eattr->type, eattr->eventid); - if (eattr->fsel) + if (filter[0].sel) return sysfs_emit(buf, "type=0x%x,eventid=0x%x,filter=0x%x\n", - eattr->type, eattr->eventid, eattr->filter); + eattr->type, eattr->eventid, filter[0].val); return sysfs_emit(buf, "type=0x%x,eventid=0x%x\n", eattr->type, eattr->eventid); @@ -849,8 +854,8 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, return attr->mode; } -#define _CMN_EVENT_DVM(_model, _name, _event, _occup, _fsel) \ - _CMN_EVENT_ATTR(_model, dn_##_name, CMN_TYPE_DVM, _event, _occup, _fsel) +#define CMN_EVENT_DVM(_model, _name, _event, _filter...) \ + CMN_EVENT_ATTR(_model, dn_##_name, CMN_TYPE_DVM, _event, ##_filter) #define CMN_EVENT_DTC(_name) \ CMN_EVENT_ATTR(CMN_ANY, dtc_##_name, CMN_TYPE_DTC, 0) #define CMN_EVENT_HNF(_model, _name, _event) \ @@ -880,32 +885,30 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, #define CMN_EVENT_HNS(_name, _event) \ CMN_EVENT_ATTR(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) -#define CMN_EVENT_DVM(_model, _name, _event) \ - _CMN_EVENT_DVM(_model, _name, _event, 0, SEL_NONE) #define CMN_EVENT_DVM_OCC(_model, _name, _event) \ - _CMN_EVENT_DVM(_model, _name##_all, _event, 0, SEL_OCCUP1_ID), \ - _CMN_EVENT_DVM(_model, _name##_dvmop, _event, 1, SEL_OCCUP1_ID), \ - _CMN_EVENT_DVM(_model, _name##_dvmsync, _event, 2, SEL_OCCUP1_ID) + CMN_EVENT_DVM(_model, _name##_all, _event, SEL_OCCUP1_ID, 0), \ + CMN_EVENT_DVM(_model, _name##_dvmop, _event, SEL_OCCUP1_ID, 1), \ + CMN_EVENT_DVM(_model, _name##_dvmsync, _event, SEL_OCCUP1_ID, 2) #define CMN_EVENT_HN_OCC(_model, _name, _type, _event) \ - _CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_OCCUP1_ID), \ - _CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 1, SEL_OCCUP1_ID), \ - _CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 2, SEL_OCCUP1_ID), \ - _CMN_EVENT_ATTR(_model, _name##_atomic, _type, _event, 3, SEL_OCCUP1_ID), \ - _CMN_EVENT_ATTR(_model, _name##_stash, _type, _event, 4, SEL_OCCUP1_ID) + CMN_EVENT_ATTR(_model, _name##_all, _type, _event, SEL_OCCUP1_ID, 0), \ + CMN_EVENT_ATTR(_model, _name##_read, _type, _event, SEL_OCCUP1_ID, 1), \ + CMN_EVENT_ATTR(_model, _name##_write, _type, _event, SEL_OCCUP1_ID, 2), \ + CMN_EVENT_ATTR(_model, _name##_atomic, _type, _event, SEL_OCCUP1_ID, 3), \ + CMN_EVENT_ATTR(_model, _name##_stash, _type, _event, SEL_OCCUP1_ID, 4) #define CMN_EVENT_HN_CLS(_model, _name, _type, _event) \ - _CMN_EVENT_ATTR(_model, _name##_class0, _type, _event, 0, SEL_CLASS_OCCUP_ID), \ - _CMN_EVENT_ATTR(_model, _name##_class1, _type, _event, 1, SEL_CLASS_OCCUP_ID), \ - _CMN_EVENT_ATTR(_model, _name##_class2, _type, _event, 2, SEL_CLASS_OCCUP_ID), \ - _CMN_EVENT_ATTR(_model, _name##_class3, _type, _event, 3, SEL_CLASS_OCCUP_ID) + CMN_EVENT_ATTR(_model, _name##_class0, _type, _event, SEL_CLASS_OCCUP_ID, 0), \ + CMN_EVENT_ATTR(_model, _name##_class1, _type, _event, SEL_CLASS_OCCUP_ID, 1), \ + CMN_EVENT_ATTR(_model, _name##_class2, _type, _event, SEL_CLASS_OCCUP_ID, 2), \ + CMN_EVENT_ATTR(_model, _name##_class3, _type, _event, SEL_CLASS_OCCUP_ID, 3) #define CMN_EVENT_HN_SNT(_model, _name, _type, _event) \ - _CMN_EVENT_ATTR(_model, _name##_all, _type, _event, 0, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_group0_read, _type, _event, 1, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_group0_write, _type, _event, 2, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_group1_read, _type, _event, 3, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_group1_write, _type, _event, 4, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_read, _type, _event, 5, SEL_CBUSY_SNTHROTTLE_SEL), \ - _CMN_EVENT_ATTR(_model, _name##_write, _type, _event, 6, SEL_CBUSY_SNTHROTTLE_SEL) + CMN_EVENT_ATTR(_model, _name##_all, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 0), \ + CMN_EVENT_ATTR(_model, _name##_group0_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 1), \ + CMN_EVENT_ATTR(_model, _name##_group0_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 2), \ + CMN_EVENT_ATTR(_model, _name##_group1_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 3), \ + CMN_EVENT_ATTR(_model, _name##_group1_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 4), \ + CMN_EVENT_ATTR(_model, _name##_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 5), \ + CMN_EVENT_ATTR(_model, _name##_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 6) #define CMN_EVENT_HNF_OCC(_model, _name, _event) \ CMN_EVENT_HN_OCC(_model, hnf_##_name, CMN_TYPE_HNF, _event) @@ -916,21 +919,21 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, #define CMN_EVENT_HNS_OCC(_name, _event) \ CMN_EVENT_HN_OCC(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_rxsnp, CMN_TYPE_HNS, _event, 5, SEL_OCCUP1_ID), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 6, SEL_OCCUP1_ID), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 7, SEL_OCCUP1_ID) + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_rxsnp, CMN_TYPE_HNS, _event, SEL_OCCUP1_ID, 5), \ + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, SEL_OCCUP1_ID, 6), \ + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, SEL_OCCUP1_ID, 7) #define CMN_EVENT_HNS_CLS( _name, _event) \ CMN_EVENT_HN_CLS(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) #define CMN_EVENT_HNS_SNT(_name, _event) \ CMN_EVENT_HN_SNT(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) #define CMN_EVENT_HNS_HBT(_name, _event) \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, 0, SEL_HBT_LBT_SEL), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, 1, SEL_HBT_LBT_SEL), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, 2, SEL_HBT_LBT_SEL) + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, SEL_HBT_LBT_SEL, 0), \ + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, SEL_HBT_LBT_SEL, 1), \ + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, SEL_HBT_LBT_SEL, 2) #define CMN_EVENT_HNS_SNH(_name, _event) \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, 0, SEL_SN_HOME_SEL), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_sn, CMN_TYPE_HNS, _event, 1, SEL_SN_HOME_SEL), \ - _CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_home, CMN_TYPE_HNS, _event, 2, SEL_SN_HOME_SEL) + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, SEL_SN_HOME_SEL, 0), \ + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_sn, CMN_TYPE_HNS, _event, SEL_SN_HOME_SEL, 1), \ + CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_home, CMN_TYPE_HNS, _event, SEL_SN_HOME_SEL, 2) #define _CMN_EVENT_XP_MESH(_name, _event) \ __CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)), \ @@ -1814,9 +1817,9 @@ done: return ret; } -static enum cmn_filter_select arm_cmn_filter_sel(const struct arm_cmn *cmn, - enum cmn_node_type type, - unsigned int eventid) +static enum cmn_filter_select arm_cmn_event_filter(const struct arm_cmn *cmn, + enum cmn_node_type type, + unsigned int eventid) { struct arm_cmn_event_attr *e; enum cmn_model model = arm_cmn_model(cmn); @@ -1824,7 +1827,7 @@ static enum cmn_filter_select arm_cmn_filter_sel(const struct arm_cmn *cmn, for (int i = 0; i < ARRAY_SIZE(arm_cmn_event_attrs) - 1; i++) { e = container_of(arm_cmn_event_attrs[i], typeof(*e), attr.attr); if (e->model & model && e->type == type && e->eventid == eventid) - return e->fsel; + return e->filter[0].sel; } return SEL_NONE; } @@ -1889,7 +1892,7 @@ static int arm_cmn_event_init(struct perf_event *event) } /* This is sufficiently annoying to recalculate, so cache it */ - hw->filter_sel = arm_cmn_filter_sel(cmn, type, eventid); + hw->filter_sel = arm_cmn_event_filter(cmn, type, eventid); bynodeid = CMN_EVENT_BYNODEID(event); nodeid = CMN_EVENT_NODEID(event); From 09178f536bb9c659953364639de6564848fa1c21 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 28 Jul 2026 16:59:24 +0100 Subject: [PATCH 84/95] perf/arm-cmn: Plumb in new filter types Add the logic to handle events with the upcoming new filter controls. Since for now we will have the sole invariant of all EVICT_STATE_SEL events having HBT_LBT_SEL as a secondary filter, for the sake of simplicity we can just special-case that, and save the complication of a full multi-filter abstraction until unavoidably necessary. Reviewed-by: Leo Yan Signed-off-by: Robin Murphy Reviewed-by: Ilkka Koskinen Signed-off-by: Will Deacon --- drivers/perf/arm-cmn.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 74bc2aae4a06..e25f46c38c23 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -166,12 +166,14 @@ #define CMN_CONFIG_FILTER GENMASK_ULL(30, 27) #define CMN_CONFIG_BYNODEID BIT_ULL(31) #define CMN_CONFIG_NODEID GENMASK_ULL(47, 32) +#define CMN_CONFIG_FILTER2 GENMASK_ULL(51, 48) #define CMN_EVENT_TYPE(event) FIELD_GET(CMN_CONFIG_TYPE, (event)->attr.config) #define CMN_EVENT_EVENTID(event) FIELD_GET(CMN_CONFIG_EVENTID, (event)->attr.config) #define CMN_EVENT_FILTER(event) FIELD_GET(CMN_CONFIG_FILTER, (event)->attr.config) #define CMN_EVENT_BYNODEID(event) FIELD_GET(CMN_CONFIG_BYNODEID, (event)->attr.config) #define CMN_EVENT_NODEID(event) FIELD_GET(CMN_CONFIG_NODEID, (event)->attr.config) +#define CMN_EVENT_FILTER2(event) FIELD_GET(CMN_CONFIG_FILTER2, (event)->attr.config) #define CMN_CONFIG_WP_COMBINE GENMASK_ULL(30, 27) #define CMN_CONFIG_WP_DEV_SEL GENMASK_ULL(50, 48) @@ -284,6 +286,9 @@ enum cmn_filter_select { SEL_CBUSY_SNTHROTTLE_SEL, SEL_HBT_LBT_SEL, SEL_SN_HOME_SEL, + SEL_SNP_VC_SEL, + SEL_ENHANCED_HBT_LBT_SEL, + SEL_EVICT_STATE_SEL, SEL_MAX }; @@ -1377,6 +1382,7 @@ static struct attribute *arm_cmn_format_attrs[] = { CMN_FORMAT_ATTR(filter, CMN_CONFIG_FILTER), CMN_FORMAT_ATTR(bynodeid, CMN_CONFIG_BYNODEID), CMN_FORMAT_ATTR(nodeid, CMN_CONFIG_NODEID), + CMN_FORMAT_ATTR(filter2, CMN_CONFIG_FILTER2), CMN_FORMAT_ATTR(wp_dev_sel, CMN_CONFIG_WP_DEV_SEL), CMN_FORMAT_ATTR(wp_chn_sel, CMN_CONFIG_WP_CHN_SEL), @@ -1745,6 +1751,8 @@ static void arm_cmn_val_add_event(struct arm_cmn *cmn, struct arm_cmn_val *val, if (sel) val->filter[dtm][sel] = CMN_EVENT_FILTER(event) + 1; + if (sel == SEL_EVICT_STATE_SEL) + val->filter[dtm][SEL_HBT_LBT_SEL] = CMN_EVENT_FILTER2(event) + 1; if (type != CMN_TYPE_WP) continue; @@ -1799,6 +1807,10 @@ static int arm_cmn_validate_group(struct arm_cmn *cmn, struct perf_event *event) val->filter[dtm][sel] != CMN_EVENT_FILTER(event) + 1) goto done; + if (sel == SEL_EVICT_STATE_SEL && val->filter[dtm][SEL_HBT_LBT_SEL] && + val->filter[dtm][SEL_HBT_LBT_SEL] != CMN_EVENT_FILTER2(event) + 1) + goto done; + if (type != CMN_TYPE_WP) continue; @@ -1943,6 +1955,8 @@ static void arm_cmn_event_clear(struct arm_cmn *cmn, struct perf_event *event, if (hw->filter_sel) hw->dn[i].filter[hw->filter_sel].count--; + if (hw->filter_sel == SEL_EVICT_STATE_SEL) + hw->dn[i].filter[SEL_HBT_LBT_SEL].count--; dtm->pmu_config_low &= ~CMN__PMEVCNT_PAIRED(dtm_idx); writel_relaxed(dtm->pmu_config_low, dtm->base + CMN_DTM_PMU_CONFIG); @@ -1961,6 +1975,11 @@ static int arm_cmn_set_event_filter(struct arm_cmn_node *dn, struct perf_event * if (fsel) ret = arm_cmn_set_event_sel_hi(dn, fsel, CMN_EVENT_FILTER(event)); + if (fsel == SEL_EVICT_STATE_SEL && !ret) { + ret = arm_cmn_set_event_sel_hi(dn, SEL_HBT_LBT_SEL, CMN_EVENT_FILTER2(event)); + if (ret) + dn->filter[fsel].count--; + } return ret; } From 21afe52e546ce7d3160dc2f502528398d40a5606 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 28 Jul 2026 16:59:25 +0100 Subject: [PATCH 85/95] perf/arm-cmn: Support CMN S3 r2 If you were disappointed at how minimal the initial CMN S3 support looked compared to previous versions, then oh boy is it time to put your party hats on... The biggest batch of incompatible changes yet comes not with a new CMN product, but a point release of an existing one. We've got new filters, loads of changes to existing events, register fields moved around for no good reason, and much, much more! On the upside, we do at least gain a means of working around the isolation feature. As such, for the sake of sanity in the driver it is easiest to split it into a distict "model" for our internal abstractions despite it bearing the same part number as r0/r1. Signed-off-by: Robin Murphy Reviewed-by: Ilkka Koskinen Signed-off-by: Will Deacon --- drivers/perf/arm-cmn.c | 270 +++++++++++++++++++++++++++++------------ 1 file changed, 193 insertions(+), 77 deletions(-) diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index e25f46c38c23..a03e2a8ca41e 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -29,6 +29,7 @@ #define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16) #define CMN_CHILD_NODE_ADDR GENMASK(29, 0) +#define CMN_CHILD_NODE_ISOLATED BIT(30) #define CMN_CHILD_NODE_EXTERNAL BIT(31) /* Some implementations use a mesh larger than the architectural max of 12 */ @@ -48,11 +49,16 @@ #define CMN_CFGM_INFO_GLOBAL 0x0900 #define CMN_INFO_MULTIPLE_DTM_EN BIT_ULL(63) +#define CMN_S3_R2_MULTIPLE_DTM_EN BIT_ULL(59) #define CMN_INFO_RSP_VC_NUM GENMASK_ULL(53, 52) #define CMN_INFO_DAT_VC_NUM GENMASK_ULL(51, 50) #define CMN_INFO_DEVICE_ISO_ENABLE BIT_ULL(44) #define CMN_CFGM_INFO_GLOBAL_1 0x0908 +#define CMN_S3_R2_RSP_VC_NUM GENMASK_ULL(11, 9) +#define CMN_S3_R2_DAT_VC_NUM GENMASK_ULL(8, 6) +#define CMN_S3_R2_SNP_VC_NUM GENMASK_ULL(5, 3) +#define CMN_S3_R2_REQ_VC_NUM GENMASK_ULL(2, 0) #define CMN_INFO_SNP_VC_NUM GENMASK_ULL(3, 2) #define CMN_INFO_REQ_VC_NUM GENMASK_ULL(1, 0) @@ -78,6 +84,16 @@ /* Technically this is 4 bits wide on DNs, but we only use 2 there anyway */ #define CMN__PMU_OCCUP1_ID GENMASK_ULL(34, 32) +/* But then... */ +#define CMN__PMU_EVICT_STATE_SEL GENMASK_ULL(54, 52) +#define CMN__PMU_ENHANCED_HBT_LBT_SEL GENMASK_ULL(51, 48) +#define CMN__PMU_SNP_VC_SEL GENMASK_ULL(47, 46) +#define CMN__S3_R2_CBUSY_SNTHROTTLE_SEL GENMASK_ULL(45, 42) +#define CMN__S3_R2_SN_HOME_SEL GENMASK_ULL(41, 40) +#define CMN__S3_R2_HBT_LBT_SEL GENMASK_ULL(39, 38) +#define CMN__S3_R2_CLASS_OCCUP_ID GENMASK_ULL(37, 36) +#define CMN__S3_R2_OCCUP1_ID GENMASK_ULL(35, 32) + /* Some types are designed to coexist with another device in the same node */ #define CMN_CCLA_PMU_EVENT_SEL 0x008 #define CMN_HNP_PMU_EVENT_SEL 0x008 @@ -202,12 +218,14 @@ enum cmn_model { CMN650 = 2, CI700 = 4, CMN700 = 8, - CMNS3 = 16, + CMNS3R01 = 16, + CMNS3R2 = 32, /* ...and then we can use bitmap tricks for commonality */ CMN_ANY = -1, NOT_CMN600 = -2, CMN_700ON = ~(CMN700 - 1), CMN_650ON = CMN_700ON | CMN650, + CMNS3 = CMNS3R01 | CMNS3R2, }; /* Actual part numbers and revision IDs defined by the hardware */ @@ -243,6 +261,10 @@ enum cmn_revision { REV_CMNS3_R0P0 = 0, REV_CMNS3_R0P1, REV_CMNS3_R1P0, + REV_CMNS3_R2P0, + REV_CMNS3_R2P1, + REV_CMNS3_R2P2, + REV_CMNS3_R2P5, REV_CI700_R0P0 = 0, REV_CI700_R1P0, REV_CI700_R2P0, @@ -429,7 +451,9 @@ static enum cmn_model arm_cmn_model(const struct arm_cmn *cmn) case PART_CI700: return CI700; case PART_CMN_S3: - return CMNS3; + if (cmn->rev >= REV_CMNS3_R2P0) + return CMNS3R2; + return CMNS3R01; default: return 0; }; @@ -609,8 +633,10 @@ enum cmn_filter_type { FILT_OCCUP1_ID, FILT_HNF_700, FILT_HNS, + FILT_HNS_S3R2, }; #define CMN_FILTER(_sel) [SEL_##_sel] = CMN__PMU_##_sel +#define CMN_FILTER_V2(_sel) [SEL_##_sel] = CMN__S3_R2_##_sel static const u64 arm_cmn_filters[][SEL_MAX] = { [FILT_NONE] = {}, @@ -632,6 +658,17 @@ static const u64 arm_cmn_filters[][SEL_MAX] = { CMN_FILTER(HBT_LBT_SEL), CMN_FILTER(SN_HOME_SEL) }, + /* Newer HN-S */ + [FILT_HNS_S3R2] = { + CMN_FILTER_V2(OCCUP1_ID), + CMN_FILTER_V2(CLASS_OCCUP_ID), + CMN_FILTER_V2(CBUSY_SNTHROTTLE_SEL), + CMN_FILTER_V2(HBT_LBT_SEL), + CMN_FILTER_V2(SN_HOME_SEL), + CMN_FILTER(SNP_VC_SEL), + CMN_FILTER(ENHANCED_HBT_LBT_SEL), + CMN_FILTER(EVICT_STATE_SEL) + } }; static enum cmn_filter_type arm_cmn_filter(enum cmn_node_type node, @@ -651,7 +688,9 @@ static enum cmn_filter_type arm_cmn_filter(enum cmn_node_type node, return FILT_OCCUP1_ID; return FILT_HNF_700; case CMN_TYPE_HNS: - return FILT_HNS; + if (model < CMNS3R2) + return FILT_HNS; + return FILT_HNS_S3R2; }; } @@ -732,7 +771,7 @@ struct arm_cmn_event_attr { enum cmn_model model; enum cmn_node_type type; u16 eventid; - struct arm_cmn_filter_attr filter[1]; + struct arm_cmn_filter_attr filter[2]; }; struct arm_cmn_format_attr { @@ -741,16 +780,16 @@ struct arm_cmn_format_attr { int config; }; -#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _fa, _fb, ...) \ +#define _CMN_EVENT_ATTR(_model, _name, _type, _eventid, _fa, _fb, _fc, _fd, ...) \ (&((struct arm_cmn_event_attr[]) {{ \ .attr = __ATTR(_name, 0444, arm_cmn_event_show, NULL), \ .model = _model, \ .type = _type, \ .eventid = _eventid, \ - .filter = {{_fa, _fb}}, \ + .filter = {{_fa, _fb}, {_fc, _fd}}, \ }})[0].attr.attr) #define CMN_EVENT_ATTR(_model, _name, _type, _eventid, _filter...) \ - _CMN_EVENT_ATTR(_model, _name, _type, _eventid, ##_filter, 0, 0) + _CMN_EVENT_ATTR(_model, _name, _type, _eventid, ##_filter, 0, 0, 0, 0) static ssize_t arm_cmn_event_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -769,6 +808,10 @@ static ssize_t arm_cmn_event_show(struct device *dev, "type=0x%x,eventid=0x%x,wp_dev_sel=?,wp_chn_sel=?,wp_grp=?,wp_val=?,wp_mask=?\n", eattr->type, eattr->eventid); + if (filter[1].sel) + return sysfs_emit(buf, "type=0x%x,eventid=0x%x,filter=0x%x,filter2=0x%x\n", + eattr->type, eattr->eventid, filter[0].val, filter[1].val); + if (filter[0].sel) return sysfs_emit(buf, "type=0x%x,eventid=0x%x,filter=0x%x\n", eattr->type, eattr->eventid, filter[0].val); @@ -887,8 +930,8 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, CMN_EVENT_ATTR(_model, ccha_##_name, CMN_TYPE_CCHA, _event) #define CMN_EVENT_CCLA(_name, _event) \ CMN_EVENT_ATTR(CMN_ANY, ccla_##_name, CMN_TYPE_CCLA, _event) -#define CMN_EVENT_HNS(_name, _event) \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) +#define _CMN_EVENT_HNS(_model, _name, _event, _filter...) \ + CMN_EVENT_ATTR(_model, hns_##_name, CMN_TYPE_HNS, _event, ##_filter) #define CMN_EVENT_DVM_OCC(_model, _name, _event) \ CMN_EVENT_DVM(_model, _name##_all, _event, SEL_OCCUP1_ID, 0), \ @@ -913,7 +956,12 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, CMN_EVENT_ATTR(_model, _name##_group1_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 3), \ CMN_EVENT_ATTR(_model, _name##_group1_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 4), \ CMN_EVENT_ATTR(_model, _name##_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 5), \ - CMN_EVENT_ATTR(_model, _name##_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 6) + CMN_EVENT_ATTR(_model, _name##_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 6), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_ccg_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 9), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_ccg_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 10), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_lbt_read, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 11), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_lbt_write, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 12), \ + CMN_EVENT_ATTR(CMNS3R2, _name##_lbt, _type, _event, SEL_CBUSY_SNTHROTTLE_SEL, 13) #define CMN_EVENT_HNF_OCC(_model, _name, _event) \ CMN_EVENT_HN_OCC(_model, hnf_##_name, CMN_TYPE_HNF, _event) @@ -922,23 +970,75 @@ static umode_t arm_cmn_event_attr_is_visible(struct kobject *kobj, #define CMN_EVENT_HNF_SNT(_model, _name, _event) \ CMN_EVENT_HN_SNT(_model, hnf_##_name, CMN_TYPE_HNF, _event) -#define CMN_EVENT_HNS_OCC(_name, _event) \ - CMN_EVENT_HN_OCC(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event), \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_rxsnp, CMN_TYPE_HNS, _event, SEL_OCCUP1_ID, 5), \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, SEL_OCCUP1_ID, 6), \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, SEL_OCCUP1_ID, 7) +#define CMN_EVENT_HNS(_name, _event) \ + _CMN_EVENT_HNS(CMN_ANY, _name, _event) +#define CMN_EVENT_HNSR0(_name, _event) \ + _CMN_EVENT_HNS(CMN700 | CMNS3R01, _name, _event) +#define _CMN_EVENT_HNS_HBT(_model, _name, _event, _sel) \ + _CMN_EVENT_HNS(_model, _name##_all, _event, _sel, 0), \ + _CMN_EVENT_HNS(_model, _name##_hbt, _event, _sel, 1), \ + _CMN_EVENT_HNS(_model, _name##_lbt, _event, _sel, 2) +#define _CMN_EVENT_HNS_HBT2(_model, _name, _event, _fsel1, f1) \ + _CMN_EVENT_HNS(_model, _name##_all, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 0), \ + _CMN_EVENT_HNS(_model, _name##_hbt, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 1), \ + _CMN_EVENT_HNS(_model, _name##_lbt, _event, _fsel1, f1, SEL_HBT_LBT_SEL, 2) + +#define CMN_EVENT_HNS_OCC(_model, _name, _event) \ + CMN_EVENT_HN_OCC(_model, hns_##_name, CMN_TYPE_HNS, _event), \ + _CMN_EVENT_HNS(_model, _name##_rxsnp, _event, SEL_OCCUP1_ID, 5), \ + _CMN_EVENT_HNS(_model, _name##_lbt, _event, SEL_OCCUP1_ID, 6), \ + _CMN_EVENT_HNS(_model, _name##_hbt, _event, SEL_OCCUP1_ID, 7), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_rnf, _event, SEL_OCCUP1_ID, 8), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_rni, _event, SEL_OCCUP1_ID, 9), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_ccglcn, _event, SEL_OCCUP1_ID, 10), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_ccgrn, _event, SEL_OCCUP1_ID, 11) #define CMN_EVENT_HNS_CLS( _name, _event) \ CMN_EVENT_HN_CLS(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) -#define CMN_EVENT_HNS_SNT(_name, _event) \ - CMN_EVENT_HN_SNT(CMN_ANY, hns_##_name, CMN_TYPE_HNS, _event) -#define CMN_EVENT_HNS_HBT(_name, _event) \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, SEL_HBT_LBT_SEL, 0), \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_hbt, CMN_TYPE_HNS, _event, SEL_HBT_LBT_SEL, 1), \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_lbt, CMN_TYPE_HNS, _event, SEL_HBT_LBT_SEL, 2) -#define CMN_EVENT_HNS_SNH(_name, _event) \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_all, CMN_TYPE_HNS, _event, SEL_SN_HOME_SEL, 0), \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_sn, CMN_TYPE_HNS, _event, SEL_SN_HOME_SEL, 1), \ - CMN_EVENT_ATTR(CMN_ANY, hns_##_name##_home, CMN_TYPE_HNS, _event, SEL_SN_HOME_SEL, 2) +#define CMN_EVENT_HNSR0_CLS( _name, _event) \ + CMN_EVENT_HN_CLS(CMN700 | CMNS3R01, hns_##_name, CMN_TYPE_HNS, _event) +#define CMN_EVENT_HNS_SNT(_model, _name, _event) \ + CMN_EVENT_HN_SNT(_model, hns_##_name, CMN_TYPE_HNS, _event) +#define CMN_EVENT_HNS_SNH(_model, _name, _event) \ + _CMN_EVENT_HNS(_model, _name##_all, _event, SEL_SN_HOME_SEL, 0), \ + _CMN_EVENT_HNS(_model, _name##_sn, _event, SEL_SN_HOME_SEL, 1), \ + _CMN_EVENT_HNS(_model, _name##_home, _event, SEL_SN_HOME_SEL, 2) +#define CMN_EVENT_HNS_VC(_name, _event) \ + CMN_EVENT_HNSR0(_name, _event), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_vc0, _event, SEL_SNP_VC_SEL, 0), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_vc1, _event, SEL_SNP_VC_SEL, 1), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_vc2, _event, SEL_SNP_VC_SEL, 2) +#define CMN_EVENT_HNS_ENHBT(_name, _event) \ + _CMN_EVENT_HNS_HBT(CMNS3R2, _name, _event, SEL_ENHANCED_HBT_LBT_SEL), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_rnf, _event, SEL_ENHANCED_HBT_LBT_SEL, 3), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_rni, _event, SEL_ENHANCED_HBT_LBT_SEL, 4), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_ccglcn, _event, SEL_ENHANCED_HBT_LBT_SEL, 5), \ + _CMN_EVENT_HNS(CMNS3R2, _name##_ccgrn, _event, SEL_ENHANCED_HBT_LBT_SEL, 6) +#define CMN_EVENT_HNS_EVICT(_model, _name, _event) \ + _CMN_EVENT_HNS_HBT2(_model, _name##_all, _event, SEL_EVICT_STATE_SEL, 0), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_eu, _event, SEL_EVICT_STATE_SEL, 1), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_en, _event, SEL_EVICT_STATE_SEL, 2), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_su, _event, SEL_EVICT_STATE_SEL, 3), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_sn, _event, SEL_EVICT_STATE_SEL, 4), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_mu, _event, SEL_EVICT_STATE_SEL, 5), \ + _CMN_EVENT_HNS_HBT2(_model, _name##_mn, _event, SEL_EVICT_STATE_SEL, 6) + +#define CMN_EVENT_HNSR0_HBT(_name, _event) \ + _CMN_EVENT_HNS_HBT(CMN700 | CMNS3R01, _name, _event, SEL_HBT_LBT_SEL) +#define CMN_EVENT_HNS_R2SNH(_name, _event) \ + CMN_EVENT_HNSR0(_name, _event), \ + CMN_EVENT_HNS_SNH(CMNS3R2, _name, _event) +#define CMN_EVENT_HNS_R2HBT(_name, _event) \ + CMN_EVENT_HNSR0(_name, _event), \ + _CMN_EVENT_HNS_HBT(CMNS3R2, _name, _event, SEL_HBT_LBT_SEL) +#define CMN_EVENT_HNS_HBT_ENHBT(_name, _event) \ + CMN_EVENT_HNSR0_HBT(_name, _event), \ + CMN_EVENT_HNS_ENHBT(_name, _event) +#define CMN_EVENT_HNS_HBT_OCC(_name, _event) \ + CMN_EVENT_HNSR0_HBT(_name, _event), \ + CMN_EVENT_HNS_OCC(CMNS3R2, _name, _event) +#define CMN_EVENT_HNS_HBT_EVICT(_name, _event) \ + CMN_EVENT_HNSR0_HBT(_name, _event), \ + CMN_EVENT_HNS_EVICT(CMNS3R2, _name, _event) #define _CMN_EVENT_XP_MESH(_name, _event) \ __CMN_EVENT_XP(e_##_name, (_event) | (0 << 2)), \ @@ -1288,65 +1388,72 @@ static struct attribute *arm_cmn_event_attrs[] = { CMN_EVENT_CCLA(pfwd_sndr_stalls_static_crd, 0x2a), CMN_EVENT_CCLA(pfwd_sndr_stalls_dynmaic_crd, 0x2b), - CMN_EVENT_HNS_HBT(cache_miss, 0x01), - CMN_EVENT_HNS_HBT(slc_sf_cache_access, 0x02), - CMN_EVENT_HNS_HBT(cache_fill, 0x03), - CMN_EVENT_HNS_HBT(pocq_retry, 0x04), - CMN_EVENT_HNS_HBT(pocq_reqs_recvd, 0x05), - CMN_EVENT_HNS_HBT(sf_hit, 0x06), - CMN_EVENT_HNS_HBT(sf_evictions, 0x07), - CMN_EVENT_HNS(dir_snoops_sent, 0x08), - CMN_EVENT_HNS(brd_snoops_sent, 0x09), - CMN_EVENT_HNS_HBT(slc_eviction, 0x0a), - CMN_EVENT_HNS_HBT(slc_fill_invalid_way, 0x0b), - CMN_EVENT_HNS(mc_retries_local, 0x0c), - CMN_EVENT_HNS_SNH(mc_reqs_local, 0x0d), + CMN_EVENT_HNS_HBT_ENHBT(cache_miss, 0x01), + CMN_EVENT_HNS_HBT_ENHBT(slc_sf_cache_access, 0x02), + CMN_EVENT_HNS_HBT_ENHBT(cache_fill, 0x03), + CMN_EVENT_HNS_HBT_OCC(pocq_retry, 0x04), + CMN_EVENT_HNS_HBT_OCC(pocq_reqs_recvd, 0x05), + CMN_EVENT_HNS_HBT_ENHBT(sf_hit, 0x06), + CMN_EVENT_HNS_HBT_EVICT(sf_evictions, 0x07), + CMN_EVENT_HNS_VC(dir_snoops_sent, 0x08), + CMN_EVENT_HNS_VC(brd_snoops_sent, 0x09), + CMN_EVENT_HNS_HBT_EVICT(slc_eviction, 0x0a), + CMN_EVENT_HNS_HBT_ENHBT(slc_fill_invalid_way, 0x0b), + CMN_EVENT_HNS_R2SNH(mc_retries_local, 0x0c), + CMN_EVENT_HNS_SNH(CMN_ANY, mc_reqs_local, 0x0d), CMN_EVENT_HNS(qos_hh_retry, 0x0e), - CMN_EVENT_HNS_OCC(qos_pocq_occupancy, 0x0f), - CMN_EVENT_HNS(pocq_addrhaz, 0x10), - CMN_EVENT_HNS(pocq_atomic_addrhaz, 0x11), - CMN_EVENT_HNS(ld_st_swp_adq_full, 0x12), - CMN_EVENT_HNS(cmp_adq_full, 0x13), + CMN_EVENT_HNS_OCC(CMN_ANY, qos_pocq_occupancy, 0x0f), + CMN_EVENT_HNS_HBT_ENHBT(pocq_addrhaz, 0x10), + CMN_EVENT_HNS_HBT_ENHBT(pocq_atomic_addrhaz, 0x11), + CMN_EVENT_HNSR0(ld_st_swp_adq_full, 0x12), + CMN_EVENT_HNSR0(cmp_adq_full, 0x13), CMN_EVENT_HNS(txdat_stall, 0x14), CMN_EVENT_HNS(txrsp_stall, 0x15), - CMN_EVENT_HNS(seq_full, 0x16), + CMN_EVENT_HNSR0(seq_full, 0x16), CMN_EVENT_HNS(seq_hit, 0x17), - CMN_EVENT_HNS(snp_sent, 0x18), - CMN_EVENT_HNS(sfbi_dir_snp_sent, 0x19), - CMN_EVENT_HNS(sfbi_brd_snp_sent, 0x1a), + CMN_EVENT_HNS_VC(snp_sent, 0x18), + CMN_EVENT_HNS_VC(sfbi_dir_snp_sent, 0x19), + CMN_EVENT_HNS_VC(sfbi_brd_snp_sent, 0x1a), CMN_EVENT_HNS(intv_dirty, 0x1c), - CMN_EVENT_HNS(stash_snp_sent, 0x1d), - CMN_EVENT_HNS(stash_data_pull, 0x1e), - CMN_EVENT_HNS(snp_fwded, 0x1f), - CMN_EVENT_HNS(atomic_fwd, 0x20), + CMN_EVENT_HNSR0(stash_snp_sent, 0x1d), + CMN_EVENT_HNSR0(stash_data_pull, 0x1e), + CMN_EVENT_HNS_VC(snp_fwded, 0x1f), + CMN_EVENT_HNSR0(atomic_fwd, 0x20), CMN_EVENT_HNS(mpam_hardlim, 0x21), CMN_EVENT_HNS(mpam_softlim, 0x22), - CMN_EVENT_HNS(snp_sent_cluster, 0x23), - CMN_EVENT_HNS(sf_imprecise_evict, 0x24), + CMN_EVENT_HNS_VC(snp_sent_cluster, 0x23), + CMN_EVENT_HNS_R2HBT(sf_imprecise_evict, 0x24), CMN_EVENT_HNS(sf_evict_shared_line, 0x25), CMN_EVENT_HNS_CLS(pocq_class_occup, 0x26), CMN_EVENT_HNS_CLS(pocq_class_retry, 0x27), CMN_EVENT_HNS_CLS(class_mc_reqs_local, 0x28), - CMN_EVENT_HNS_CLS(class_cgnt_cmin, 0x29), - CMN_EVENT_HNS_SNT(sn_throttle, 0x2a), - CMN_EVENT_HNS_SNT(sn_throttle_min, 0x2b), + CMN_EVENT_HNSR0_CLS(class_cgnt_cmin, 0x29), + CMN_EVENT_HNS_SNT(CMN_ANY, sn_throttle, 0x2a), + CMN_EVENT_HNS_SNT(CMN_ANY, sn_throttle_min, 0x2b), CMN_EVENT_HNS(sf_precise_to_imprecise, 0x2c), CMN_EVENT_HNS(snp_intv_cln, 0x2d), CMN_EVENT_HNS(nc_excl, 0x2e), - CMN_EVENT_HNS(excl_mon_ovfl, 0x2f), + CMN_EVENT_HNSR0(excl_mon_ovfl, 0x2f), CMN_EVENT_HNS(snp_req_recvd, 0x30), CMN_EVENT_HNS(snp_req_byp_pocq, 0x31), CMN_EVENT_HNS(dir_ccgha_snp_sent, 0x32), CMN_EVENT_HNS(brd_ccgha_snp_sent, 0x33), - CMN_EVENT_HNS(ccgha_snp_stall, 0x34), + CMN_EVENT_HNSR0(ccgha_snp_stall, 0x34), CMN_EVENT_HNS(lbt_req_hardlim, 0x35), CMN_EVENT_HNS(hbt_req_hardlim, 0x36), CMN_EVENT_HNS(sf_reupdate, 0x37), - CMN_EVENT_HNS(excl_sf_imprecise, 0x38), + CMN_EVENT_HNS_R2HBT(excl_sf_imprecise, 0x38), CMN_EVENT_HNS(snp_pocq_addrhaz, 0x39), - CMN_EVENT_HNS(mc_retries_remote, 0x3a), - CMN_EVENT_HNS_SNH(mc_reqs_remote, 0x3b), + CMN_EVENT_HNS_R2SNH(mc_retries_remote, 0x3a), + CMN_EVENT_HNS_SNH(CMN_ANY, mc_reqs_remote, 0x3b), CMN_EVENT_HNS_CLS(class_mc_reqs_remote, 0x3c), + CMN_EVENT_HNS_ENHBT(readonce_hazard_detected, 0x3d), + CMN_EVENT_HNS_ENHBT(readonce_fwd_data_completed, 0x3e), + CMN_EVENT_HNS_SNT(CMNS3R2, cbusy00, 0x40), + CMN_EVENT_HNS_SNT(CMNS3R2, cbusy01, 0x41), + CMN_EVENT_HNS_SNT(CMNS3R2, cbusy10, 0x42), + CMN_EVENT_HNS_SNT(CMNS3R2, cbusy11, 0x43), + CMN_EVENT_HNS_ENHBT(ro_rnsd_new_alloc_hint, 0x44), NULL }; @@ -2431,21 +2538,33 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) /* * With the device isolation feature, if firmware has neglected to enable * an XP port then we risk locking up if we try to access anything behind - * it; however we also have no way to tell from Non-Secure whether any - * given port is disabled or not, so the only way to win is not to play... + * it; however prior to CMN S3 r2p0 we also have no way to tell from + * Non-Secure whether any given port is disabled or not, so in that case + * the only way to win is not to play... */ reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL); - if (reg & CMN_INFO_DEVICE_ISO_ENABLE) { + if (reg & CMN_INFO_DEVICE_ISO_ENABLE && model == CMNS3R01) { dev_err(cmn->dev, "Device isolation enabled, not continuing due to risk of lockup\n"); return -ENODEV; } - cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; - cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); - cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); + if (model < CMNS3R2) { + cmn->multi_dtm = reg & CMN_INFO_MULTIPLE_DTM_EN; + cmn->rsp_vc_num = FIELD_GET(CMN_INFO_RSP_VC_NUM, reg); + cmn->dat_vc_num = FIELD_GET(CMN_INFO_DAT_VC_NUM, reg); + } else { + cmn->multi_dtm = reg & CMN_S3_R2_MULTIPLE_DTM_EN; + } reg = readq_relaxed(cfg_region + CMN_CFGM_INFO_GLOBAL_1); - cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg); - cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg); + if (model < CMNS3R2) { + cmn->snp_vc_num = FIELD_GET(CMN_INFO_SNP_VC_NUM, reg); + cmn->req_vc_num = FIELD_GET(CMN_INFO_REQ_VC_NUM, reg); + } else { + cmn->rsp_vc_num = FIELD_GET(CMN_S3_R2_RSP_VC_NUM, reg); + cmn->dat_vc_num = FIELD_GET(CMN_S3_R2_DAT_VC_NUM, reg); + cmn->snp_vc_num = FIELD_GET(CMN_S3_R2_SNP_VC_NUM, reg); + cmn->req_vc_num = FIELD_GET(CMN_S3_R2_REQ_VC_NUM, reg); + } reg = readq_relaxed(cfg_region + CMN_CHILD_INFO); child_count = FIELD_GET(CMN_CI_CHILD_COUNT, reg); @@ -2545,15 +2664,12 @@ static int arm_cmn_discover(struct arm_cmn *cmn, unsigned int rgn_offset) reg = readq_relaxed(xp_region + child_poff + j * 8); /* * Don't even try to touch anything external, since in general - * we haven't a clue how to power up arbitrary CHI requesters. - * As of CMN-600r1 these could only be RN-SAMs or CXLAs, - * neither of which have any PMU events anyway. - * (Actually, CXLAs do seem to have grown some events in r1p2, - * but they don't go to regular XP DTMs, and they depend on - * secure configuration which we can't easily deal with) + * we haven't a clue how to power up arbitrary CHI requesters, + * and none of them have standard PMU events anyway. Isolated + * nodes effectively just do not exist at all from our PoV. */ - if (reg & CMN_CHILD_NODE_EXTERNAL) { - dev_dbg(cmn->dev, "ignoring external node %llx\n", reg); + if (reg & (CMN_CHILD_NODE_EXTERNAL | CMN_CHILD_NODE_ISOLATED)) { + dev_dbg(cmn->dev, "ignoring external/isolated node %llx\n", reg); continue; } /* From d8fc0793cf6862813d9bde224230e0edeafab88c Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 6 Aug 2026 16:31:28 +0000 Subject: [PATCH 86/95] iommu/arm-smmu-v3-sva: Use system_supports_bbml3() to detect CPU feature Commit 94104e3cfa80 ("arm64: cpufeature: Rename BBML2_NOABORT as BBML3") renamed the cpu_supports_bbml2_noabort() helper to cpu_supports_bbml3(), as the Linux-defined "noabort" semantics have now been incorporated into the architecture under the BBML3 feature. Update the caller in the SMMUv3 SVA driver to use the new function. There is a slightly oddity in that the SMMUv3 architecture already defined BBML2 in such a way that aborts were prohibited, so we use the BBML3 feature on the CPU to enable BBML2 in the SMMU. Fixes: 94104e3cfa80 ("arm64: cpufeature: Rename BBML2_NOABORT as BBML3") Signed-off-by: Will Deacon --- drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 1ed8a6f29dc4..01492fb52659 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -208,7 +208,7 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) feat_mask |= ARM_SMMU_FEAT_VAX; } - if (system_supports_bbml2_noabort()) + if (system_supports_bbml3()) feat_mask |= ARM_SMMU_FEAT_BBML2; if ((smmu->features & feat_mask) != feat_mask) From fb0f3ef5601a67877cee38257e872c21db9d7f77 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 3 Aug 2026 18:33:47 +0200 Subject: [PATCH 87/95] arm64: mm: Unmap kernel data/bss entirely from the linear map The linear aliases of the kernel text and rodata are also mapped read-only in the linear map. Given that the contents of these regions are mostly identical to the version in the loadable image, mapping them read-only and leaving their contents visible is a reasonable hardening measure. Data and bss, however, are now also mapped read-only but the contents of these regions are more likely to contain data that we'd rather not leak. So let's unmap these entirely in the linear map when the kernel is running normally. When going into hibernation or waking up from it, these regions need to be mapped, so map the region initially, and toggle the valid bit so map/unmap the region as needed. Doing so is required because pages covering the kernel image are marked as PageReserved, and therefore disregarded for snapshotting by the hibernate logic unless they are mapped. Cc: Ryan Roberts Cc: Anshuman Khandual Cc: Kevin Brodsky Cc: Liz Prucka Cc: Seth Jenkins Cc: Kees Cook Cc: David Hildenbrand Cc: Jann Horn Signed-off-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/mm/mmu.c | 46 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index f51566316ba4..abe53881553f 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1062,6 +1063,29 @@ static void __init __map_memblock(phys_addr_t start, phys_addr_t end, end - start, prot, early_pgtable_alloc, flags); } +static void mark_linear_data_alias_valid(bool valid) +{ + set_memory_valid((unsigned long)lm_alias(__init_end), + (unsigned long)(__bss_stop - __init_end) / PAGE_SIZE, + valid); +} + +static int arm64_hibernate_pm_notify(struct notifier_block *nb, + unsigned long mode, void *unused) +{ + switch (mode) { + default: + break; + case PM_POST_HIBERNATION: + mark_linear_data_alias_valid(false); + break; + case PM_HIBERNATION_PREPARE: + mark_linear_data_alias_valid(true); + break; + } + return 0; +} + void __init mark_linear_text_alias_ro(void) { /* @@ -1070,6 +1094,21 @@ void __init mark_linear_text_alias_ro(void) update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text), (unsigned long)__init_begin - (unsigned long)_text, PAGE_KERNEL_RO); + + /* + * Register a PM notifier to remap the linear alias of data/bss as + * valid read/write before hibernation. This is needed because the + * snapshot logic disregards PageReserved pages (such as the ones + * covering the kernel image) unless they are mapped in the linear + * map. + */ + if (IS_ENABLED(CONFIG_HIBERNATION) && rodata_enabled) { + static struct notifier_block nb = { + .notifier_call = arm64_hibernate_pm_notify + }; + + register_pm_notifier(&nb); + } } #ifdef CONFIG_KFENCE @@ -1231,11 +1270,8 @@ void mark_rodata_ro(void) (unsigned long)_stext - (unsigned long)_text, PAGE_KERNEL_RO); - /* Map the kernel data/bss read-only in the linear map */ - update_mapping_prot(__pa_symbol(__init_end), - (unsigned long)lm_alias(__init_end), - (unsigned long)__bss_stop - (unsigned long)__init_end, - PAGE_KERNEL_RO); + /* Map the kernel data/bss as invalid in the linear map */ + mark_linear_data_alias_valid(false); } static void __init declare_vma(struct vm_struct *vma, From 7c3b63386c27bed8d59a4b4c283d02860420eb0a Mon Sep 17 00:00:00 2001 From: James Clark Date: Fri, 7 Aug 2026 10:14:59 +0100 Subject: [PATCH 88/95] perf: arm_pmuv3: Zero initialize hw_id branch stack field PERF_SAMPLE_BRANCH_HW_INDEX is supported by BRBE so hw_id is passed to userspace, but it's never set by the BRBE driver. Zero initialize it as it should be according to the docs: * For the architectures whose raw branch records are * already stored in age order, the hw_idx should be 0. It's probably too risky to remove PERF_SAMPLE_BRANCH_HW_INDEX from BRBE now in case anyone is setting it and reading the value, but zero initializing the whole struct also protects against the same issue with new fields that are added in the future. Fixes: 58074a0fce66 ("perf: arm_pmuv3: Add support for the Branch Record Buffer Extension (BRBE)") Signed-off-by: James Clark Reviewed-by: Anshuman Khandual Signed-off-by: Will Deacon --- drivers/perf/arm_pmuv3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 6d4d57342352..03359e078301 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -1407,7 +1407,7 @@ static int branch_records_alloc(struct arm_pmu *armpmu) struct pmu_hw_events *events_cpu; events_cpu = per_cpu_ptr(armpmu->hw_events, cpu); - events_cpu->branch_stack = kmalloc(size, GFP_KERNEL); + events_cpu->branch_stack = kzalloc(size, GFP_KERNEL); if (!events_cpu->branch_stack) return -ENOMEM; } From fc996c39689bb15aa80e5366809434f7258fb703 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Thu, 6 Aug 2026 15:46:30 +0100 Subject: [PATCH 89/95] arm_mpam: Fix a NULL pointer dereference on unbinding after an error interrupt If a user unbinds an MSC after mpam_disable() has been run in response to an error interrupt then a dereference of a NULL pointer occurs as mpam_disable() sets the drvdata to NULL. Add an early return to the driver remove callback to avoid this. Fixes: f04046f2577a ("arm_mpam: Add probe/remove for mpam msc driver and kbuild boiler plate") Signed-off-by: Ben Horgan Signed-off-by: Will Deacon --- drivers/resctrl/mpam_devices.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 53942b331269..677ee118d839 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -2017,6 +2017,9 @@ static void mpam_msc_drv_remove(struct platform_device *pdev) { struct mpam_msc *msc = platform_get_drvdata(pdev); + if (!msc) + return; + mutex_lock(&mpam_list_lock); mpam_msc_destroy(msc); mutex_unlock(&mpam_list_lock); From bb1a0f582d519c0461b0940116e2b52c5ba31459 Mon Sep 17 00:00:00 2001 From: Ben Horgan Date: Thu, 6 Aug 2026 15:46:31 +0100 Subject: [PATCH 90/95] arm_mpam: Disable driver unbind to avoid UAF When a user unbinds an MSC and that MSC is the only MSC left for a component then the corresponding mpam_component will be freed. If the user then goes on to read the schemata file in the resctrl filesystem then the mpam_component will be accessed from resctrl_arch_get_config() leading to a use after free. As the MPAM driver is not a module the unbind sysfs interface is the only way to trigger the remove. Instead of dealing with the complexity of allowing some unused MSC to unbind just remove the unbind sysfs interface. Fixes: f04046f2577a ("arm_mpam: Add probe/remove for mpam msc driver and kbuild boiler plate") Signed-off-by: Ben Horgan Signed-off-by: Will Deacon --- drivers/resctrl/mpam_devices.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c index 677ee118d839..87975ed4158c 100644 --- a/drivers/resctrl/mpam_devices.c +++ b/drivers/resctrl/mpam_devices.c @@ -2133,6 +2133,7 @@ static int mpam_msc_drv_probe(struct platform_device *pdev) static struct platform_driver mpam_msc_driver = { .driver = { .name = "mpam_msc", + .suppress_bind_attrs = true, }, .probe = mpam_msc_drv_probe, .remove = mpam_msc_drv_remove, From 5eaec4cf41a8f5ac1a0c69a607cdc5035a797f73 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 7 Aug 2026 13:37:32 +0000 Subject: [PATCH 91/95] arm64: Disable KCSAN instrumentation in delay.o KCSAN relies on udelay() for injecting delays. To avoid recursively triggering a watchpoint, where KCSAN sets up watchpoint on an address that is accessed by udelay() in the same thread, disable instrumentation in arm64's delay implementation. Paul found a manifestation of this as follows: | BUG: KCSAN: data-race in __delay / set_need_resched_current | | read (marked) to 0xffff000005899b48 of 8 bytes by interrupt on cpu 8: | __delay+0xb0/0x378 | __udelay+0x4c/0x60 | kcsan_setup_watchpoint+0x3b4/0x820 | __tsan_unaligned_write4+0x228/0x26c | set_need_resched_current+0x138/0x1a8 | rcu_exp_handler+0x418/0x4a0 | __flush_smp_call_function_queue+0x36c/0x4a0 | generic_smp_call_function_single_interrupt+0x20/0x30 | ipi_handler+0xec/0x558 | handle_percpu_devid_irq+0x220/0x2a0 | generic_handle_domain_irq+0x84/0xb4 | gic_handle_irq+0x64/0x144 | call_on_irq_stack+0x30/0x48 | do_interrupt_handler+0x80/0xb8 | el1_interrupt+0x3c/0x60 | el1h_64_irq_handler+0x18/0x24 | el1h_64_irq+0x6c/0x70 | smp_call_function_single+0x18c/0x25c | sync_rcu_exp_select_node_cpus+0x534/0x8bc | rcu_exp_sel_wait_wake+0x358/0xef4 | wait_rcu_exp_gp+0x30/0x44 | kthread_worker_fn+0x1b4/0x5dc | kthread+0x1d8/0x204 | ret_from_fork+0x10/0x20 | | write to 0xffff000005899b4c of 4 bytes by interrupt on cpu 8: | set_need_resched_current+0x138/0x1a8 | [...] This matches what is already done in arch/x86/lib/Makefile. Reported-by: "Paul E. McKenney" Fixes: dd03762ab608 ("arm64: Enable KCSAN") Signed-off-by: Marco Elver Signed-off-by: Will Deacon --- arch/arm64/lib/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index 448c917494f3..b33e1ca4a781 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,4 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 + +# KCSAN uses udelay for introducing watchpoint delay; avoid recursion. +KCSAN_SANITIZE_delay.o := n + lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_page.o \ clear_page.o csum.o insn.o memchr.o memcpy.o \ From 067f029c6463ab3bc980053bd0d50f98d7edfa3e Mon Sep 17 00:00:00 2001 From: Ada Couprie Diaz Date: Mon, 27 Jul 2026 17:34:16 +0100 Subject: [PATCH 92/95] irqchip/gic-v3: make the unmasking of pseudo-NMIs explicit when handling IRQs `gic_arch_enable_irqs()` is only used when handling IRQs (which could be pseudo-NMIs) and unmasking pseudo-NMIs. The chain of `gic_pmr_mask_irqs()` and `gic_arch_enable_irqs()` for it is slightly confusing without further explanation. Remove `gic_arch_enable_irqs()` and instead do the whole pseudo-NMI umasking in `gic_unmask_pnmis()`, making the operation explicit. Signed-off-by: Ada Couprie Diaz Signed-off-by: Vladimir Murzin Reviewed-by: Jinjie Ruan Reviewed-by: Marc Zyngier Signed-off-by: Will Deacon --- arch/arm/include/asm/arch_gicv3.h | 6 +----- arch/arm64/include/asm/arch_gicv3.h | 7 +++++-- arch/arm64/include/asm/entry-common.h | 2 +- drivers/irqchip/irq-gic-v3.c | 5 +---- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h index 847590df7551..d4ac8d3271b1 100644 --- a/arch/arm/include/asm/arch_gicv3.h +++ b/arch/arm/include/asm/arch_gicv3.h @@ -246,11 +246,7 @@ static inline void gic_pmr_mask_irqs(void) WARN_ON_ONCE(true); } -static inline void gic_arch_enable_irqs(void) -{ - /* Should not get called. */ - WARN_ON_ONCE(true); -} +static inline void gic_unmask_pnmis(void) {} static inline bool gic_has_relaxed_pmr_sync(void) { diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index d20b03931a8d..3dcb7b8309d9 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -178,9 +178,12 @@ static inline void gic_pmr_mask_irqs(void) gic_write_pmr(GIC_PRIO_IRQOFF); } -static inline void gic_arch_enable_irqs(void) +static inline void gic_unmask_pnmis(void) { - asm volatile ("msr daifclr, #3" : : : "memory"); + if (gic_prio_masking_enabled()) { + gic_pmr_mask_irqs(); + asm volatile ("msr daifclr, #3" : : : "memory"); + } } static inline bool gic_has_relaxed_pmr_sync(void) diff --git a/arch/arm64/include/asm/entry-common.h b/arch/arm64/include/asm/entry-common.h index cab8cd78f693..1905765159aa 100644 --- a/arch/arm64/include/asm/entry-common.h +++ b/arch/arm64/include/asm/entry-common.h @@ -32,7 +32,7 @@ static inline bool arch_irqentry_exit_need_resched(void) /* * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC * priority masking is used the GIC irqchip driver will clear DAIF.IF - * using gic_arch_enable_irqs() for normal IRQs. If anything is set in + * in gic_unmask_pnmis() for normal IRQs. If anything is set in * DAIF we must have handled an NMI, so skip preemption. */ if (system_uses_irq_prio_masking() && read_sysreg(daif)) diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 99444a1b2ffa..94c6a3f2b009 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -867,10 +867,7 @@ static void __gic_handle_irq_from_irqson(struct pt_regs *regs) nmi_exit(); } - if (gic_prio_masking_enabled()) { - gic_pmr_mask_irqs(); - gic_arch_enable_irqs(); - } + gic_unmask_pnmis(); if (!is_nmi) __gic_handle_irq(irqnr, regs); From 39aebe0e89469c2904e60b1e977e0d4dbf33326b Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 27 Jul 2026 17:34:17 +0100 Subject: [PATCH 93/95] arm64: entry: Avoid unnecessary local_irq_disable() on kernel exit Currently, when exiting to kernel mode, we attempt involuntary preemption. The preemption logic expects IRQs to be disabled, which is why we call local_irq_disable() before attempting preemption. However, depending on the context, local_irq_disable() may be unnecessary: - __el1_irq(), the non-NMI EL1 IRQ path, already has IRQs disabled, so local_irq_disable() is redundant. - irqentry_exit_to_kernel_mode_preempt() immediately returns when exiting from an NMI-like context, so calling local_irq_disable() beforehand is unnecessary work. Furthermore, it confuses the pNMI state tracking when we are in a context with interrupts disabled and the GIC_PRIO_PSR_I_SET bit is set in the PMR, leading to a warning when CONFIG_ARM64_DEBUG_PRIORITY_MASKING=y: WARNING: ./arch/arm64/include/asm/irqflags.h:63 at arm64_exit_to_kernel_mode+0xb8/0xc0, CPU#40: retsnoop/31805 CPU: 40 UID: 0 PID: 31805 Comm: retsnoop Not tainted 7.2.0-rc6-next-20260805 #7 PREEMPTLAZY pstate: 234013c9 (nzCv DAIF +PAN -UAO +TCO +DIT +SSBS BTYPE=--) pc : arm64_exit_to_kernel_mode (arch/arm64/kernel/entry-common.c:63) lr : el1_abort (arch/arm64/kernel/entry-common.c:323) pmr: 000000f0 Call trace: arm64_exit_to_kernel_mode (arch/arm64/kernel/entry-common.c:63) (P) el1_abort (arch/arm64/kernel/entry-common.c:323) el1h_64_sync_handler (arch/arm64/kernel/entry-common.c:449) el1h_64_sync (arch/arm64/kernel/entry.S:589) [...] Split arm64_exit_to_kernel_mode() into preempt, non-preempt, and dispatch parts so that we can avoid this extra work where it is not needed and avoid breaking the pNMI tracking logic. Reported-by: Breno Leitao Fixes: ae654112eac0 ("arm64: entry: Use split preemption logic") Link: https://lore.kernel.org/all/20260807-arm64_fix-v1-1-d069ccf9d71b@debian.org/ Reviewed-by: Jinjie Ruan Signed-off-by: Vladimir Murzin Signed-off-by: Will Deacon --- arch/arm64/kernel/entry-common.c | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 2be42d7f4eaa..72c03ccea59f 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -52,16 +52,36 @@ static noinstr irqentry_state_t arm64_enter_from_kernel_mode(struct pt_regs *reg * After this function returns it is not safe to call regular kernel code, * instrumentable code, or any code which may trigger an exception. */ -static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, - irqentry_state_t state) +static void noinstr __arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) { - local_irq_disable(); - irqentry_exit_to_kernel_mode_preempt(regs, state); local_daif_mask(); mte_check_tfsr_exit(); irqentry_exit_to_kernel_mode_after_preempt(regs, state); } +/* + * We are returning from the context which allows involuntary kernel preemption + */ +static void noinstr arm64_exit_to_kernel_mode_preempt(struct pt_regs *regs, + irqentry_state_t state) +{ + irqentry_exit_to_kernel_mode_preempt(regs, state); + __arm64_exit_to_kernel_mode(regs, state); +} + +static void noinstr arm64_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) +{ + if (!regs_irqs_disabled(regs)) { + local_irq_disable(); + arm64_exit_to_kernel_mode_preempt(regs, state); + return; + } + + __arm64_exit_to_kernel_mode(regs, state); +} + static __always_inline void arm64_syscall_enter_from_user_mode(struct pt_regs *regs) { enter_from_user_mode(regs); @@ -510,7 +530,7 @@ static __always_inline void __el1_irq(struct pt_regs *regs, do_interrupt_handler(regs, handler); irq_exit_rcu(); - arm64_exit_to_kernel_mode(regs, state); + arm64_exit_to_kernel_mode_preempt(regs, state); } static void noinstr el1_interrupt(struct pt_regs *regs, void (*handler)(struct pt_regs *)) From d3359af21fc9e7a47577ff90e821b6510ec34dee Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Tue, 11 Aug 2026 14:11:44 +0000 Subject: [PATCH 94/95] arm64: bti: Disable in-kernel BTI with recent versions of Clang The following BTI exception was seen when loading a livepatch module: Internal error: Oops - BTI: 0000000036000001 [#1] SMP pstate: 634004c9 (nZCv daIF +PAN -UAO +TCO +DIT -SSBS BTYPE=jc) pc : kill_orphaned_pgrp+0x0/0x150 lr : do_exit+0x498/0xaf0 [livepatch_combined] The problem is that the patch module's do_exit() is branching to a static function in vmlinux using a module PLT veneer (indirect branch), but the target function doesn't have a BTI landing pad. Clang 21+ omits the landing pad for static functions which can only be reached by a direct branch. That's normally fine for ordinary modules which only branch to global exported functions, but Mark Brown points out [1] that this isn't guaranteed if the module branches between sections. Futhermore, livepatch modules use klp relocations to reference arbitrary kernel symbols, so with CONFIG_RANDOMIZE_MODULE_REGION_FULL the module is far enough from the kernel that every R_AARCH64_CALL26 needs a PLT. Put Clang 21+ in the naughty corner alongside GCC, which suffers from the same issue, by disabling CONFIG_ARM64_BTI_KERNEL until we have a version of the toolchain with the problem resolved. Cc: Ard Biesheuvel Link: https://lore.kernel.org/r/da06bbd3-d04b-4d0f-b331-f5b91bc373a5@sirena.org.uk [1] Fixes: fd1e0fd71f65 ("arm64: Implement HAVE_LIVEPATCH") Signed-off-by: Josh Poimboeuf [will: Stitched together commit message, diff and bug number] Signed-off-by: Will Deacon --- arch/arm64/Kconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b3afe0688919..fc57d90d92c1 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -2116,6 +2116,8 @@ config ARM64_BTI_KERNEL depends on !CC_IS_GCC || GCC_VERSION >= 100100 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106671 depends on !CC_IS_GCC + # https://github.com/llvm/llvm-project/issues/215547 + depends on !CC_IS_CLANG || CLANG_VERSION < 210000 depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS) help Build the kernel with Branch Target Identification annotations From e98a9d0146372b046d863164025a66ab4488b972 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 11 Aug 2026 15:04:29 +0100 Subject: [PATCH 95/95] arm64/efi: Avoid voluntary preemption with efi_mm installed Gus reports a bad kernel memory access when using software PAN (CONFIG_ARM64_SW_TTBR0_PAN=y) on a machine with support for EFI runtime services: Unable to handle kernel access to user memory outside uaccess routines at virtual address 00000000f322ff30 Mem abort info: ESR = 0x0000000096000004 FSC = 0x04: level 0 translation fault Internal error: Oops: 0000000096000004 [#1] SMP Workqueue: efi_rts_wq efi_call_rts pstate: 80400005 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--) pc : efi_call_rts+0xd8/0x288 Call trace: efi_call_rts+0xd8/0x288 (P) process_one_work+0x178/0x4f8 worker_thread+0x194/0x328 This is because the fpsimd context management code called from __efi_fpsimd_begin() can preempt voluntarily, returning later to the EFI code with an incorrect value for TTBR0_EL1 thanks to the deferred mm switching used by the software PAN implementation. Since EFI runtime services cannot preempt voluntarily and because the fpsimd switching code does not rely on the TTBR0_EL1 mappings, simply reorder the fpsimd switch so that it occurs before we change the page-table. Cc: Ard Biesheuvel Reported-by: Gus Bourg Tested-by: Gus Bourg Fixes: a5baf582f4c0 ("arm64/efi: Call EFI runtime services without disabling preemption") Link: https://lore.kernel.org/all/20260806000144.3388823-1-gus@bourg.net/ Reviewed-by: Ard Biesheuvel Signed-off-by: Will Deacon --- arch/arm64/kernel/efi.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kernel/efi.c b/arch/arm64/kernel/efi.c index 30cd7f804398..0ec90fd1754e 100644 --- a/arch/arm64/kernel/efi.c +++ b/arch/arm64/kernel/efi.c @@ -184,6 +184,8 @@ void arch_efi_call_virt_setup(void) efi_virtmap_load(); } + __efi_fpsimd_begin(); + /* * Enable access to the valid TTBR0_EL1 and invoke the errata * workaround directly since there is no return from exception when @@ -191,8 +193,6 @@ void arch_efi_call_virt_setup(void) */ uaccess_ttbr0_enable(); post_ttbr_update_workaround(); - - __efi_fpsimd_begin(); } void arch_efi_call_virt_teardown(void)