From 224041412693b0d60acee0573eee18cbcd281d64 Mon Sep 17 00:00:00 2001 From: Yao Sang Date: Wed, 10 Jun 2026 11:28:46 +0800 Subject: [PATCH 001/241] nvme-multipath: revalidate zones for namespace heads Zoned multipath namespace heads get BLK_FEAT_ZONED and their limits are refreshed from the paths, but the zone state for the head disk is never initialized. The previous nr_zones assignment only updated a single field and did not allocate or populate the block layer's per-zone state. The failure was found with xfstests xfs/643 and xfs/646 on an NVMe ZNS multipath namespace. Tracing showed regular REQ_OP_WRITE I/O being submitted to sequential zones through the multipath head. That leaves the head disk without valid zone condition information. Code using the head device, such as bdev_zone_is_seq(), can then treat a sequential zone as non-sequential and submit regular writes to it. Add a small helper to run blk_revalidate_disk_zones() for a live zoned namespace head after the path limits have been committed and when a path becomes live. Return the error to the namespace update path, and keep the live path transition as a warning-only update. Drop the nr_zones copy, as blk_revalidate_disk_zones() updates it together with the rest of the zoned disk state. Signed-off-by: Yao Sang Reviewed-by: Damien Le Moal Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 4 ++++ drivers/nvme/host/multipath.c | 24 ++++++++++++++++++++---- drivers/nvme/host/nvme.h | 9 +++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 453c1f0b2dd0..db0c8ad4628a 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2592,11 +2592,15 @@ static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info) lim.max_write_streams = ns_lim->max_write_streams; lim.write_stream_granularity = ns_lim->write_stream_granularity; ret = queue_limits_commit_update(ns->head->disk->queue, &lim); + if (ret) + goto unfreeze_head_queue; set_capacity_and_notify(ns->head->disk, get_capacity(ns->disk)); set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info)); nvme_mpath_revalidate_paths(ns->head); + ret = nvme_mpath_revalidate_zones(ns->head); +unfreeze_head_queue: blk_mq_unfreeze_queue(ns->head->disk->queue, memflags); } diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 9b9a657fa330..7e9fb7227300 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -288,6 +288,25 @@ void nvme_mpath_revalidate_paths(struct nvme_ns_head *head) kblockd_schedule_work(&head->requeue_work); } +#ifdef CONFIG_BLK_DEV_ZONED +int nvme_mpath_revalidate_zones(struct nvme_ns_head *head) +{ + struct gendisk *disk = head->disk; + int ret; + + if (!disk || !blk_queue_is_zoned(disk->queue) || + !test_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) + return 0; + + ret = blk_revalidate_disk_zones(disk); + if (ret) + dev_warn_ratelimited(disk_to_dev(disk), + "failed to revalidate zoned namespace head: %d\n", + ret); + return ret; +} +#endif /* CONFIG_BLK_DEV_ZONED */ + static bool nvme_path_is_disabled(struct nvme_ns *ns) { enum nvme_ctrl_state state = nvme_ctrl_state(ns->ctrl); @@ -819,6 +838,7 @@ static void nvme_mpath_set_live(struct nvme_ns *ns) mutex_unlock(&head->lock); synchronize_srcu(&head->srcu); + nvme_mpath_revalidate_zones(head); kblockd_schedule_work(&head->requeue_work); } @@ -1375,10 +1395,6 @@ void nvme_mpath_add_disk(struct nvme_ns *ns, __le32 anagrpid) nvme_mpath_set_live(ns); } -#ifdef CONFIG_BLK_DEV_ZONED - if (blk_queue_is_zoned(ns->queue) && ns->head->disk) - ns->head->disk->nr_zones = ns->disk->nr_zones; -#endif } void nvme_mpath_remove_disk(struct nvme_ns_head *head) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 824651cc898d..a679a4c61462 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -1184,6 +1184,15 @@ static inline bool nvme_mpath_queue_if_no_path(struct nvme_ns_head *head) } #endif /* CONFIG_NVME_MULTIPATH */ +#if defined(CONFIG_NVME_MULTIPATH) && defined(CONFIG_BLK_DEV_ZONED) +int nvme_mpath_revalidate_zones(struct nvme_ns_head *head); +#else +static inline int nvme_mpath_revalidate_zones(struct nvme_ns_head *head) +{ + return 0; +} +#endif + int nvme_ns_get_unique_id(struct nvme_ns *ns, u8 id[16], enum blk_unique_id type); From f61c934aa084b7440fec681be3f4b481eb5a8609 Mon Sep 17 00:00:00 2001 From: Gui-Dong Han Date: Thu, 18 Jun 2026 10:15:43 +0800 Subject: [PATCH 002/241] nvme-apple: Use acquire/release for queue enabled state apple_nvme_init_queue() initializes queue state and then marks the queue enabled. The interrupt and request paths check enabled before using that queue state. The old wmb() after WRITE_ONCE(enabled, true) does not publish the earlier initialization before enabled becomes visible. Use a release store when enabling the queue and acquire loads when testing it. Although the shutdown-side enabled accesses are not used for publishing queue initialization, use helpers for them as well for consistency. Fixes: 5bd2927aceba ("nvme-apple: Add initial Apple SoC NVMe driver") Signed-off-by: Gui-Dong Han Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/apple.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index be3b91b43ea5..2723bc1a7d8a 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -151,6 +151,23 @@ struct apple_nvme_queue { bool enabled; }; +static inline bool apple_nvme_queue_enabled(struct apple_nvme_queue *q) +{ + /* Pair with apple_nvme_enable_queue(). */ + return smp_load_acquire(&q->enabled); +} + +static inline void apple_nvme_enable_queue(struct apple_nvme_queue *q) +{ + /* Publish queue initialization before setting q->enabled. */ + smp_store_release(&q->enabled, true); +} + +static inline void apple_nvme_disable_queue(struct apple_nvme_queue *q) +{ + WRITE_ONCE(q->enabled, false); +} + /* * The apple_nvme_iod describes the data in an I/O. * @@ -677,7 +694,7 @@ static bool apple_nvme_handle_cq(struct apple_nvme_queue *q, bool force) bool found; DEFINE_IO_COMP_BATCH(iob); - if (!READ_ONCE(q->enabled) && !force) + if (!apple_nvme_queue_enabled(q) && !force) return false; found = apple_nvme_poll_cq(q, &iob); @@ -780,7 +797,7 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx, * We should not need to do this, but we're still using this to * ensure we can drain requests on a dying queue. */ - if (unlikely(!READ_ONCE(q->enabled))) + if (unlikely(!apple_nvme_queue_enabled(q))) return BLK_STS_IOERR; if (!nvme_check_ready(&anv->ctrl, req, true)) @@ -863,7 +880,7 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_quiesce_io_queues(&anv->ctrl); if (!dead) { - if (READ_ONCE(anv->ioq.enabled)) { + if (apple_nvme_queue_enabled(&anv->ioq)) { apple_nvme_remove_sq(anv); apple_nvme_remove_cq(anv); } @@ -887,8 +904,8 @@ static void apple_nvme_disable(struct apple_nvme *anv, bool shutdown) nvme_disable_ctrl(&anv->ctrl, false); } - WRITE_ONCE(anv->ioq.enabled, false); - WRITE_ONCE(anv->adminq.enabled, false); + apple_nvme_disable_queue(&anv->ioq); + apple_nvme_disable_queue(&anv->adminq); mb(); /* ensure that nvme_queue_rq() sees that enabled is cleared */ nvme_quiesce_admin_queue(&anv->ctrl); @@ -1016,8 +1033,7 @@ static void apple_nvme_init_queue(struct apple_nvme_queue *q) memset(q->tcbs, 0, anv->hw->max_queue_depth * sizeof(struct apple_nvmmu_tcb)); memset(q->cqes, 0, depth * sizeof(struct nvme_completion)); - WRITE_ONCE(q->enabled, true); - wmb(); /* ensure the first interrupt sees the initialization */ + apple_nvme_enable_queue(q); } static void apple_nvme_reset_work(struct work_struct *work) From 4fe024eeba34b87b7e7388139d7836cde9928c6a Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Mon, 22 Jun 2026 11:32:54 +0800 Subject: [PATCH 003/241] nvme: fix typos in reservation related constants Fix the following spelling errors: - NVMET_PR_NOTIFI_MASK_ALL -> NVMET_PR_NOTIFY_MASK_ALL - NVME_PR_LOG_RESERVATOIN_PREEMPTED -> NVME_PR_LOG_RESERVATION_PREEMPTED - NVME_AEN_RESV_LOG_PAGE_AVALIABLE -> NVME_AEN_RESV_LOG_PAGE_AVAILABLE Signed-off-by: Guixin Liu Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/target/pr.c | 10 +++++----- include/linux/nvme.h | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/target/pr.c b/drivers/nvme/target/pr.c index c71ae46244ff..5dd2f3553d8c 100644 --- a/drivers/nvme/target/pr.c +++ b/drivers/nvme/target/pr.c @@ -8,7 +8,7 @@ #include #include "nvmet.h" -#define NVMET_PR_NOTIFI_MASK_ALL \ +#define NVMET_PR_NOTIFY_MASK_ALL \ (1 << NVME_PR_NOTIFY_BIT_REG_PREEMPTED | \ 1 << NVME_PR_NOTIFY_BIT_RESV_RELEASED | \ 1 << NVME_PR_NOTIFY_BIT_RESV_PREEMPTED) @@ -44,7 +44,7 @@ u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask) unsigned long idx; u16 status; - if (mask & ~(NVMET_PR_NOTIFI_MASK_ALL)) { + if (mask & ~(NVMET_PR_NOTIFY_MASK_ALL)) { req->error_loc = offsetof(struct nvme_common_command, cdw11); return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; } @@ -169,7 +169,7 @@ static void nvmet_pr_resv_released(struct nvmet_pr *pr, uuid_t *hostid) nvmet_pr_add_resv_log(ctrl, NVME_PR_LOG_RESERVATION_RELEASED, ns->nsid); nvmet_add_async_event(ctrl, NVME_AER_CSS, - NVME_AEN_RESV_LOG_PAGE_AVALIABLE, + NVME_AEN_RESV_LOG_PAGE_AVAILABLE, NVME_LOG_RESERVATION); } } @@ -188,7 +188,7 @@ static void nvmet_pr_send_event_to_host(struct nvmet_pr *pr, uuid_t *hostid, if (uuid_equal(hostid, &ctrl->hostid)) { nvmet_pr_add_resv_log(ctrl, log_type, ns->nsid); nvmet_add_async_event(ctrl, NVME_AER_CSS, - NVME_AEN_RESV_LOG_PAGE_AVALIABLE, + NVME_AEN_RESV_LOG_PAGE_AVAILABLE, NVME_LOG_RESERVATION); } } @@ -201,7 +201,7 @@ static void nvmet_pr_resv_preempted(struct nvmet_pr *pr, uuid_t *hostid) return; nvmet_pr_send_event_to_host(pr, hostid, - NVME_PR_LOG_RESERVATOIN_PREEMPTED); + NVME_PR_LOG_RESERVATION_PREEMPTED); } static void nvmet_pr_registration_preempted(struct nvmet_pr *pr, diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 041f30931a90..91ce434a7e8d 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -2272,14 +2272,14 @@ struct nvme_completion { #define NVME_TERTIARY(ver) ((ver) & 0xff) enum { - NVME_AEN_RESV_LOG_PAGE_AVALIABLE = 0x00, + NVME_AEN_RESV_LOG_PAGE_AVAILABLE = 0x00, }; enum { NVME_PR_LOG_EMPTY_LOG_PAGE = 0x00, NVME_PR_LOG_REGISTRATION_PREEMPTED = 0x01, NVME_PR_LOG_RESERVATION_RELEASED = 0x02, - NVME_PR_LOG_RESERVATOIN_PREEMPTED = 0x03, + NVME_PR_LOG_RESERVATION_PREEMPTED = 0x03, }; enum { From 09c9062d4f62199a634a45e2bb9e6e8e572cc78c Mon Sep 17 00:00:00 2001 From: Xixin Liu Date: Thu, 25 Jun 2026 10:00:00 +0800 Subject: [PATCH 004/241] nvme: zns: cap zone report nr_zones by DMA buffer size With Partial Report (PR=1), the Number of Zones (NZ) field in the report header must equal the number of zone descriptors fully transferred in the DMA buffer (ZNS Command Set Specification Rev 1.2, section 3.4.2). nvme_ns_report_zones() does not cap the parse loop by max_in_buf derived from buflen. Cap nz with min3() over the device-reported count, nr_zones - zone_idx, and max_in_buf. Signed-off-by: Xixin Liu Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/zns.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/zns.c b/drivers/nvme/host/zns.c index 8ed1b6a33454..03a2528b7192 100644 --- a/drivers/nvme/host/zns.c +++ b/drivers/nvme/host/zns.c @@ -178,7 +178,7 @@ int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, struct nvme_zone_report *report; struct nvme_command c = { }; int ret, zone_idx = 0; - unsigned int nz, i; + unsigned int max_in_buf, nz, i; size_t buflen; if (ns->head->ids.csi != NVME_CSI_ZNS) @@ -188,6 +188,9 @@ int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, if (!report) return -ENOMEM; + max_in_buf = (buflen - sizeof(struct nvme_zone_report)) / + sizeof(struct nvme_zone_descriptor); + c.zmr.opcode = nvme_cmd_zone_mgmt_recv; c.zmr.nsid = cpu_to_le32(ns->head->ns_id); c.zmr.numd = cpu_to_le32(nvme_bytes_to_numd(buflen)); @@ -207,7 +210,8 @@ int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, goto out_free; } - nz = min((unsigned int)le64_to_cpu(report->nr_zones), nr_zones); + nz = min3((unsigned int)le64_to_cpu(report->nr_zones), + nr_zones - zone_idx, max_in_buf); if (!nz) break; From 3456b525528967456a8837b5dc166507d877a476 Mon Sep 17 00:00:00 2001 From: Xixin Liu Date: Thu, 25 Jun 2026 10:00:00 +0800 Subject: [PATCH 005/241] nvme: zns: include zone index in invalid zone type error Include the zone index when reporting an invalid zone type during zone descriptor parsing. Signed-off-by: Xixin Liu Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/zns.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/zns.c b/drivers/nvme/host/zns.c index 03a2528b7192..2a152e87bd76 100644 --- a/drivers/nvme/host/zns.c +++ b/drivers/nvme/host/zns.c @@ -155,7 +155,8 @@ static int nvme_zone_parse_entry(struct nvme_ns *ns, struct blk_zone zone = { }; if ((entry->zt & 0xf) != NVME_ZONE_TYPE_SEQWRITE_REQ) { - dev_err(ns->ctrl->device, "invalid zone type %#x\n", entry->zt); + dev_err(ns->ctrl->device, "invalid zone type %#x at zone %u\n", + entry->zt, idx); return -EINVAL; } From ba6e9472b453ccea313c7dad5bf2ad98d6ad13f5 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Thu, 25 Jun 2026 10:59:11 -0700 Subject: [PATCH 006/241] nvme-auth: Avoid C=1 warning in nvme_auth_derive_tls_psk() The following works fine with gcc and clang, but sparse warns about label_len not being an actual constant expression: const size_t label_len = sizeof(label) - 1; ... static_assert(label_len <= 255); Avoid this by giving label an explicit length and using sizeof(label) instead of label_len. Reported-by: John Garry Closes: https://lore.kernel.org/linux-nvme/965a37dd-f698-46b6-9623-1099a13f7e60@oracle.com Fixes: d126cbaa7d9a ("nvme-auth: common: use crypto library in nvme_auth_derive_tls_psk()") Signed-off-by: Eric Biggers Reviewed-by: Hannes Reinecke Reviewed-by: John Garry Signed-off-by: Keith Busch --- drivers/nvme/common/auth.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c index 77f1d22512f8..e2e0c736540a 100644 --- a/drivers/nvme/common/auth.c +++ b/drivers/nvme/common/auth.c @@ -692,8 +692,7 @@ int nvme_auth_derive_tls_psk(int hmac_id, const u8 *psk, size_t psk_len, const char *psk_digest, u8 **ret_psk) { static const u8 default_salt[NVME_AUTH_MAX_DIGEST_SIZE]; - static const char label[] = "tls13 nvme-tls-psk"; - const size_t label_len = sizeof(label) - 1; + static const char label[18] = "tls13 nvme-tls-psk"; u8 prk[NVME_AUTH_MAX_DIGEST_SIZE]; size_t hash_len, ctx_len; u8 *hmac_data = NULL, *tls_key; @@ -729,7 +728,7 @@ int nvme_auth_derive_tls_psk(int hmac_id, const u8 *psk, size_t psk_len, */ hmac_data = kmalloc(/* output length */ 2 + - /* label */ 1 + label_len + + /* label */ 1 + sizeof(label) + /* context (max) */ 1 + 3 + 1 + strlen(psk_digest) + /* counter */ 1, GFP_KERNEL); @@ -743,10 +742,10 @@ int nvme_auth_derive_tls_psk(int hmac_id, const u8 *psk, size_t psk_len, hmac_data[i++] = hash_len; /* label */ - static_assert(label_len <= 255); - hmac_data[i] = label_len; - memcpy(&hmac_data[i + 1], label, label_len); - i += 1 + label_len; + static_assert(sizeof(label) <= 255); + hmac_data[i] = sizeof(label); + memcpy(&hmac_data[i + 1], label, sizeof(label)); + i += 1 + sizeof(label); /* context */ ctx_len = sprintf(&hmac_data[i + 1], "%02d %s", hmac_id, psk_digest); From f4254b18d48af66a99101092b1d72f58c4c69b23 Mon Sep 17 00:00:00 2001 From: Surabhi Gogte Date: Fri, 26 Jun 2026 22:15:50 -0600 Subject: [PATCH 007/241] nvme-rdma: refactor nvme_rdma_alloc_queue() to take a queue pointer Callers are responsible for initializing queue->ctrl and queue->queue_size before calling nvme_rdma_alloc_queue(), which now derives ctrl and idx from the queue pointer directly. This removes redundant assignments inside the function and simplifies the interface. Signed-off-by: Surabhi Gogte Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/rdma.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 6909e3542794..6b0b0a3dea62 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -566,16 +566,14 @@ out_put_dev: return ret; } -static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl, - int idx, size_t queue_size) +static int nvme_rdma_alloc_queue(struct nvme_rdma_queue *queue) { - struct nvme_rdma_queue *queue; + struct nvme_rdma_ctrl *ctrl = queue->ctrl; + int idx = nvme_rdma_queue_idx(queue); struct sockaddr *src_addr = NULL; int ret; - queue = &ctrl->queues[idx]; mutex_init(&queue->queue_lock); - queue->ctrl = ctrl; if (idx && ctrl->ctrl.max_integrity_segments) queue->pi_support = true; else @@ -587,8 +585,6 @@ static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl, else queue->cmnd_capsule_len = sizeof(struct nvme_command); - queue->queue_size = queue_size; - queue->cm_id = rdma_create_id(&init_net, nvme_rdma_cm_handler, queue, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(queue->cm_id)) { @@ -736,8 +732,9 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl) nvmf_set_io_queues(opts, nr_io_queues, ctrl->io_queues); for (i = 1; i < ctrl->ctrl.queue_count; i++) { - ret = nvme_rdma_alloc_queue(ctrl, i, - ctrl->ctrl.sqsize + 1); + ctrl->queues[i].ctrl = ctrl; + ctrl->queues[i].queue_size = ctrl->ctrl.sqsize + 1; + ret = nvme_rdma_alloc_queue(&ctrl->queues[i]); if (ret) goto out_free_queues; } @@ -783,7 +780,9 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl, bool pi_capable = false; int error; - error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); + ctrl->queues[0].ctrl = ctrl; + ctrl->queues[0].queue_size = NVME_AQ_DEPTH; + error = nvme_rdma_alloc_queue(&ctrl->queues[0]); if (error) return error; From 2a8513091d2f0b9a1e94b1843c48c712e7c17301 Mon Sep 17 00:00:00 2001 From: Surabhi Gogte Date: Fri, 26 Jun 2026 22:15:51 -0600 Subject: [PATCH 008/241] nvme-rdma: parallelize I/O queue allocation and startup Refactor nvme rdma I/O queue setup to use async API, combining allocation and startup into a single parallel operation per queue. This reduces connection and reconnection setup time when there are delays in establishing connections, which is especially important for high-core-count hosts. Key changes: - Use async API to facilitate parallel calls for io queue setup. - Add nvme_rdma_setup_ctx for propagating errors from async workers. - Remove nvme_rdma_alloc_io_queues() and nvme_rdma_start_io_queues(); their logic is folded into nvme_rdma_setup_io_queues() and nvme_rdma_configure_io_queues(). - Move queue count negotiation (nvme_set_queue_count, nvmf_set_io_queues) from the removed nvme_rdma_alloc_io_queues() into nvme_rdma_configure_io_queues(). Testing on a 64-core host with 64 IO-queues shows nvme-rdma connection time reduced from ~1.4s to 416ms. Signed-off-by: Surabhi Gogte Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/rdma.c | 122 ++++++++++++++++++++++++--------------- 1 file changed, 76 insertions(+), 46 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 6b0b0a3dea62..52933d11ea03 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -100,6 +101,11 @@ struct nvme_rdma_queue { struct mutex queue_lock; }; +struct nvme_rdma_setup_ctx { + struct nvme_rdma_queue *queue; + int *err; +}; + struct nvme_rdma_ctrl { /* read only in the hot path */ struct nvme_rdma_queue *queues; @@ -690,60 +696,68 @@ static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx) return ret; } -static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl, - int first, int last) +static void nvme_rdma_setup_queue_async(void *data, async_cookie_t cookie) { - int i, ret = 0; + struct nvme_rdma_setup_ctx *ctx = data; + struct nvme_rdma_queue *queue; + int ret; - for (i = first; i < last; i++) { - ret = nvme_rdma_start_queue(ctrl, i); - if (ret) - goto out_stop_queues; - } + queue = ctx->queue; + ret = nvme_rdma_alloc_queue(queue); + if (ret) + goto out_err; - return 0; + ret = nvme_rdma_start_queue(queue->ctrl, nvme_rdma_queue_idx(queue)); + if (ret) + goto out_err; -out_stop_queues: - for (i--; i >= first; i--) - nvme_rdma_stop_queue(&ctrl->queues[i]); - return ret; + return; +out_err: + WRITE_ONCE(*ctx->err, ret); } -static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl) +static int nvme_rdma_setup_io_queues(struct nvme_rdma_ctrl *ctrl, + unsigned int first, unsigned int last, size_t queue_size) { - struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; - unsigned int nr_io_queues; - int i, ret; + ASYNC_DOMAIN_EXCLUSIVE(queue_domain); + struct nvme_rdma_setup_ctx *ctxs; + int nr_queues = last - first; + int err = 0, i, ret; - nr_io_queues = nvmf_nr_io_queues(opts); - ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); - if (ret) - return ret; - - if (nr_io_queues == 0) { - dev_err(ctrl->ctrl.device, - "unable to set any I/O queues\n"); + ctxs = kmalloc_objs(*ctxs, nr_queues); + if (!ctxs) return -ENOMEM; + + for (i = 0; i < nr_queues; i++) { + struct nvme_rdma_queue *queue = &ctrl->queues[first + i]; + + queue->ctrl = ctrl; + queue->queue_size = queue_size; + + ctxs[i].queue = queue; + ctxs[i].err = &err; + async_schedule_domain(nvme_rdma_setup_queue_async, &ctxs[i], + &queue_domain); } - ctrl->ctrl.queue_count = nr_io_queues + 1; - dev_info(ctrl->ctrl.device, - "creating %d I/O queues.\n", nr_io_queues); + async_synchronize_full_domain(&queue_domain); + kfree(ctxs); - nvmf_set_io_queues(opts, nr_io_queues, ctrl->io_queues); - for (i = 1; i < ctrl->ctrl.queue_count; i++) { - ctrl->queues[i].ctrl = ctrl; - ctrl->queues[i].queue_size = ctrl->ctrl.sqsize + 1; - ret = nvme_rdma_alloc_queue(&ctrl->queues[i]); - if (ret) - goto out_free_queues; - } + ret = READ_ONCE(err); + if (ret) + goto out_free_queues; return 0; - out_free_queues: - for (i--; i >= 1; i--) - nvme_rdma_free_queue(&ctrl->queues[i]); + for (i = 0; i < nr_queues; i++) { + struct nvme_rdma_queue *queue = + &ctrl->queues[first + i]; + + if (test_bit(NVME_RDMA_Q_LIVE, &queue->flags)) + nvme_rdma_stop_queue(queue); + if (test_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags)) + nvme_rdma_free_queue(queue); + } return ret; } @@ -862,12 +876,23 @@ out_free_queue: static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) { + unsigned int nr_io_queues; int ret, nr_queues; - ret = nvme_rdma_alloc_io_queues(ctrl); + nr_io_queues = nvmf_nr_io_queues(ctrl->ctrl.opts); + ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); if (ret) return ret; + if (nr_io_queues == 0) { + dev_err(ctrl->ctrl.device, "unable to set any I/O queues\n"); + return -ENOMEM; + } + + ctrl->ctrl.queue_count = nr_io_queues + 1; + dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues); + nvmf_set_io_queues(ctrl->ctrl.opts, nr_io_queues, ctrl->io_queues); + if (new) { ret = nvme_rdma_alloc_tag_set(&ctrl->ctrl); if (ret) @@ -880,7 +905,9 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) * queue number might have changed. */ nr_queues = min(ctrl->tag_set.nr_hw_queues + 1, ctrl->ctrl.queue_count); - ret = nvme_rdma_start_io_queues(ctrl, 1, nr_queues); + ret = nvme_rdma_setup_io_queues(ctrl, 1, nr_queues, + ctrl->ctrl.sqsize + 1); + if (ret) goto out_cleanup_tagset; @@ -904,12 +931,15 @@ static int nvme_rdma_configure_io_queues(struct nvme_rdma_ctrl *ctrl, bool new) /* * If the number of queues has increased (reconnect case) - * start all new queues now. + * setup all new queues now. */ - ret = nvme_rdma_start_io_queues(ctrl, nr_queues, - ctrl->tag_set.nr_hw_queues + 1); - if (ret) - goto out_wait_freeze_timed_out; + if (ctrl->tag_set.nr_hw_queues + 1 > nr_queues) { + ret = nvme_rdma_setup_io_queues(ctrl, nr_queues, + ctrl->tag_set.nr_hw_queues + 1, + ctrl->ctrl.sqsize + 1); + if (ret) + goto out_wait_freeze_timed_out; + } return 0; From 90096175473f7c86e39c3f74f10343f965f5a05d Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Mon, 29 Jun 2026 14:15:27 +0900 Subject: [PATCH 009/241] nvmet-rdma: factor out response resource cleanup Move the RDMA read/write context teardown and the request SGL freeing out of nvmet_rdma_release_rsp() into a new helper function nvmet_rdma_free_rsp_resources(). This is a refactoring with no functional change, in preparation for the following patch that uses nvmet_rdma_free_rsp_resources(). Signed-off-by: Shin'ichiro Kawasaki Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/target/rdma.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index ea1185b8267e..3c34f235e542 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -657,18 +657,25 @@ static void nvmet_rdma_rw_ctx_destroy(struct nvmet_rdma_rsp *rsp) req->sg, req->sg_cnt, nvmet_data_dir(req)); } -static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +static void nvmet_rdma_free_rsp_resources(struct nvmet_rdma_rsp *rsp) { struct nvmet_rdma_queue *queue = rsp->queue; - atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); - if (rsp->n_rdma) nvmet_rdma_rw_ctx_destroy(rsp); if (rsp->req.sg < rsp->cmd->inline_sg || rsp->req.sg >= rsp->cmd->inline_sg + queue->dev->inline_page_count) nvmet_req_free_sgls(&rsp->req); +} + +static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) +{ + struct nvmet_rdma_queue *queue = rsp->queue; + + atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); + + nvmet_rdma_free_rsp_resources(rsp); if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) nvmet_rdma_process_wr_wait_list(queue); From 0114dd303b373522dea06053aabae34bdd33a7c4 Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Mon, 29 Jun 2026 14:15:28 +0900 Subject: [PATCH 010/241] nvmet-rdma: fix response resource leak on queue teardown When an nvme target with rdma transport is removed while I/Os are in flight, a response can be posted but its send completion is never delivered before the connection is torn down. As a result nvmet_rdma_send_done() and nvmet_rdma_release_rsp() are never called for the response, and this leaks the allocated RDMA read/write context and request SGLs. These leaks are recreated by running blktests nvme/061 with the rdma transport and the siw driver. Kernel kmemleak feature reports them as follows: unreferenced object 0xffff88812bc490c0 (size 32): comm "kworker/2:1H", pid 409, jiffies 4307744490 backtrace (crc 89afd339): __kmalloc_noprof+0x5f9/0x890 sgl_alloc_order+0x7b/0x380 nvmet_req_alloc_sgls+0x290/0x4f0 [nvmet] nvmet_rdma_map_sgl_keyed+0x241/0x12e0 [nvmet_rdma] nvmet_rdma_handle_command+0x73e/0xb80 [nvmet_rdma] __ib_process_cq+0x149/0x4c0 [ib_core] ib_cq_poll_work+0x49/0x160 [ib_core] process_one_work+0x8b2/0x1640 worker_thread+0x5fd/0xfe0 kthread+0x367/0x460 ret_from_fork+0x655/0x9d0 ret_from_fork_asm+0x1a/0x30 unreferenced object 0xffff88814bd05e80 (size 64): comm "kworker/3:1H", pid 148, jiffies 4295195428 backtrace (crc e35510cb): __kmalloc_noprof+0x5f9/0x890 rdma_rw_ctx_init+0x333/0x1fa0 [ib_core] nvmet_rdma_map_sgl_keyed+0x5c8/0x12e0 [nvmet_rdma] nvmet_rdma_handle_command+0x73e/0xb80 [nvmet_rdma] __ib_process_cq+0x149/0x4c0 [ib_core] ib_cq_poll_work+0x49/0x160 [ib_core] process_one_work+0x8b2/0x1640 worker_thread+0x5fd/0xfe0 kthread+0x367/0x460 ret_from_fork+0x655/0x9d0 ret_from_fork_asm+0x1a/0x30 To avoid the memory leaks, reclaim the memory of the in-flight responses when the queue QP is torn down. Call nvmet_rdma_free_rsp_resources() that frees up the RDMA read/write context and the request SGLs of such responses. Fixes: 8f000cac6e7a ("nvmet-rdma: add a NVMe over Fabrics RDMA target driver") Signed-off-by: Shin'ichiro Kawasaki Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/target/rdma.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 3c34f235e542..de5a88fbb233 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -1345,9 +1345,27 @@ err_destroy_cq: goto out; } +static bool nvmet_rdma_reclaim_rsp(struct sbitmap *sb, unsigned int bitnr, + void *data) +{ + struct nvmet_rdma_queue *queue = data; + + nvmet_rdma_free_rsp_resources(&queue->rsps[bitnr]); + + return true; +} + static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue) { ib_drain_qp(queue->qp); + + /* + * Reclaim resources of a response that is still in-flight when the + * queue is being torn down. This happens when the connection was + * forcefully disconnected while an I/O is in flight. + */ + sbitmap_for_each_set(&queue->rsp_tags, nvmet_rdma_reclaim_rsp, queue); + if (queue->cm_id) rdma_destroy_id(queue->cm_id); ib_destroy_qp(queue->qp); From dd516cd7624648c71d338d99368587f10ecc9f0b Mon Sep 17 00:00:00 2001 From: John Garry Date: Tue, 30 Jun 2026 10:27:17 +0000 Subject: [PATCH 011/241] nvme: handle positive error codes in nuse_show() Function __nvme_submit_sync_cmd() returns a positive error code for NVMe errors. Otherwise, we get 0 for success or a negative error code for a kernel error. In nuse_show() -> ns_{head}_update_nuse() -> nvme_identify_ns() -> nvme_submit_sync_cmd() -> __nvme_submit_sync_cmd(), we then may get a positive error code returned. Function nuse_show() - being a device attr handler - should return the number of bytes written to the buffer or a negative error code. Convert any positive NVMe error code to -EIO. Signed-off-by: John Garry Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/sysfs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c index 75b2d69b5957..abf8edaae371 100644 --- a/drivers/nvme/host/sysfs.c +++ b/drivers/nvme/host/sysfs.c @@ -240,8 +240,10 @@ static ssize_t nuse_show(struct device *dev, struct device_attribute *attr, ret = ns_head_update_nuse(head); else ret = ns_update_nuse(disk->private_data); - if (ret) + if (ret < 0) return ret; + else if (ret > 0) + return -EIO; return sysfs_emit(buf, "%llu\n", head->nuse); } From 3a6d89836ab09d53f4c32c57588689a378ce4363 Mon Sep 17 00:00:00 2001 From: Xixin Liu Date: Wed, 1 Jul 2026 14:30:00 +0800 Subject: [PATCH 012/241] nvme-auth: use crypto_memneq for DH-HMAC-CHAP response comparison DH-HMAC-CHAP authentication compares HMAC response digests with memcmp(). Standard memcmp() may stop at the first differing byte, which can leak timing information to a remote attacker and allow incremental recovery of the expected digest. Use crypto_memneq() for constant-time comparison on both the host path that validates the controller Success1 response and the target path that validates the host Reply digest. Other memcmp() uses in the NVMe auth code (e.g. fixed string prefix checks) are not security-sensitive and are left unchanged. Signed-off-by: Xixin Liu Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/auth.c | 3 ++- drivers/nvme/target/fabrics-cmd-auth.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/auth.c b/drivers/nvme/host/auth.c index 16de4499a8e7..e55920642f2c 100644 --- a/drivers/nvme/host/auth.c +++ b/drivers/nvme/host/auth.c @@ -8,6 +8,7 @@ #include #include #include +#include #include "nvme.h" #include "fabrics.h" #include @@ -361,7 +362,7 @@ static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, return 0; /* Validate controller response */ - if (memcmp(chap->response, data->rval, data->hl)) { + if (crypto_memneq(chap->response, data->rval, data->hl)) { dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n", __func__, chap->qid, (int)chap->hash_len, data->rval); dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n", diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 45820a12750d..03529e19698b 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "nvmet.h" static void nvmet_auth_expired_work(struct work_struct *work) @@ -177,7 +178,7 @@ static u8 nvmet_auth_reply(struct nvmet_req *req, void *d, u32 tl) return NVME_AUTH_DHCHAP_FAILURE_FAILED; } - if (memcmp(data->rval, response, data->hl)) { + if (crypto_memneq(data->rval, response, data->hl)) { pr_info("ctrl %d qid %d host response mismatch\n", ctrl->cntlid, req->sq->qid); pr_debug("ctrl %d qid %d rval %*ph\n", From 3ddcfb013322aa37eaa7a0d344b73079c38dfa21 Mon Sep 17 00:00:00 2001 From: Bryam Vargas Date: Thu, 2 Jul 2026 03:45:14 -0500 Subject: [PATCH 013/241] nvmet-auth: zero the AUTH_RECEIVE response buffer nvmet_execute_auth_receive() allocates the response buffer with kmalloc() sized by the host-supplied AUTH_RECEIVE allocation length, but the DH-HMAC-CHAP builders write only a fixed-size message into it. The full allocation length is then copied to the wire by nvmet_copy_to_sgl(), so a remote initiator receives the bytes past the built message -- up to nearly a page of uninitialized slab -- during the pre-authentication handshake. Allocate the buffer with kzalloc() so the unwritten tail is zeroed before it is sent; conforming responses are unaffected. Fixes: db1312dd9548 ("nvmet: implement basic In-Band Authentication") Cc: stable@vger.kernel.org Signed-off-by: Bryam Vargas Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/target/fabrics-cmd-auth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index 03529e19698b..d1b39e64d877 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -558,7 +558,7 @@ void nvmet_execute_auth_receive(struct nvmet_req *req) return; } - d = kmalloc(al, GFP_KERNEL); + d = kzalloc(al, GFP_KERNEL); if (!d) { status = NVME_SC_INTERNAL; goto done; From 627e8bb91a8fdec02cfb68a98d360cec4bafa122 Mon Sep 17 00:00:00 2001 From: John Garry Date: Tue, 7 Jul 2026 14:57:44 +0100 Subject: [PATCH 014/241] nvme: swap synchronization ordering in nvme_remove_head() sashiko bot reported a potential issue in the requeue handling in [0] - the code there is same as the NVMe driver. The issue is that when we schedule the requeue work, if a bio is added to the requeue list afterwards in nvme_ns_head_submit_bio(), it is missed by the requeue worker. This issue can be recreated by hacking a large delay in the bio submission requeue path: } else if (nvme_available_path(head)) { dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); + msleep(30000); spin_lock_irq(&head->requeue_lock); bio_list_add(&head->requeue_list, bio); spin_unlock_irq(&head->requeue_lock); Then if we issue a write after removing all paths, a hang can be seen: # echo 20 > /sys/devices/virtual/nvme-subsystem/nvme-subsys1/nvme1n1/delayed_removal_secs # # ./ini_nvme_teardown.sh [ 25.877224] nvme nvme1: Removing ctrl: NQN "nvme-test-target" [ 25.939569] nvme nvme2: Removing ctrl: NQN "nvme-test-target" # # xfs_io -d -C "pwrite -b 64k -V 1 -D 0 64k" /dev/nvme1n1p1 [ 29.883653] block nvme1n1: no usable path - requeuing I/O Fix by re-ordering the SRCU synchronization and scheduling the requeue work. [0] https://lore.kernel.org/linux-scsi/20260703102918.3723667-1-john.g.garry@oracle.com/T/#m72af1f29deb0ebfb2973464207f201f1be1f660c Reviewed-by: Christoph Hellwig Reviewed-by: Nilay Shroff Signed-off-by: John Garry Signed-off-by: Keith Busch --- drivers/nvme/host/multipath.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 7e9fb7227300..56587ae59c7f 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -711,14 +711,15 @@ static void nvme_remove_head(struct nvme_ns_head *head) { if (test_and_clear_bit(NVME_NSHEAD_DISK_LIVE, &head->flags)) { /* - * requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared - * to allow multipath to fail all I/O. + * Requeue I/O after NVME_NSHEAD_DISK_LIVE has been cleared + * to allow multipath to fail all I/O. First synchronize to + * add any bios to the requeue list. */ + synchronize_srcu(&head->srcu); kblockd_schedule_work(&head->requeue_work); if (test_and_clear_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags)) nvme_cdev_del(&head->cdev, &head->cdev_device); - synchronize_srcu(&head->srcu); del_gendisk(head->disk); } nvme_put_ns_head(head); From 77c57daf98a2ad95ec9ac1371caeff8939cc1f58 Mon Sep 17 00:00:00 2001 From: John Garry Date: Mon, 6 Jul 2026 12:54:02 +0000 Subject: [PATCH 015/241] nvme: don't reference NS after unlocking in nvme_ns_head_ctrl_ioctl() In nvme_ns_head_ctrl_ioctl(), once we drop the SRCU read lock we should not reference the NS to lookup the controller, so use the available controller pointer directly. Reviewed-by: Christoph Hellwig Signed-off-by: John Garry Signed-off-by: Keith Busch --- drivers/nvme/host/ioctl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index 664216eece4a..d5a8f375953b 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -699,7 +699,7 @@ static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, nvme_get_ctrl(ns->ctrl); srcu_read_unlock(&head->srcu, srcu_idx); - ret = nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write); + ret = nvme_ctrl_ioctl(ctrl, cmd, argp, open_for_write); nvme_put_ctrl(ctrl); return ret; From 8f82aaf16f1c620558c3e0f3a76528c6787835bc Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 7 Jul 2026 20:11:13 +0800 Subject: [PATCH 016/241] nvmet: add namespace-level debugfs directory Add per-namespace debugfs directory support under the subsystem debugfs directory. Each enabled namespace gets a ns/ directory created during nvmet_ns_enable() and removed during nvmet_ns_disable(). This provides the infrastructure for exposing namespace-specific debug information in subsequent patches. Reviewed-by: Daniel Wagner Reviewed-by: Christoph Hellwig Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- drivers/nvme/target/core.c | 2 ++ drivers/nvme/target/debugfs.c | 21 +++++++++++++++++++++ drivers/nvme/target/debugfs.h | 5 +++++ drivers/nvme/target/nvmet.h | 3 +++ 4 files changed, 31 insertions(+) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 4477c4d6b1ee..a2403a808360 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -616,6 +616,7 @@ int nvmet_ns_enable(struct nvmet_ns *ns) nvmet_ns_changed(subsys, ns->nsid); ns->enabled = true; xa_set_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED); + nvmet_debugfs_ns_setup(ns); ret = 0; out_unlock: mutex_unlock(&subsys->lock); @@ -642,6 +643,7 @@ void nvmet_ns_disable(struct nvmet_ns *ns) ns->enabled = false; xa_clear_mark(&subsys->namespaces, ns->nsid, NVMET_NS_ENABLED); + nvmet_debugfs_ns_free(ns); list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) pci_dev_put(radix_tree_delete(&ctrl->p2p_ns_map, ns->nsid)); diff --git a/drivers/nvme/target/debugfs.c b/drivers/nvme/target/debugfs.c index 5dcbd5aa86e1..e6f51eb59010 100644 --- a/drivers/nvme/target/debugfs.c +++ b/drivers/nvme/target/debugfs.c @@ -153,6 +153,27 @@ static int nvmet_ctrl_tls_concat_show(struct seq_file *m, void *p) NVMET_DEBUGFS_ATTR(nvmet_ctrl_tls_concat); #endif +void nvmet_debugfs_ns_setup(struct nvmet_ns *ns) +{ + char name[16]; + struct dentry *parent = ns->subsys->debugfs_dir; + + if (!parent) + return; + snprintf(name, sizeof(name), "ns%u", ns->nsid); + ns->debugfs_dir = debugfs_create_dir(name, parent); + if (IS_ERR(ns->debugfs_dir)) { + ns->debugfs_dir = NULL; + return; + } +} + +void nvmet_debugfs_ns_free(struct nvmet_ns *ns) +{ + debugfs_remove_recursive(ns->debugfs_dir); + ns->debugfs_dir = NULL; +} + int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) { char name[32]; diff --git a/drivers/nvme/target/debugfs.h b/drivers/nvme/target/debugfs.h index cfb8bbf6a297..b559d254fc2a 100644 --- a/drivers/nvme/target/debugfs.h +++ b/drivers/nvme/target/debugfs.h @@ -14,6 +14,8 @@ int nvmet_debugfs_subsys_setup(struct nvmet_subsys *subsys); void nvmet_debugfs_subsys_free(struct nvmet_subsys *subsys); int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl); void nvmet_debugfs_ctrl_free(struct nvmet_ctrl *ctrl); +void nvmet_debugfs_ns_setup(struct nvmet_ns *ns); +void nvmet_debugfs_ns_free(struct nvmet_ns *ns); int __init nvmet_init_debugfs(void); void nvmet_exit_debugfs(void); @@ -30,6 +32,9 @@ static inline int nvmet_debugfs_ctrl_setup(struct nvmet_ctrl *ctrl) } static inline void nvmet_debugfs_ctrl_free(struct nvmet_ctrl *ctrl) {} +static inline void nvmet_debugfs_ns_setup(struct nvmet_ns *ns) {} +static inline void nvmet_debugfs_ns_free(struct nvmet_ns *ns) {} + static inline int __init nvmet_init_debugfs(void) { return 0; diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index aaba745e3c21..c672c9bf3053 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -128,6 +128,9 @@ struct nvmet_ns { u8 csi; struct nvmet_pr pr; struct xarray pr_per_ctrl_refs; +#ifdef CONFIG_NVME_TARGET_DEBUGFS + struct dentry *debugfs_dir; +#endif }; static inline struct nvmet_ns *to_nvmet_ns(struct config_item *item) From 1511516478bb9d718ae5c9deceb19cfffd1113da Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 7 Jul 2026 20:11:14 +0800 Subject: [PATCH 017/241] nvmet: expose reservation state through debugfs Add a 'reservation' debugfs file under each namespace directory that shows the persistent reservation state, including enable status, generation counter, notify mask, current holder info, and the full registrant list with hostid and reservation key. Each attribute is emitted as a single "key=value" line so the output is easy to parse from scripts. The registrant list is emitted as repeated "reg=" lines. The notify mask is emitted as a comma-separated list of masked notification names. Empty values are reported as "none". Example output: enable=1 generation=2 notify_mask=reg_preempted,resv_released,resv_preempted rtype=write_exclusive holder=11111111-1111-1111-1111-111111111111,0x1111 reg=11111111-1111-1111-1111-111111111111,0x1111 reg=22222222-2222-2222-2222-222222222222,0x2222 When reservation is not enabled only "enable=0" is printed. The output uses rcu_read_lock() for safe access to the holder and registrant_list, consistent with other PR read paths. Reviewed-by: Daniel Wagner Reviewed-by: Christoph Hellwig Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- drivers/nvme/target/debugfs.c | 82 +++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/drivers/nvme/target/debugfs.c b/drivers/nvme/target/debugfs.c index e6f51eb59010..e85fe1d4c9f8 100644 --- a/drivers/nvme/target/debugfs.c +++ b/drivers/nvme/target/debugfs.c @@ -153,6 +153,86 @@ static int nvmet_ctrl_tls_concat_show(struct seq_file *m, void *p) NVMET_DEBUGFS_ATTR(nvmet_ctrl_tls_concat); #endif +static const char *const nvmet_pr_type_names[] = { + [NVME_PR_WRITE_EXCLUSIVE] = "write_exclusive", + [NVME_PR_EXCLUSIVE_ACCESS] = "exclusive_access", + [NVME_PR_WRITE_EXCLUSIVE_REG_ONLY] = "write_exclusive_reg_only", + [NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY] = "exclusive_access_reg_only", + [NVME_PR_WRITE_EXCLUSIVE_ALL_REGS] = "write_exclusive_all_regs", + [NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS] = "exclusive_access_all_regs", +}; + +static const char *nvmet_pr_type_to_str(enum nvme_pr_type type) +{ + if (type < ARRAY_SIZE(nvmet_pr_type_names) && + nvmet_pr_type_names[type]) + return nvmet_pr_type_names[type]; + return "unknown"; +} + +static const char *const nvmet_pr_notify_names[] = { + [NVME_PR_NOTIFY_BIT_REG_PREEMPTED] = "reg_preempted", + [NVME_PR_NOTIFY_BIT_RESV_RELEASED] = "resv_released", + [NVME_PR_NOTIFY_BIT_RESV_PREEMPTED] = "resv_preempted", +}; + +static void nvmet_pr_notify_mask_to_str(struct seq_file *m, unsigned long mask) +{ + bool sep = false; + int i; + + if (!mask) { + seq_puts(m, "none"); + return; + } + + for (i = 0; i < ARRAY_SIZE(nvmet_pr_notify_names); i++) { + if (!test_bit(i, &mask) || !nvmet_pr_notify_names[i]) + continue; + if (sep) + seq_putc(m, ','); + seq_puts(m, nvmet_pr_notify_names[i]); + sep = true; + } +} + +static int nvmet_ns_pr_show(struct seq_file *m, void *p) +{ + struct nvmet_ns *ns = m->private; + struct nvmet_pr *pr = &ns->pr; + struct nvmet_pr_registrant *holder, *reg; + + seq_printf(m, "enable=%d\n", pr->enable); + if (!pr->enable) + return 0; + + seq_printf(m, "generation=%u\n", atomic_read(&pr->generation)); + seq_puts(m, "notify_mask="); + nvmet_pr_notify_mask_to_str(m, pr->notify_mask); + seq_putc(m, '\n'); + + rcu_read_lock(); + holder = rcu_dereference(pr->holder); + if (holder) { + seq_printf(m, "rtype=%s\n", + nvmet_pr_type_to_str(holder->rtype)); + seq_printf(m, "holder=%pUb,0x%llx\n", + &holder->hostid, holder->rkey); + } else { + seq_puts(m, "rtype=none\n"); + seq_puts(m, "holder=none\n"); + } + + list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { + seq_printf(m, "reg=%pUb,0x%llx\n", + ®->hostid, reg->rkey); + } + rcu_read_unlock(); + + return 0; +} +NVMET_DEBUGFS_ATTR(nvmet_ns_pr); + void nvmet_debugfs_ns_setup(struct nvmet_ns *ns) { char name[16]; @@ -166,6 +246,8 @@ void nvmet_debugfs_ns_setup(struct nvmet_ns *ns) ns->debugfs_dir = NULL; return; } + debugfs_create_file("reservation", 0400, ns->debugfs_dir, ns, + &nvmet_ns_pr_fops); } void nvmet_debugfs_ns_free(struct nvmet_ns *ns) From 1c4635cf4de92564ead2b4be402c405dcc375f01 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 7 Jul 2026 20:12:25 +0800 Subject: [PATCH 018/241] nvme: add ABI documentation for host sysfs interfaces Add Documentation/ABI/stable/sysfs-nvme documenting all NVMe host sysfs attributes, covering controller attributes under /sys/class/nvme/nvmeX/, namespace attributes under /sys/block/nvmeXnY/, and subsystem attributes under /sys/class/nvme-subsystem/nvme-subsysX/. Each entry has been traced to its original introducing commit to provide accurate Date, KernelVersion, and Contact information. Reviewed-by: Hannes Reinecke Reviewed-by: Nilay Shroff Reviewed-by: Christoph Hellwig Reviewed-by: Daniel Wagner Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- Documentation/ABI/stable/sysfs-nvme | 453 +++++++++++++++++++++++++++ Documentation/ABI/testing/sysfs-nvme | 13 - 2 files changed, 453 insertions(+), 13 deletions(-) create mode 100644 Documentation/ABI/stable/sysfs-nvme delete mode 100644 Documentation/ABI/testing/sysfs-nvme diff --git a/Documentation/ABI/stable/sysfs-nvme b/Documentation/ABI/stable/sysfs-nvme new file mode 100644 index 000000000000..a2f5d0710db4 --- /dev/null +++ b/Documentation/ABI/stable/sysfs-nvme @@ -0,0 +1,453 @@ +What: /sys/class/nvme/nvmeX/model +What: /sys/class/nvme/nvmeX/serial +What: /sys/class/nvme/nvmeX/firmware_rev +Date: January 2016 +KernelVersion: 4.5 +Contact: Keith Busch +Description: + Shows the model, serial number, or firmware revision string + of the NVMe controller, as reported in the Identify + Controller data structure. + +What: /sys/class/nvme/nvmeX/cntlid +Date: February 2016 +KernelVersion: 4.6 +Contact: Ming Lin +Description: + Shows the controller identifier assigned by the NVMe + subsystem. + +What: /sys/class/nvme/nvmeX/cntrltype +What: /sys/class/nvme/nvmeX/dctype +Date: February 2022 +KernelVersion: 5.18 +Contact: Martin Belanger +Description: + cntrltype: Shows the controller type. Possible values: "io", + "discovery", "admin", "reserved". + + dctype: Shows the discovery controller type. Possible values: + "none", "ddc", "cdc", "reserved". + +What: /sys/class/nvme/nvmeX/reset_controller +Date: November 2015 +KernelVersion: 4.5 +Contact: Christoph Hellwig +Description: + Write-only. Writing any value triggers a synchronous + controller reset. + +What: /sys/class/nvme/nvmeX/rescan_controller +Date: April 2016 +KernelVersion: 4.7 +Contact: Keith Busch +Description: + Write-only. Writing any value triggers a namespace rescan + on this controller. + +What: /sys/class/nvme/nvmeX/transport +What: /sys/class/nvme/nvmeX/subsysnqn +What: /sys/class/nvme/nvmeX/address +What: /sys/class/nvme/nvmeX/delete_controller +What: /sys/class/nvme/nvmeX/reconnect_delay +What: /sys/class/nvme/nvmeX/ctrl_loss_tmo +Date: June 2016 +KernelVersion: 4.8 +Contact: Ming Lin +Description: + Fabrics controller attributes added with NVMe-oF support. + + transport: Shows the transport type string. Possible values: + "pcie", "tcp", "rdma", "fc", "loop". + + subsysnqn: Shows the NVMe Qualified Name (NQN) of the + subsystem this controller belongs to. + + address: Shows the transport-specific address string. Only + available for fabrics controllers. + + delete_controller: Write-only. Triggers deletion of this + fabrics controller. + + reconnect_delay: Shows or sets the reconnect delay in + seconds. Reading returns the delay value, or "off" if + disabled. + + ctrl_loss_tmo: Shows or sets the controller loss timeout in + seconds. Reading returns the timeout value, or "off" if + infinite reconnects are allowed. Writing a negative value + disables the timeout. + +What: /sys/class/nvme/nvmeX/hostnqn +What: /sys/class/nvme/nvmeX/hostid +Date: February 2020 +KernelVersion: 5.7 +Contact: Sagi Grimberg +Description: + hostnqn: Shows the host NQN used by this fabrics controller. + + hostid: Shows the host identifier (UUID format) used by this + fabrics controller. + + Only available for fabrics controllers. + +What: /sys/class/nvme/nvmeX/fast_io_fail_tmo +Date: November 2020 +KernelVersion: 5.11 +Contact: Victor Gladkov +Description: + Shows or sets the fast I/O fail timeout in seconds. Reading + returns the timeout value, or "off" if disabled. Writing a + negative value disables the fast I/O fail. Only available + for fabrics controllers. + +What: /sys/class/nvme/nvmeX/kato +Date: April 2021 +KernelVersion: 5.13 +Contact: Hannes Reinecke +Description: + Shows the Keep Alive Timeout value in milliseconds for + this controller. + +What: /sys/class/nvme/nvmeX/cmb +Date: October 2016 +KernelVersion: 4.9 +Contact: Stephen Bates +Description: + Shows the Controller Memory Buffer (CMB) register values + in format "cmbloc : 0x%08x\ncmbsz : 0x%08x\n". Only + visible when the controller has a CMB (cmbsz != 0). + PCI transport only. + +What: /sys/class/nvme/nvmeX/cmbloc +What: /sys/class/nvme/nvmeX/cmbsz +What: /sys/class/nvme/nvmeX/hmb +Date: July 2021 +KernelVersion: 5.15 +Contact: Keith Busch +Description: + cmbloc: Shows the CMBLOC register value. + + cmbsz: Shows the CMBSZ register value. + + cmbloc and cmbsz are only visible when the controller has + a CMB. PCI transport only. + + hmb: Shows or sets whether the Host Memory Buffer (HMB) is + enabled. Reading returns 1 (enabled) or 0 (disabled). + Writing 1 enables HMB; writing 0 disables it. Only + visible when the controller supports HMB (hmpre != 0). + PCI transport only. + +What: /sys/class/nvme/nvmeX/state +Date: November 2016 +KernelVersion: 4.11 +Contact: Sagi Grimberg +Description: + Shows the current state of the controller. Possible values: + "new", "live", "resetting", "connecting", "deleting", + "deleting (no IO)", "dead". + +What: /sys/class/nvme/nvmeX/numa_node +Date: November 2018 +KernelVersion: 5.0 +Contact: Hannes Reinecke +Description: + Shows the NUMA node the controller is attached to. + +What: /sys/class/nvme/nvmeX/queue_count +What: /sys/class/nvme/nvmeX/sqsize +Date: September 2019 +KernelVersion: 5.4 +Contact: James Smart +Description: + queue_count: Shows the total number of queues (admin + I/O) + for this controller. + + sqsize: Shows the submission queue size for this controller. + +What: /sys/class/nvme/nvmeX/dhchap_secret +What: /sys/class/nvme/nvmeX/dhchap_ctrl_secret +Date: June 2022 +KernelVersion: 6.0 +Contact: Hannes Reinecke +Description: + dhchap_secret: Shows or sets the host DH-HMAC-CHAP secret + for this controller. Reading returns "none" if not set. + Writing must use the "DHHC-1:" key format and triggers + re-authentication. + + dhchap_ctrl_secret: Shows or sets the controller + DH-HMAC-CHAP secret for bidirectional authentication. + Same format as dhchap_secret. + + Only available when CONFIG_NVME_HOST_AUTH is enabled and + for fabrics controllers. + +What: /sys/class/nvme/nvmeX/tls_key +Date: August 2023 +KernelVersion: 6.7 +Contact: Hannes Reinecke +Description: + Shows the serial of the currently active TLS PSK as hex. + Returns empty if no TLS key is active. Only available for + TCP controllers with TLS or secure concatenation enabled + (CONFIG_NVME_TCP_TLS). + +What: /sys/class/nvme/nvmeX/tls_configured_key +Date: July 2024 +KernelVersion: 6.12 +Contact: Hannes Reinecke +Description: + Shows the serial of the configured TLS key. Writing 0 + triggers a PSK reauthentication (REPLACETLSPSK) with + the target. After reauthentication the returned serial + will be the new key. Only available for TCP controllers + with secure concatenation enabled (CONFIG_NVME_TCP_TLS). + +What: /sys/class/nvme/nvmeX/tls_keyring +Date: July 2024 +KernelVersion: 6.12 +Contact: Hannes Reinecke +Description: + Shows the TLS keyring description. Only available for TCP + controllers with a keyring configured (CONFIG_NVME_TCP_TLS). + +What: /sys/class/nvme/nvmeX/tls_mode +Date: April 2026 +KernelVersion: 7.1 +Contact: Daniel Wagner +Description: + Shows the TLS mode: "tls" for direct TLS or "concat" for + secure concatenation. Only available for TCP controllers + with TLS or secure concatenation enabled + (CONFIG_NVME_TCP_TLS). + +What: /sys/class/nvme/nvmeX/passthru_err_log_enabled +Date: January 2024 +KernelVersion: 6.8 +Contact: Alan Adamson +Description: + Shows or sets whether admin passthrough error logging is + enabled for this controller. Reading returns "on" or "off". + Writing accepts a boolean value. + +What: /sys/class/nvme/nvmeX/quirks +Date: November 2025 +KernelVersion: 7.0 +Contact: Maurizio Lombardi +Description: + Shows the active quirk names for this controller, one per + line. Shows "none" if no quirks are active. + +What: /sys/class/nvme/nvmeX/admin_timeout +What: /sys/class/nvme/nvmeX/io_timeout +Date: May 2026 +KernelVersion: 7.2 +Contact: Maurizio Lombardi +Description: + admin_timeout: Shows or sets the admin command timeout in + milliseconds. + + io_timeout: Shows or sets the I/O command timeout in + milliseconds. Changes are propagated to all namespace + request queues. + + The value must be nonzero. Only writable after the + controller has been started at least once. + +What: /sys/block/nvmeXnY/uuid +What: /sys/block/nvmeXnY/eui +What: /sys/block/nvmeXnY/nsid +Date: December 2015 +KernelVersion: 4.5 +Contact: Keith Busch +Description: + Namespace identification attributes. + + uuid: Shows the UUID for this namespace. Falls back to + showing the NGUID for backward compatibility. Hidden if + both are all zeros. + + eui: Shows the IEEE Extended Unique Identifier (EUI-64). + Hidden if all zeros. + + nsid: Shows the namespace identifier (NSID). + +What: /sys/block/nvmeXnY/wwid +Date: February 2016 +KernelVersion: 4.6 +Contact: Keith Busch +Description: + Shows the World Wide Identifier for this namespace. The + format depends on available identifiers (in priority + order): "uuid.{UUID}", "eui.{NGUID}", "eui.{EUI64}", or + "nvme.{VID}-{SERIAL}-{MODEL}-{NSID}". + +What: /sys/block/nvmeXnY/nguid +Date: June 2017 +KernelVersion: 4.13 +Contact: Johannes Thumshirn +Description: + Shows the Namespace Globally Unique Identifier (NGUID). + Hidden if the NGUID is all zeros. + +What: /sys/block/nvmeXcYnZ/ana_grpid +What: /sys/block/nvmeXcYnZ/ana_state +Date: May 2018 +KernelVersion: 4.19 +Contact: Christoph Hellwig +Description: + ana_grpid: Shows the ANA Group ID for this namespace + path device. + + ana_state: Shows the ANA state. Possible values: + "optimized", "non-optimized", "inaccessible", + "persistent-loss", "change". + + Only visible when the controller supports ANA. + Requires CONFIG_NVME_MULTIPATH. + +What: /sys/block/nvmeXcYnZ/queue_depth +Date: June 2024 +KernelVersion: 6.11 +Contact: Thomas Song +Description: + Shows the current active I/O count on this path's + controller. Returns empty if iopolicy is not "queue-depth". + Requires CONFIG_NVME_MULTIPATH. + +What: /sys/block/nvmeXcYnZ/numa_nodes +Date: January 2025 +KernelVersion: 6.15 +Contact: Nilay Shroff +Description: + Shows the NUMA node mask for which this path is the + currently selected path. Returns empty if iopolicy is not + "numa". Requires CONFIG_NVME_MULTIPATH. + +What: /sys/block/nvmeXnY/delayed_removal_secs +Date: May 2025 +KernelVersion: 6.16 +Contact: Nilay Shroff +Description: + Shows or sets the delayed removal timeout in seconds for + the multipath head device. When nonzero, I/O is queued + instead of failed when all paths are gone, and head removal + is deferred. Only visible on multipath head devices. + Requires CONFIG_NVME_MULTIPATH. + +What: /sys/block/nvmeXnY/csi +What: /sys/block/nvmeXnY/metadata_bytes +What: /sys/block/nvmeXnY/nuse +Date: December 2023 +KernelVersion: 6.8 +Contact: Daniel Wagner +Description: + csi: Shows the Command Set Identifier for this namespace. + + metadata_bytes: Shows the metadata size in bytes. + + nuse: Shows the Namespace Utilization (NUSE) value. Reading + triggers an Identify Namespace command to refresh the + value (rate-limited to avoid excessive commands). + +What: /sys/block/nvmeXnY/passthru_err_log_enabled +Date: January 2024 +KernelVersion: 6.8 +Contact: Alan Adamson +Description: + Shows or sets whether I/O passthrough error logging is + enabled for this namespace. Reading returns "on" or "off". + Writing accepts a boolean value. + +What: /sys/class/nvme/nvmeX/diag/command_error_count +What: /sys/class/nvme/nvmeX/diag/reset_count +What: /sys/class/nvme/nvmeX/diag/reconnect_count +Date: May 2026 +KernelVersion: 7.2 +Contact: Nilay Shroff +Description: + Controller diagnostic counters. + + command_error_count: Admin command error counter. + + reset_count: Controller reset counter. + + reconnect_count: Accumulated reconnect counter. Only + available for fabrics controllers. + + All counters can be reset by writing a value. + +What: /sys/block/nvmeXnY/diag/command_retries_count +What: /sys/block/nvmeXnY/diag/command_error_count +Date: May 2026 +KernelVersion: 7.2 +Contact: Nilay Shroff +Description: + Namespace diagnostic counters for non-multipath + configurations (when CONFIG_NVME_MULTIPATH is not + configured). + + command_retries_count: I/O command retry counter. + + command_error_count: I/O command error counter. + + All counters can be reset by writing any value. + +What: /sys/block/nvmeXcYnZ/diag/command_retries_count +What: /sys/block/nvmeXcYnZ/diag/command_error_count +What: /sys/block/nvmeXcYnZ/diag/multipath_failover_count +What: /sys/block/nvmeXnY/diag/io_requeue_no_usable_path_count +What: /sys/block/nvmeXnY/diag/io_fail_no_available_path_count +Date: May 2026 +KernelVersion: 7.2 +Contact: Nilay Shroff +Description: + Namespace diagnostic counters for multipath + configurations (when CONFIG_NVME_MULTIPATH is + configured). + + command_retries_count: I/O command retry counter. + + command_error_count: I/O command error counter. + + multipath_failover_count: Multipath failover counter. + + io_requeue_no_usable_path_count: Counter of I/Os + requeued because no usable path was available. + + io_fail_no_available_path_count: Counter of I/Os + failed because no available path existed. + + All counters can be reset by writing any value. + +What: /sys/class/nvme-subsystem/nvme-subsysX/model +What: /sys/class/nvme-subsystem/nvme-subsysX/serial +What: /sys/class/nvme-subsystem/nvme-subsysX/firmware_rev +What: /sys/class/nvme-subsystem/nvme-subsysX/subsysnqn +Date: November 2017 +KernelVersion: 4.15 +Contact: Hannes Reinecke +Description: + Shows the model, serial number, firmware revision, or NQN + of the NVMe subsystem. + +What: /sys/class/nvme-subsystem/nvme-subsysX/iopolicy +Date: February 2019 +KernelVersion: 5.1 +Contact: Hannes Reinecke +Description: + Shows or sets the multipath I/O path selection policy for + this subsystem. Accepted values: "numa", "round-robin", + "queue-depth". Changing the policy clears all current path + selections. Only available when CONFIG_NVME_MULTIPATH is + enabled. + +What: /sys/class/nvme-subsystem/nvme-subsysX/subsystype +Date: September 2021 +KernelVersion: 5.16 +Contact: Hannes Reinecke +Description: + Shows the subsystem type. Possible values: "discovery", + "nvm", "reserved". diff --git a/Documentation/ABI/testing/sysfs-nvme b/Documentation/ABI/testing/sysfs-nvme deleted file mode 100644 index 499d5f843cd4..000000000000 --- a/Documentation/ABI/testing/sysfs-nvme +++ /dev/null @@ -1,13 +0,0 @@ -What: /sys/devices/virtual/nvme-fabrics/ctl/.../tls_configured_key -Date: November 2025 -KernelVersion: 6.19 -Contact: Linux NVMe mailing list -Description: - The file is avaliable when using a secure concatanation - connection to a NVMe target. Reading the file will return - the serial of the currently negotiated key. - - Writing 0 to the file will trigger a PSK reauthentication - (REPLACETLSPSK) with the target. After a reauthentication - the value returned by tls_configured_key will be the new - serial. From 5d92321c83b63c6c548bbd8a7fca4405ff30add3 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 7 Jul 2026 20:12:26 +0800 Subject: [PATCH 019/241] nvmet: add ABI documentation for target configfs interfaces Add Documentation/ABI/stable/configfs-nvmet documenting all NVMe target configfs attributes, covering port attributes, subsystem attributes, namespace attributes, host authentication, passthrough mode, and ANA configuration. Each entry has been traced to its original introducing commit to provide accurate Date, KernelVersion, and Contact information. Reviewed-by: Hannes Reinecke Reviewed-by: Nilay Shroff Reviewed-by: Christoph Hellwig Reviewed-by: Daniel Wagner Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- Documentation/ABI/stable/configfs-nvmet | 352 ++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 Documentation/ABI/stable/configfs-nvmet diff --git a/Documentation/ABI/stable/configfs-nvmet b/Documentation/ABI/stable/configfs-nvmet new file mode 100644 index 000000000000..36b587404ee7 --- /dev/null +++ b/Documentation/ABI/stable/configfs-nvmet @@ -0,0 +1,352 @@ +What: /config/nvmet/ports/N/addr_adrfam +What: /config/nvmet/ports/N/addr_portid +What: /config/nvmet/ports/N/addr_traddr +What: /config/nvmet/ports/N/addr_trsvcid +What: /config/nvmet/ports/N/addr_trtype +What: /config/nvmet/ports/N/addr_treq +Date: June 2016 +KernelVersion: 4.8 +Contact: Christoph Hellwig +Description: + Address attributes for an NVMe-oF target port. + + addr_adrfam: Shows or sets the address family. Accepted + values: "pcie", "ipv4", "ipv6", "ib", "fc", "pci", "loop". + + addr_portid: Shows or sets the port identifier (u16). + + addr_traddr: Shows or sets the transport address string. + + addr_trsvcid: Shows or sets the transport service identifier. + + addr_trtype: Shows or sets the transport type. Accepted + values: "rdma", "fc", "tcp", "pci", "loop". Also + initializes default TSAS values. + + addr_treq: Shows or sets the transport security requirements. + Accepted values: "not specified", "required", + "not required". For TCP with TLS1.3, "not specified" is + rejected. + + All attributes require the port to be disabled before + modification. + +What: /config/nvmet/ports/N/referrals/NAME/addr_adrfam +What: /config/nvmet/ports/N/referrals/NAME/addr_portid +What: /config/nvmet/ports/N/referrals/NAME/addr_traddr +What: /config/nvmet/ports/N/referrals/NAME/addr_trsvcid +What: /config/nvmet/ports/N/referrals/NAME/addr_trtype +What: /config/nvmet/ports/N/referrals/NAME/addr_treq +What: /config/nvmet/ports/N/referrals/NAME/enable +Date: June 2016 +KernelVersion: 4.8 +Contact: Christoph Hellwig +Description: + Address attributes and enable control for a referral entry + under a port. The addr_* attributes have the same semantics + as the corresponding port-level attributes. The enable + attribute shows or sets whether this referral is enabled + (boolean). + +What: /config/nvmet/ports/N/param_inline_data_size +Date: June 2018 +KernelVersion: 4.19 +Contact: Steve Wise +Description: + Shows or sets the inline data size for this port. Default + is -1 which lets the transport choose. The port must be + disabled before modification. + +What: /config/nvmet/ports/N/ana_groups/ID/ana_state +Date: June 2018 +KernelVersion: 4.19 +Contact: Christoph Hellwig +Description: + Shows or sets the ANA (Asymmetric Namespace Access) state + for this group on this port. Accepted values: "optimized", + "non-optimized", "inaccessible", "persistent-loss", + "change". Changes trigger an ANA change event. + +What: /config/nvmet/ports/N/param_pi_enable +Date: May 2020 +KernelVersion: 5.8 +Contact: Israel Rukshin +Description: + Shows or sets whether protection information (PI) is + enabled/supported for this port. Accepts boolean value. + Only available when CONFIG_BLK_DEV_INTEGRITY is enabled. + The port must be disabled before modification. + +What: /config/nvmet/ports/N/addr_tsas +Date: August 2023 +KernelVersion: 6.7 +Contact: Hannes Reinecke +Description: + Shows or sets the transport-specific address subtype. For + TCP transport, accepted values: "none", "tls1.3" (requires + CONFIG_NVME_TARGET_TCP_TLS). For RDMA transport, shows the + QP type: "connected" or "datagram". The port must be + disabled before modification. + +What: /config/nvmet/ports/N/param_max_queue_size +Date: January 2024 +KernelVersion: 6.9 +Contact: Max Gurtovoy +Description: + Shows or sets the maximum queue size for this port. Default + is -1 which lets the transport choose. The port must be + disabled before modification. + +What: /config/nvmet/ports/N/param_mdts +Date: April 2026 +KernelVersion: 7.1 +Contact: Aurelien Aptel +Description: + Shows or sets the maximum data transfer size for this port. + Default is -1 which lets the transport choose. The port + must be disabled before modification. + +What: /config/nvmet/subsystems/NAME/attr_allow_any_host +Date: June 2016 +KernelVersion: 4.8 +Contact: Christoph Hellwig +Description: + Shows or sets whether any host is allowed to connect. + Accepts boolean value. Cannot be set to 1 if explicit + hosts are linked in the allowed_hosts/ directory. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/device_path +What: /config/nvmet/subsystems/NAME/namespaces/NSID/device_nguid +What: /config/nvmet/subsystems/NAME/namespaces/NSID/enable +Date: June 2016 +KernelVersion: 4.8 +Contact: Christoph Hellwig +Description: + Namespace attributes added with the initial NVMe target. + + device_path: Shows or sets the backend block device path. + The namespace must be disabled before modification. + + device_nguid: Shows or sets the NGUID (128-bit identifier). + Accepts 32 hex digits with optional "-" or ":" separators. + The namespace must be disabled before modification. + + enable: Shows or sets whether this namespace is enabled + (boolean). + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/device_uuid +Date: June 2017 +KernelVersion: 4.13 +Contact: Johannes Thumshirn +Description: + Shows or sets the UUID for this namespace. The namespace + must be disabled before modification. + +What: /config/nvmet/subsystems/NAME/attr_version +What: /config/nvmet/subsystems/NAME/attr_serial +Date: July 2017 +KernelVersion: 4.13 +Contact: Johannes Thumshirn +Description: + attr_version: Shows or sets the NVMe version reported by + this subsystem. Format: "major.minor" or + "major.minor.tertiary". Cannot be changed after the + subsystem has been discovered. + + attr_serial: Shows or sets the serial number. Must be a + 1-20 byte ASCII string (characters 0x20-0x7e). Cannot be + changed after the subsystem has been discovered. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/ana_grpid +Date: June 2018 +KernelVersion: 4.19 +Contact: Christoph Hellwig +Description: + Shows or sets the ANA (Asymmetric Namespace Access) Group + ID for this namespace. Must be between 1 and 128. Changing + triggers an ANA event notification. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/buffered_io +Date: June 2018 +KernelVersion: 4.19 +Contact: Chaitanya Kulkarni +Description: + Shows or sets whether buffered I/O is used for this + namespace. Accepts boolean value. The namespace must be + disabled before modification. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/p2pmem +Date: October 2018 +KernelVersion: 4.20 +Contact: Logan Gunthorpe +Description: + Shows or sets the P2P DMA memory device for this namespace. + Accepts a PCI device BDF, "auto", or "none". The namespace + must be disabled before modification. Only available when + CONFIG_PCI_P2PDMA is enabled. + +What: /config/nvmet/subsystems/NAME/attr_cntlid_min +What: /config/nvmet/subsystems/NAME/attr_cntlid_max +Date: January 2020 +KernelVersion: 5.7 +Contact: Chaitanya Kulkarni +Description: + attr_cntlid_min: Shows or sets the minimum controller ID + (u16). Must be nonzero and not greater than attr_cntlid_max. + + attr_cntlid_max: Shows or sets the maximum controller ID + (u16). Must be nonzero and not less than attr_cntlid_min. + +What: /config/nvmet/subsystems/NAME/attr_model +Date: January 2020 +KernelVersion: 5.7 +Contact: Mark Ruijter +Description: + Shows or sets the model number for this subsystem. Must + be a 1-40 byte ASCII string (characters 0x20-0x7e). + Cannot be changed after the subsystem has been discovered. + +What: /config/nvmet/subsystems/NAME/attr_pi_enable +Date: May 2020 +KernelVersion: 5.8 +Contact: Israel Rukshin +Description: + Shows or sets whether protection information (PI) is + enabled/supported for this subsystem. Accepts boolean + value. Only available when CONFIG_BLK_DEV_INTEGRITY is + enabled. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/revalidate_size +Date: May 2020 +KernelVersion: 5.8 +Contact: Chaitanya Kulkarni +Description: + Write-only. Writing 1 triggers namespace size revalidation. + If the size has changed, a namespace changed AEN is sent. + The namespace must be enabled. + +What: /config/nvmet/subsystems/NAME/passthru/device_path +What: /config/nvmet/subsystems/NAME/passthru/enable +Date: July 2020 +KernelVersion: 5.9 +Contact: Logan Gunthorpe +Description: + Passthrough mode attributes. + + device_path: Shows or sets the NVMe controller character + device path (e.g., /dev/nvme0). Cannot be changed while + the passthrough controller is active. + + enable: Shows or sets whether passthrough mode is enabled + (boolean). + + Only available when CONFIG_NVME_TARGET_PASSTHRU is enabled. + +What: /config/nvmet/subsystems/NAME/passthru/admin_timeout +What: /config/nvmet/subsystems/NAME/passthru/io_timeout +Date: November 2020 +KernelVersion: 5.11 +Contact: Chaitanya Kulkarni +Description: + admin_timeout: Shows or sets the admin command timeout for + passthrough mode, in jiffies. + + io_timeout: Shows or sets the I/O command timeout for + passthrough mode, in jiffies. + + Only available when CONFIG_NVME_TARGET_PASSTHRU is enabled. + +What: /config/nvmet/subsystems/NAME/passthru/clear_ids +Date: June 2022 +KernelVersion: 5.19 +Contact: Alan Adamson +Description: + Shows or sets whether to clear identify data IDs in + passthrough mode. Only available when + CONFIG_NVME_TARGET_PASSTHRU is enabled. + +What: /config/nvmet/subsystems/NAME/attr_qid_max +Date: August 2022 +KernelVersion: 6.1 +Contact: Daniel Wagner +Description: + Shows or sets the maximum queue ID (number of I/O queues, + u16). Must be between 1 and 128. Changing this value + forces reconnection of all connected controllers. + +What: /config/nvmet/subsystems/NAME/attr_ieee_oui +Date: November 2022 +KernelVersion: 6.2 +Contact: Aleksandr Miloserdov +Description: + Shows or sets the IEEE OUI for this subsystem. Displayed + in "0x%06x" format. Must be a 24-bit value. Cannot be + changed after the subsystem has been discovered. + +What: /config/nvmet/subsystems/NAME/attr_firmware +Date: November 2022 +KernelVersion: 6.2 +Contact: Aleksandr Miloserdov +Description: + Shows or sets the firmware revision string for this + subsystem. Must be a 1-8 byte ASCII string (characters + 0x20-0x7e). Cannot be changed after the subsystem has + been discovered. + +What: /config/nvmet/subsystems/NAME/namespaces/NSID/resv_enable +Date: November 2024 +KernelVersion: 6.13 +Contact: Guixin Liu +Description: + Shows or sets whether persistent reservation support is + enabled for this namespace. Accepts boolean value. The + namespace must be disabled before modification. + +What: /config/nvmet/subsystems/NAME/attr_vendor_id +What: /config/nvmet/subsystems/NAME/attr_subsys_vendor_id +Date: January 2025 +KernelVersion: 6.14 +Contact: Damien Le Moal +Description: + attr_vendor_id: Shows or sets the PCI vendor ID reported + by this subsystem. Displayed in "0x%x" format. + + attr_subsys_vendor_id: Shows or sets the PCI subsystem + vendor ID. Displayed in "0x%x" format. + +What: /config/nvmet/hosts/HOSTNQN/dhchap_key +What: /config/nvmet/hosts/HOSTNQN/dhchap_ctrl_key +What: /config/nvmet/hosts/HOSTNQN/dhchap_hash +What: /config/nvmet/hosts/HOSTNQN/dhchap_dhgroup +Date: June 2022 +KernelVersion: 6.0 +Contact: Hannes Reinecke +Description: + DH-HMAC-CHAP authentication attributes. + + dhchap_key: Shows or sets the host secret key. Accepts a + key string in "DHHC-1:" format. + + dhchap_ctrl_key: Shows or sets the controller secret key + for bidirectional authentication. Same format as dhchap_key. + + dhchap_hash: Shows or sets the HMAC hash algorithm. + Accepted values: "hmac(sha256)", "hmac(sha384)", + "hmac(sha512)". + + dhchap_dhgroup: Shows or sets the Diffie-Hellman group for + DH-HMAC-CHAP key exchange. Accepted values: "null", + "ffdhe2048", "ffdhe3072", "ffdhe4096", "ffdhe6144". + Non-null groups require the corresponding KPP crypto + algorithm to be available. + + Only available when CONFIG_NVME_TARGET_AUTH is enabled. + +What: /config/nvmet/discovery_nqn +Date: April 2024 +KernelVersion: 6.9 +Contact: Hannes Reinecke +Description: + Shows or sets the NQN of the discovery subsystem. The + value must be unique and not duplicate any existing + subsystem name. From cdf9a65e80ec874b630502946d269fac38dc5de8 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 7 Jul 2026 20:12:27 +0800 Subject: [PATCH 020/241] MAINTAINERS: add missing NVMe documentation files Add documentation file entries that were missing from the NVM EXPRESS DRIVER and NVM EXPRESS TARGET DRIVER sections, so patches touching these files are properly routed to the NVMe mailing list and maintainers. Reviewed-by: Hannes Reinecke Reviewed-by: Nilay Shroff Reviewed-by: Christoph Hellwig Reviewed-by: Daniel Wagner Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- MAINTAINERS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 3d6db8cb608f..5bbc5b49d36a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -19207,6 +19207,9 @@ L: linux-nvme@lists.infradead.org S: Supported W: http://git.infradead.org/nvme.git T: git git://git.infradead.org/nvme.git +F: Documentation/ABI/stable/sysfs-nvme +F: Documentation/admin-guide/nvme-multipath.rst +F: Documentation/fault-injection/nvme-fault-injection.rst F: Documentation/nvme/ F: drivers/nvme/common/ F: drivers/nvme/host/ @@ -19249,6 +19252,7 @@ L: linux-nvme@lists.infradead.org S: Supported W: http://git.infradead.org/nvme.git T: git git://git.infradead.org/nvme.git +F: Documentation/ABI/stable/configfs-nvmet F: drivers/nvme/target/ NVMEM FRAMEWORK From 3c568b35a0d309acb40746552bec2af24cd550ef Mon Sep 17 00:00:00 2001 From: Hari Mishal Date: Thu, 9 Jul 2026 14:30:32 +0200 Subject: [PATCH 021/241] nvme: bound ns descriptor header and body to identify buffer nvme_identify_ns_descs() allocates a buffer and gives it to the controller, which populates it and then iterates the buffer with variable byte increments that vary by type and body size. But, there is no bounds check inside the iteration itself except the loop bound itself. Fix this by checking and stopping iteration if the next header or its declared body would go past the buffer itself. Assisted-by: gkh_clanker_t1000 Reviewed-by: Christoph Hellwig Signed-off-by: Hari Mishal Signed-off-by: Greg Kroah-Hartman Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index db0c8ad4628a..0b8330c79b1a 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1583,8 +1583,12 @@ static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) { struct nvme_ns_id_desc *cur = data + pos; + if (pos + sizeof(*cur) > NVME_IDENTIFY_DATA_SIZE) + break; if (cur->nidl == 0) break; + if (pos + sizeof(*cur) + cur->nidl > NVME_IDENTIFY_DATA_SIZE) + break; len = nvme_process_ns_desc(ctrl, &info->ids, cur, &csi_seen); if (len < 0) From 29261f8bb41662f2a660c479e5cf592942b53f78 Mon Sep 17 00:00:00 2001 From: Hari Mishal Date: Thu, 9 Jul 2026 14:30:33 +0200 Subject: [PATCH 022/241] nvme: clamp FDP nruhsd to allocated RUH status descriptor count nvme_query_fdp_info() allocates the RUH status buffer for at most S8_MAX - 1 descriptors, and then copies ruhs->ruhsd[] into head->plids[] using the controller reported ruhs->nruhsd directly as the loop bound. However, that count wasn't taken into account for the actual buffer's size, so there was a chance for a controller reporting a larger nruhsd to cause the copy to overflow the buffer. Clamp nr_plids to the same bound used for the allocation. Assisted-by: gkh_clanker_t1000 Reviewed-by: Christoph Hellwig Signed-off-by: Hari Mishal Signed-off-by: Greg Kroah-Hartman Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0b8330c79b1a..cdb16e949e2a 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2361,7 +2361,7 @@ static int nvme_query_fdp_info(struct nvme_ns *ns, struct nvme_ns_info *info) goto free; } - head->nr_plids = le16_to_cpu(ruhs->nruhsd); + head->nr_plids = min(le16_to_cpu(ruhs->nruhsd), S8_MAX - 1); if (!head->nr_plids) goto free; From a11e0a4cb4189c468683a2f384f0c266c26a497f Mon Sep 17 00:00:00 2001 From: John Garry Date: Mon, 13 Jul 2026 10:42:37 +0000 Subject: [PATCH 023/241] nvme: add nvme_get_ns_head() Add a wrapper for getting a reference to the NS head. This would be used in scenarios when we know that getting a reference would not fail. Reviewed-by: Christoph Hellwig Reviewed-by: Nilay Shroff Signed-off-by: John Garry Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 5 +++++ drivers/nvme/host/multipath.c | 2 +- drivers/nvme/host/nvme.h | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index cdb16e949e2a..882920b7327b 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -693,6 +693,11 @@ static void nvme_free_ns_head(struct kref *ref) kfree(head); } +void nvme_get_ns_head(struct nvme_ns_head *head) +{ + kref_get(&head->ref); +} + bool nvme_tryget_ns_head(struct nvme_ns_head *head) { return kref_get_unless_zero(&head->ref); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 56587ae59c7f..8cb417036fe1 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -797,7 +797,7 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) set_bit(GD_SUPPRESS_PART_SCAN, &head->disk->state); sprintf(head->disk->disk_name, "nvme%dn%d", ctrl->subsys->instance, head->instance); - nvme_tryget_ns_head(head); + nvme_get_ns_head(head); return 0; } diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index a679a4c61462..2e9dea6420da 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -995,6 +995,7 @@ int nvme_delete_ctrl(struct nvme_ctrl *ctrl); void nvme_queue_scan(struct nvme_ctrl *ctrl); int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi, void *log, size_t size, u64 offset); +void nvme_get_ns_head(struct nvme_ns_head *head); bool nvme_tryget_ns_head(struct nvme_ns_head *head); void nvme_put_ns_head(struct nvme_ns_head *head); int nvme_cdev_add(const char *name, struct cdev *cdev, From 9952f3882ecba709092379b71e02b09762b26f96 Mon Sep 17 00:00:00 2001 From: John Garry Date: Mon, 13 Jul 2026 10:42:38 +0000 Subject: [PATCH 024/241] nvme: fix cdev lifetime Sashiko bot reported a potential problem for the cdev lifetime in [0] - the code there is heavily based on the NVMe code. Currently the NS head .open and .release file_operations methods take and put a reference to the nvme_ns_head to ensure that this structure does not disappear while we open fds for that cdev. In multipath mode, when we teardown the NS head, we call nvme_cdev_del() -> cdev_device_del() -> cdev_del(). However after cdev_del() returns, cdevs already open will remain and their fops will still be callable. As such, we can still reference the cdev after the nvme_ns_head reference count drops to 0 (and is freed). This can be shown with an application which delays between opening the cdev and issuing the ioctl while the NS head is being torn down: # ./ioctl_file /dev/ng1n1 & # waiting 10 seconds .... # ./ini_nvme_teardown.sh [ 21.221718] nvme nvme1: Removing ctrl: NQN "nvme-test-target" [ 21.274609] nvme nvme2: Removing ctrl: NQN "nvme-test-target" # now going to issue ioctl .... [ 26.549285] ================================================================== [ 26.550841] BUG: KASAN: slab-use-after-free in cdev_put.part.0+0x3d/0x40 [ 26.552352] Read of size 8 at addr ffff88811e7fa170 by task ioctl_file/237 [ 26.553805] [ 26.554227] CPU: 3 UID: 0 PID: 237 Comm: ioctl_file Not tainted 7.2.0-rc1-00004-g6852a10e32d4 #921 PREEMPT(lazy) [ 26.554236] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS 1.16.3-debian-1.16.3-2 04/01/2014 [ 26.554241] Call Trace: [ 26.554245] [ 26.554248] dump_stack_lvl+0x68/0xa0 [ 26.554266] print_report+0x10d/0x5d0 [ 26.554276] ? __virt_addr_valid+0x21d/0x3f0 [ 26.554287] ? cdev_put.part.0+0x3d/0x40 [ 26.554292] kasan_report+0x96/0xd0 [ 26.554300] ? cdev_put.part.0+0x3d/0x40 [ 26.554307] cdev_put.part.0+0x3d/0x40 [ 26.554313] __fput+0x7bc/0xa70 [ 26.554322] fput_close_sync+0xd8/0x190 [ 26.554328] ? __pfx_fput_close_sync+0x10/0x10 [ 26.554337] __x64_sys_close+0x79/0xd0 [ 26.554344] do_syscall_64+0x117/0x6b0 [ 26.554351] entry_SYSCALL_64_after_hwframe+0x77/0x7f [ 26.554358] RIP: 0033:0x7f938c067727 [ 26.554364] Code: 48 89 fa 4c 89 df e8 28 ad 00 00 8b 93 08 03 00 00 59 5e 48 83 f8 fc 74 1a 5b c3 0f 1f 84 00 00 00 00 00 48 8b 44 24 10 0f 05 <5bf [ 26.554369] RSP: 002b:00007fff49f05980 EFLAGS: 00000202 ORIG_RAX: 0000000000000003 [ 26.554376] RAX: ffffffffffffffda RBX: 00007f938bfd7780 RCX: 00007f938c067727 [ 26.554380] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000003 [ 26.554383] RBP: 00007fff49f05a10 R08: 0000000000000000 R09: 0000000000000000 [ 26.554386] R10: 0000000000000000 R11: 0000000000000202 R12: 0000000000000000 [ 26.554389] R13: 00007fff49f05b40 R14: 00007f938c207000 R15: 000055c148471d78 [ 26.554397] [ 26.554399] [ 26.575612] Allocated by task 100: [ 26.575871] kasan_save_stack+0x24/0x50 [ 26.576157] kasan_save_track+0x14/0x30 [ 26.576418] __kasan_kmalloc+0x7f/0x90 [ 26.576668] __kmalloc_noprof+0x281/0x6c0 [ 26.576938] nvme_alloc_ns+0x7f7/0x3170 [ 26.577206] nvme_scan_ns+0x508/0x880 [ 26.577449] async_run_entry_fn+0x8c/0x350 [ 26.577723] process_scheduled_works+0xb6f/0x1a00 [ 26.578034] worker_thread+0x4ad/0xb40 [ 26.578283] kthread+0x34f/0x450 [ 26.578501] ret_from_fork+0x563/0x800 [ 26.578752] ret_from_fork_asm+0x1a/0x30 [ 26.579012] [ 26.579124] Freed by task 237: [ 26.579335] kasan_save_stack+0x24/0x50 [ 26.579596] kasan_save_track+0x14/0x30 [ 26.579855] kasan_save_free_info+0x3a/0x60 [ 26.580131] __kasan_slab_free+0x43/0x70 [ 26.580388] kfree+0x321/0x500 [ 26.580591] nvme_ns_head_chr_release+0x39/0x50 [ 26.580883] __fput+0x352/0xa70 [ 26.581095] fput_close_sync+0xd8/0x190 [ 26.581350] __x64_sys_close+0x79/0xd0 [ 26.581595] do_syscall_64+0x117/0x6b0 [ 26.581842] entry_SYSCALL_64_after_hwframe+0x77/0x7f [ 26.582176] [ 26.582286] Last potentially related work creation: [ 26.582593] kasan_save_stack+0x24/0x50 [ 26.582841] kasan_record_aux_stack+0x89/0xa0 [ 26.583210] insert_work+0x22/0x170 [ 26.583442] __queue_work+0x7b1/0xfa0 [ 26.583682] queue_work_on+0x77/0x80 [ 26.583921] kblockd_schedule_work+0x18/0x20 [ 26.584207] nvme_mpath_put_disk+0x42/0xa0 [ 26.584632] nvme_free_ns_head+0x1c/0x160 [ 26.584904] nvme_ns_head_chr_release+0x39/0x50 [ 26.585208] __fput+0x352/0xa70 [ 26.585420] fput_close_sync+0xd8/0x190 [ 26.585677] __x64_sys_close+0x79/0xd0 [ 26.585924] do_syscall_64+0x117/0x6b0 [ 26.586173] entry_SYSCALL_64_after_hwframe+0x77/0x7f [ 26.586505] [ 26.586616] Second to last potentially related work creation: [ 26.586991] kasan_save_stack+0x24/0x50 [ 26.587246] kasan_record_aux_stack+0x89/0xa0 [ 26.587538] insert_work+0x22/0x170 [ 26.587770] __queue_work+0x7b1/0xfa0 [ 26.588010] queue_work_on+0x77/0x80 [ 26.588247] kblockd_schedule_work+0x18/0x20 [ 26.588529] nvme_remove_head+0x3d/0xb0 [ 26.588787] nvme_ns_remove+0x4b2/0x930 [ 26.589040] nvme_remove_namespaces+0x29c/0x410 [ 26.589340] nvme_do_delete_ctrl+0xf3/0x190 [ 26.589611] nvme_delete_ctrl_sync+0x71/0x90 [ 26.589889] nvme_sysfs_delete+0x91/0xb0 [ 26.590151] kernfs_fop_write_iter+0x2fb/0x4a0 [ 26.590452] vfs_write+0x929/0xfc0 [ 26.590688] ksys_write+0xf2/0x1d0 [ 26.590923] do_syscall_64+0x117/0x6b0 [ 26.591171] entry_SYSCALL_64_after_hwframe+0x77/0x7f [ 26.591498] [ 26.591605] The buggy address belongs to the object at ffff88811e7fa000 [ 26.591605] which belongs to the cache kmalloc-4k of size 4096 [ 26.592409] The buggy address is located 368 bytes inside of [ 26.592409] freed 4096-byte region [ffff88811e7fa000, ffff88811e7fb000) [ 26.593205] [ 26.593321] The buggy address belongs to the physical page: [ 26.593701] page: refcount:0 mapcount:0 mapping:0000000000000000 index:0x0 pfn:0x11e7f8 [ 26.594250] head: order:3 mapcount:0 entire_mapcount:0 nr_pages_mapped:0 pincount:0 [ 26.594743] flags: 0x200000000000040(head|node=0|zone=2) [ 26.595093] page_type: f5(slab) [ 26.595312] raw: 0200000000000040 ffff888100043040 dead000000000122 0000000000000000 [ 26.595810] raw: 0000000000000000 0000000000040004 00000000f5000000 0000000000000000 [ 26.596309] head: 0200000000000040 ffff888100043040 dead000000000122 0000000000000000 [ 26.596806] head: 0000000000000000 0000000000040004 00000000f5000000 0000000000000000 [ 26.597313] head: 0200000000000003 fffffffffffffe01 00000000ffffffff 00000000ffffffff [ 26.597813] head: 0000000000000000 0000000000000000 00000000ffffffff 0000000000000000 [ 26.598317] page dumped because: kasan: bad access detected [ 26.598676] [ 26.598784] Memory state around the buggy address: [ 26.599093] ffff88811e7fa000: fa fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 26.599559] ffff88811e7fa080: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 26.600023] >ffff88811e7fa100: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 26.600485] ^ [ 26.600921] ffff88811e7fa180: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 26.601390] ffff88811e7fa200: fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb fb [ 26.601855] ================================================================== [ 26.602374] Disabling lock debugging due to kernel taint When all fds for the cdev disappear, the cdev removal path puts a reference to the parent object, which is the nvme_ns_head.cdev_device - see cdev_default_release() -> kobject_put(parent). Fix the lifetime for the cdev by making adding the cdev add take a reference to the NS head and drop that reference in the nvme_ns_head.cdev_device release function. The same problem exists for the NS cdev lifetime, so resolve that issue through a similar method by taking a reference to the NS for the lifetime of the cdev. Note that nvme_ns_chr_open() -> nvme_ns_open() also takes a reference to the NS. Now that should not be needed, but that code is common to bdev ioctl, so keep as is. [0] https://lore.kernel.org/linux-scsi/20260703102918.3723667-1-john.g.garry@oracle.com/T/#m67265e2906d617acd2743c0a00809246d0cfc506 Reviewed-by: Christoph Hellwig Reviewed-by: Nilay Shroff Signed-off-by: John Garry Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 7 +++++++ drivers/nvme/host/multipath.c | 22 ++-------------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 882920b7327b..b7293fe66540 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3903,6 +3903,11 @@ static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys, static void nvme_cdev_rel(struct device *dev) { ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt)); + if (dev->parent->class == &nvme_class) + nvme_put_ns(container_of(dev, struct nvme_ns, cdev_device)); + else + nvme_put_ns_head(container_of(dev, struct nvme_ns_head, + cdev_device)); } void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device) @@ -3968,10 +3973,12 @@ static void nvme_add_ns_cdev(struct nvme_ns *ns) snprintf(name, sizeof(name), "ng%dn%d", ns->ctrl->instance, ns->head->instance); + nvme_get_ns(ns); /* Undone in nvme_cdev_rel() */ if (nvme_cdev_add(name, &ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops, ns->ctrl->ops->module)) { dev_err(ns->ctrl->device, "Unable to create the %s device\n", name); + nvme_put_ns(ns); return; } set_bit(NVME_NS_CDEV_LIVE, &ns->flags); diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 8cb417036fe1..c850a4bf7380 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -630,28 +630,8 @@ const struct block_device_operations nvme_ns_head_ops = { .pr_ops = &nvme_pr_ops, }; -static inline struct nvme_ns_head *cdev_to_ns_head(struct cdev *cdev) -{ - return container_of(cdev, struct nvme_ns_head, cdev); -} - -static int nvme_ns_head_chr_open(struct inode *inode, struct file *file) -{ - if (!nvme_tryget_ns_head(cdev_to_ns_head(inode->i_cdev))) - return -ENXIO; - return 0; -} - -static int nvme_ns_head_chr_release(struct inode *inode, struct file *file) -{ - nvme_put_ns_head(cdev_to_ns_head(inode->i_cdev)); - return 0; -} - static const struct file_operations nvme_ns_head_chr_fops = { .owner = THIS_MODULE, - .open = nvme_ns_head_chr_open, - .release = nvme_ns_head_chr_release, .unlocked_ioctl = nvme_ns_head_chr_ioctl, .compat_ioctl = compat_ptr_ioctl, .uring_cmd = nvme_ns_head_chr_uring_cmd, @@ -666,10 +646,12 @@ static void nvme_add_ns_head_cdev(struct nvme_ns_head *head) snprintf(name, sizeof(name), "ng%dn%d", head->subsys->instance, head->instance); + nvme_get_ns_head(head); /* Undone in nvme_cdev_rel() */ if (nvme_cdev_add(name, &head->cdev, &head->cdev_device, &nvme_ns_head_chr_fops, THIS_MODULE)) { dev_err(disk_to_dev(head->disk), "Unable to create the %s device\n", name); + nvme_put_ns_head(head); return; } set_bit(NVME_NSHEAD_CDEV_LIVE, &head->flags); From 737a3b535247226f6e1a7988fd9d6e63e7d6fc71 Mon Sep 17 00:00:00 2001 From: Greg Kroah-Hartman Date: Mon, 27 Jul 2026 22:03:31 +0200 Subject: [PATCH 025/241] nvmet-tcp: Do not WARN on remotely-controlled oversized SGL allocations When fuzzing the nvme target code, I tripped a kernel warning in nvmet_tcp_map_data() because the length passed into the allocator is controlled by the remote initiator. A remote initiator that sends a command with an SGL claiming a huge number, can create a scatterlist and iovec allocation of over 1 million entries, which causes the backing kmalloc call to exceed MAX_PAGE_ORDER and then the page allocator will trip on a WARN_ON_ONCE_GFP() message: WARNING: mm/page_alloc.c:5280 __alloc_frozen_pages_noprof Workqueue: nvmet_tcp_wq nvmet_tcp_io_work ... sgl_alloc_order nvmet_tcp_map_data nvmet_tcp_try_recv_pdu As it's never good to trip a kernel warning remotely due to many systems having panic-on-warn enabled, let's silence it by just add GFP_NOWARN to the allocation flags. Assisted-by: gkh_clanker_2000 Cc: stable Signed-off-by: Greg Kroah-Hartman Signed-off-by: Keith Busch --- drivers/nvme/target/tcp.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 75a276d73be3..cb6d37798d74 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -433,13 +433,15 @@ static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) } cmd->req.transfer_len += len; - cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt); + cmd->req.sg = sgl_alloc(len, GFP_KERNEL | __GFP_NOWARN, + &cmd->req.sg_cnt); if (!cmd->req.sg) return NVME_SC_INTERNAL; cmd->cur_sg = cmd->req.sg; if (nvmet_tcp_has_data_in(cmd)) { - cmd->iov = kmalloc_objs(*cmd->iov, cmd->req.sg_cnt); + cmd->iov = kmalloc_objs(*cmd->iov, cmd->req.sg_cnt, + GFP_KERNEL | __GFP_NOWARN); if (!cmd->iov) goto err; } From a7609033629624fcbc2032431cbe8c4a84a3ac34 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:02 +0530 Subject: [PATCH 026/241] list: introduce LIST_HEAD_GUARDED Introduce LIST_HEAD_GUARDED(name, lock) to define a struct list_head annotated with __guarded_by(lock). This provides a convenient shorthand for defining lock-protected list heads and allows compiler context analysis to validate accesses to the list against the associated lock. The new helper also reduces boilerplate and improves consistency across callers that annotate struct list_head objects with __guarded_by(). This is a preparatory change for subsequent patches that annotate LIST_HEAD() instances with their protecting lock. Suggested-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- include/linux/list.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/linux/list.h b/include/linux/list.h index 09d979976b3b..f6f22c8b06f7 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -33,6 +33,14 @@ #define LIST_HEAD(name) \ struct list_head name = LIST_HEAD_INIT(name) +/** + * LIST_HEAD_GUARDED - define a &struct list_head annotated with __guarded_by() + * @name: name of the list_head + * @lock: lock protecting the list + */ +#define LIST_HEAD_GUARDED(name, lock) \ + __guarded_by(&(lock)) LIST_HEAD(name) + /** * INIT_LIST_HEAD - Initialize a list_head structure * @list: list_head structure to be initialized. From 2b58c94ea7ac6d26ee44b4734f7dc0e5d773ff70 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 13 Jul 2026 17:24:03 +0530 Subject: [PATCH 027/241] list: Permit context-unguarded access with list_empty_careful() With Context Analysis (viz. Clang's Thread Safety Analysis), list_heads that are __guarded_by(..) require holding the appropriate context lock when accessing and manipulating them via the list API. Because Clang's warning diagnostics do not perform inter-procedural analysis, this is enforced by Clang with -Wthread-safety-pointer in the caller at the call boundary; a warning is produced when passing a pointer to a guarded variable without holding the appropriate context locks: warning: passing pointer to variable 'list' requires holding [...] [-Wthread-safety-pointer] if (list_empty(&ctrl->list)) An exception is list_empty_careful(), which is like list_empty(), except that it is permitted to use without holding any context lock (carefully). Mark list_empty_careful() __context_unsafe, which disables context analysis within list_empty_careful(), but also suppresses warnings generated in callers related to its pointer arguments. Reviewed-by: Christoph Hellwig Signed-off-by: Marco Elver Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- include/linux/list.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/list.h b/include/linux/list.h index f6f22c8b06f7..19212bfc3f6d 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -444,6 +444,7 @@ static inline void list_del_init_careful(struct list_head *entry) * if another CPU could re-list_add() it. */ static inline int list_empty_careful(const struct list_head *head) + __context_unsafe(/* intentional lockless access to @head */) { struct list_head *next = smp_load_acquire(&head->next); return list_is_head(next, head) && (next == READ_ONCE(head->prev)); From f6f7849c1655ff012d6396c408cf9d4712307fdb Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:04 +0530 Subject: [PATCH 028/241] nvme: update nvme_passthru_end() signature Change nvme_passthru_end() to return the command effects value passed to it. This is a preparatory change for Clang's context/thread-safety analysis support. The conditional release annotations (__cond_releases()) model lock release based on a function's return value. Returning the existing effects value allows a subsequent patch to annotate nvme_passthru_end() as conditionally releasing locks acquired by nvme_passthru_start(). No functional change intended. A follow-up patch will add the corresponding context analysis annotations. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 6 ++++-- drivers/nvme/host/nvme.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index b7293fe66540..9cb32beae028 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -1273,7 +1273,7 @@ u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode) } EXPORT_SYMBOL_NS_GPL(nvme_passthru_start, "NVME_TARGET_PASSTHRU"); -void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, +u32 nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, struct nvme_command *cmd, int status) { if (effects & NVME_CMD_EFFECTS_CSE_MASK) { @@ -1294,7 +1294,7 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, flush_work(&ctrl->scan_work); } if (ns) - return; + return effects; switch (cmd->common.opcode) { case nvme_admin_set_features: @@ -1315,6 +1315,8 @@ void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, default: break; } + + return effects; } EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, "NVME_TARGET_PASSTHRU"); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 2e9dea6420da..ebff6b45e976 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -1302,7 +1302,7 @@ u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); int nvme_execute_rq(struct request *rq, bool at_head); -void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, +u32 nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, struct nvme_command *cmd, int status); struct nvme_ctrl *nvme_ctrl_from_file(struct file *file); struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid); From a6732bd8003ad1ea9283c204b9ea6443d98bbc26 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:05 +0530 Subject: [PATCH 029/241] nvme: add context annotations for nvme_passthru_{start|stop} Annotate nvme_passthru_start() and nvme_passthru_end() for Clang context/thread-safety analysis. The __cond_acquires() and __cond_releases() annotations model conditional lock acquisition and release based on a function's return value. Use a nonzero return value as the abstract condition denoting that the associated locks have been acquired or released. This allows the analyzer to track the lock state across the nvme_passthru_start() / nvme_passthru_end() pair and verify correct locking semantics. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/nvme.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index ebff6b45e976..26859aea3e2d 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -1300,10 +1300,16 @@ static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {}; u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); -u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode); +u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode) + __cond_acquires(nonzero, &ctrl->subsys->lock) + __cond_acquires(nonzero, &ctrl->scan_lock); + int nvme_execute_rq(struct request *rq, bool at_head); u32 nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects, - struct nvme_command *cmd, int status); + struct nvme_command *cmd, int status) + __cond_releases(nonzero, &ctrl->scan_lock) + __cond_releases(nonzero, &ctrl->subsys->lock); + struct nvme_ctrl *nvme_ctrl_from_file(struct file *file); struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid); bool nvme_get_ns(struct nvme_ns *ns); From 86f9536c2d8f4496f1e45cb0a70ca1b3a7d89106 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:06 +0530 Subject: [PATCH 030/241] nvme: add context annotations for nvme_ns_head::srcu Add Clang lock context annotations for helpers that operate under head->srcu read-side protection. The path selection helpers invoked by nvme_find_path() access SRCU- protected data through srcu_dereference() or list APIs which iterate through rcu protected list and therefore require the caller to hold head->srcu. Annotate these helpers and nvme_find_path() with __must_hold_shared(&head->srcu) so that Clang's lock context analysis can verify the SRCU locking requirements across the call chain. Also update nvme_ns_head_ctrl_ioctl() to use __releases_shared() to match the shared SRCU read-side lock acquired through srcu_read_lock(). Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/ioctl.c | 2 +- drivers/nvme/host/multipath.c | 6 ++++++ drivers/nvme/host/nvme.h | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index d5a8f375953b..bae52bd5bdd2 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -692,7 +692,7 @@ int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd, static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd, void __user *argp, struct nvme_ns_head *head, int srcu_idx, bool open_for_write) - __releases(&head->srcu) + __releases_shared(&head->srcu) { struct nvme_ctrl *ctrl = ns->ctrl; int ret; diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index c850a4bf7380..b9bb9777da96 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -325,6 +325,7 @@ static bool nvme_path_is_disabled(struct nvme_ns *ns) } static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) + __must_hold_shared(&head->srcu) { int found_distance = INT_MAX, fallback_distance = INT_MAX, distance; struct nvme_ns *found = NULL, *fallback = NULL, *ns; @@ -367,6 +368,7 @@ static struct nvme_ns *__nvme_find_path(struct nvme_ns_head *head, int node) static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, struct nvme_ns *ns) + __must_hold_shared(&head->srcu) { ns = list_next_or_null_rcu(&head->list, &ns->siblings, struct nvme_ns, siblings); @@ -376,6 +378,7 @@ static struct nvme_ns *nvme_next_ns(struct nvme_ns_head *head, } static struct nvme_ns *nvme_round_robin_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu) { struct nvme_ns *ns, *found = NULL; int node = numa_node_id(); @@ -424,6 +427,7 @@ out: } static struct nvme_ns *nvme_queue_depth_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu) { struct nvme_ns *best_opt = NULL, *best_nonopt = NULL, *ns; unsigned int min_depth_opt = UINT_MAX, min_depth_nonopt = UINT_MAX; @@ -467,6 +471,7 @@ static inline bool nvme_path_is_optimized(struct nvme_ns *ns) } static struct nvme_ns *nvme_numa_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu) { int node = numa_node_id(); struct nvme_ns *ns; @@ -492,6 +497,7 @@ inline struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) } static bool nvme_available_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu) { struct nvme_ns *ns; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 26859aea3e2d..ec9dea4d7fb9 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -1033,7 +1033,8 @@ extern const struct attribute_group *nvme_dev_attr_groups[]; extern const struct block_device_operations nvme_bdev_ops; void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl); -struct nvme_ns *nvme_find_path(struct nvme_ns_head *head); +struct nvme_ns *nvme_find_path(struct nvme_ns_head *head) + __must_hold_shared(&head->srcu); #ifdef CONFIG_NVME_MULTIPATH static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) { From 499d05d5d10eb381eac87a83626e7141a5823a09 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:07 +0530 Subject: [PATCH 031/241] nvme: remove redundant initialization of nvme_ns_head::requeue_list bio_list_init() is a no-op for zero-initialized objects. Remove the redundant initialization of nvme_ns_head::requeue_list from nvme_mpath_alloc_disk(). Besides simplifying the code, this also avoids a false positive from Clang's context analysis once nvme_ns_head::requeue_list is annotated with __guarded_by(&requeue_lock). Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/multipath.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index b9bb9777da96..fac6ea2311c1 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -736,7 +736,6 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) struct queue_limits lim; mutex_init(&head->lock); - bio_list_init(&head->requeue_list); spin_lock_init(&head->requeue_lock); INIT_WORK(&head->requeue_work, nvme_requeue_work); INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work); From aa5d8dda3a455c8b0c06a9ab360a52c6a6fae0b6 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:08 +0530 Subject: [PATCH 032/241] nvme: add context annotations for nvme_ns_head::requeue_list nvme_ns_head::requeue_list is protected by nvme_ns_head::requeue_lock. Annotate requeue_list with __guarded_by(&requeue_lock) so that Clang's context analysis can validate accesses to the list. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/nvme.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index ec9dea4d7fb9..27023648cbd1 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -561,7 +561,8 @@ struct nvme_ns_head { u16 nr_plids; u16 *plids; #ifdef CONFIG_NVME_MULTIPATH - struct bio_list requeue_list; + struct bio_list requeue_list + __guarded_by(&requeue_lock); spinlock_t requeue_lock; struct work_struct requeue_work; struct work_struct partition_scan_work; From 696d2aeb77513eb474eb3557efc32be763289e4f Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:09 +0530 Subject: [PATCH 033/241] nvme: add context annotations for nvme_ns_head::current_path Annotate nvme_ns_head::current_path[] with __rcu_guarded so that Clang's context analysis can validate accesses to the SRCU/RCU protected pointer. Cc: Paul E. McKenney Reviewed-by: Paul E. McKenney Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/nvme.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 27023648cbd1..51221ba0f1ad 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -575,7 +575,7 @@ struct nvme_ns_head { #define NVME_NSHEAD_DISK_LIVE 0 #define NVME_NSHEAD_QUEUE_IF_NO_PATH 1 #define NVME_NSHEAD_CDEV_LIVE 2 - struct nvme_ns __rcu *current_path[]; + struct nvme_ns __rcu_guarded *current_path[]; #endif }; From 4258bf237e7f26cbbd44c9dada11b87cda18041c Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:10 +0530 Subject: [PATCH 034/241] nvme: add context annotations for nvme_dev::shutdown_lock nvme_setup_io_queues_trylock() conditionally acquires dev->shutdown_lock using mutex_trylock(). The function returns 0 when the lock is successfully acquired and a negative error code otherwise. Annotate the function with __cond_acquires(0, &dev->shutdown_lock) so that Clang's lock context analysis can track the lock state based on the return value and verify correct lock usage at call sites. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 8438c904ec49..8c6d169f2c38 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2199,6 +2199,7 @@ static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) * Try getting shutdown_lock while setting up IO queues. */ static int nvme_setup_io_queues_trylock(struct nvme_dev *dev) + __cond_acquires(0, &dev->shutdown_lock) { /* * Give up if the lock is being held by nvme_dev_disable. From 9c65eeeb26b1d614787deec36faec81e45b8f8e8 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:11 +0530 Subject: [PATCH 035/241] nvme: remove redundant initialization of delayed_removal_secs nvme_ns_head is allocated with kzalloc(), so explicitly initializing nvme_ns_head::delayed_removal_secs to 0 in nvme_mpath_alloc_disk() is redundant. Removing the redundant initialization also avoids a false positive from Clang's context analysis once nvme_ns_head::delayed_removal_secs is annotated with __guarded_by(nvme_subsystem::lock). Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/multipath.c | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index fac6ea2311c1..091aceb9b1d8 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -740,7 +740,6 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) INIT_WORK(&head->requeue_work, nvme_requeue_work); INIT_WORK(&head->partition_scan_work, nvme_partition_scan_work); INIT_DELAYED_WORK(&head->remove_work, nvme_remove_head_work); - head->delayed_removal_secs = 0; /* * If "multipath_always_on" is enabled, a multipath node is added From d1fdf49b5f7fce5f65ae0d11d484bd7e31cedbb1 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:12 +0530 Subject: [PATCH 036/241] nvme: add context annotations for nvme_subsystem::lock Several helpers access or traverse data structures protected by nvme_subsystem::lock and therefore require callers to hold the lock. Annotate nvme_mpath_unfreeze(), nvme_mpath_wait_freeze(), nvme_mpath_start_freeze(), nvme_find_ns_head(), nvme_alloc_ns_head() and nvme_subsys_check_duplicate_ids() with __must_hold(&subsys->lock) so that Clang's lock context analysis can validate the locking requirements at compile time. Also annotate nvme_subsystem::nsheads and nvme_ns_head::delayed_removal_secs with __guarded_by(&subsys->lock), as both are protected by the subsystem lock. Annotate nvme_init_subsystem() with __context_unsafe(), as it initializes these lock-protected members before the object is published, suppressing a false positive from Clang's context analysis. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 4 ++++ drivers/nvme/host/nvme.h | 15 ++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 9cb32beae028..178ac655aa2b 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3287,6 +3287,7 @@ static bool nvme_validate_cntlid(struct nvme_subsystem *subsys, } static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) + __context_unsafe(/* initialize unpublished/lock-guarded variables */) { struct nvme_subsystem *subsys, *found; int ret; @@ -3858,6 +3859,7 @@ static const struct file_operations nvme_dev_fops = { static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl, unsigned nsid) + __must_hold(&ctrl->subsys->lock) { struct nvme_ns_head *h; @@ -3880,6 +3882,7 @@ static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl, static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys, struct nvme_ns_ids *ids) + __must_hold(&subsys->lock) { bool has_uuid = !uuid_is_null(&ids->uuid); bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid)); @@ -3988,6 +3991,7 @@ static void nvme_add_ns_cdev(struct nvme_ns *ns) static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl, struct nvme_ns_info *info) + __must_hold(&ctrl->subsys->lock) { struct nvme_ns_head *head; size_t size = sizeof(*head); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 51221ba0f1ad..fac4acbbd85d 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -496,7 +496,8 @@ struct nvme_subsystem { struct list_head entry; struct mutex lock; struct list_head ctrls; - struct list_head nsheads; + struct list_head nsheads + __guarded_by(&lock); char subnqn[NVMF_NQN_SIZE]; char serial[20]; char model[40]; @@ -569,7 +570,8 @@ struct nvme_ns_head { struct mutex lock; unsigned long flags; struct delayed_work remove_work; - unsigned int delayed_removal_secs; + unsigned int delayed_removal_secs + __guarded_by(&subsys->lock); atomic_long_t io_requeue_no_usable_path_count; atomic_long_t io_fail_no_available_path_count; #define NVME_NSHEAD_DISK_LIVE 0 @@ -1042,9 +1044,12 @@ static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl) return ctrl->ana_log_buf != NULL; } -void nvme_mpath_unfreeze(struct nvme_subsystem *subsys); -void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys); -void nvme_mpath_start_freeze(struct nvme_subsystem *subsys); +void nvme_mpath_unfreeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); +void nvme_mpath_wait_freeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); +void nvme_mpath_start_freeze(struct nvme_subsystem *subsys) + __must_hold(&subsys->lock); void nvme_mpath_default_iopolicy(struct nvme_subsystem *subsys); void nvme_failover_req(struct request *req); void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl); From ca0058e8b599ae75a30e7f53025a46b62905bea3 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:13 +0530 Subject: [PATCH 037/241] nvme: add context annotations for nvme_ctrl::ana_lock nvme_parse_ana_log() accesses ANA state protected by ctrl->ana_lock and therefore requires callers to hold the lock. Annotate nvme_parse_ana_log() with __must_hold(&ctrl->ana_lock) so that Clang's lock context analysis can verify the locking requirement at compile time. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/multipath.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 091aceb9b1d8..75dbb58286a3 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -832,6 +832,7 @@ static void nvme_mpath_set_live(struct nvme_ns *ns) static int nvme_parse_ana_log(struct nvme_ctrl *ctrl, void *data, int (*cb)(struct nvme_ctrl *ctrl, struct nvme_ana_group_desc *, void *)) + __must_hold(&ctrl->ana_lock) { void *base = ctrl->ana_log_buf; size_t offset = sizeof(struct nvme_ana_rsp_hdr); From 8aa68dba25f53f011ea39939af76171d4bf481ea Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:14 +0530 Subject: [PATCH 038/241] nvme: add context annotations for nvme_subsystems_lock The global nvme_subsystems list, nvme_subsystem::entry, nvme_subsystem::ctrls, and nvme_ctrl::subsys_entry are protected by nvme_subsystems_lock. Annotate these objects with __guarded_by(&nvme_subsystems_lock) so that Clang's context analysis can validate accesses to them. __nvme_find_get_subsystem() and nvme_validate_cntlid() traverse the global subsystem list and subsystem controller list and therefore require callers to hold nvme_subsystems_lock. Annotate both helpers with __must_hold(&nvme_subsystems_lock). Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 4 +++- drivers/nvme/host/nvme.h | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 178ac655aa2b..cb93ada4376a 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -126,8 +126,8 @@ EXPORT_SYMBOL_GPL(nvme_reset_wq); struct workqueue_struct *nvme_delete_wq; EXPORT_SYMBOL_GPL(nvme_delete_wq); -static LIST_HEAD(nvme_subsystems); DEFINE_MUTEX(nvme_subsystems_lock); +static LIST_HEAD_GUARDED(nvme_subsystems, nvme_subsystems_lock); static DEFINE_IDA(nvme_instance_ida); static dev_t nvme_ctrl_base_chr_devt; @@ -3213,6 +3213,7 @@ static void nvme_put_subsystem(struct nvme_subsystem *subsys) } static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn) + __must_hold(&nvme_subsystems_lock) { struct nvme_subsystem *subsys; @@ -3257,6 +3258,7 @@ static inline bool nvme_is_io_ctrl(struct nvme_ctrl *ctrl) static bool nvme_validate_cntlid(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) + __must_hold(&nvme_subsystems_lock) { struct nvme_ctrl *tmp; diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index fac4acbbd85d..862464301d01 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -361,7 +361,8 @@ struct nvme_ctrl { wait_queue_head_t state_wq; struct nvme_subsystem *subsys; - struct list_head subsys_entry; + struct list_head subsys_entry + __guarded_by(&nvme_subsystems_lock); struct opal_dev *opal_dev; @@ -493,9 +494,11 @@ struct nvme_subsystem { * a separate refcount. */ struct kref ref; - struct list_head entry; + struct list_head entry + __guarded_by(&nvme_subsystems_lock); struct mutex lock; - struct list_head ctrls; + struct list_head ctrls + __guarded_by(&nvme_subsystems_lock); struct list_head nsheads __guarded_by(&lock); char subnqn[NVMF_NQN_SIZE]; From 50be6cb15f15006477332a20c4a4adba59a55163 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:15 +0530 Subject: [PATCH 039/241] nvme: add context annotations in fabric.c The global nvmf_transports list is protected by nvmf_transports_rwsem and the global nvmf_hosts list is protected by nvmf_hosts_mutex. Define both lists using LIST_HEAD_GUARDED() so that Clang's context analysis can validate accesses to the lists against the corresponding locking requirements. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/fabrics.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c index ac3d4f400601..fd5abd04e080 100644 --- a/drivers/nvme/host/fabrics.c +++ b/drivers/nvme/host/fabrics.c @@ -14,11 +14,11 @@ #include "fabrics.h" #include -static LIST_HEAD(nvmf_transports); static DECLARE_RWSEM(nvmf_transports_rwsem); +static LIST_HEAD_GUARDED(nvmf_transports, nvmf_transports_rwsem); -static LIST_HEAD(nvmf_hosts); static DEFINE_MUTEX(nvmf_hosts_mutex); +static LIST_HEAD_GUARDED(nvmf_hosts, nvmf_hosts_mutex); static struct nvmf_host *nvmf_default_host; From 1f3d29bdca645edd5a623639604328b8ec2193d6 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:16 +0530 Subject: [PATCH 040/241] nvme: add context annotations for nvme_queue::sq_lock nvme_queue::sq_tail, nvme_queue::last_sq_tail and nvme_queue::sq_cmds are protected by nvme_queue::sq_lock. Annotate each field with __guarded_by(&sq_lock) and annotate helpers that access them with __must_hold(&sq_lock) so that Clang's context analysis can validate the locking requirements. Access to nvme_queue::sq_tail used solely for tracing is annotated with data_race(), as they only require a lockless snapshot of the value. nvme_init_queue() initializes nvme_queue::sq_tail and nvme_queue::last_sq_tail before the queue is published and thus do not require nvme_queue::sq_lock protection. So annotate nvme_init_queue() with context_unsafe() to suppress false positive context analyzer warning. nvme_free_queue() operate on queues which are no longer reachable, and therefore do not require nvme_queue::sq_lock protection. Similarly, nvme_alloc_sq_cmds() allocates memory for nvme_queue::sq_cmds for the queue which is not yet published or in use and hence it's safe to annotate all these helpers using context_unsafe. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/pci.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 8c6d169f2c38..0bce364c7874 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -366,7 +366,8 @@ struct nvme_queue { struct nvme_dev *dev; struct nvme_descriptor_pools descriptor_pools; spinlock_t sq_lock; - void *sq_cmds; + void *sq_cmds + __guarded_by(&sq_lock); /* only used for poll queues: */ spinlock_t cq_poll_lock ____cacheline_aligned_in_smp; struct nvme_completion *cqes; @@ -375,9 +376,11 @@ struct nvme_queue { u32 __iomem *q_db; u32 q_depth; u16 cq_vector; - u16 sq_tail; - u16 last_sq_tail; u16 cq_head; + u16 sq_tail + __guarded_by(&sq_lock); + u16 last_sq_tail + __guarded_by(&sq_lock); u16 qid; u8 cq_phase; u8 sqes; @@ -716,6 +719,7 @@ static void nvme_pci_map_queues(struct blk_mq_tag_set *set) * Write sq tail if we are asked to, or if the next command would wrap. */ static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq) + __must_hold(&nvmeq->sq_lock) { if (!write_sq) { u16 next_tail = nvmeq->sq_tail + 1; @@ -734,6 +738,7 @@ static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq) static inline void nvme_sq_copy_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd) + __must_hold(&nvmeq->sq_lock) { memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes), absolute_pointer(cmd), sizeof(*cmd)); @@ -1586,7 +1591,12 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, return; } - trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail); + /* + * Tracing only; annotate a lockless snapshot of nvmeq->sq_tail using + * data_race(). This would also help suppress context analysis warning + * while accessing nvmeq->sq_tail without acquiring ->sq_lock. + */ + trace_nvme_sq(req, cqe->sq_head, data_race(nvmeq->sq_tail)); if (!nvme_try_complete_req(req, cqe->status, cqe->result) && !blk_mq_add_to_batch(req, iob, nvme_req(req)->status != NVME_SC_SUCCESS, @@ -2013,6 +2023,7 @@ disable: } static void nvme_free_queue(struct nvme_queue *nvmeq) + __context_unsafe(/* frees queue which is no longer in use */) { dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes, nvmeq->cq_dma_addr); @@ -2107,6 +2118,7 @@ static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq, int qid) + __context_unsafe(/* safe to allocate sq_cmds without any protection */) { struct pci_dev *pdev = to_pci_dev(dev->dev); @@ -2181,6 +2193,7 @@ static int queue_request_irq(struct nvme_queue *nvmeq) } static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid) + __context_unsafe(/* initialize unpublished/lock-guarded variables */) { struct nvme_dev *dev = nvmeq->dev; From 5de3b73cea44d2c5be7ae9ba9602755897ac2588 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:17 +0530 Subject: [PATCH 041/241] nvme: add context annotations in rdma.c device_list and nvme_rdma_device::entry are protected by device_list_mutex. Define device_list using LIST_HEAD_GUARDED(device_list, device_list_mutex) and annotate nvme_rdma_device::entry with __guarded_by(&device_list_mutex) so that Clang's context analysis can validate accesses against the corresponding locking requirements. Similarly, nvme_rdma_ctrl_list and nvme_rdma_ctrl::list are protected by nvme_rdma_ctrl_mutex. Define nvme_rdma_ctrl_list using LIST_HEAD_GUARDED(nvme_rdma_ctrl_list, nvme_rdma_ctrl_mutex) and annotate nvme_rdma_ctrl::list with __guarded_by(&nvme_rdma_ctrl_mutex). It is safe to initialize nvme_rdma_ctrl::list while allocating the controller object because the list entry has not yet been added to nvme_rdma_ctrl_list. Annotate the initialization with context_unsafe() to suppress the corresponding Clang context analysis warning. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/rdma.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 52933d11ea03..9111d58f9871 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -40,11 +40,18 @@ #define NVME_RDMA_METADATA_SGL_SIZE \ (sizeof(struct scatterlist) * NVME_INLINE_METADATA_SG_CNT) +static DEFINE_MUTEX(device_list_mutex); +static LIST_HEAD_GUARDED(device_list, device_list_mutex); + +static DEFINE_MUTEX(nvme_rdma_ctrl_mutex); +static LIST_HEAD_GUARDED(nvme_rdma_ctrl_list, nvme_rdma_ctrl_mutex); + struct nvme_rdma_device { struct ib_device *dev; struct ib_pd *pd; struct kref ref; - struct list_head entry; + struct list_head entry + __guarded_by(&device_list_mutex); unsigned int num_inline_segments; }; @@ -118,7 +125,8 @@ struct nvme_rdma_ctrl { struct delayed_work reconnect_work; - struct list_head list; + struct list_head list + __guarded_by(&nvme_rdma_ctrl_mutex); struct blk_mq_tag_set admin_tag_set; struct nvme_rdma_device *device; @@ -138,12 +146,6 @@ static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl) return container_of(ctrl, struct nvme_rdma_ctrl, ctrl); } -static LIST_HEAD(device_list); -static DEFINE_MUTEX(device_list_mutex); - -static LIST_HEAD(nvme_rdma_ctrl_list); -static DEFINE_MUTEX(nvme_rdma_ctrl_mutex); - /* * Disabling this option makes small I/O goes faster, but is fundamentally * unsafe. With it turned off we will have to register a global rkey that @@ -2283,7 +2285,10 @@ static struct nvme_rdma_ctrl *nvme_rdma_alloc_ctrl(struct device *dev, if (!ctrl) return ERR_PTR(-ENOMEM); ctrl->ctrl.opts = opts; - INIT_LIST_HEAD(&ctrl->list); + /* + * Safe to init list while allocating ctrl object. + */ + context_unsafe(INIT_LIST_HEAD(&ctrl->list)); if (!(opts->mask & NVMF_OPT_TRSVCID)) { opts->trsvcid = From e906dc2a33de221b4cb2b2b7ba2e03835f3ee40e Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:18 +0530 Subject: [PATCH 042/241] nvme: fix context analysis warning in rdma.c After adding Clang lock context annotations in rdma.c, Clang reports the following warning when context analysis is enabled: drivers/nvme/host/rdma.c:972:24: warning: passing pointer to variable 'list' requires holding mutex 'nvme_rdma_ctrl_mutex' [-Wthread-safety-pointer] 972 | if (list_empty(&ctrl->list)) | ^ The warning is triggered because ctrl->list is annotated as being protected by nvme_rdma_ctrl_mutex, but list_empty(&ctrl->list) is invoked without holding that mutex. Replace list_empty() with list_empty_careful(), which is intended for lockless inspection of list heads during teardown when no concurrent list modifications are expected. This suppresses the corresponding context analysis warning while preserving the existing behavior. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/rdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 9111d58f9871..01743ae01466 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -1000,7 +1000,7 @@ static void nvme_rdma_free_ctrl(struct nvme_ctrl *nctrl) { struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl); - if (list_empty(&ctrl->list)) + if (list_empty_careful(&ctrl->list)) goto free_ctrl; mutex_lock(&nvme_rdma_ctrl_mutex); From 27a75a6290d610b40e4ef8024bef2acf7e3268ce Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:19 +0530 Subject: [PATCH 043/241] nvme: add context annotations in tcp.c The nvme_tcp_ctrl_list and nvme_tcp_ctrl::list are protected by nvme_tcp_ctrl_mutex. Define nvme_tcp_ctrl_list using LIST_HEAD_GUARDED(nvme_tcp_ctrl_list, nvme_tcp_ctrl_mutex) and annotate nvme_tcp_ctrl::list using __guarded_by(&nvme_tcp_ctrl_mutex) so that Clang's context analysis can validate accesses against the corresponding locking requirements. It is safe to initialize nvme_tcp_ctrl::list while allocating the controller object because the list entry has not yet been added to nvme_tcp_ctrl_list. Annotate the initialization with context_unsafe() to suppress the corresponding Clang warning. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index ba5c7b3e2a7c..8d2fbfc7cd8d 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -149,13 +149,17 @@ struct nvme_tcp_queue { #endif }; +static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); +static LIST_HEAD_GUARDED(nvme_tcp_ctrl_list, nvme_tcp_ctrl_mutex); + struct nvme_tcp_ctrl { /* read only in the hot path */ struct nvme_tcp_queue *queues; struct blk_mq_tag_set tag_set; /* other member variables */ - struct list_head list; + struct list_head list + __guarded_by(&nvme_tcp_ctrl_mutex); struct blk_mq_tag_set admin_tag_set; struct sockaddr_storage addr; struct sockaddr_storage src_addr; @@ -167,8 +171,6 @@ struct nvme_tcp_ctrl { u32 io_queues[HCTX_MAX_TYPES]; }; -static LIST_HEAD(nvme_tcp_ctrl_list); -static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); static struct workqueue_struct *nvme_tcp_wq; static const struct blk_mq_ops nvme_tcp_mq_ops; static const struct blk_mq_ops nvme_tcp_admin_mq_ops; @@ -2919,7 +2921,10 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev, if (!ctrl) return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&ctrl->list); + /* + * Safe to init list while allocating ctrl object. + */ + context_unsafe(INIT_LIST_HEAD(&ctrl->list)); ctrl->ctrl.opts = opts; ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + opts->nr_poll_queues + 1; From 521b1587de93650950d59b59b058bf5c24230973 Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:20 +0530 Subject: [PATCH 044/241] nvme: fix context analysis warning in tcp.c After adding Clang context annotations, compiling tcp.c reports the following warning while context analysis is enabled: drivers/nvme/host/tcp.c:2572:24: warning: passing pointer to variable 'list' requires holding mutex 'nvme_tcp_ctrl_mutex' [-Wthread-safety-pointer] 2572 | if (list_empty(&ctrl->list)) | ^ The above warning is triggered because ctrl->list is guarded with mutex nvme_tcp_ctrl_mutex but when list_empty(&ctrl->list) is invoked it doesn't acquire nvme_tcp_ctrl_mutex. Replace list_empty() with list_empty_careful(), which is intended for lockless inspection of list heads during teardown when no concurrent list modifications are expected. This suppresses the corresponding Clang context analysis warning while preserving the existing behavior. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 8d2fbfc7cd8d..87d8067f3283 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -2579,7 +2579,7 @@ static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) { struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); - if (list_empty(&ctrl->list)) + if (list_empty_careful(&ctrl->list)) goto free_ctrl; mutex_lock(&nvme_tcp_ctrl_mutex); From fccada336f6d29344e3853d44b96684807dd7d7d Mon Sep 17 00:00:00 2001 From: Nilay Shroff Date: Mon, 13 Jul 2026 17:24:21 +0530 Subject: [PATCH 045/241] nvme: enable context analysis support for nvme host driver Update nvme host driver makefile to enable support for the Clang's context anaysis. Reviewed-by: Christoph Hellwig Signed-off-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile index 6414ec968f99..67563a69f7dc 100644 --- a/drivers/nvme/host/Makefile +++ b/drivers/nvme/host/Makefile @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS := y ccflags-y += -I$(src) obj-$(CONFIG_NVME_CORE) += nvme-core.o From 08660a5c8d497f43191635d97efd31cd35051f15 Mon Sep 17 00:00:00 2001 From: Myeonghun Pak Date: Wed, 15 Jul 2026 16:44:59 +0900 Subject: [PATCH 046/241] nvme-pci: disable controller on admin queue IRQ setup failure nvme_pci_configure_admin_queue() enables the controller and then requests the admin queue interrupt. If queue_request_irq() fails it returns without disabling the controller, and no caller compensates: nvme_pci_enable() only frees the IRQ vectors and calls pci_disable_device(), after which nvme_dev_disable() treats the controller as dead and skips nvme_disable_ctrl(). The controller is left enabled (CC.EN set) on this error path. Disable it in the failure path, while the PCI device is still enabled so the CC.EN clear handshake completes. This issue was identified during our ongoing static-analysis research while reviewing kernel code. Fixes: b60503ba432b ("NVMe: New driver") Cc: stable@vger.kernel.org Reviewed-by: Christoph Hellwig Co-developed-by: Ijae Kim Signed-off-by: Ijae Kim Signed-off-by: Myeonghun Pak Signed-off-by: Keith Busch --- drivers/nvme/host/pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 0bce364c7874..16d42e5138c8 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2414,6 +2414,7 @@ static int nvme_pci_configure_admin_queue(struct nvme_dev *dev) result = queue_request_irq(nvmeq); if (result) { dev->online_queues--; + nvme_disable_ctrl(&dev->ctrl, false); return result; } From 13330446caef56de008ecb9ad2a1545ce2fedd0a Mon Sep 17 00:00:00 2001 From: Pan Chuang Date: Thu, 16 Jul 2026 11:33:01 +0800 Subject: [PATCH 047/241] nvme-apple: Remove redundant dev_err_probe() Since commit 55b48e23f5c4 ("genirq/devres: Add error handling in devm_request_*_irq()"), devm_request_irq() automatically logs detailed error messages on failure. Remove the now-redundant driver-specific dev_err_probe() calls. Reviewed-by: Christoph Hellwig Signed-off-by: Pan Chuang Signed-off-by: Keith Busch --- drivers/nvme/host/apple.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 2723bc1a7d8a..09eb2295ceee 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1583,10 +1583,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) ret = devm_request_irq(anv->dev, anv->irq, apple_nvme_irq, 0, "nvme-apple", anv); - if (ret) { - dev_err_probe(dev, ret, "Failed to request IRQ"); + if (ret) goto put_dev; - } anv->rtk = devm_apple_rtkit_init(dev, anv, NULL, 0, &apple_nvme_rtkit_ops); From b53d495c7f0db46b6748b5ade48371a10dd5d3bc Mon Sep 17 00:00:00 2001 From: Yang Xiuwei Date: Mon, 20 Jul 2026 14:03:05 +0800 Subject: [PATCH 048/241] nvme/ioctl: check SUBMIT_IO with nvme_cmd_allowed() Unlike IO_CMD / IO64_CMD, NVME_IOCTL_SUBMIT_IO never calls nvme_cmd_allowed(). Unprivileged callers can thus issue I/O on a partition device or write through a read-only file descriptor. Pass flags and open_for_write through and reject disallowed commands with -EACCES. Reviewed-by: Christoph Hellwig Signed-off-by: Yang Xiuwei Signed-off-by: Keith Busch --- drivers/nvme/host/ioctl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index bae52bd5bdd2..f4ea52d11945 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -202,7 +202,8 @@ out_free_req: return ret; } -static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) +static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio, + unsigned int flags, bool open_for_write) { struct nvme_user_io io; struct nvme_command c; @@ -260,6 +261,9 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio) c.rw.lbat = cpu_to_le16(io.apptag); c.rw.lbatm = cpu_to_le16(io.appmask); + if (!nvme_cmd_allowed(ns, &c, flags, open_for_write)) + return -EACCES; + return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata, meta_len, NULL, 0, 0); } @@ -595,7 +599,7 @@ static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd, case NVME_IOCTL_SUBMIT_IO32: #endif case NVME_IOCTL_SUBMIT_IO: - return nvme_submit_io(ns, argp); + return nvme_submit_io(ns, argp, flags, open_for_write); case NVME_IOCTL_IO64_CMD_VEC: flags |= NVME_IOCTL_VEC; fallthrough; From 581d8bb556dd3e5567bcf322aa5e3e4b6a200c08 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 29 Jul 2026 18:56:07 +0800 Subject: [PATCH 049/241] nvmet: fix return status of RMI log page on allocation failure nvmet_execute_get_log_page_rmi() leaves 'status' holding NVME_SC_SUCCESS (set by the successful nvmet_req_find_ns() call) when the kzalloc() for the log buffer fails. It then jumps to the out label and completes the request with a success status, so the host is told the command succeeded while no data was transferred. Initialize 'status' to NVME_SC_INTERNAL, matching the smart log handler, so an allocation failure is reported as an internal error. Fixes: 5fd075cdaf36 ("nvmet: implement rotational media information log") Reviewed-by: Christoph Hellwig Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- drivers/nvme/target/admin-cmd.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 01b799e92ae6..0b24d31f966d 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -309,8 +309,10 @@ static void nvmet_execute_get_log_page_rmi(struct nvmet_req *req) } log = kzalloc_obj(*log); - if (!log) + if (!log) { + status = NVME_SC_INTERNAL; goto out; + } log->endgid = req->cmd->get_log_page.lsi; disk = req->ns->bdev->bd_disk; From f49d0c3a8d56a7cda1628ae17341a4a42063563c Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 29 Jul 2026 18:58:46 +0800 Subject: [PATCH 050/241] nvme-fc: unmap cmd_iu DMA on rsp_iu mapping failure in init_request __nvme_fc_init_request() maps cmd_iu and then rsp_iu for DMA. If the rsp_iu mapping fails, the original code only recorded the error and fell through: it left the already-mapped cmd_iu unmapped and still marked the op as FCPOP_STATE_IDLE before returning. Since blk-mq does not call .exit_request() when .init_request() fails, the cmd_iu mapping is leaked for every op whose rsp_iu mapping fails. Jump to an error path on rsp_iu mapping failure that unmaps cmd_iu and returns the error without marking the op idle, so it stays in the FCPOP_STATE_UNINIT state set by the initial memset(). Fixes: e399441de911 ("nvme-fabrics: Add host support for FC transport") Reviewed-by: Christoph Hellwig Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- drivers/nvme/host/fc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 04363b9c4489..40f9da2833ff 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2100,9 +2100,15 @@ __nvme_fc_init_request(struct nvme_fc_ctrl *ctrl, dev_err(ctrl->dev, "FCP Op failed - rspiu dma mapping failed.\n"); ret = -EFAULT; + goto out_unmap; } atomic_set(&op->state, FCPOP_STATE_IDLE); + return 0; + +out_unmap: + fc_dma_unmap_single(ctrl->lport->dev, op->fcp_req.cmddma, + sizeof(op->cmd_iu), DMA_TO_DEVICE); out_on_error: return ret; } From df74eaad001cf669c332dc67ef91996532e6b52c Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 29 Jul 2026 18:59:58 +0800 Subject: [PATCH 051/241] nvme-pci: return error when parsing a quirk string fails quirks_param_set() reuses 'err', which param_set_copystring() left as 0, as the return value of the whole function. When nvme_parse_quirk_entry() fails to parse a field, the code jumps to out_free_qlist and returns that stale 0, so a malformed quirks= parameter is silently accepted as valid. Set err to -EINVAL before jumping out on a parse failure. Fixes: 7bb8c40f5ad8 ("nvme: add support for dynamic quirk configuration via module parameter") Reviewed-by: Christoph Hellwig Reviewed-by: Daniel Wagner Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- drivers/nvme/host/pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 16d42e5138c8..375e7a1fc91d 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -213,6 +213,7 @@ static int quirks_param_set(const char *value, const struct kernel_param *kp) if (nvme_parse_quirk_entry(field, &qlist[i])) { pr_err("nvme: failed to parse quirk string %s\n", value); + err = -EINVAL; goto out_free_qlist; } From bf881dd20062db5e951a0d0703cb476df8c9fdee Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Wed, 29 Jul 2026 19:02:31 +0800 Subject: [PATCH 052/241] nvmet: reject out-of-range mdts values in configfs store nvmet_param_mdts_store() accepts any integer that kstrtoint() can parse and stores it directly into port->mdts. The value is only range-checked later, when the port is enabled: nvmet_enable_port() silently resets port->mdts to 0 if it is negative or greater than NVMET_MAX_MDTS. As a result, writing e.g. "mdts=1000" succeeds and reading the attribute back returns 1000, yet enabling the port quietly turns it into 0. This is confusing and hides the invalid input from the user. Validate the value against [0, NVMET_MAX_MDTS] in the store handler and reject anything out of range with -EINVAL, so the error is reported at write time and port->mdts never holds a value the port cannot use. Fixes: 0a5a94648627 ("nvmet: introduce new mdts configuration entry") Reviewed-by: Christoph Hellwig Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- drivers/nvme/target/configfs.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c index 2b69ffcfc8df..413ee2d16d29 100644 --- a/drivers/nvme/target/configfs.c +++ b/drivers/nvme/target/configfs.c @@ -312,15 +312,17 @@ static ssize_t nvmet_param_mdts_store(struct config_item *item, const char *page, size_t count) { struct nvmet_port *port = to_nvmet_port(item); - int ret; + int ret, mdts; if (nvmet_is_port_enabled(port, __func__)) return -EACCES; - ret = kstrtoint(page, 0, &port->mdts); - if (ret) { - pr_err("Invalid value '%s' for mdts\n", page); + ret = kstrtoint(page, 0, &mdts); + if (ret || mdts < 0 || mdts > NVMET_MAX_MDTS) { + pr_err("Invalid value '%s' for mdts, should be 0-%d\n", + page, NVMET_MAX_MDTS); return -EINVAL; } + port->mdts = mdts; return count; } From f565925810cb8bc799421485770e15d922ef766a Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Mon, 22 Jun 2026 20:46:49 +0800 Subject: [PATCH 053/241] md/raid5: protect bitmap batch counters aka seq_flush/seq_write consistency kcsan detect race : - raid5d() closes the current bitmap batch by updating conf->seq_flush under conf->device_lock. - __add_stripe_bio() read conf->seq_flush without that lock when assigning sh->bm_seq. so, protect seq_flush/seq_write consistency for multiple CPUs by READ_ONCE()/WRITE_ONCE() under the path without held device_lock. re-explain the stripe batch sequence number update flow: 1. sh->bm_seq declare which batch number the stripe belongs to when perform bitmap-related write. ==> bm_seq = seq_flush+1 2. stripe be handled, * if sh->bm_seq - conf->seq_write > 0, means the batch stripes **newer than** the last written batch, it cannot proceed yet, queued on bitmap_list. * otherwise , has already proceed. 3. raid5d() `++seq_flush` to closes the current batch, means * no more stripes join that old batch * just-closed batch ready to write-out to disk 4. raid5d() calls bitmap hooks unplug() or writeout, then, `++seq_write` to the same as bm_seq. - seq_flush - for producer, to close batches. - seq_write - for consumer, the checkpoint number. the report: ==================================== BUG: KCSAN: data-race in __add_stripe_bio / raid5d write to 0xffff88ba5625d470 of 4 bytes by task 82401 on cpu 0: raid5d+0x1d9/0xba0 [.....] read to 0xffff88ba5625d470 of 4 bytes by task 82421 on cpu 8: __add_stripe_bio+0x332/0x400 raid5_make_request+0x6ac/0x2930 md_handle_request+0x4a2/0xa40 md_submit_bio+0x109/0x1a0 __submit_bio+0x2ec/0x390 [.....] Fixes: 7c13edc87510 ("md: incorporate new plugging into raid5.") v1 -> v2: - remove WRITE_ONCE(conf->seq_write) in held device_lock path. - remove READ_ONCE(conf->seq_flush) in held device_lock path. Signed-off-by: Chen Cheng Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260622124649.1780233-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index ffb5fcde54a9..a6c52fb1fe68 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -3553,7 +3553,7 @@ static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi, sh->dev[dd_idx].sector); if (conf->mddev->bitmap && firstwrite && !sh->batch_head) { - sh->bm_seq = conf->seq_flush+1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } } @@ -5799,7 +5799,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) } spin_unlock_irq(&sh->stripe_lock); if (conf->mddev->bitmap) { - sh->bm_seq = conf->seq_flush + 1; + sh->bm_seq = READ_ONCE(conf->seq_flush) + 1; set_bit(STRIPE_BIT_DELAY, &sh->state); } @@ -6849,12 +6849,14 @@ static void raid5d(struct md_thread *thread) if ( !list_empty(&conf->bitmap_list)) { /* Now is a good time to flush some bitmap updates */ - conf->seq_flush++; + int seq = conf->seq_flush + 1; + + WRITE_ONCE(conf->seq_flush, seq); spin_unlock_irq(&conf->device_lock); if (md_bitmap_enabled(mddev, true)) mddev->bitmap_ops->unplug(mddev, true); spin_lock_irq(&conf->device_lock); - conf->seq_write = conf->seq_flush; + conf->seq_write = seq; activate_bit_delay(conf, conf->temp_inactive_list); } raid5_activate_delayed(conf); From 371f7a1b392edc8b7cf449cc7713179b588f2d0e Mon Sep 17 00:00:00 2001 From: Sajal Gupta Date: Mon, 22 Jun 2026 19:36:03 +0530 Subject: [PATCH 054/241] md/raid5-ppl: fix use-after-free in ppl_do_flush() The loop in ppl_do_flush() continues iterating after calling ppl_io_unit_finished(), touching io->pending_flushes and leading to a use-after-free. Add a break statement to stop the loop once io is freed. Fixes: 1532d9e87e8b ("raid5-ppl: PPL support for disks with write-back cache enabled") Reported-by: Dan Carpenter Closes: https://lore.kernel.org/all/ajJF2wKYWRk4GGCK@stanley.mountain/ Signed-off-by: Sajal Gupta Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260622142146.56637-1-sajal2005gupta@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid5-ppl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index 7be1648c4e4f..7f8a9d3fd578 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -643,8 +643,10 @@ static void ppl_do_flush(struct ppl_io_unit *io) log->disk_flush_bitmap = 0; for (i = flushed_disks ; i < raid_disks; i++) { - if (atomic_dec_and_test(&io->pending_flushes)) + if (atomic_dec_and_test(&io->pending_flushes)) { ppl_io_unit_finished(io); + break; + } } } From e12e619c2e2d0c3f42b14e9ef1ab778e696ffffd Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Tue, 23 Jun 2026 15:59:40 +0800 Subject: [PATCH 055/241] md/raid1: protect sequential read hints for read balance The patch just suppress KCSAN noise. No functional change. KCSAN reports a race, point to update_read_sectors() update next_seq_sect vs. read next_seq_sect. Protect next_seq_sect and seq_start with READ_ONCE/WRITE_ONCE, otherwise, read balance see stale sequential-read hints. KCSAN report: ============== BUG: KCSAN: data-race in raid1_read_request / raid1_read_request write to 0xffff8e3a2d6736d0 of 8 bytes by task 593784 on cpu 10: raid1_read_request+0xe5a/0x19f0 raid1_make_request+0xdf/0x1990 md_handle_request+0x4a2/0xa40 [...] read to 0xffff8e3a2d6736d0 of 8 bytes by task 593776 on cpu 11: raid1_read_request+0xe3f/0x19f0 raid1_make_request+0xdf/0x1990 md_handle_request+0x4a2/0xa40 [...] value changed: 0x0000000000356368 -> 0x0000000000356370 Signed-off-by: Chen Cheng Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260623075940.2476255-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid1.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index afe2ca96ad8c..4d7f33a2faaa 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -604,9 +604,9 @@ static void update_read_sectors(struct r1conf *conf, int disk, struct raid1_info *info = &conf->mirrors[disk]; atomic_inc(&info->rdev->nr_pending); - if (info->next_seq_sect != this_sector) - info->seq_start = this_sector; - info->next_seq_sect = this_sector + len; + if (READ_ONCE(info->next_seq_sect) != this_sector) + WRITE_ONCE(info->seq_start, this_sector); + WRITE_ONCE(info->next_seq_sect, this_sector + len); } static int choose_first_rdev(struct r1conf *conf, struct r1bio *r1_bio, @@ -735,8 +735,7 @@ static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio, static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) { - /* TODO: address issues with this check and concurrency. */ - return conf->mirrors[disk].next_seq_sect == r1_bio->sector || + return READ_ONCE(conf->mirrors[disk].next_seq_sect) == r1_bio->sector || READ_ONCE(conf->mirrors[disk].head_position) == r1_bio->sector; } @@ -747,15 +746,18 @@ static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) static bool should_choose_next(struct r1conf *conf, int disk) { struct raid1_info *mirror = &conf->mirrors[disk]; + sector_t seq_start, next_seq_sect; int opt_iosize; if (!test_bit(Nonrot, &mirror->rdev->flags)) return false; opt_iosize = bdev_io_opt(mirror->rdev->bdev) >> 9; - return opt_iosize > 0 && mirror->seq_start != MaxSector && - mirror->next_seq_sect > opt_iosize && - mirror->next_seq_sect - opt_iosize >= mirror->seq_start; + seq_start = READ_ONCE(mirror->seq_start); + next_seq_sect = READ_ONCE(mirror->next_seq_sect); + return opt_iosize > 0 && seq_start != MaxSector && + next_seq_sect > opt_iosize && + next_seq_sect - opt_iosize >= seq_start; } static bool rdev_readable(struct md_rdev *rdev, struct r1bio *r1_bio) From 6cb6ab75bdf2f49c0adb0fd6971886b082932eaa Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Wed, 24 Jun 2026 10:40:42 +0800 Subject: [PATCH 056/241] md/raid5: fix lockless max_nr_stripes reads max_nr_stripes is updated under cache_size_mutex in the stripe cache grow/shrink paths, while is_inactive_blocked() and raid5_end_read_request() read it without that lock. Use READ_ONCE() for those reads in lockless path to match the WRITE_ONCE() updates and avoid KCSAN data race reports. A similar issue was previously fixed in commit-id: dfd2bf436709b2bccb78c2dda550dde93700efa7. Fixes: 0009fad03337 ("raid5 improve too many read errors msg by adding limits") Fixes: 3514da58be9c ("md/raid5: Make is_inactive_blocked() helper") KCSAN report: ================= BUG: KCSAN: data-race in grow_one_stripe / is_inactive_blocked write (marked) to 0xffff8f01f0b5a268 of 4 bytes by task 12616 on cpu 9: grow_one_stripe+0x2d8/0x320 raid5d+0xb57/0xba0 md_thread+0x15a/0x2d0 [..........] read to 0xffff8f01f0b5a268 of 4 bytes by task 12670 on cpu 11: is_inactive_blocked+0x97/0xc0 raid5_get_active_stripe+0x2fd/0xa70 raid5_make_request+0x4aa/0x2940 [..........] value changed: 0x000003b9 -> 0x000003ba Signed-off-by: Chen Cheng Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260624024042.2561803-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index a6c52fb1fe68..992d0b14822e 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -801,7 +801,7 @@ static bool is_inactive_blocked(struct r5conf *conf, int hash) return true; return (atomic_read(&conf->active_stripes) < - (conf->max_nr_stripes * 3 / 4)); + (READ_ONCE(conf->max_nr_stripes) * 3 / 4)); } struct stripe_head *raid5_get_active_stripe(struct r5conf *conf, @@ -2785,6 +2785,7 @@ static void raid5_end_read_request(struct bio * bi) } else { int retry = 0; int set_bad = 0; + int max_nr_stripes = READ_ONCE(conf->max_nr_stripes); clear_bit(R5_UPTODATE, &sh->dev[i].flags); if (!(bi->bi_status == BLK_STS_PROTECTION)) @@ -2810,13 +2811,12 @@ static void raid5_end_read_request(struct bio * bi) mdname(conf->mddev), (unsigned long long)s, rdev->bdev); - } else if (atomic_read(&rdev->read_errors) - > conf->max_nr_stripes) { + } else if (atomic_read(&rdev->read_errors) > max_nr_stripes) { if (!test_bit(Faulty, &rdev->flags)) { pr_warn("md/raid:%s: %d read_errors > %d stripes\n", mdname(conf->mddev), atomic_read(&rdev->read_errors), - conf->max_nr_stripes); + max_nr_stripes); pr_warn("md/raid:%s: Too many read errors, failing device %pg.\n", mdname(conf->mddev), rdev->bdev); } From 788e4139463f74b945600237f7186411015d996e Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Wed, 24 Jun 2026 15:58:24 +0800 Subject: [PATCH 057/241] md/raid5: fix reshape deadlock while failed devices more than max degraded reshape stripe lifetime: - start reshape ==> reshape_request(): * get destination stripe, - if need to copy source data chunks, set STRIPE_EXPANDING; - or, if new regions past the old end of the array, zero-filled, no need source data, set STRIPE_EXPANDING | STRIPE_READY * get source stripe, - set STRIPE_EXPAND_SOURCE - handle expand stripe ==> handle_stripe(): reshape use reconstruct-write to construct stripe, four stages: 1. prepare source data chunks for old geometry stripe - fill source stripe data by read or compute 2. move data from old geometry source stripe to new geometry destination stripe - source stripe clear STRIPE_EXPAND_SOURCE - drain data from source to destination stripe - mark stripe chunk as R5_Expanded|R5_UPTODATE when the drain from source chunk to destination chunk is completed - all stripe chunks drain are completed, then mark STRIPE_EXPAND_READY 3. calculate p/q chunks for destination stripe - if destination stripe doesn't depends on source dstripe, then we can clear STRIPE_EXPANDING 4. write-out to disks and release - set R5_Wantwrite|R5_Locked, writeout to disk - if write-out succeeded, clear STRIPE_EXPAND_READY, and decrement reshape_stripe, call md_done_sync() to report reshape progress. 1. cleanup the following kinds of **destination stripe** when failed device more than max degraded: - new regions past the old end of the array, zero-filled in place, requires no source data. (STRIPE_EXPANDING | STRIPE_EXPAND_READY) - prepare source data chunks already done, and writeout failed (STRIPE_EXPAND_READY) 2. destination stripes that need source data (STRIPE_EXPANDING, no STRIPE_HANDLE) - these kind of stripes sit idle in the stripe cache and are never seen by handle_stripe(). So clean up indirectly when their source stripe (type 3) is processed. 3. source stripes (STRIPE_EXPAND_SOURCE) - hit handle_stripe() after their member disks are marked Faulty. - clear STRIPE_EXPAND_SOURCE, finds and cleanup all dependent destination stripes that were waiting for data. - walks the source's data disks, compute the corresponding destination sector, looks up the destination stripe, and do cleanup(clear flags, dec counters, call md_done_sync()) Reproducer: - Create a 4-disk RAID5 with mdadm on top of 5 disposable test disks wrapped by dm targets. - Add the 5th device as a spare and start a 4 -> 5 reshape. - Wait until /sys/block/mdX/md/sync_action reports "reshape". - Inject failures on two members so reshape exceeds max_degraded. - After a few seconds, write "frozen" to /sys/block/mdX/md/sync_action. Before this fix, the write blocks indefinitely. Read-error variant: - Use dm-dust on /dev/sd[b-f]. - Preload bad blocks on two source members, e.g. dust0 and dust1: dmsetup message dust0 0 addbadblock dmsetup message dust1 0 addbadblock - Start reshape: mdadm -C /dev/mdX -e 1.2 -l 5 -n 4 -c 64 \ --assume-clean /dev/mapper/dust{0..3} mdadm --manage /dev/mdX --add /dev/mapper/dust4 mdadm --grow /dev/mdX -n 5 --backup-file=/tmp/grow.backup & - Once reshape starts, enable the injected read failures: dmsetup message dust0 0 enable dmsetup message dust1 0 enable - Then: echo frozen > /sys/block/mdX/md/sync_action hangs forever before the fix. Write-error variant: - Use dm-flakey on /dev/sd[b-f]. - Start the same 4 -> 5 reshape on flakey0..flakey4. - Once reshape starts, switch two members, e.g. flakey3 and flakey4, to error_writes. - Then: echo frozen > /sys/block/mdX/md/sync_action hangs forever before the fix. md_do_sync() exits its main loop on MD_RECOVERY_INTR but then blocks forever at: wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active)); After the fix recovery_active drains to zero, md_do_sync() prints md/raid:md0: Cannot continue operation (2/5 failed). md: md0: reshape interrupted. Signed-off-by: Chen Cheng Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260624075824.2601110-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 992d0b14822e..83f8deefd03b 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -3745,6 +3745,79 @@ handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, md_sync_error(conf->mddev); } +/* + * handle_failed_reshape - handle failed stripes when reshape failed and + * degraded devices >= max_degraded + * + * handle following kinds of stripe: + * 1. cleanup the following kinds of destination stripe: + * - new regions past the old end of the array, zero-filled in place, + * requires no source data. + * (STRIPE_EXPANDING | STRIPE_EXPAND_READY) + * - prepare source data chunks already done, and writeout failed + * (STRIPE_EXPAND_READY) + * 2. dest stripes that need source data (STRIPE_EXPANDING, no STRIPE_HANDLE) + * - these kind of stripes sit idle in the stripe cache and are never seen + * by handle_stripe(). So clean up indirectly when their source stripe + * (type 3) is processed. + * 3. src stripes (STRIPE_EXPAND_SOURCE) + * - hit handle_stripe() after their member disks are marked Faulty. + * - clear STRIPE_EXPAND_SOURCE, finds and cleanup all dependent destination + * stripes that were waiting for data. + * - walks the source's data disks, compute the corresponding destination + * sector, looks up the destination stripe, and do cleanup(clear flags, + * dec counters, call md_done_sync()) + */ +static void handle_failed_reshape(struct r5conf *conf, struct stripe_head *sh, + struct stripe_head_state *s) +{ + int i; + bool was_expanding = test_and_clear_bit(STRIPE_EXPANDING, &sh->state); + bool was_ready = test_and_clear_bit(STRIPE_EXPAND_READY, &sh->state); + + if (was_expanding || was_ready) { + atomic_dec(&conf->reshape_stripes); + wake_up(&conf->wait_for_reshape); + md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf)); + } + + s->expanded = 0; + s->expanding = 0; + + /* release the destination stripes that are waiting to be filled */ + if (test_and_clear_bit(STRIPE_EXPAND_SOURCE, &sh->state)) { + for (i = 0; i < sh->disks; i++) { + int dd_idx; + struct stripe_head *sh2; + sector_t bn, sec; + + if (i == sh->pd_idx) + continue; + if (conf->level == 6 && i == sh->qd_idx) + continue; + + bn = raid5_compute_blocknr(sh, i, 1); + sec = raid5_compute_sector(conf, bn, 0, &dd_idx, NULL); + sh2 = raid5_get_active_stripe(conf, NULL, sec, + R5_GAS_NOBLOCK | + R5_GAS_NOQUIESCE); + if (!sh2) + continue; + + if (test_and_clear_bit(STRIPE_EXPANDING, &sh2->state)) { + atomic_dec(&conf->reshape_stripes); + wake_up(&conf->wait_for_reshape); + md_done_sync(conf->mddev, + RAID5_STRIPE_SECTORS(conf)); + } + + clear_bit(STRIPE_EXPAND_READY, &sh2->state); + + raid5_release_stripe(sh2); + } + } +} + static int want_replace(struct stripe_head *sh, int disk_idx) { struct md_rdev *rdev; @@ -5025,6 +5098,8 @@ static void handle_stripe(struct stripe_head *sh) handle_failed_stripe(conf, sh, &s, disks); if (s.syncing + s.replacing) handle_failed_sync(conf, sh, &s); + if (s.expanding + s.expanded) + handle_failed_reshape(conf, sh, &s); } /* Now we check to see if any write operations have recently From a47431dfb3538a1485f65b68a0605a05307b5b2d Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Sat, 27 Jun 2026 18:25:19 +0800 Subject: [PATCH 058/241] md/raid5: protect lockless recovery_offset accesses during reshape During reshape: - reshape_request() advances rdev->recovery_offset for non-In_sync devices locklessly. - analyse_stripe() reads rdev->recovery_offset locklessly to decide: a. use a replacement device to read ? b. a device can already be treated as in-sync for the current stripe ? one possible scenario is: CPU1 CPU2 reshape_request() -> mddev->curr_resync_completed = sector_nr -> if (!mddev->reshape_backwards) -> rdev->recovery_offset = sector_nr analyse_stripe(sh) -> rdev = conf->disks[i].replacement -> if (rdev->recovery_offset >= sh->sector + stripe_sectors) set_bit(R5_ReadRepl) -> or -> if (sh->sector + stripe_sectors <= rdev->recovery_offset) set_bit(R5_Insync) And it could be: - reading from a replacement before it is recovered far enough; or - treating a not-yet-recovered device as in-sync for the current stripe. Fixes: db0505d32066 ("md: be cautious about using ->curr_resync_completed for ->recovery_offset") The race report: ================================================================== BUG: KCSAN: data-race in ops_run_io / reshape_request write to 0xffff8bdee168b270 of 8 bytes by task 1704 on cpu 10: reshape_request+0x1292/0x17b0 raid5_sync_request+0x815/0xa00 md_do_sync.cold+0xf8d/0x1516 [......] read to 0xffff8bdee168b270 of 8 bytes by task 1696 on cpu 9: ops_run_io+0xc25/0x1960 handle_stripe+0x2273/0x4570 handle_active_stripes.isra.0+0x6e0/0xa50 raid5d+0x7d5/0xb90 [......] value changed: 0x0000000000091a00 -> 0x0000000000091b00 ================================================================== Signed-off-by: Chen Cheng Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260627102519.136940-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 83f8deefd03b..4f967574bb1f 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -3824,11 +3824,10 @@ static int want_replace(struct stripe_head *sh, int disk_idx) int rv = 0; rdev = sh->raid_conf->disks[disk_idx].replacement; - if (rdev - && !test_bit(Faulty, &rdev->flags) - && !test_bit(In_sync, &rdev->flags) - && (rdev->recovery_offset <= sh->sector - || rdev->mddev->resync_offset <= sh->sector)) + if (rdev && !test_bit(Faulty, &rdev->flags) && + !test_bit(In_sync, &rdev->flags) && + (READ_ONCE(rdev->recovery_offset) <= sh->sector || + rdev->mddev->resync_offset <= sh->sector)) rv = 1; return rv; } @@ -4745,7 +4744,8 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) */ rdev = conf->disks[i].replacement; if (rdev && !test_bit(Faulty, &rdev->flags) && - rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) && + READ_ONCE(rdev->recovery_offset) >= + sh->sector + RAID5_STRIPE_SECTORS(conf) && !rdev_has_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf))) set_bit(R5_ReadRepl, &dev->flags); @@ -4787,7 +4787,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) } else if (test_bit(In_sync, &rdev->flags)) set_bit(R5_Insync, &dev->flags); else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= - rdev->recovery_offset) { + READ_ONCE(rdev->recovery_offset)) { /* * in sync if: * - normal IO, or @@ -5533,13 +5533,13 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio) rdev = conf->disks[dd_idx].replacement; if (!rdev || test_bit(Faulty, &rdev->flags) || - rdev->recovery_offset < end_sector) { + READ_ONCE(rdev->recovery_offset) < end_sector) { rdev = conf->disks[dd_idx].rdev; if (!rdev) return 0; if (test_bit(Faulty, &rdev->flags) || !(test_bit(In_sync, &rdev->flags) || - rdev->recovery_offset >= end_sector)) + READ_ONCE(rdev->recovery_offset) >= end_sector)) return 0; } @@ -6502,8 +6502,8 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); @@ -6611,8 +6611,8 @@ finish: if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && - rdev->recovery_offset < sector_nr) - rdev->recovery_offset = sector_nr; + READ_ONCE(rdev->recovery_offset) < sector_nr) + WRITE_ONCE(rdev->recovery_offset, sector_nr); conf->reshape_checkpoint = jiffies; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); md_wakeup_thread(mddev->thread); @@ -8133,9 +8133,9 @@ static int raid5_run(struct mddev *mddev) /* Hack because v0.91 doesn't store recovery_offset properly. */ if (mddev->major_version == 0 && mddev->minor_version > 90) - rdev->recovery_offset = reshape_offset; + WRITE_ONCE(rdev->recovery_offset, reshape_offset); - if (rdev->recovery_offset < reshape_offset) { + if (READ_ONCE(rdev->recovery_offset) < reshape_offset) { /* We need to check old and new layout */ if (!only_parity(rdev->raid_disk, conf->algorithm, @@ -8290,10 +8290,10 @@ static int raid5_spare_active(struct mddev *mddev) for (i = 0; i < conf->raid_disks; i++) { rdev = conf->disks[i].rdev; replacement = conf->disks[i].replacement; - if (replacement - && replacement->recovery_offset == MaxSector - && !test_bit(Faulty, &replacement->flags) - && !test_and_set_bit(In_sync, &replacement->flags)) { + if (replacement && + READ_ONCE(replacement->recovery_offset) == MaxSector && + !test_bit(Faulty, &replacement->flags) && + !test_and_set_bit(In_sync, &replacement->flags)) { /* Replacement has just become active. */ if (!rdev || !test_and_clear_bit(In_sync, &rdev->flags)) @@ -8308,10 +8308,10 @@ static int raid5_spare_active(struct mddev *mddev) rdev->sysfs_state); } sysfs_notify_dirent_safe(replacement->sysfs_state); - } else if (rdev - && rdev->recovery_offset == MaxSector - && !test_bit(Faulty, &rdev->flags) - && !test_and_set_bit(In_sync, &rdev->flags)) { + } else if (rdev && + READ_ONCE(rdev->recovery_offset) == MaxSector && + !test_bit(Faulty, &rdev->flags) && + !test_and_set_bit(In_sync, &rdev->flags)) { count++; sysfs_notify_dirent_safe(rdev->sysfs_state); } @@ -8680,7 +8680,7 @@ static int raid5_start_reshape(struct mddev *mddev) >= conf->previous_raid_disks) set_bit(In_sync, &rdev->flags); else - rdev->recovery_offset = 0; + WRITE_ONCE(rdev->recovery_offset, 0); /* Failure here is OK */ sysfs_link_rdev(mddev, rdev); @@ -8732,7 +8732,7 @@ static void end_reshape(struct r5conf *conf) if (rdev->raid_disk >= 0 && !test_bit(Journal, &rdev->flags) && !test_bit(In_sync, &rdev->flags)) - rdev->recovery_offset = MaxSector; + WRITE_ONCE(rdev->recovery_offset, MaxSector); spin_unlock_irq(&conf->device_lock); wake_up(&conf->wait_for_reshape); From 3fe5b7c9fb72ccc29bfd0f955b124892af7e3674 Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Sun, 28 Jun 2026 14:27:37 +0000 Subject: [PATCH 059/241] md: remove REQ_NOWAIT support from raid1/10/456 REQ_NOWAIT support in md personalities that can block internally is fundamentally incomplete. While reads can avoid some blocking paths, write requests can still encounter cases where one mirror succeeds while another returns -EAGAIN. At that point md cannot distinguish queue pressure from a real device failure, so it can neither record a bad block nor safely retry the write without REQ_NOWAIT, leaving mirrors with divergent data. Rather than continue advertising REQ_NOWAIT support for personalities that cannot implement it correctly, remove it from raid1, raid10 and raid456. Keep REQ_NOWAIT for linear and raid0, which only remap bios to their underlying devices; stacked limits will still clear the feature if any component device lacks REQ_NOWAIT support. Fixes: bf2c411bb1cf ("md: raid456 add nowait support") Fixes: c9aa889b035f ("md: raid10 add nowait support") Fixes: 5aa705039c4f ("md: raid1 add nowait support") Fixes: f51d46d0e7cb ("md: add support for REQ_NOWAIT") Suggested-by: Yu Kuai Signed-off-by: Abd-Alrhman Masalkhi Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260628142737.1051059-1-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/md-bitmap.c | 9 +--- drivers/md/md-bitmap.h | 2 +- drivers/md/md-linear.c | 1 + drivers/md/md-llbitmap.c | 10 +--- drivers/md/md.c | 6 +-- drivers/md/raid0.c | 1 + drivers/md/raid1-10.c | 8 ++-- drivers/md/raid1.c | 97 +++++++++------------------------------ drivers/md/raid10.c | 98 +++++++++++----------------------------- drivers/md/raid5.c | 13 ------ 10 files changed, 60 insertions(+), 185 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 0f02e2956398..7d778fe1c47c 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2064,23 +2064,18 @@ static void bitmap_end_behind_write(struct mddev *mddev) bitmap->mddev->bitmap_info.max_write_behind); } -static bool bitmap_wait_behind_writes(struct mddev *mddev, bool nowait) +static void bitmap_wait_behind_writes(struct mddev *mddev) { struct bitmap *bitmap = mddev->bitmap; /* wait for behind writes to complete */ if (bitmap && atomic_read(&bitmap->behind_writes) > 0) { - if (nowait) - return false; - pr_debug("md:%s: behind writes in progress - waiting to stop.\n", mdname(mddev)); /* need to kick something here to make sure I/O goes? */ wait_event(bitmap->behind_wait, atomic_read(&bitmap->behind_writes) == 0); } - - return true; } static void bitmap_destroy(struct mddev *mddev) @@ -2090,7 +2085,7 @@ static void bitmap_destroy(struct mddev *mddev) if (!bitmap) /* there was no bitmap */ return; - bitmap_wait_behind_writes(mddev, false); + bitmap_wait_behind_writes(mddev); if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags)) mddev_destroy_serial_pool(mddev, NULL); diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index f46674bdfeb9..214f623c7e79 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -98,7 +98,7 @@ struct bitmap_operations { void (*start_behind_write)(struct mddev *mddev); void (*end_behind_write)(struct mddev *mddev); - bool (*wait_behind_writes)(struct mddev *mddev, bool nowait); + void (*wait_behind_writes)(struct mddev *mddev); md_bitmap_fn *start_write; md_bitmap_fn *end_write; diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c index fdff250d0d51..73b367b61b87 100644 --- a/drivers/md/md-linear.c +++ b/drivers/md/md-linear.c @@ -71,6 +71,7 @@ static int linear_set_limits(struct mddev *mddev) int err; md_init_stacking_limits(&lim); + lim.features |= BLK_FEAT_NOWAIT; lim.max_hw_sectors = mddev->chunk_sectors; lim.logical_block_size = mddev->logical_block_size; lim.max_write_zeroes_sectors = mddev->chunk_sectors; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 5a4e2abaa757..2a2b38c663c3 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1574,19 +1574,13 @@ static void llbitmap_end_behind_write(struct mddev *mddev) wake_up(&llbitmap->behind_wait); } -static bool llbitmap_wait_behind_writes(struct mddev *mddev, bool nowait) +static void llbitmap_wait_behind_writes(struct mddev *mddev) { struct llbitmap *llbitmap = mddev->bitmap; - if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) { - if (nowait) - return false; - + if (llbitmap && atomic_read(&llbitmap->behind_writes) > 0) wait_event(llbitmap->behind_wait, atomic_read(&llbitmap->behind_writes) == 0); - } - - return true; } static ssize_t bits_show(struct mddev *mddev, char *page) diff --git a/drivers/md/md.c b/drivers/md/md.c index d1465bcd86c8..997c26568b9e 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -6283,7 +6283,7 @@ void md_init_stacking_limits(struct queue_limits *lim) { blk_set_stacking_limits(lim); lim->features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | - BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT; + BLK_FEAT_IO_STAT; } EXPORT_SYMBOL_GPL(md_init_stacking_limits); @@ -6631,7 +6631,6 @@ int md_run(struct mddev *mddev) int err; struct md_rdev *rdev; struct md_personality *pers; - bool nowait = true; if (list_empty(&mddev->disks)) /* cannot run an array with no devices.. */ @@ -6702,7 +6701,6 @@ int md_run(struct mddev *mddev) } } sysfs_notify_dirent_safe(rdev->sysfs_state); - nowait = nowait && bdev_nowait(rdev->bdev); } pers = get_pers(mddev->level, mddev->clevel); @@ -7050,7 +7048,7 @@ EXPORT_SYMBOL_GPL(md_stop_writes); static void mddev_detach(struct mddev *mddev) { if (md_bitmap_enabled(mddev, false)) - mddev->bitmap_ops->wait_behind_writes(mddev, false); + mddev->bitmap_ops->wait_behind_writes(mddev); if (mddev->pers && mddev->pers->quiesce && !is_md_suspended(mddev)) { mddev->pers->quiesce(mddev, 1); mddev->pers->quiesce(mddev, 0); diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index 2c000b3a5f49..35e103f0c2c3 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -385,6 +385,7 @@ static int raid0_set_limits(struct mddev *mddev) int err; md_init_stacking_limits(&lim); + lim.features |= BLK_FEAT_NOWAIT; lim.max_hw_sectors = mddev->chunk_sectors; lim.max_write_zeroes_sectors = mddev->chunk_sectors; lim.max_hw_wzeroes_unmap_sectors = mddev->chunk_sectors; diff --git a/drivers/md/raid1-10.c b/drivers/md/raid1-10.c index 56a56a4da4f8..3b0e230692ba 100644 --- a/drivers/md/raid1-10.c +++ b/drivers/md/raid1-10.c @@ -290,9 +290,8 @@ static inline bool raid1_should_read_first(struct mddev *mddev, } /* - * bio with REQ_RAHEAD or REQ_NOWAIT can fail at anytime, before such IO is - * submitted to the underlying disks, hence don't record badblocks or retry - * in this case. + * bio with REQ_RAHEAD can fail at anytime, before such IO is submitted to the + * underlying disks, hence don't record badblocks or retry in this case. * * BLK_STS_INVAL means the bio was not valid for the underlying device. This * is a user error, not a device failure, so retrying or recording bad blocks @@ -300,6 +299,5 @@ static inline bool raid1_should_read_first(struct mddev *mddev, */ static inline bool raid1_should_handle_error(struct bio *bio) { - return !(bio->bi_opf & (REQ_RAHEAD | REQ_NOWAIT)) && - bio->bi_status != BLK_STS_INVAL; + return !(bio->bi_opf & REQ_RAHEAD) && bio->bi_status != BLK_STS_INVAL; } diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 4d7f33a2faaa..4dfd95f28e7f 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1053,10 +1053,8 @@ static void lower_barrier(struct r1conf *conf, sector_t sector_nr) wake_up(&conf->wait_barrier); } -static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) +static void _wait_barrier(struct r1conf *conf, int idx) { - bool ret = true; - /* * We need to increase conf->nr_pending[idx] very early here, * then raise_barrier() can be blocked when it waits for @@ -1087,7 +1085,7 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) */ if (!READ_ONCE(conf->array_frozen) && !atomic_read(&conf->barrier[idx])) - return ret; + return; /* * After holding conf->resync_lock, conf->nr_pending[idx] @@ -1106,26 +1104,18 @@ static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) wake_up_barrier(conf); /* Wait for the barrier in same barrier unit bucket to drop. */ - /* Return false when nowait flag is set */ - if (nowait) { - ret = false; - } else { - wait_event_lock_irq(conf->wait_barrier, - !conf->array_frozen && - !atomic_read(&conf->barrier[idx]), - conf->resync_lock); - atomic_inc(&conf->nr_pending[idx]); - } + wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen && + !atomic_read(&conf->barrier[idx]), + conf->resync_lock); + atomic_inc(&conf->nr_pending[idx]); atomic_dec(&conf->nr_waiting[idx]); spin_unlock_irq(&conf->resync_lock); - return ret; } -static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) +static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr) { int idx = sector_to_idx(sector_nr); - bool ret = true; /* * Very similar to _wait_barrier(). The difference is, for read @@ -1137,7 +1127,7 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa atomic_inc(&conf->nr_pending[idx]); if (!READ_ONCE(conf->array_frozen)) - return ret; + return; spin_lock_irq(&conf->resync_lock); atomic_inc(&conf->nr_waiting[idx]); @@ -1149,27 +1139,19 @@ static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowa wake_up_barrier(conf); /* Wait for array to be unfrozen */ - /* Return false when nowait flag is set */ - if (nowait) { - /* Return false when nowait flag is set */ - ret = false; - } else { - wait_event_lock_irq(conf->wait_barrier, - !conf->array_frozen, - conf->resync_lock); - atomic_inc(&conf->nr_pending[idx]); - } + wait_event_lock_irq(conf->wait_barrier, !conf->array_frozen, + conf->resync_lock); + atomic_inc(&conf->nr_pending[idx]); atomic_dec(&conf->nr_waiting[idx]); spin_unlock_irq(&conf->resync_lock); - return ret; } -static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) +static void wait_barrier(struct r1conf *conf, sector_t sector_nr) { int idx = sector_to_idx(sector_nr); - return _wait_barrier(conf, idx, nowait); + _wait_barrier(conf, idx); } static void _allow_barrier(struct r1conf *conf, int idx) @@ -1344,7 +1326,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, int max_sectors; int rdisk; bool r1bio_existed = !!r1_bio; - bool nowait = bio->bi_opf & REQ_NOWAIT; /* * An md cloned bio indicates we are in the error path. @@ -1364,16 +1345,7 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, * Still need barrier for READ in case that whole * array is frozen. */ - if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, nowait)) { - bio_wouldblock_error(bio); - - if (r1bio_existed) { - set_bit(R1BIO_Returned, &r1_bio->state); - raid_end_bio_io(r1_bio); - } - - return; - } + wait_read_barrier(conf, bio->bi_iter.bi_sector); if (!r1_bio) r1_bio = alloc_r1bio(mddev, bio); @@ -1408,14 +1380,10 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, md_bitmap_enabled(mddev, false)) { /* * Reading from a write-mostly device must take care not to - * over-take any writes that are 'behind' - */ - mddev_add_trace_msg(mddev, "raid1 wait behind writes"); - if (!mddev->bitmap_ops->wait_behind_writes(mddev, nowait)) { - bio_wouldblock_error(bio); - set_bit(R1BIO_Returned, &r1_bio->state); - goto err_handle; - } + * over-take any writes that are 'behind' + */ + mddev_add_trace_msg(mddev, "raid1 wait behind writes"); + mddev->bitmap_ops->wait_behind_writes(mddev); } if (max_sectors < bio_sectors(bio)) { @@ -1437,7 +1405,6 @@ static void raid1_read_request(struct mddev *mddev, struct bio *bio, } read_bio = bio_alloc_clone(mirror->rdev->bdev, bio, gfp, &mddev->bio_set); - read_bio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[rdisk] = read_bio; read_bio->bi_iter.bi_sector = r1_bio->sector + @@ -1456,7 +1423,7 @@ err_handle: raid_end_bio_io(r1_bio); } -static bool wait_blocked_rdev(struct mddev *mddev, struct bio *bio) +static void wait_blocked_rdev(struct mddev *mddev, struct bio *bio) { struct r1conf *conf = mddev->private; int disks = conf->raid_disks * 2; @@ -1476,9 +1443,6 @@ retry: set_bit(BlockedBadBlocks, &rdev->flags); if (rdev_blocked(rdev)) { - if (bio->bi_opf & REQ_NOWAIT) - return false; - mddev_add_trace_msg(rdev->mddev, "raid1 wait rdev %d blocked", rdev->raid_disk); atomic_inc(&rdev->nr_pending); @@ -1486,8 +1450,6 @@ retry: goto retry; } } - - return true; } static void raid1_start_write_behind(struct mddev *mddev, struct r1bio *r1_bio, @@ -1523,18 +1485,12 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, unsigned long flags; int first_clone; bool write_behind = false; - bool nowait = bio->bi_opf & REQ_NOWAIT; bool is_discard = op_is_discard(bio->bi_opf); sector_t sector = bio->bi_iter.bi_sector; if (mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, bio_end_sector(bio))) { - - if (nowait) { - bio_wouldblock_error(bio); - return false; - } wait_event_idle(conf->wait_barrier, !mddev->cluster_ops->area_resyncing(mddev, WRITE, sector, @@ -1546,15 +1502,9 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, * thread has put up a bar for new requests. * Continue immediately if no resync is active currently. */ - if (!wait_barrier(conf, sector, nowait)) { - bio_wouldblock_error(bio); - return false; - } + wait_barrier(conf, sector); - if (!wait_blocked_rdev(mddev, bio)) { - bio_wouldblock_error(bio); - goto err_allow_barrier; - } + wait_blocked_rdev(mddev, bio); r1_bio = alloc_r1bio(mddev, bio); r1_bio->sectors = max_sectors; @@ -1683,7 +1633,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, wait_for_serialization(rdev, r1_bio); } - mbio->bi_opf &= ~REQ_NOWAIT; r1_bio->bios[i] = mbio; mbio->bi_iter.bi_sector = sector + rdev->data_offset; @@ -1722,8 +1671,6 @@ err_dec_pending: } free_r1bio(r1_bio); - -err_allow_barrier: allow_barrier(conf, sector); return false; @@ -1852,7 +1799,7 @@ static void close_sync(struct r1conf *conf) int idx; for (idx = 0; idx < BARRIER_BUCKETS_NR; idx++) { - _wait_barrier(conf, idx, false); + _wait_barrier(conf, idx); _allow_barrier(conf, idx); } diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 0a3cfdd3f5df..4b702e832f06 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1002,32 +1002,22 @@ static bool wait_barrier_nolock(struct r10conf *conf) return false; } -static bool wait_barrier(struct r10conf *conf, bool nowait) +static void wait_barrier(struct r10conf *conf) { - bool ret = true; - if (wait_barrier_nolock(conf)) - return true; + return; write_seqlock_irq(&conf->resync_lock); if (conf->barrier) { - /* Return false when nowait flag is set */ - if (nowait) { - ret = false; - } else { - conf->nr_waiting++; - mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); - wait_event_barrier(conf, stop_waiting_barrier(conf)); - conf->nr_waiting--; - } + conf->nr_waiting++; + mddev_add_trace_msg(conf->mddev, "raid10 wait barrier"); + wait_event_barrier(conf, stop_waiting_barrier(conf)); + conf->nr_waiting--; if (!conf->nr_waiting) wake_up(&conf->wait_barrier); } - /* Only increment nr_pending when we wait */ - if (ret) - atomic_inc(&conf->nr_pending); + atomic_inc(&conf->nr_pending); write_sequnlock_irq(&conf->resync_lock); - return ret; } static void allow_barrier(struct r10conf *conf) @@ -1119,30 +1109,22 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) * currently. * 2. If IO spans the reshape position. Need to wait for reshape to pass. */ -static bool regular_request_wait(struct mddev *mddev, struct r10conf *conf, +static void regular_request_wait(struct mddev *mddev, struct r10conf *conf, struct bio *bio, sector_t sectors) { - /* Bail out if REQ_NOWAIT is set for the bio */ - if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { - bio_wouldblock_error(bio); - return false; - } + wait_barrier(conf); + while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && bio->bi_iter.bi_sector < conf->reshape_progress && bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { allow_barrier(conf); - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return false; - } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape"); wait_event(conf->wait_barrier, conf->reshape_progress <= bio->bi_iter.bi_sector || conf->reshape_progress >= bio->bi_iter.bi_sector + sectors); - wait_barrier(conf, false); + wait_barrier(conf); } - return true; } static void raid10_read_request(struct mddev *mddev, struct bio *bio, @@ -1191,10 +1173,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, } } - if (!regular_request_wait(mddev, conf, bio, r10_bio->sectors)) { - free_r10bio(r10_bio); - return; - } + regular_request_wait(mddev, conf, bio, r10_bio->sectors); rdev = read_balance(conf, r10_bio, &max_sectors); if (!rdev) { @@ -1215,7 +1194,7 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, allow_barrier(conf); bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); - wait_barrier(conf, false); + wait_barrier(conf); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; @@ -1231,7 +1210,6 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, r10_bio->master_bio = bio; } read_bio = bio_alloc_clone(rdev->bdev, bio, gfp, &mddev->bio_set); - read_bio->bi_opf &= ~REQ_NOWAIT; r10_bio->devs[slot].bio = read_bio; r10_bio->devs[slot].rdev = rdev; @@ -1265,7 +1243,6 @@ static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio, conf->mirrors[devnum].rdev; mbio = bio_alloc_clone(rdev->bdev, bio, GFP_NOIO, &mddev->bio_set); - mbio->bi_opf &= ~REQ_NOWAIT; if (replacement) r10_bio->devs[n_copy].repl_bio = mbio; else @@ -1344,7 +1321,7 @@ retry_wait: "raid10 %s wait rdev %d blocked", __func__, blocked_rdev->raid_disk); md_wait_for_blocked_rdev(blocked_rdev, mddev); - wait_barrier(conf, false); + wait_barrier(conf); goto retry_wait; } } @@ -1361,28 +1338,14 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, mddev->cluster_ops->area_resyncing(mddev, WRITE, bio->bi_iter.bi_sector, bio_end_sector(bio)))) { - DEFINE_WAIT(w); - /* Bail out if REQ_NOWAIT is set for the bio */ - if (bio->bi_opf & REQ_NOWAIT) { - bio_wouldblock_error(bio); - return false; - } - for (;;) { - prepare_to_wait(&conf->wait_barrier, - &w, TASK_IDLE); - if (!mddev->cluster_ops->area_resyncing(mddev, WRITE, - bio->bi_iter.bi_sector, bio_end_sector(bio))) - break; - schedule(); - } - finish_wait(&conf->wait_barrier, &w); + wait_event_idle(conf->wait_barrier, + !mddev->cluster_ops->area_resyncing(mddev, WRITE, + bio->bi_iter.bi_sector, + bio_end_sector(bio))); } sectors = r10_bio->sectors; - if (!regular_request_wait(mddev, conf, bio, sectors)) { - free_r10bio(r10_bio); - return false; - } + regular_request_wait(mddev, conf, bio, sectors); if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && (mddev->reshape_backwards @@ -1395,11 +1358,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, set_mask_bits(&mddev->sb_flags, 0, BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); md_wakeup_thread(mddev->thread); - if (bio->bi_opf & REQ_NOWAIT) { - allow_barrier(conf); - bio_wouldblock_error(bio); - return false; - } mddev_add_trace_msg(conf->mddev, "raid10 wait reshape metadata"); wait_event(mddev->sb_wait, @@ -1494,7 +1452,7 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, allow_barrier(conf); bio = bio_submit_split_bioset(bio, r10_bio->sectors, &conf->bio_split); - wait_barrier(conf, false); + wait_barrier(conf); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; @@ -1637,11 +1595,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) return -EAGAIN; - if (!wait_barrier(conf, bio->bi_opf & REQ_NOWAIT)) { - bio_wouldblock_error(bio); - md_write_end(mddev); - return 0; - } + wait_barrier(conf); /* * Check reshape again to avoid reshape happens after checking @@ -1692,7 +1646,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) allow_barrier(conf); /* Resend the fist split part */ submit_bio_noacct(split); - wait_barrier(conf, false); + wait_barrier(conf); } div_u64_rem(bio_end, stripe_size, &remainder); if (remainder) { @@ -1712,7 +1666,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) /* Resend the second split part */ submit_bio_noacct(bio); bio = split; - wait_barrier(conf, false); + wait_barrier(conf); } bio_start = bio->bi_iter.bi_sector; @@ -1870,7 +1824,7 @@ retry_discard: end_disk_offset += geo->stride; atomic_inc(&first_r10bio->remaining); raid_end_discard_bio(r10_bio); - wait_barrier(conf, false); + wait_barrier(conf); goto retry_discard; } @@ -2069,7 +2023,7 @@ static void print_conf(struct r10conf *conf) static void close_sync(struct r10conf *conf) { - wait_barrier(conf, false); + wait_barrier(conf); allow_barrier(conf); mempool_exit(&conf->r10buf_pool); @@ -4702,7 +4656,7 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, if (need_flush || time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { /* Need to update reshape_position in metadata */ - wait_barrier(conf, false); + wait_barrier(conf); mddev->reshape_position = conf->reshape_progress; if (mddev->reshape_backwards) mddev->curr_resync_completed = raid10_size(mddev, 0, 0) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 4f967574bb1f..552624bbec91 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -5793,10 +5793,6 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) struct bio *orig_bi = bi; int stripe_sectors; - /* We need to handle this when io_uring supports discard/trim */ - if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT)) - return; - if (mddev->reshape_position != MaxSector) /* Skip discard while reshape is happening */ return; @@ -6266,15 +6262,6 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) pr_debug("raid456: %s, logical %llu to %llu\n", __func__, bi->bi_iter.bi_sector, ctx->last_sector); - /* Bail out if conflicts with reshape and REQ_NOWAIT is set */ - if ((bi->bi_opf & REQ_NOWAIT) && - get_reshape_loc(mddev, conf, logical_sector) == LOC_INSIDE_RESHAPE) { - bio_wouldblock_error(bi); - if (rw == WRITE) - md_write_end(mddev); - mempool_free(ctx, conf->ctx_pool); - return true; - } md_account_bio(mddev, &bi); /* From c7d34d17ea43ebc86b45d439ebb435e11ca44bca Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Wed, 8 Jul 2026 11:20:03 +0000 Subject: [PATCH 060/241] md: recheck spare changes before starting sync remove_spares() and remove_and_add_spares() modify the array's rdev configuration. These operations are only safe after the array has been suspended. md_start_sync() checks whether spare configuration changes are needed before taking reconfig_mutex. However, the rdev state can change before the mutex is acquired, so the initial check can become stale. In that case, md_choose_sync_action() may remove or replace rdevs while normal I/O is still accessing them. The race can occur as follows: raid10d Worker Normal IO ____________ _______________________ ______________________ raid10_write_request() wait_blocked_dev() set Blocked set Faulty Skip Faulty rdev rrdev->nr_pending++ .repl_bio = bio removeable_rdev = false . array not suspended . lock mddev goto err_handle lock mddev (wait) . update sb . clear Blocked . . unlock mddev . lock mddev (acquires) remove_spares() removeable_rdev = true raid10_remove_disk() rdev = replacement replacement = NULL rdev_dec_pending(NULL) unlock mddev (NULL)->nr_pending-- In this case, rdev_dec_pending() is called with a NULL pointer, resulting in a NULL pointer dereference when attempting to decrement nr_pending. Fix this by suspending the array when spare configuration changes are needed, including for non-read-write arrays, and checking again after taking reconfig_mutex. If the array was not already suspended and a change is now needed, release the mutex, suspend the array, and reacquire the mutex before continuing. Fixes: bc08041b32ab ("md: suspend array in md_start_sync() if array need reconfiguration") Reported-by: sashiko-bot Closes: https://sashiko.dev/#/patchset/20260628142420.1051027-1-abd.masalkhi@gmail.com?part=3 Signed-off-by: Abd-Alrhman Masalkhi Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260708112003.474537-1-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/md.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 997c26568b9e..dc848a24f592 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -10179,13 +10179,25 @@ static void md_start_sync(struct work_struct *ws) * If reshape is still in progress, spares won't be added or removed * from conf until reshape is done. */ - if (mddev->reshape_position == MaxSector && + if ((mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && md_spares_need_change(mddev)) { suspend = true; mddev_suspend(mddev, false); } mddev_lock_nointr(mddev); + + /* + * The spare configuration can change before reconfig_mutex is acquired. + * Recheck while holding the lock and suspend if needed. + */ + if (!suspend && (mddev->reshape_position == MaxSector || !md_is_rdwr(mddev)) && + md_spares_need_change(mddev)) { + mddev_unlock(mddev); + mddev_suspend_and_lock_nointr(mddev); + suspend = true; + } + if (!md_is_rdwr(mddev)) { /* * On a read-only array we can: From 86d801e895b853667a886918998e1628fcc3174e Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Fri, 10 Jul 2026 10:15:16 +0000 Subject: [PATCH 061/241] md/raid1: restrict atomic write limits and handle runtime constraints Restrict the RAID1 atomic write limits by setting chunk_sectors to BARRIER_UNIT_SECTOR_SIZE so that atomic writes never straddle a barrier unit. A bio that passes block-layer validation may still become unserviceable within RAID1 due to bad blocks or write-behind constraints. In the former case, complete the bio with EIO. In the latter case, disable write-behind rather than failing the bio with EIO. Fixes: f2a38abf5f1c ("md/raid1: Atomic write support") Fixes: a4c55c902670 ("md/raid1: simplify raid1_write_request() error handling") Reviewed-by: John Garry Signed-off-by: Abd-Alrhman Masalkhi Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260710101521.1714-3-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid1.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 4dfd95f28e7f..e9baba7b241f 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1485,6 +1485,7 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, unsigned long flags; int first_clone; bool write_behind = false; + bool atomic = bio->bi_opf & REQ_ATOMIC; bool is_discard = op_is_discard(bio->bi_opf); sector_t sector = bio->bi_iter.bi_sector; @@ -1531,6 +1532,8 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, */ if (!is_discard && rdev && test_bit(WriteMostly, &rdev->flags)) write_behind = true; + if (atomic && max_sectors > BIO_MAX_VECS * (PAGE_SIZE >> 9)) + write_behind = false; r1_bio->bios[i] = NULL; if (!rdev || test_bit(Faulty, &rdev->flags)) @@ -1556,19 +1559,6 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) { - bio->bi_status = BLK_STS_NOTSUPP; - bio_endio(bio); - goto err_dec_pending; - } - good_sectors = first_bad - sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1589,6 +1579,11 @@ static bool raid1_write_request(struct mddev *mddev, struct bio *bio, max_sectors = min_t(int, max_sectors, BIO_MAX_VECS * (PAGE_SIZE >> 9)); if (max_sectors < bio_sectors(bio)) { + if (atomic) { + bio_io_error(bio); + goto err_dec_pending; + } + bio = bio_submit_split_bioset(bio, max_sectors, &conf->bio_split); if (!bio) @@ -3177,6 +3172,7 @@ static int raid1_set_limits(struct mddev *mddev) md_init_stacking_limits(&lim); lim.max_write_zeroes_sectors = 0; lim.max_hw_wzeroes_unmap_sectors = 0; + lim.chunk_sectors = BARRIER_UNIT_SECTOR_SIZE; lim.logical_block_size = mddev->logical_block_size; lim.features |= BLK_FEAT_ATOMIC_WRITES; lim.features |= BLK_FEAT_PCI_P2PDMA; From 3409bf2f9678d769a4c33bd232a3571c51fac481 Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Fri, 10 Jul 2026 10:15:17 +0000 Subject: [PATCH 062/241] md/raid10: consistently fail atomic writes that require splitting RAID10 currently handles one badblock path explicitly by failing atomic writes with EIO. However, another badblock path can also reduce the writable range and force the bio through bio_submit_split_bioset(), which implicitly completes the bio with EINVAL. Fix this by handling atomic writes in the common split check. If RAID10 determines that an atomic write would require splitting, complete the bio with EIO. Fixes: a1d9b4fd42d9 ("md/raid10: Atomic write support") Signed-off-by: Abd-Alrhman Masalkhi Reviewed-by: Yu Kuai Reviewed-by: John Garry Link: https://patch.msgid.link/20260710101521.1714-4-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 4b702e832f06..d08a4ad76115 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1333,6 +1333,7 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, int i, k; sector_t sectors; int max_sectors; + bool atomic = bio->bi_opf & REQ_ATOMIC; if ((mddev_is_clustered(mddev) && mddev->cluster_ops->area_resyncing(mddev, WRITE, @@ -1420,16 +1421,6 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, if (is_bad) { int good_sectors; - /* - * We cannot atomically write this, so just - * error in that case. It could be possible to - * atomically write other mirrors, but the - * complexity of supporting that is not worth - * the benefit. - */ - if (bio->bi_opf & REQ_ATOMIC) - goto err_handle; - good_sectors = first_bad - dev_sector; if (good_sectors < max_sectors) max_sectors = good_sectors; @@ -1449,6 +1440,9 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, r10_bio->sectors = max_sectors; if (r10_bio->sectors < bio_sectors(bio)) { + if (atomic) + goto err_handle; + allow_barrier(conf); bio = bio_submit_split_bioset(bio, r10_bio->sectors, &conf->bio_split); From addb977450a662e1961d272b6ebfcb477d115044 Mon Sep 17 00:00:00 2001 From: Abd-Alrhman Masalkhi Date: Fri, 10 Jul 2026 10:15:18 +0000 Subject: [PATCH 063/241] md/raid10: remove unnecessary barrier around bio_submit_split_bioset() raid10_write_request() drops the barrier before calling bio_submit_split_bioset() and reacquires it afterwards. This is no longer necessary because the split bio cannot re-enter raid10_write_request() while the barrier is held. The allow_barrier()/wait_barrier() pair was introduced by commit e820d55cb99d ("md: fix raid10 hang issue caused by barrier") when submit_flushes() called md_handle_request() directly, allowing re-entry into raid10_write_request(). Since v5.2, submit_flushes() has instead gone through submit_bio(), eliminating that recursion. submit_flushes() was later removed entirely by commit b75197e86e6d ("md: Remove flush handling"). Currently, raid10_write_request() is only entered from the bio submission path, so the split bio submitted by bio_submit_split_bioset() cannot recurse back into wait_barrier(). Remove the redundant allow_barrier()/wait_barrier() pair around bio_submit_split_bioset(). Signed-off-by: Abd-Alrhman Masalkhi Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260710101521.1714-5-abd.masalkhi@gmail.com Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index d08a4ad76115..3e45013d9607 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1443,10 +1443,8 @@ static bool raid10_write_request(struct mddev *mddev, struct bio *bio, if (atomic) goto err_handle; - allow_barrier(conf); bio = bio_submit_split_bioset(bio, r10_bio->sectors, &conf->bio_split); - wait_barrier(conf); if (!bio) { set_bit(R10BIO_Returned, &r10_bio->state); goto err_handle; From 6bc3deb600ee8c234204c6f4ea703ee04a11de2f Mon Sep 17 00:00:00 2001 From: Hiroshi Nishida Date: Fri, 10 Jul 2026 06:23:29 -0700 Subject: [PATCH 064/241] md: widen badblock sectors param from int to sector_t The badblocks core API -- badblocks_set(), badblocks_clear() and badblocks_check() -- and the is_badblock() helper all take the range length as sector_t. The md wrappers rdev_set_badblocks(), rdev_clear_badblocks() and rdev_has_badblock(), however, declared the same length as int, narrowing sector_t to int and back again in the middle of an otherwise 64-bit clean path. Change the sectors parameter to sector_t in these three wrappers so it matches the core API and is_badblock(). No functional change: current callers pass per-I/O or per-resync-chunk lengths well within int range. This just removes a gratuitous truncation point and keeps the type consistent end to end. Signed-off-by: Hiroshi Nishida Link: https://patch.msgid.link/20260710132329.7273-3-nishidafmly@gmail.com Signed-off-by: Yu Kuai --- drivers/md/md.c | 4 ++-- drivers/md/md.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index dc848a24f592..c6b9b4705c94 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -10563,7 +10563,7 @@ EXPORT_SYMBOL(md_finish_reshape); /* Bad block management */ /* Returns true on success, false on failure */ -bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors, +bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors, int is_new) { struct mddev *mddev = rdev->mddev; @@ -10603,7 +10603,7 @@ bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors, } EXPORT_SYMBOL_GPL(rdev_set_badblocks); -void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors, +void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors, int is_new) { if (is_new) diff --git a/drivers/md/md.h b/drivers/md/md.h index d8daf0f75cbb..1b47af09c4e2 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -311,7 +311,7 @@ static inline int is_badblock(struct md_rdev *rdev, sector_t s, sector_t sectors } static inline int rdev_has_badblock(struct md_rdev *rdev, sector_t s, - int sectors) + sector_t sectors) { sector_t first_bad; sector_t bad_sectors; @@ -319,9 +319,9 @@ static inline int rdev_has_badblock(struct md_rdev *rdev, sector_t s, return is_badblock(rdev, s, sectors, &first_bad, &bad_sectors); } -extern bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, int sectors, +extern bool rdev_set_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors, int is_new); -extern void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors, +extern void rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, sector_t sectors, int is_new); struct md_cluster_info; struct md_cluster_operations; From 798d79a7e4b04819a8ee575e5e1911215489e2ae Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Sat, 11 Jul 2026 18:03:50 +0800 Subject: [PATCH 065/241] md: suspend array when sync_action=reshape raid10 needs to resize/swap r10bio_pool when reshape changes raid_disks, and, don't let new requests keep allocating r10bio objects from the old pool while that transition is in progress. suspend and lock array before mddev_start_reshape(), and resume it on exit. Other sync_action ops are unchanged. Signed-off-by: Chen Cheng Link: https://patch.msgid.link/20260711100352.425177-2-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/md.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index c6b9b4705c94..addf3aeec85f 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -5263,21 +5263,28 @@ action_store(struct mddev *mddev, const char *page, size_t len) if (!mddev->pers || !mddev->pers->sync_request) return -EINVAL; + action = md_sync_action_by_name(page); + if (action == ACTION_RESHAPE) { + ret = mddev_suspend(mddev, true); + if (ret) + return ret; + } retry: if (work_busy(&mddev->sync_work)) flush_work(&mddev->sync_work); ret = mddev_lock(mddev); - if (ret) + if (ret) { + if (action == ACTION_RESHAPE) + mddev_resume(mddev); return ret; + } if (work_busy(&mddev->sync_work)) { mddev_unlock(mddev); goto retry; } - action = md_sync_action_by_name(page); - /* TODO: mdadm rely on "idle" to start sync_thread. */ if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) { switch (action) { @@ -5347,6 +5354,8 @@ retry: out: mddev_unlock(mddev); + if (action == ACTION_RESHAPE) + mddev_resume(mddev); return ret; } From 8e9171decb5ed5c3fe0430a559f45ca556e94d26 Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Sat, 11 Jul 2026 18:03:51 +0800 Subject: [PATCH 066/241] md/raid10: resize r10bio_pool for reshape When reshape grows raid_disks, the pool must also switch to new geometry object size , and allocate a new geometry size pool and replace the old. But not for shrinking reshape, because regular I/O can still use the prev geo for sectors that have not crossed reshape_progress yet. Signed-off-by: Chen Cheng Link: https://patch.msgid.link/20260711100352.425177-3-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 46 ++++++++++++++++++++++++++++++++------------- drivers/md/raid10.h | 2 +- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 3e45013d9607..3f2da07676e7 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -103,13 +103,23 @@ static inline struct r10bio *get_resync_r10bio(struct bio *bio) return get_resync_pages(bio)->raid_bio; } -static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data) +static inline int calc_r10bio_size(unsigned int raid_disks) { - struct r10conf *conf = data; - int size = offsetof(struct r10bio, devs[conf->geo.raid_disks]); + return offsetof(struct r10bio, devs[raid_disks]); +} - /* allocate a r10bio with room for raid_disks entries in the - * bios array */ +static mempool_t *create_r10bio_pool(unsigned int raid_disks) +{ + int size = calc_r10bio_size(raid_disks); + + return mempool_create_kmalloc_pool(NR_RAID_BIOS, size); +} + +static struct r10bio *alloc_r10bio(unsigned int raid_disks, gfp_t gfp_flags) +{ + int size = calc_r10bio_size(raid_disks); + + /* allocate a r10bio sized for current geometry */ return kzalloc(size, gfp_flags); } @@ -137,7 +147,7 @@ static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data) int nalloc, nalloc_rp; struct resync_pages *rps; - r10_bio = r10bio_pool_alloc(gfp_flags, conf); + r10_bio = alloc_r10bio(conf->geo.raid_disks, gfp_flags); if (!r10_bio) return NULL; @@ -277,7 +287,7 @@ static void free_r10bio(struct r10bio *r10_bio) struct r10conf *conf = r10_bio->mddev->private; put_all_bios(conf, r10_bio); - mempool_free(r10_bio, &conf->r10bio_pool); + mempool_free(r10_bio, conf->r10bio_pool); } static void put_buf(struct r10bio *r10_bio) @@ -1492,7 +1502,7 @@ static bool __make_request(struct mddev *mddev, struct bio *bio, int sectors) struct r10conf *conf = mddev->private; struct r10bio *r10_bio; - r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); + r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); r10_bio->master_bio = bio; r10_bio->sectors = sectors; @@ -1688,7 +1698,7 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio) (last_stripe_index << geo->chunk_shift); retry_discard: - r10_bio = mempool_alloc(&conf->r10bio_pool, GFP_NOIO); + r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO); r10_bio->mddev = mddev; r10_bio->state = 0; r10_bio->sectors = 0; @@ -3790,7 +3800,7 @@ static void raid10_free_conf(struct r10conf *conf) if (!conf) return; - mempool_exit(&conf->r10bio_pool); + mempool_destroy(conf->r10bio_pool); kfree(conf->mirrors); kfree(conf->mirrors_old); kfree(conf->mirrors_new); @@ -3837,9 +3847,8 @@ static struct r10conf *setup_conf(struct mddev *mddev) conf->geo = geo; conf->copies = copies; - err = mempool_init(&conf->r10bio_pool, NR_RAID_BIOS, r10bio_pool_alloc, - rbio_pool_free, conf); - if (err) + conf->r10bio_pool = create_r10bio_pool(conf->geo.raid_disks); + if (!conf->r10bio_pool) goto out; err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0); @@ -4333,6 +4342,7 @@ static int raid10_start_reshape(struct mddev *mddev) struct md_rdev *rdev; int spares = 0; int ret; + mempool_t *new_pool = NULL; if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) return -EBUSY; @@ -4369,6 +4379,11 @@ static int raid10_start_reshape(struct mddev *mddev) return -EINVAL; conf->offset_diff = min_offset_diff; + if (mddev->delta_disks > 0) { + new_pool = create_r10bio_pool(new.raid_disks); + if (!new_pool) + return -ENOMEM; + } spin_lock_irq(&conf->device_lock); if (conf->mirrors_new) { memcpy(conf->mirrors_new, conf->mirrors, @@ -4469,6 +4484,10 @@ out: mddev->raid_disks = conf->geo.raid_disks; mddev->reshape_position = conf->reshape_progress; set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); + if (new_pool) { + mempool_destroy(conf->r10bio_pool); + conf->r10bio_pool = new_pool; + } clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); @@ -4491,6 +4510,7 @@ abort: conf->reshape_safe = MaxSector; mddev->reshape_position = MaxSector; spin_unlock_irq(&conf->device_lock); + mempool_destroy(new_pool); return ret; } diff --git a/drivers/md/raid10.h b/drivers/md/raid10.h index ec79d87fb92f..b711626a5db7 100644 --- a/drivers/md/raid10.h +++ b/drivers/md/raid10.h @@ -87,7 +87,7 @@ struct r10conf { */ wait_queue_head_t wait_barrier; - mempool_t r10bio_pool; + mempool_t *r10bio_pool; mempool_t r10buf_pool; struct page *tmppage; struct bio_set bio_split; From fe8d6b0187469c91d57dbc25ece5b503bfb3bc26 Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Sat, 11 Jul 2026 18:03:52 +0800 Subject: [PATCH 067/241] md/raid10: free r10bio before ending master_bio in raid_end_bio_io() and raid_end_discard_bio() origin flow: bio_endio(master_bio); /* may drop active_io to zero */ allow_barrier(conf); free_r10bio(r10_bio); /* reads conf->geo, returns to pool */ one scenario is: CPU A (softirq, raid_end_bio_io) CPU B (action_store) --> reshape ================================ =============================== bio_endio(master_bio) md_end_clone_io percpu_ref_put -> 0 wait_event wakeup, and, mddev_suspend return raid10_start_reshape: setup_geo(&conf->geo, new) ... mempool_destroy(old_pool) conf->r10bio_pool = new_pool allow_barrier(conf) free_r10bio(r10_bio) put_all_bios: for (i=0; igeo.raid_disks; i++) ==> old obj, new geo, OOB mempool_free(r10_bio, conf->r10bio_pool) ==> old-geometry obj freed into new pool so .. fix by reorder the flow: free_r10bio(r10_bio) bio_endio(master_bio) allow_barrier(conf) raid_end_discard_bio() is exactly the same. Signed-off-by: Chen Cheng Link: https://patch.msgid.link/20260711100352.425177-4-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 3f2da07676e7..ed3c6fbe65f7 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -331,20 +331,24 @@ static void raid_end_bio_io(struct r10bio *r10_bio) { struct bio *bio = r10_bio->master_bio; struct r10conf *conf = r10_bio->mddev->private; + bool returned = true; if (!test_and_set_bit(R10BIO_Returned, &r10_bio->state)) { if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) bio->bi_status = BLK_STS_IOERR; - bio_endio(bio); + returned = false; } + free_r10bio(r10_bio); + + if (!returned) + bio_endio(bio); + /* * Wake up any possible resync thread that waits for the device * to go idle. */ allow_barrier(conf); - - free_r10bio(r10_bio); } /* @@ -1537,9 +1541,11 @@ static void raid_end_discard_bio(struct r10bio *r10bio) free_r10bio(r10bio); r10bio = first_r10bio; } else { + struct bio *master_bio = r10bio->master_bio; + md_write_end(r10bio->mddev); - bio_endio(r10bio->master_bio); free_r10bio(r10bio); + bio_endio(master_bio); break; } } From 75ae18ca942674f9d5b55d7e8a2975485125f715 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 20 Jul 2026 13:10:53 -0700 Subject: [PATCH 068/241] block: use blkdev_iov_iter_get_pages status for errors blkdev_iov_iter_get_pages() can return various error values, including EIO, EFAULT, and ENOMEM. Set the actual reported status so user space can know why an operation failed. Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch Link: https://patch.msgid.link/20260720201057.1862857-2-kbusch@meta.com Signed-off-by: Jens Axboe --- block/fops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/fops.c b/block/fops.c index 15783a6180de..0827bb884d47 100644 --- a/block/fops.c +++ b/block/fops.c @@ -218,7 +218,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, ret = blkdev_iov_iter_get_pages(bio, iter, bdev); if (unlikely(ret)) { - bio_endio_status(bio, BLK_STS_IOERR); + bio_endio_status(bio, errno_to_blk_status(ret)); break; } if (iocb->ki_flags & IOCB_NOWAIT) { From 702a2a9f3dfe066a7481698c858371112f3cb697 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 20 Jul 2026 13:10:54 -0700 Subject: [PATCH 069/241] block: fix dio leak on metadata mapping error A failed integrity mapping holds a dio reference, so we need to go through the full bio ending in case there were previously submitted bio's in the sequence. Fixes: 2729a60bbfb92 ("block: don't silently ignore metadata for sync read/write") Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch Link: https://patch.msgid.link/20260720201057.1862857-3-kbusch@meta.com Signed-off-by: Jens Axboe --- block/fops.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/block/fops.c b/block/fops.c index 0827bb884d47..0098a90a956e 100644 --- a/block/fops.c +++ b/block/fops.c @@ -238,8 +238,10 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, } if (iocb->ki_flags & IOCB_HAS_METADATA) { ret = bio_integrity_map_iter(bio, iocb->private); - if (unlikely(ret)) - goto fail; + if (unlikely(ret)) { + bio_endio_status(bio, errno_to_blk_status(ret)); + break; + } } if (is_read) { From 6c8dec275ccc35e8f86cb9287283d31c5d8e9ab7 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 20 Jul 2026 13:10:55 -0700 Subject: [PATCH 070/241] loop: set dma_alignment from the backing file for direct I/O Direct I/O user pages are forwarded to the backing file unchanged, so the backing's DMA alignment requirement applies to them. Track the backing's dio_mem_align and advertise it as the loop device's dma_alignment so we advertise proper limits and misaligned I/O is rejected here instead of being dispatched to the backend. Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch Link: https://patch.msgid.link/20260720201057.1862857-4-kbusch@meta.com Signed-off-by: Jens Axboe --- drivers/block/loop.c | 46 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 1faecef33009..26d7130c3f55 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -54,6 +54,7 @@ struct loop_device { struct file *lo_backing_file; unsigned int lo_min_dio_size; + unsigned int lo_dio_mem_align; struct block_device *lo_device; gfp_t old_gfp_mask; @@ -447,26 +448,37 @@ static void loop_reread_partitions(struct loop_device *lo) __func__, lo->lo_number, lo->lo_file_name, rc); } -static unsigned int loop_query_min_dio_size(struct loop_device *lo) +static void loop_update_dio_alignment(struct loop_device *lo) { struct file *file = lo->lo_backing_file; struct block_device *sb_bdev = file->f_mapping->host->i_sb->s_bdev; struct kstat st; /* - * Use the minimal dio alignment of the file system if provided. + * Use the dio alignment of the file system if provided. The incomoing + * request's bio_vec is forwarded to the backing file unchanged, so its + * required memory alignment becomes the device's dma_alignment when + * used for direct-io. */ if (!vfs_getattr(&file->f_path, &st, STATX_DIOALIGN, 0) && - (st.result_mask & STATX_DIOALIGN)) - return st.dio_offset_align; + (st.result_mask & STATX_DIOALIGN)) { + lo->lo_min_dio_size = st.dio_offset_align; + lo->lo_dio_mem_align = st.dio_mem_align - 1; + return; + } /* * In a perfect world this wouldn't be needed, but as of Linux 6.13 only * a handful of file systems support the STATX_DIOALIGN flag. */ - if (sb_bdev) - return bdev_logical_block_size(sb_bdev); - return SECTOR_SIZE; + if (sb_bdev) { + lo->lo_min_dio_size = bdev_logical_block_size(sb_bdev); + lo->lo_dio_mem_align = bdev_dma_alignment(sb_bdev); + return; + } + + lo->lo_min_dio_size = SECTOR_SIZE; + lo->lo_dio_mem_align = SECTOR_SIZE - 1; } static inline int is_loop_device(struct file *file) @@ -509,7 +521,7 @@ static void loop_assign_backing_file(struct loop_device *lo, struct file *file) lo->old_gfp_mask & ~(__GFP_IO | __GFP_FS)); if (lo->lo_backing_file->f_flags & O_DIRECT) lo->lo_flags |= LO_FLAGS_DIRECT_IO; - lo->lo_min_dio_size = loop_query_min_dio_size(lo); + loop_update_dio_alignment(lo); } static int loop_check_backing_file(struct file *file) @@ -940,6 +952,19 @@ static unsigned int loop_default_blocksize(struct loop_device *lo) return SECTOR_SIZE; } +static void loop_set_dma_limit(struct loop_device *lo, struct queue_limits *lim) +{ + /* + * Direct I/O forwards the user pages to the backing file unchanged, so + * track the backing's DMA alignment requirement as the mode is toggled. + */ + if (lo->lo_flags & LO_FLAGS_DIRECT_IO) + lim->dma_alignment = max_t(unsigned int, lo->lo_dio_mem_align, + SECTOR_SIZE - 1); + else + lim->dma_alignment = SECTOR_SIZE - 1; +} + static void loop_update_limits(struct loop_device *lo, struct queue_limits *lim, unsigned int bsize) { @@ -961,6 +986,7 @@ static void loop_update_limits(struct loop_device *lo, struct queue_limits *lim, lim->logical_block_size = bsize; lim->physical_block_size = bsize; lim->io_min = bsize; + loop_set_dma_limit(lo, lim); lim->features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_ROTATIONAL); if (file->f_op->fsync && !(lo->lo_flags & LO_FLAGS_READ_ONLY)) lim->features |= BLK_FEAT_WRITE_CACHE; @@ -1412,6 +1438,7 @@ static int loop_set_dio(struct loop_device *lo, unsigned long arg) { bool use_dio = !!arg; unsigned int memflags; + struct queue_limits lim; if (lo->lo_state != Lo_bound) return -ENXIO; @@ -1430,6 +1457,9 @@ static int loop_set_dio(struct loop_device *lo, unsigned long arg) lo->lo_flags |= LO_FLAGS_DIRECT_IO; else lo->lo_flags &= ~LO_FLAGS_DIRECT_IO; + lim = queue_limits_start_update(lo->lo_queue); + loop_set_dma_limit(lo, &lim); + queue_limits_commit_update(lo->lo_queue, &lim); blk_mq_unfreeze_queue(lo->lo_queue, memflags); return 0; } From c5059c1af2bd22bc1435b99d27d800164162cb72 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 20 Jul 2026 13:10:56 -0700 Subject: [PATCH 071/241] zloop: set dma_alignment from the backing files for direct I/O Direct I/O request's use pages handed to the backing files unchanged, so the backing's DMA alignment requirement applies. Track dio_mem_align and advertise it as the device's dma_alignment so we communicate proper limits and misaligned I/O is rejected here instead of reaching the backend. Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch Link: https://patch.msgid.link/20260720201057.1862857-5-kbusch@meta.com Signed-off-by: Jens Axboe --- drivers/block/zloop.c | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 55eeb6aac0ea..f97a20cfdb7c 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -144,6 +144,7 @@ struct zloop_device { unsigned int nr_conv_zones; unsigned int max_open_zones; unsigned int block_size; + unsigned int dio_mem_align; spinlock_t open_zones_lock; struct list_head open_zones_lru_list; @@ -1037,20 +1038,30 @@ static int zloop_get_block_size(struct zloop_device *zlo, struct kstat st; /* - * If the FS block size is lower than or equal to 4K, use that as the - * device block size. Otherwise, fallback to the FS direct IO alignment - * constraint if that is provided, and to the FS underlying device - * physical block size if the direct IO alignment is unknown. + * Use the dio alignment of the file system if provided. The incoming + * request's bio_vec is forwarded to the backing file unchanged, so its + * required memory alignment becomes the device's dma_alignment when + * used for direct-io. + */ + if (!vfs_getattr(&zone->file->f_path, &st, STATX_DIOALIGN, 0) && + (st.result_mask & STATX_DIOALIGN)) { + zlo->block_size = st.dio_offset_align; + zlo->dio_mem_align = st.dio_mem_align - 1; + } else if (sb_bdev) { + zlo->block_size = bdev_physical_block_size(sb_bdev); + zlo->dio_mem_align = bdev_dma_alignment(sb_bdev); + } else { + zlo->block_size = SECTOR_SIZE; + zlo->dio_mem_align = SECTOR_SIZE - 1; + } + + /* + * Prefer the FS block size for the device block size when it is no + * larger than 4K; otherwise keep the direct I/O / physical block size + * selected above. */ if (file_inode(zone->file)->i_sb->s_blocksize <= SZ_4K) zlo->block_size = file_inode(zone->file)->i_sb->s_blocksize; - else if (!vfs_getattr(&zone->file->f_path, &st, STATX_DIOALIGN, 0) && - (st.result_mask & STATX_DIOALIGN)) - zlo->block_size = st.dio_offset_align; - else if (sb_bdev) - zlo->block_size = bdev_physical_block_size(sb_bdev); - else - zlo->block_size = SECTOR_SIZE; if (zlo->zone_capacity & ((zlo->block_size >> SECTOR_SHIFT) - 1)) { pr_err("Zone capacity is not aligned to block size %u\n", @@ -1279,6 +1290,10 @@ static int zloop_ctl_add(struct zloop_options *opts) lim.physical_block_size = zlo->block_size; lim.logical_block_size = zlo->block_size; + /* Direct I/O forwards the request pages to the backing files as-is. */ + if (!opts->buffered_io) + lim.dma_alignment = max_t(unsigned int, zlo->dio_mem_align, + SECTOR_SIZE - 1); if (zlo->zone_append) lim.max_hw_zone_append_sectors = lim.max_hw_sectors; lim.max_open_zones = zlo->max_open_zones; From 85764f475f3b3abd956bf8eaeaa643b367676cf4 Mon Sep 17 00:00:00 2001 From: Genjian Zhang Date: Sun, 12 Jul 2026 00:13:26 +0800 Subject: [PATCH 072/241] md/raid5: complete discard bios while reshape is active make_discard_request() returns without completing the bio when reshape is in progress. Discard callers block in submit_bio_wait() waiting for a completion that never arrives. The caller hangs in uninterruptible sleep, and this does not resolve when reshape finishes. Complete the bio with BLK_STS_AGAIN so userspace can retry after reshape, consistent with the existing policy of not processing discard during reshape. Tested on a loop-backed RAID5 array during mdadm --grow: without this patch, blkdiscard hangs in bio_await() and remains in uninterruptible sleep after md reports "reshape done"; with this patch it returns -EAGAIN instead. Signed-off-by: Genjian Zhang Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260711161326.962336-1-zhanggenjian@126.com Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 552624bbec91..e5348cebf12d 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -5794,8 +5794,7 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) int stripe_sectors; if (mddev->reshape_position != MaxSector) - /* Skip discard while reshape is happening */ - return; + goto complete_again; if (!raid5_discard_limits(mddev, bi)) return; @@ -5882,6 +5881,11 @@ static void make_discard_request(struct mddev *mddev, struct bio *bi) } bio_endio(bi); + return; + +complete_again: + /* Skip discard while reshape is happening */ + bio_endio_status(bi, BLK_STS_AGAIN); } static bool ahead_of_reshape(struct mddev *mddev, sector_t sector, From 2911cd0a0f4366a7e06832bc5f0a7fdcc138e4dc Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Sat, 18 Jul 2026 11:42:36 +0800 Subject: [PATCH 073/241] md/bitmap: resume array on backlog_store() error path backlog_store() suspends the array before checking whether a write-mostly device exists. If no such device exists, the error path only unlocks reconfig_mutex and leaves the array suspended, blocking subsequent I/O. Use mddev_unlock_and_resume() to release both states. Fixes: 58226942ad3d ("md: use new apis to suspend array before mddev_create/destroy_serial_pool") Signed-off-by: Chen Cheng Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260718034236.4119093-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/md-bitmap.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 7d778fe1c47c..9730aab9bcff 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2857,7 +2857,7 @@ backlog_store(struct mddev *mddev, const char *buf, size_t len) if (!has_write_mostly) { pr_warn_ratelimited("%s: can't set backlog, no write mostly device available\n", mdname(mddev)); - mddev_unlock(mddev); + mddev_unlock_and_resume(mddev); return -EINVAL; } From bace2010dd7ac07bc980575afb135c406730a7fe Mon Sep 17 00:00:00 2001 From: Chen Cheng Date: Sat, 18 Jul 2026 16:42:18 +0800 Subject: [PATCH 074/241] md: scope memalloc_noio to allocation critical sections Storing a memalloc_noio_save() token in mddev->noio_flags lets one task save the token and another task restore it. With concurrent suspend sysfs writes, task A can enter PF_MEMALLOC_NOIO, return to userspace still in that scope, and later task B can restore A's saved token. Avoid tying the token lifetime to mddev. Keep mddev_suspend() and mddev_resume() only responsible for array suspension, and enter PF_MEMALLOC_NOIO only in the MD paths that allocate memory after the array has been suspended. Restore the token before resuming the array. A reproducer repeatedly writes suspend_lo and suspend_hi from concurrent workers and checks each worker's /proc/self/stat flags before and after the sysfs write. Link: https://github.com/chencheng-fnnas/reproducer/blob/main/repro-md-noio-token-leak.sh Fixes: 78f57ef9d50a ("md: use memalloc scope APIs in mddev_suspend()/mddev_resume()") Signed-off-by: Chen Cheng Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260718084218.417895-1-chencheng@fnnas.com Signed-off-by: Yu Kuai --- drivers/md/md-bitmap.c | 3 +++ drivers/md/md.c | 53 ++++++++++++++++++++++++++++-------------- drivers/md/md.h | 1 - drivers/md/raid5.c | 14 +++++++---- 4 files changed, 48 insertions(+), 23 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 9730aab9bcff..7e4fbca93ccb 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -2624,10 +2624,12 @@ static ssize_t location_store(struct mddev *mddev, const char *buf, size_t len) { int rv; + unsigned int noio_flags; rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (mddev->recovery || mddev->sync_thread) { @@ -2714,6 +2716,7 @@ location_store(struct mddev *mddev, const char *buf, size_t len) } rv = 0; out: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (rv) return rv; diff --git a/drivers/md/md.c b/drivers/md/md.c index addf3aeec85f..3d6357f8fc04 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -233,23 +233,21 @@ static int rdev_need_serial(struct md_rdev *rdev) void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) { int ret = 0; + unsigned int noio_flags; if (rdev && !rdev_need_serial(rdev) && !test_bit(CollisionCheck, &rdev->flags)) return; + noio_flags = memalloc_noio_save(); if (!rdev) ret = rdevs_init_serial(mddev); else ret = rdev_init_serial(rdev); if (ret) - return; + goto out; if (mddev->serial_info_pool == NULL) { - /* - * already in memalloc noio context by - * mddev_suspend() - */ mddev->serial_info_pool = mempool_create_kmalloc_pool(NR_SERIAL_INFOS, sizeof(struct serial_info)); @@ -258,6 +256,8 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) pr_err("can't alloc memory pool for serialization\n"); } } +out: + memalloc_noio_restore(noio_flags); } /* @@ -516,9 +516,6 @@ int mddev_suspend(struct mddev *mddev, bool interruptible) */ WRITE_ONCE(mddev->suspended, mddev->suspended + 1); - /* restrict memory reclaim I/O during raid array is suspend */ - mddev->noio_flag = memalloc_noio_save(); - mutex_unlock(&mddev->suspend_mutex); return 0; } @@ -535,9 +532,6 @@ static void __mddev_resume(struct mddev *mddev, bool recovery_needed) return; } - /* entred the memalloc scope from mddev_suspend() */ - memalloc_noio_restore(mddev->noio_flag); - percpu_ref_resurrect(&mddev->active_io); wake_up(&mddev->sb_wait); @@ -4047,6 +4041,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) char clevel[16]; ssize_t rv; size_t slen = len; + unsigned int noio_flags; struct md_personality *pers, *oldpers; long level; void *priv, *oldpriv; @@ -4058,6 +4053,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) rv = mddev_suspend_and_lock(mddev); if (rv) return rv; + noio_flags = memalloc_noio_save(); if (mddev->pers == NULL) { memcpy(mddev->clevel, buf, slen); @@ -4233,6 +4229,7 @@ level_store(struct mddev *mddev, const char *buf, size_t len) md_new_event(); rv = len; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return rv; } @@ -4412,6 +4409,7 @@ static ssize_t raid_disks_store(struct mddev *mddev, const char *buf, size_t len) { unsigned int n; + unsigned int noio_flags; int err; err = kstrtouint(buf, 10, &n); @@ -4421,6 +4419,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->pers) { if (n != mddev->raid_disks) err = update_raid_disks(mddev, n); @@ -4444,6 +4443,7 @@ raid_disks_store(struct mddev *mddev, const char *buf, size_t len) } else mddev->raid_disks = n; out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ? err : len; } @@ -4824,6 +4824,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) int minor; dev_t dev; struct md_rdev *rdev; + unsigned int noio_flags; int err; if (!*buf || *e != ':' || !e[1] || e[1] == '\n') @@ -4839,6 +4840,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) err = mddev_suspend_and_lock(mddev); if (err) return err; + noio_flags = memalloc_noio_save(); if (mddev->persistent) { rdev = md_import_device(dev, mddev->major_version, mddev->minor_version); @@ -4857,6 +4859,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) rdev = md_import_device(dev, -1, -1); if (IS_ERR(rdev)) { + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return PTR_ERR(rdev); } @@ -4864,6 +4867,7 @@ new_dev_store(struct mddev *mddev, const char *buf, size_t len) out: if (err) export_rdev(rdev); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); if (!err) md_new_event(); @@ -8331,8 +8335,10 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, unsigned int cmd, unsigned long arg) { int err = 0; + unsigned int noio_flags = 0; void __user *argp = (void __user *)arg; struct mddev *mddev = NULL; + bool suspend; err = md_ioctl_valid(cmd); if (err) @@ -8382,13 +8388,15 @@ static int md_ioctl(struct block_device *bdev, blk_mode_t mode, if (!md_is_rdwr(mddev)) flush_work(&mddev->sync_work); - err = md_ioctl_need_suspend(cmd) ? mddev_suspend_and_lock(mddev) : - mddev_lock(mddev); + suspend = md_ioctl_need_suspend(cmd); + err = suspend ? mddev_suspend_and_lock(mddev) : mddev_lock(mddev); if (err) { pr_debug("md: ioctl lock interrupted, reason %d, cmd %d\n", err, cmd); goto out; } + if (suspend) + noio_flags = memalloc_noio_save(); if (cmd == SET_ARRAY_INFO) { err = __md_set_array_info(mddev, argp); @@ -8513,8 +8521,12 @@ unlock: err != -EINVAL) mddev->hold_active = 0; - md_ioctl_need_suspend(cmd) ? mddev_unlock_and_resume(mddev) : - mddev_unlock(mddev); + if (suspend) { + memalloc_noio_restore(noio_flags); + mddev_unlock_and_resume(mddev); + } else { + mddev_unlock(mddev); + } out: if (cmd == STOP_ARRAY_RO || (err && cmd == STOP_ARRAY)) @@ -10182,6 +10194,7 @@ static void md_start_sync(struct work_struct *ws) struct mddev *mddev = container_of(ws, struct mddev, sync_work); int spares = 0; bool suspend = false; + unsigned int noio_flags = 0; char *name; /* @@ -10192,6 +10205,7 @@ static void md_start_sync(struct work_struct *ws) md_spares_need_change(mddev)) { suspend = true; mddev_suspend(mddev, false); + noio_flags = memalloc_noio_save(); } mddev_lock_nointr(mddev); @@ -10205,6 +10219,7 @@ static void md_start_sync(struct work_struct *ws) mddev_unlock(mddev); mddev_suspend_and_lock_nointr(mddev); suspend = true; + noio_flags = memalloc_noio_save(); } if (!md_is_rdwr(mddev)) { @@ -10250,8 +10265,10 @@ static void md_start_sync(struct work_struct *ws) * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } md_wakeup_thread(mddev->sync_thread); sysfs_notify_dirent_safe(mddev->sysfs_action); md_new_event(); @@ -10270,8 +10287,10 @@ not_running: * https://bugzilla.kernel.org/show_bug.cgi?id=218200 * Therefore, use __mddev_resume(mddev, false). */ - if (suspend) + if (suspend) { + memalloc_noio_restore(noio_flags); __mddev_resume(mddev, false); + } wake_up(&resync_wait); if (test_and_clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery) && diff --git a/drivers/md/md.h b/drivers/md/md.h index 1b47af09c4e2..bb2eb5f39914 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -621,7 +621,6 @@ struct mddev { struct md_cluster_info *cluster_info; struct md_cluster_operations *cluster_ops; unsigned int good_device_nr; /* good device num within cluster raid */ - unsigned int noio_flag; /* for memalloc scope API */ /* * Temporarily store rdev that will be finally removed when diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index e5348cebf12d..e2c5a7072aca 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -2471,11 +2471,6 @@ static int scribble_alloc(struct raid5_percpu *percpu, sizeof(unsigned int) * (num + 2); void *scribble; - /* - * If here is in raid array suspend context, it is in memalloc noio - * context as well, there is no potential recursive memory reclaim - * I/Os with the GFP_KERNEL flag. - */ scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL); if (!scribble) return -ENOMEM; @@ -2490,6 +2485,7 @@ static int scribble_alloc(struct raid5_percpu *percpu, static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) { unsigned long cpu; + unsigned int noio_flags; int err = 0; /* Never shrink. */ @@ -2498,6 +2494,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) return 0; raid5_quiesce(conf->mddev, true); + noio_flags = memalloc_noio_save(); cpus_read_lock(); for_each_present_cpu(cpu) { @@ -2511,6 +2508,7 @@ static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) } cpus_read_unlock(); + memalloc_noio_restore(noio_flags); raid5_quiesce(conf->mddev, false); if (!err) { @@ -7107,6 +7105,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) { struct r5conf *conf; unsigned long new; + unsigned int noio_flags = 0; int err; int size; @@ -7147,6 +7146,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) goto out_unlock; } + noio_flags = memalloc_noio_save(); mutex_lock(&conf->cache_size_mutex); size = conf->max_nr_stripes; @@ -7163,6 +7163,7 @@ raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len) mutex_unlock(&conf->cache_size_mutex); out_unlock: + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err ?: len; } @@ -9043,6 +9044,7 @@ static void *raid6_takeover(struct mddev *mddev) static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) { struct r5conf *conf; + unsigned int noio_flags; int err; err = mddev_suspend_and_lock(mddev); @@ -9054,6 +9056,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) return -ENODEV; } + noio_flags = memalloc_noio_save(); if (strncmp(buf, "ppl", 3) == 0) { /* ppl only works with RAID 5 */ if (!raid5_has_ppl(conf) && conf->level == 5) { @@ -9093,6 +9096,7 @@ static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf) if (!err) md_update_sb(mddev, 1); + memalloc_noio_restore(noio_flags); mddev_unlock_and_resume(mddev); return err; From 35d522bd32462afcf1981dab6da8a9256c26c1e0 Mon Sep 17 00:00:00 2001 From: Coly Li Date: Mon, 20 Jul 2026 19:14:00 +0800 Subject: [PATCH 075/241] md: do overflow check for sb->bblog_shift in super_1_load() In super_1_load(), sb->bblog_shift is an __u8 type value loaded from on- disk superblock. It is used for badblocks API badblocks_set() by the following sequence, 1930 rdev->badblocks.shift = sb->bblog_shift; 1931 for (i = 0 ; i < (sectors << (9-3)) ; i++, bbp++) { 1932 u64 bb = le64_to_cpu(*bbp); 1933 int count = bb & (0x3ff); 1934 u64 sector = bb >> 10; 1935 sector <<= sb->bblog_shift; 1936 count <<= sb->bblog_shift; 1937 if (bb + 1 == 0) 1938 break; 1939 if (!badblocks_set(&rdev->badblocks, sector, count, 1)) 1940 return -EINVAL; 1941 } bb->bblog_shit is in range of 0-255, variable sector is 64bit width, for an invalid bb->bblog_shit, it is possible to make sector be overflowed by the following calculation, 1935 sector <<= sb->bblog_shift; Then in turn when call badblocks_set() at line 1939 with the invalid rdev->badblocks.shift set at line 1930, may result an overflow inside _badblocks_clear() in block/badblocks.c. Although there are many places to call badblocks APIs, the non-zero shift value is only used in super_1_load(), other places always use 0 as the shift value. Therefore it is unnecessary to do a general shift value overflow check inside badblock API, and just check here as the caller. This may avoid unnecessary check, make the badblocks API code more simple and elegant. Fixes: 2699b67223ac ("md: load/store badblock list from v1.x metadata") Fixes: 1726c7746783 ("badblocks: improve badblocks_set() for multiple ranges handling") Cc: stable@vger.kernel.org Cc: Ramesh Adhikari Signed-off-by: Coly Li Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260720111400.2120834-1-colyli@fygo.io Signed-off-by: Yu Kuai --- drivers/md/md.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/md/md.c b/drivers/md/md.c index 3d6357f8fc04..f280664a8b32 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -1914,6 +1914,13 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ rdev->bb_page, REQ_OP_READ, true)) return -EIO; bbp = (__le64 *)page_address(rdev->bb_page); + + /* check for badblocks api. */ + if (sb->bblog_shift >= BITS_PER_TYPE(sector_t)) { + pr_err("md: %pg: bogus bblog_shift %u for badblocks.\n", + rdev->bdev, sb->bblog_shift); + return -EINVAL; + } rdev->badblocks.shift = sb->bblog_shift; for (i = 0 ; i < (sectors << (9-3)) ; i++, bbp++) { u64 bb = le64_to_cpu(*bbp); From 140234b2380ffb8ffb0cfc46fee0e822f43adef7 Mon Sep 17 00:00:00 2001 From: Martin Wilck Date: Thu, 23 Jul 2026 13:27:41 +0200 Subject: [PATCH 076/241] md/raid1: create serial pool adding rdev to array with serialize_policy=1 The following bug has been observed with kernel 7.1.3 after adding a new rdev to an existing RAID1 array with serialize_policy enabled: Oops: 0002 [#1] CPU: 0 UID: 0 PID: 19639 Comm: ext4lazyinit Not tainted 7.1.3-1-default RIP: _raw_spin_lock_irqsave+0x27/0x50 CR2: 0000000000004960 Call Trace: wait_for_serialization+0xb9/0x260 [raid1] raid1_make_request+0x762/0xaff [raid1] md_handle_request+0x1c9/0x2e0 [md_mod] The raid1.c code calls wait_for_serialization() if the MD_SERIALIZE_POLICY is set, and wait_for_serialization assumes that rdev->serial is initialized. Normally this will be the case for arrays that have the serialize_policy sysfs attribute set to 1. But when a new rdev is added to an existing array in bind_rdev_to_array(), the condition at mddev_create_serial_pool() causes creation of rdev->serial to be skipped. Fix it. Fixes: 69b00b5bb235 ("md: introduce a new struct for IO serialization") Signed-off-by: Martin Wilck Reviewed-by: Mykola Marzhan Link: https://patch.msgid.link/20260723112741.1206836-1-mwilck@suse.com Signed-off-by: Yu Kuai --- drivers/md/md.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index f280664a8b32..51b620edbef7 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -235,7 +235,8 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev) int ret = 0; unsigned int noio_flags; - if (rdev && !rdev_need_serial(rdev) && + if (!test_bit(MD_SERIALIZE_POLICY, &mddev->flags) && + rdev && !rdev_need_serial(rdev) && !test_bit(CollisionCheck, &rdev->flags)) return; From 14b007e178811db72fbb1ebb3535160db6ec1e6a Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Mon, 20 Jul 2026 13:10:57 -0700 Subject: [PATCH 077/241] block: validate user space vectors during extraction The bio-based drivers don't necessarily check the alignment split, and stacking block drivers don't always handle a misalignment detected after submitting the bio. Validate user vectors against the device's dma_alignment as the bio is built from the iov_iter, rejecting misaligned early with -EINVAL. Cc: stable@vger.kernel.org Fixes: 5ff3f74e145a ("block: simplify direct io validity check") Fixes: 7eac33186957 ("iomap: simplify direct io validity check") Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch Link: https://patch.msgid.link/20260720201057.1862857-6-kbusch@meta.com Signed-off-by: Jens Axboe --- block/bio.c | 56 +++++++++++++++++++++++++++++++++++++++++--- block/blk-map.c | 2 +- block/fops.c | 2 +- fs/iomap/direct-io.c | 1 + include/linux/bio.h | 2 +- include/linux/uio.h | 10 +++++++- lib/iov_iter.c | 9 ++++++- 7 files changed, 74 insertions(+), 8 deletions(-) diff --git a/block/bio.c b/block/bio.c index 6a2f6fc3413e..5018a6fc2f36 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1221,10 +1221,45 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, return 0; } +#ifdef CONFIG_DEBUG_KERNEL +static inline bool bio_iov_bvec_aligned(const struct bio *bio, + unsigned mem_align_mask) +{ + struct bvec_iter iter; + struct bio_vec bv; + + /* + * Correct callers never break the alignment requirements, so this + * exhaustive check is only paid for in debug builds. + */ + for_each_mp_bvec(bv, bio->bi_io_vec, iter, bio->bi_iter) + if ((bv.bv_offset | bv.bv_len) & mem_align_mask) + return false; + return true; +} +#else +static inline bool bio_iov_bvec_aligned(const struct bio *bio, + unsigned mem_align_mask) +{ + /* + * We forward the bio_vec as-is, so ITER_BVEC callers must provide + * segments already aligned to the device's DMA alignment. The only + * unchecked user-controllable offset that reaches here is an io_uring + * registered buffer where just the first segment can be unaligned + * (the rest is virtually contiguous), so checking only that one is + * sufficient to know if the entire vector is valid. + */ + return !(mp_bvec_iter_offset(bio->bi_io_vec, bio->bi_iter) & + mem_align_mask); +} +#endif + /** * bio_iov_iter_get_pages - add user or kernel pages to a bio * @bio: bio to add pages to * @iter: iov iterator describing the region to be added + * @mem_align_mask: the mask the source address and length must be aligned to, + * 0 for no requirement * @len_align_mask: the mask to align the total size to, 0 for any length * * This takes either an iterator pointing to user memory, or one pointing to @@ -1243,7 +1278,7 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter, * is returned only if 0 pages could be pinned. */ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, - unsigned len_align_mask) + unsigned mem_align_mask, unsigned len_align_mask) { iov_iter_extraction_t flags = 0; @@ -1252,6 +1287,10 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, if (iov_iter_is_bvec(iter)) { bio_iov_bvec_set(bio, iter); + + if (!bio_iov_bvec_aligned(bio, mem_align_mask)) + return -EINVAL; + iov_iter_advance(iter, bio->bi_iter.bi_size); return 0; } @@ -1266,8 +1305,19 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec, BIO_MAX_SIZE - bio->bi_iter.bi_size, - &bio->bi_vcnt, bio->bi_max_vecs, flags); + &bio->bi_vcnt, bio->bi_max_vecs, + mem_align_mask, flags); if (ret <= 0) { + /* + * A misaligned vector fails the whole I/O. Release any + * pages pinned by earlier iterations before returning + * since this bio won't be submitted to release them. + */ + if (ret == -EINVAL) { + bio_release_pages(bio, false); + bio_clear_flag(bio, BIO_PAGE_PINNED); + bio->bi_vcnt = 0; + } if (!bio->bi_vcnt) return ret; break; @@ -1380,7 +1430,7 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter, do { ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len, - &bio->bi_vcnt, bio->bi_max_vecs - 1, 0); + &bio->bi_vcnt, bio->bi_max_vecs - 1, 0, 0); if (ret <= 0) { if (!bio->bi_vcnt) goto out_folio_put; diff --git a/block/blk-map.c b/block/blk-map.c index d1d6bbe0ecf1..615d29bb840e 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -274,7 +274,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, * No alignment requirements on our part to support arbitrary * passthrough commands. */ - ret = bio_iov_iter_get_pages(bio, iter, 0); + ret = bio_iov_iter_get_pages(bio, iter, 0, 0); if (ret) goto out_put; ret = blk_rq_append_bio(rq, bio); diff --git a/block/fops.c b/block/fops.c index 0098a90a956e..e519d7f43b31 100644 --- a/block/fops.c +++ b/block/fops.c @@ -46,7 +46,7 @@ static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb, static inline int blkdev_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, struct block_device *bdev) { - return bio_iov_iter_get_pages(bio, iter, + return bio_iov_iter_get_pages(bio, iter, bdev_dma_alignment(bdev), bdev_logical_block_size(bdev) - 1); } diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index e2cd5f92babe..01daed391856 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -358,6 +358,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter, iomap_max_bio_size(&iter->iomap), alignment); else ret = bio_iov_iter_get_pages(bio, dio->submit.iter, + bdev_dma_alignment(bio->bi_bdev), alignment - 1); if (unlikely(ret)) goto out_put_bio; diff --git a/include/linux/bio.h b/include/linux/bio.h index 8f33f717b14f..ce34ea49ef35 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -477,7 +477,7 @@ int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, size_t len, enum req_op op); int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, - unsigned len_align_mask); + unsigned mem_align_mask, unsigned len_align_mask); void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter); void __bio_release_pages(struct bio *bio, bool mark_dirty); diff --git a/include/linux/uio.h b/include/linux/uio.h index a9bc5b3067e3..fe2e985d74d2 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -389,9 +389,17 @@ ssize_t iov_iter_extract_pages(struct iov_iter *i, struct page ***pages, size_t maxsize, unsigned int maxpages, iov_iter_extraction_t extraction_flags, size_t *offset0); +/* + * Block-layer consumers (e.g. bio_iov_iter_get_pages()) require that the + * segments of an ITER_BVEC iterator are already aligned to the target device's + * DMA alignment, and forward them as-is. In-kernel users that build their own + * bvecs must not create sub-aligned segments; iov_iter_extract_bvecs() enforces + * the same for the segments it extracts via @mem_align_mask. + */ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, - unsigned short max_vecs, iov_iter_extraction_t extraction_flags); + unsigned short max_vecs, unsigned mem_align_mask, + iov_iter_extraction_t extraction_flags); /** * iov_iter_extract_will_pin - Indicate how pages from the iterator will be retained diff --git a/lib/iov_iter.c b/lib/iov_iter.c index c2484551a4e8..34a52e9ba9e1 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1904,6 +1904,8 @@ static unsigned int get_contig_folio_len(struct page **pages, * @max_size: maximum size to extract from @iter * @nr_vecs: number of vectors in @bv (on in and output) * @max_vecs: maximum vectors in @bv, including those filled before calling + * @mem_align_mask: reject with -EINVAL if the source address or + * length is not aligned to this mask * @extraction_flags: flags to qualify request * * Like iov_iter_extract_pages(), but returns physically contiguous ranges @@ -1915,14 +1917,19 @@ static unsigned int get_contig_folio_len(struct page **pages, */ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, - unsigned short max_vecs, iov_iter_extraction_t extraction_flags) + unsigned short max_vecs, unsigned mem_align_mask, + iov_iter_extraction_t extraction_flags) { + unsigned long start = (unsigned long)iter_iov_addr(iter); unsigned short entries_left = max_vecs - *nr_vecs; unsigned short nr_pages, i = 0; size_t left, offset, len; struct page **pages; ssize_t size; + if ((start | iter_iov_len(iter)) & mem_align_mask) + return -EINVAL; + /* * Move page array up in the allocated memory for the bio vecs as far as * possible so that we can start filling biovecs from the beginning From 8ed0831b5263e1aa48d99063795a1f20e4e69b50 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Thu, 30 Jul 2026 02:57:51 -0400 Subject: [PATCH 078/241] block: introduce bio_in_atomic() Move the atomic context detection logic from erofs's z_erofs_in_atomic() into the block layer as bio_in_atomic(). This helper returns true when the current context is unsafe for sleeping bio completion handlers (e.g., hard/soft IRQ, preempt-disabled). The logic was originally added to erofs in commit c99fab6e80b7 ("erofs: fix atomic context detection when !CONFIG_DEBUG_LOCK_ALLOC"). A subsequent patch will use it in the block layer's bio completion infrastructure, so move it to include/linux/bio.h where both subsystems can share it. Convert erofs to call the new bio_in_atomic() directly. Suggested-by: Christoph Hellwig Signed-off-by: Tal Zussman Reviewed-by: Jan Kara Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/20260730-blk-dontcache-v7-1-3e8e6850068d@columbia.edu Signed-off-by: Jens Axboe --- fs/erofs/zdata.c | 11 +---------- include/linux/bio.h | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 74520e910259..f796bb3ef53b 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -1427,15 +1427,6 @@ static void z_erofs_decompressqueue_kthread_work(struct kthread_work *work) } #endif -/* Use (kthread_)work in atomic contexts to minimize scheduling overhead */ -static inline bool z_erofs_in_atomic(void) -{ - if (IS_ENABLED(CONFIG_PREEMPTION) && rcu_preempt_depth()) - return true; - if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) - return true; - return !preemptible(); -} static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, int bios) @@ -1452,7 +1443,7 @@ static void z_erofs_decompress_kickoff(struct z_erofs_decompressqueue *io, if (atomic_add_return(bios, &io->pending_bios)) return; - if (z_erofs_in_atomic()) { + if (bio_in_atomic()) { /* See `sync_decompress` in sysfs-fs-erofs for more details */ if (sbi->sync_decompress == EROFS_SYNC_DECOMPRESS_AUTO) sbi->sync_decompress = EROFS_SYNC_DECOMPRESS_FORCE_ON; diff --git a/include/linux/bio.h b/include/linux/bio.h index ce34ea49ef35..e8f5b7938a7f 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -368,6 +368,21 @@ static inline struct bio *bio_alloc(struct block_device *bdev, void submit_bio(struct bio *bio); +/** + * bio_in_atomic - check if the current context is unsafe for bio completion + * + * Return: %true in atomic contexts (e.g. hard/soft IRQ, preempt-disabled); + * %false when a bio can be safely completed in the current context. + */ +static inline bool bio_in_atomic(void) +{ + if (IS_ENABLED(CONFIG_PREEMPTION) && rcu_preempt_depth()) + return true; + if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) + return true; + return !preemptible(); +} + extern void bio_endio(struct bio *); /** From 77e73fa61b3888c9765a549af02c928372522cc9 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Thu, 30 Jul 2026 02:57:52 -0400 Subject: [PATCH 079/241] block: add task-context bio completion infrastructure Some bio completion handlers need to run from preemptible task context, but bio_endio() may be called from IRQ context (e.g., buffer_head writeback). Callers need a way to ensure their callback eventually runs from a sleepable context. Add infrastructure for that, in two forms: 1. BIO_COMPLETE_IN_TASK, a bio flag the submitter sets when it knows in advance that its callback needs task context (e.g., dropbehind writeback). bio_endio() sees the flag and offloads completion to a worker automatically. 2. bio_complete_in_task(), a helper that completion callbacks can invoke from within bi_end_io() when the deferral decision is dynamic (e.g., fserror reporting). Both share a per-CPU list drained by a work item on a WQ_PERCPU workqueue. Producers push the bio onto the local CPU's list and schedule the work item, which then dispatches each bio's bi_end_io() from task context. Both methods are gated on bio_in_atomic(), which returns true in any context where a sleeping bi_end_io() is unsafe, including non-preemptible task context. Two CPU hotplug callbacks are used to drain remaining bios from the departing CPU's batch, while maintaining the per-CPU behavior. The CPUHP_AP_ONLINE_DYN callback disables the per-CPU work item while the CPU is still online, preventing it from running on an unbound worker later. CPUHP_BP_PREPARE_DYN then drains any bios added between disabling the work item and CPU offline. Link: https://lore.kernel.org/all/20260409160243.1008358-1-hch@lst.de/ Suggested-by: Matthew Wilcox Suggested-by: Christoph Hellwig Signed-off-by: Tal Zussman Reviewed-by: Jan Kara Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/20260730-blk-dontcache-v7-2-3e8e6850068d@columbia.edu Signed-off-by: Jens Axboe --- block/bio.c | 132 +++++++++++++++++++++++++++++++++++++- include/linux/bio.h | 24 +++++++ include/linux/blk_types.h | 1 + 3 files changed, 156 insertions(+), 1 deletion(-) diff --git a/block/bio.c b/block/bio.c index 5018a6fc2f36..500389f332d9 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1791,6 +1791,61 @@ defer: schedule_work(&bio_dirty_work); } +/* + * Infrastructure for deferring bio completions to task-context via a per-CPU + * workqueue. Triggered either by the BIO_COMPLETE_IN_TASK bio flag (static + * decision at submit time) or by calling bio_complete_in_task() from + * bi_end_io() (dynamic decision at completion time). + */ + +struct bio_complete_batch { + struct bio_list list; + struct work_struct work; + int cpu; +}; + +static DEFINE_PER_CPU(struct bio_complete_batch, bio_complete_batch); +static struct workqueue_struct *bio_complete_wq; + +static void bio_complete_work_fn(struct work_struct *w) +{ + struct bio_complete_batch *batch = + container_of(w, struct bio_complete_batch, work); + + while (1) { + struct bio_list list; + struct bio *bio; + + local_irq_disable(); + list = batch->list; + bio_list_init(&batch->list); + local_irq_enable(); + + if (bio_list_empty(&list)) + break; + + while ((bio = bio_list_pop(&list))) + bio->bi_end_io(bio); + } +} + +void __bio_complete_in_task(struct bio *bio) +{ + struct bio_complete_batch *batch; + unsigned long flags; + bool was_empty; + + local_irq_save(flags); + batch = this_cpu_ptr(&bio_complete_batch); + was_empty = bio_list_empty(&batch->list); + bio_list_add(&batch->list, bio); + local_irq_restore(flags); + + if (was_empty) + queue_work_on(batch->cpu, bio_complete_wq, &batch->work); +} +EXPORT_SYMBOL_GPL(__bio_complete_in_task); + static inline bool bio_remaining_done(struct bio *bio) { /* @@ -1865,7 +1920,9 @@ again: } #endif - if (bio->bi_end_io) + if (bio_flagged(bio, BIO_COMPLETE_IN_TASK) && bio_in_atomic()) + __bio_complete_in_task(bio); + else if (bio->bi_end_io) bio->bi_end_io(bio); } EXPORT_SYMBOL(bio_endio); @@ -2051,6 +2108,55 @@ bad: } EXPORT_SYMBOL(bioset_init); +static int bio_complete_batch_cpu_online(unsigned int cpu) +{ + struct bio_complete_batch *batch = &per_cpu(bio_complete_batch, cpu); + + enable_work(&batch->work); + if (!bio_list_empty(&batch->list)) + queue_work_on(cpu, bio_complete_wq, &batch->work); + return 0; +} + +/* + * Disable this CPU's work item so that it cannot run on an unbound worker + * after the CPU is offlined. + */ +static int bio_complete_batch_cpu_down_prep(unsigned int cpu) +{ + disable_work_sync(&per_cpu(bio_complete_batch, cpu).work); + return 0; +} + +/* + * Drain a dead CPU's deferred bio completions. The CPU is dead and the worker + * is canceled so no locking is needed. + */ +static int bio_complete_batch_cpu_dead(unsigned int cpu) +{ + struct bio_complete_batch *batch = + per_cpu_ptr(&bio_complete_batch, cpu); + struct bio *bio; + + while ((bio = bio_list_pop(&batch->list))) + bio->bi_end_io(bio); + + return 0; +} + +static void __init bio_complete_batch_init(int cpu) +{ + struct bio_complete_batch *batch = + per_cpu_ptr(&bio_complete_batch, cpu); + + bio_list_init(&batch->list); + INIT_WORK(&batch->work, bio_complete_work_fn); + batch->cpu = cpu; + + if (!cpu_online(cpu)) + disable_work_sync(&batch->work); +} + static int __init init_bio(void) { int i; @@ -2065,6 +2171,30 @@ static int __init init_bio(void) SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); } + for_each_possible_cpu(i) + bio_complete_batch_init(i); + + bio_complete_wq = alloc_workqueue("bio_complete", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); + if (!bio_complete_wq) + panic("bio: can't allocate bio_complete workqueue\n"); + + /* + * bio task-context completion draining on hot-unplugged CPUs: + * + * 1. Stop the per-CPU work item while the CPU is still online, so + * that it cannot run on an unbound worker later. + * 2. Drain leftover bios added between worker disabling and CPU + * offlining. + */ + cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, + "block/bio:complete:online", + bio_complete_batch_cpu_online, + bio_complete_batch_cpu_down_prep); + cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, + "block/bio:complete:dead", + NULL, bio_complete_batch_cpu_dead); + cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL, bio_cpu_dead); diff --git a/include/linux/bio.h b/include/linux/bio.h index e8f5b7938a7f..0445ecba3b24 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -383,6 +383,30 @@ static inline bool bio_in_atomic(void) return !preemptible(); } +void __bio_complete_in_task(struct bio *bio); + +/** + * bio_complete_in_task - ensure a bio is completed in preemptible task context + * @bio: bio to complete + * + * If called from non-task context, offload the bio completion to a worker + * thread and return %true. Else return %false and do nothing. + * + * Uses BIO_COMPLETE_IN_TASK as a sentinel: if set, the bio was already + * deferred and we are running in the worker — return %false so the + * callback proceeds instead of re-deferring. + */ +static inline bool bio_complete_in_task(struct bio *bio) +{ + if (bio_flagged(bio, BIO_COMPLETE_IN_TASK)) + return false; + if (!bio_in_atomic()) + return false; + bio_set_flag(bio, BIO_COMPLETE_IN_TASK); + __bio_complete_in_task(bio); + return true; +} + extern void bio_endio(struct bio *); /** diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e73c..d49d97a050d0 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -322,6 +322,7 @@ enum { BIO_REMAPPED, BIO_ZONE_WRITE_PLUGGING, /* bio handled through zone write plugging */ BIO_EMULATES_ZONE_APPEND, /* bio emulates a zone append operation */ + BIO_COMPLETE_IN_TASK, /* complete bi_end_io() in task context */ BIO_FLAG_LAST }; From efbde6f9f449da3306f5b5d32f08829954ffb44d Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Thu, 30 Jul 2026 02:57:53 -0400 Subject: [PATCH 080/241] iomap: use BIO_COMPLETE_IN_TASK for dropbehind writeback Set BIO_COMPLETE_IN_TASK on iomap writeback bios when a dropbehind folio is added. This ensures that bi_end_io runs in task context, where folio_end_dropbehind() can safely invalidate folios. With the bio layer now handling task-context deferral generically, IOMAP_IOEND_DONTCACHE is no longer needed, as XFS no longer needs to route DONTCACHE ioends through its completion workqueue. Remove the flag and its NOMERGE entry. Without the NOMERGE, regular I/Os that get merged with a dropbehind folio will also have their completion deferred to task context. Reviewed-by: Christoph Hellwig Signed-off-by: Tal Zussman Link: https://patch.msgid.link/20260730-blk-dontcache-v7-3-3e8e6850068d@columbia.edu Signed-off-by: Jens Axboe --- fs/iomap/ioend.c | 5 +++-- fs/xfs/xfs_aops.c | 4 ---- include/linux/iomap.h | 5 +---- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/fs/iomap/ioend.c b/fs/iomap/ioend.c index 30468d51b5ad..1ae8a8fb8503 100644 --- a/fs/iomap/ioend.c +++ b/fs/iomap/ioend.c @@ -238,8 +238,6 @@ ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, if (wpc->iomap.flags & IOMAP_F_SHARED) ioend_flags |= IOMAP_IOEND_SHARED; - if (folio_test_dropbehind(folio)) - ioend_flags |= IOMAP_IOEND_DONTCACHE; if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY)) ioend_flags |= IOMAP_IOEND_BOUNDARY; @@ -256,6 +254,9 @@ new_ioend: if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff)) goto new_ioend; + if (folio_test_dropbehind(folio)) + bio_set_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK); + /* * Clamp io_offset and io_size to the incore EOF so that ondisk * file size updates in the ioend completion are byte-accurate. diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index 51293b6f331f..cd8de8c82d78 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -522,10 +522,6 @@ xfs_ioend_needs_wq_completion( if (ioend->io_flags & (IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_SHARED)) return true; - /* Page cache invalidation cannot be done in irq context. */ - if (ioend->io_flags & IOMAP_IOEND_DONTCACHE) - return true; - return false; } diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 56b43d594e6e..68af4935a107 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -404,16 +404,13 @@ sector_t iomap_bmap(struct address_space *mapping, sector_t bno, #define IOMAP_IOEND_BOUNDARY (1U << 2) /* is direct I/O */ #define IOMAP_IOEND_DIRECT (1U << 3) -/* is DONTCACHE I/O */ -#define IOMAP_IOEND_DONTCACHE (1U << 4) /* * Flags that if set on either ioend prevent the merge of two ioends. * (IOMAP_IOEND_BOUNDARY also prevents merges, but only one-way) */ #define IOMAP_IOEND_NOMERGE_FLAGS \ - (IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT | \ - IOMAP_IOEND_DONTCACHE) + (IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT) /* * Structure for writeback I/O completions. From a2c924c240e74dc2dd14ff763245dc86b93db714 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Thu, 30 Jul 2026 02:57:54 -0400 Subject: [PATCH 081/241] buffer: set BIO_COMPLETE_IN_TASK for dropbehind writeback Set BIO_COMPLETE_IN_TASK in __bh_submit() for write bios when the folio has dropbehind set, so that buffer_head writeback completions get deferred to task context where folio_end_dropbehind() can safely invalidate folios. Read completions are not deferred since dropbehind invalidation for reads is handled synchronously by the reader. Reviewed-by: Christoph Hellwig Signed-off-by: Tal Zussman Reviewed-by: Jan Kara Link: https://patch.msgid.link/20260730-blk-dontcache-v7-4-3e8e6850068d@columbia.edu Signed-off-by: Jens Axboe --- fs/buffer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fs/buffer.c b/fs/buffer.c index 9af5f061a1f8..6f099847240e 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -1203,6 +1203,9 @@ static void __bh_submit(struct buffer_head *bh, blk_opf_t opf, bio = bio_alloc(bh->b_bdev, 1, opf, GFP_NOIO); + if (folio_test_dropbehind(bh->b_folio) && op_is_write(opf)) + bio_set_flag(bio, BIO_COMPLETE_IN_TASK); + if (IS_ENABLED(CONFIG_FS_ENCRYPTION)) buffer_set_crypto_ctx(bio, bh, GFP_NOIO); From 8b5ffb43ae9d27eed9165e1962963d685d879219 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Thu, 30 Jul 2026 02:57:55 -0400 Subject: [PATCH 082/241] block: enable RWF_DONTCACHE for block devices Block device buffered reads and writes already pass through filemap_read() and iomap_file_buffered_write() respectively, both of which handle IOCB_DONTCACHE. Enable RWF_DONTCACHE for block device files by setting FOP_DONTCACHE in def_blk_fops. For CONFIG_BUFFER_HEAD=y, writeback goes through buffer_head's __bh_submit() which sets BIO_COMPLETE_IN_TASK on dropbehind folios. For CONFIG_BUFFER_HEAD=n, writeback goes through iomap which handles it via BIO_COMPLETE_IN_TASK on the ioend bio. This support is useful for databases that operate on raw block devices, among other userspace applications. Reviewed-by: Christoph Hellwig Signed-off-by: Tal Zussman Reviewed-by: Jan Kara Link: https://patch.msgid.link/20260730-blk-dontcache-v7-5-3e8e6850068d@columbia.edu Signed-off-by: Jens Axboe --- block/fops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/fops.c b/block/fops.c index e519d7f43b31..3c2099dfef1d 100644 --- a/block/fops.c +++ b/block/fops.c @@ -943,7 +943,7 @@ const struct file_operations def_blk_fops = { .splice_write = iter_file_splice_write, .fallocate = blkdev_fallocate, .uring_cmd = blkdev_uring_cmd, - .fop_flags = FOP_BUFFER_RASYNC, + .fop_flags = FOP_BUFFER_RASYNC | FOP_DONTCACHE, }; static __init int blkdev_init(void) From 3be7ad35f10e8e19f618f67ddca0bc0e247f6505 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 3 Aug 2026 21:34:00 +0800 Subject: [PATCH 083/241] block/blk-cgroup-rwstat: use data_race() for online test blkg_rwstat_recursive_sum() reads pos_blkg->online without the queue lock that its doc comment requires, since blkcg_print_blkgs() stopped holding it in 56cc24f59c14. Concurrent blkg_create/destroy flips ->online, tripping KCSAN. The race is harmless (RCU-protected, stale online only causes minor stat noise). Use data_race() to annotate the intentional lockless read. Also update the stale doc comment that still requires the queue lock. Reviewed-by: Yu Kuai Signed-off-by: Tao Cui Acked-by: Tejun Heo Link: https://patch.msgid.link/20260803133400.137906-1-cui.tao@linux.dev Signed-off-by: Jens Axboe --- block/blk-cgroup-rwstat.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/block/blk-cgroup-rwstat.c b/block/blk-cgroup-rwstat.c index aae910713814..2e9bf081cce2 100644 --- a/block/blk-cgroup-rwstat.c +++ b/block/blk-cgroup-rwstat.c @@ -88,8 +88,7 @@ EXPORT_SYMBOL_GPL(blkg_prfill_rwstat); * @sum: blkg_rwstat_sample structure containing the results * * Collect the blkg_rwstat specified by @blkg, @pol and @off and all its - * online descendants and their aux counts. The caller must be holding the - * queue lock for online tests. + * online descendants and their aux counts. * * If @pol is NULL, blkg_rwstat is at @off bytes into @blkg; otherwise, it * is at @off bytes into @blkg's blkg_policy_data of the policy. @@ -107,7 +106,7 @@ void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol, blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) { struct blkg_rwstat *rwstat; - if (!pos_blkg->online) + if (!data_race(pos_blkg->online)) continue; if (pol) { From 9a916798946e5107472cdc714079c0167b8cd251 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Sun, 2 Aug 2026 19:25:17 +0800 Subject: [PATCH 084/241] blk-cgroup: protect q->blkg_list iteration in blkg_destroy_all() with blkcg_mutex blkg_destroy_all() iterates q->blkg_list without holding blkcg_mutex, which can race with blkg_free_workfn() that removes blkgs from the list while holding blkcg_mutex. Add blkcg_mutex protection around the q->blkg_list iteration to prevent potential list corruption or use-after-free issues. Reviewed-by: Tang Yizhou Signed-off-by: Yu Kuai Reviewed-by: Tao Cui Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260802112525.3933753-2-yukuai@kernel.org Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index d9676126c5b5..eb0cfb10b859 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -569,6 +569,7 @@ static void blkg_destroy_all(struct gendisk *disk) int i; restart: + mutex_lock(&q->blkcg_mutex); spin_lock_irq(&q->queue_lock); list_for_each_entry(blkg, &q->blkg_list, q_node) { struct blkcg *blkcg = blkg->blkcg; @@ -587,6 +588,7 @@ restart: if (!(--count)) { count = BLKG_DESTROY_BATCH_SIZE; spin_unlock_irq(&q->queue_lock); + mutex_unlock(&q->blkcg_mutex); cond_resched(); goto restart; } @@ -606,6 +608,7 @@ restart: q->root_blkg = NULL; spin_unlock_irq(&q->queue_lock); + mutex_unlock(&q->blkcg_mutex); wake_up_var(&q->root_blkg); } From 5313d4d41739b0cb63000747c97bb1217ac45f3e Mon Sep 17 00:00:00 2001 From: Zheng Qixing Date: Sun, 2 Aug 2026 19:25:18 +0800 Subject: [PATCH 085/241] blk-cgroup: fix race between policy activation and blkg destruction When switching an IO scheduler on a block device, blkcg_activate_policy() allocates blkg_policy_data (pd) for all blkgs attached to the queue. However, blkcg_activate_policy() may race with concurrent blkcg deletion, leading to use-after-free and memory leak issues. The use-after-free occurs in the following race: T1 (blkcg_activate_policy): - Successfully allocates pd for blkg1 (loop0->queue, blkcgA) - Fails to allocate pd for blkg2 (loop0->queue, blkcgB) - Enters the enomem rollback path to release blkg1 resources T2 (blkcg deletion): - blkcgA is deleted concurrently - blkg1 is freed via blkg_free_workfn() - blkg1->pd is freed T1 (continued): - Rollback path accesses blkg1->pd->online after pd is freed - Triggers use-after-free In addition, blkg_free_workfn() frees pd before removing the blkg from q->blkg_list. This allows blkcg_activate_policy() to allocate a new pd for a blkg that is being destroyed, leaving the newly allocated pd unreachable when the blkg is finally freed. Fix these races by extending blkcg_mutex coverage to serialize blkcg_activate_policy() rollback and blkg destruction, ensuring pd lifecycle is synchronized with blkg list visibility. Fixes: f1c006f1c685 ("blk-cgroup: synchronize pd_free_fn() from blkg_free_workfn() and blkcg_deactivate_policy()") Signed-off-by: Zheng Qixing Reviewed-by: Tang Yizhou Signed-off-by: Yu Kuai Reviewed-by: Tao Cui Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260802112525.3933753-3-yukuai@kernel.org Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index eb0cfb10b859..047bb42c282b 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1566,6 +1566,8 @@ int blkcg_activate_policy(struct gendisk *disk, const struct blkcg_policy *pol) if (queue_is_mq(q)) memflags = blk_mq_freeze_queue(q); + + mutex_lock(&q->blkcg_mutex); retry: spin_lock_irq(&q->queue_lock); @@ -1628,6 +1630,7 @@ retry: spin_unlock_irq(&q->queue_lock); out: + mutex_unlock(&q->blkcg_mutex); if (queue_is_mq(q)) blk_mq_unfreeze_queue(q, memflags); if (pinned_blkg) From 5e9220389920f33b6a804d50c548cd0cd1b04634 Mon Sep 17 00:00:00 2001 From: Zheng Qixing Date: Sun, 2 Aug 2026 19:25:19 +0800 Subject: [PATCH 086/241] blk-cgroup: skip dying blkg in blkcg_activate_policy() When switching IO schedulers on a block device, blkcg_activate_policy() can race with concurrent blkcg deletion, leading to a use-after-free in rcu_accelerate_cbs. T1: T2: blkg_destroy kill(&blkg->refcnt) // blkg->refcnt=1->0 blkg_release // call_rcu(__blkg_release) ... blkg_free_workfn ->pd_free_fn(pd) elv_iosched_store elevator_switch ... iterate blkg list blkg_get(blkg) // blkg->refcnt=0->1 list_del_init(&blkg->q_node) blkg_put(pinned_blkg) // blkg->refcnt=1->0 blkg_release // call_rcu again rcu_accelerate_cbs // uaf Fix this by checking hlist_unhashed(&blkg->blkcg_node) before getting a reference to the blkg. This is the same check used in blkg_destroy() to detect if a blkg has already been destroyed. If the blkg is already unhashed, skip processing it since it's being destroyed. Fixes: f1c006f1c685 ("blk-cgroup: synchronize pd_free_fn() from blkg_free_workfn() and blkcg_deactivate_policy()") Signed-off-by: Zheng Qixing Reviewed-by: Tang Yizhou Signed-off-by: Yu Kuai Reviewed-by: Tao Cui Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260802112525.3933753-4-yukuai@kernel.org Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 047bb42c282b..d1895bc60fcf 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1577,6 +1577,8 @@ retry: if (blkg->pd[pol->plid]) continue; + if (hlist_unhashed(&blkg->blkcg_node)) + continue; /* If prealloc matches, use it; otherwise try GFP_NOWAIT */ if (blkg == pinned_blkg) { From d3f775718a66e4242b3b344d6b3fc836a390785b Mon Sep 17 00:00:00 2001 From: Zheng Qixing Date: Sun, 2 Aug 2026 19:25:20 +0800 Subject: [PATCH 087/241] blk-cgroup: factor policy pd teardown loop into helper Move the teardown sequence which offlines and frees per-policy blkg_policy_data (pd) into a helper for readability. No functional change intended. Signed-off-by: Zheng Qixing Reviewed-by: Christoph Hellwig Reviewed-by: Tang Yizhou Signed-off-by: Yu Kuai Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260802112525.3933753-5-yukuai@kernel.org Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 57 ++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index d1895bc60fcf..354637f3b158 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1529,6 +1529,31 @@ struct cgroup_subsys io_cgrp_subsys = { }; EXPORT_SYMBOL_GPL(io_cgrp_subsys); +/* + * Tear down per-blkg policy data for @pol on @q. + */ +static void blkcg_policy_teardown_pds(struct request_queue *q, + const struct blkcg_policy *pol) +{ + struct blkcg_gq *blkg; + + list_for_each_entry(blkg, &q->blkg_list, q_node) { + struct blkcg *blkcg = blkg->blkcg; + struct blkg_policy_data *pd; + + spin_lock(&blkcg->lock); + pd = blkg->pd[pol->plid]; + if (pd) { + if (pd->online && pol->pd_offline_fn) + pol->pd_offline_fn(pd); + pd->online = false; + pol->pd_free_fn(pd); + WRITE_ONCE(blkg->pd[pol->plid], NULL); + } + spin_unlock(&blkcg->lock); + } +} + /** * blkcg_activate_policy - activate a blkcg policy on a gendisk * @disk: gendisk of interest @@ -1644,21 +1669,7 @@ out: enomem: /* alloc failed, take down everything */ spin_lock_irq(&q->queue_lock); - list_for_each_entry(blkg, &q->blkg_list, q_node) { - struct blkcg *blkcg = blkg->blkcg; - struct blkg_policy_data *pd; - - spin_lock(&blkcg->lock); - pd = blkg->pd[pol->plid]; - if (pd) { - if (pd->online && pol->pd_offline_fn) - pol->pd_offline_fn(pd); - pd->online = false; - pol->pd_free_fn(pd); - WRITE_ONCE(blkg->pd[pol->plid], NULL); - } - spin_unlock(&blkcg->lock); - } + blkcg_policy_teardown_pds(q, pol); spin_unlock_irq(&q->queue_lock); ret = -ENOMEM; goto out; @@ -1677,7 +1688,6 @@ void blkcg_deactivate_policy(struct gendisk *disk, const struct blkcg_policy *pol) { struct request_queue *q = disk->queue; - struct blkcg_gq *blkg; unsigned int memflags; if (!blkcg_policy_enabled(q, pol)) @@ -1690,20 +1700,7 @@ void blkcg_deactivate_policy(struct gendisk *disk, spin_lock_irq(&q->queue_lock); __clear_bit(pol->plid, q->blkcg_pols); - - list_for_each_entry(blkg, &q->blkg_list, q_node) { - struct blkcg *blkcg = blkg->blkcg; - - spin_lock(&blkcg->lock); - if (blkg->pd[pol->plid]) { - if (blkg->pd[pol->plid]->online && pol->pd_offline_fn) - pol->pd_offline_fn(blkg->pd[pol->plid]); - pol->pd_free_fn(blkg->pd[pol->plid]); - blkg->pd[pol->plid] = NULL; - } - spin_unlock(&blkcg->lock); - } - + blkcg_policy_teardown_pds(q, pol); spin_unlock_irq(&q->queue_lock); mutex_unlock(&q->blkcg_mutex); From 9d617828cfc4d9a4d385daa2cd61f9db0592c53f Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 20 Jul 2026 17:37:23 +0800 Subject: [PATCH 088/241] block/blk-stat: drain per-cpu callback stats over possible CPUs blk_stat_timer_fn() sums and resets a callback's per-cpu buckets using for_each_online_cpu(). A CPU that goes offline with pending samples is skipped, so its samples are neither accumulated into the window nor cleared; they sit in the bucket until the CPU comes back online, at which point the stale values are flushed into whatever window is then running. This silently corrupts the latency picture that consumers (notably writeback throttling via wbt, and blk-mq latency tracking) base decisions on around CPU hotplug: under-counting while the CPU is offline, then a burst of stale data on re-online. Fixes: 34dbad5d26e2 ("blk-stat: convert to callback-based statistics reporting") Signed-off-by: Tao Cui Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260720093726.28965-2-cui.tao@linux.dev Signed-off-by: Jens Axboe --- block/blk-stat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/blk-stat.c b/block/blk-stat.c index de126e1ea5ac..d57c2fc6bf06 100644 --- a/block/blk-stat.c +++ b/block/blk-stat.c @@ -83,7 +83,7 @@ static void blk_stat_timer_fn(struct timer_list *t) for (bucket = 0; bucket < cb->buckets; bucket++) blk_rq_stat_init(&cb->stat[bucket]); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct blk_rq_stat *cpu_stat; cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu); From e0698304bd4a833d6d5a7b6582515e36b1ec3ea7 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 20 Jul 2026 17:37:24 +0800 Subject: [PATCH 089/241] block/blk-iolatency: account per-cpu latency stats over possible CPUs iolatency_check_latencies() and iolatency_ssd_stat() iterate a blkg's per-cpu latency stats with for_each_online_cpu(). When a CPU that has accumulated io.latency samples goes offline, its bucket is skipped: the check loop (which also resets) neither sums nor clears it, and the show path under-reports. On re-online the stranded samples are flushed into a later check window, which can trigger a spurious throttle/scale adjustment. Fixes: d70675121546 ("block: introduce blk-iolatency io controller") Fixes: 1fa2840e56f9 ("blk-iolatency: use a percentile approache for ssd's") Signed-off-by: Tao Cui Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260720093726.28965-3-cui.tao@linux.dev Signed-off-by: Jens Axboe --- block/blk-iolatency.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index cef02b6c5fa9..9eb69010c34e 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -523,7 +523,7 @@ static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now) latency_stat_init(iolat, &stat); preempt_disable(); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct latency_stat *s; s = per_cpu_ptr(iolat->stats, cpu); latency_stat_sum(iolat, &stat, s); @@ -925,7 +925,7 @@ static void iolatency_ssd_stat(struct iolatency_grp *iolat, struct seq_file *s) latency_stat_init(iolat, &stat); preempt_disable(); - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct latency_stat *s; s = per_cpu_ptr(iolat->stats, cpu); latency_stat_sum(iolat, &stat, s); From 4e050c5b92c1600415b2cd452583e543036f3d73 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 20 Jul 2026 17:37:25 +0800 Subject: [PATCH 090/241] block/blk-iocost: collect per-cpu latency stats over possible CPUs ioc_lat_stat() walks ioc->pcpu_stat with for_each_online_cpu() to compute missed-ppm and rq_wait deltas. An offlined CPU is skipped, so its delta is dropped from the period and its last_* watermark is not advanced; on re-online the next collection sees a delta spanning the whole offline interval, corrupting the latency/vrate picture. Fixes: 7caa47151ab2 ("blkcg: implement blk-iocost") Signed-off-by: Tao Cui Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260720093726.28965-4-cui.tao@linux.dev Signed-off-by: Jens Axboe --- block/blk-iocost.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/blk-iocost.c b/block/blk-iocost.c index 8b2aeba2e1e3..b60625613e09 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -1592,7 +1592,7 @@ static void ioc_lat_stat(struct ioc *ioc, u32 *missed_ppm_ar, u32 *rq_wait_pct_p u64 rq_wait_ns = 0; int cpu, rw; - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct ioc_pcpu_stat *stat = per_cpu_ptr(ioc->pcpu_stat, cpu); u64 this_rq_wait_ns; From 482fc257de95ab181688e9d1dfcc6b6a58857b1e Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 20 Jul 2026 17:37:26 +0800 Subject: [PATCH 091/241] block/kyber-iosched: flush per-cpu latency buckets over possible CPUs kyber_timer_fn() sums the per-cpu latency histograms with for_each_online_cpu(). A CPU that goes offline mid-interval leaves its bucket un-flushed; the samples are lost from the current decision and re-appear (stale) when the CPU is onlined again. Fixes: 6e25cb01ea20 ("kyber: implement improved heuristics") Signed-off-by: Tao Cui Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260720093726.28965-5-cui.tao@linux.dev Signed-off-by: Jens Axboe --- block/kyber-iosched.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 971818bcdc9d..2ee552ab8135 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -275,7 +275,7 @@ static void kyber_timer_fn(struct timer_list *t) bool bad = false; /* Sum all of the per-cpu latency histograms. */ - for_each_online_cpu(cpu) { + for_each_possible_cpu(cpu) { struct kyber_cpu_latency *cpu_latency; cpu_latency = per_cpu_ptr(kqd->cpu_latency, cpu); From 19f6bbd753f5d09eadc4a127b89ffac823d90b26 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Fri, 17 Jul 2026 13:48:55 +0800 Subject: [PATCH 092/241] blk-throttle: remove dead field last_check_time from throtl_grp The last_check_time field in throtl_grp was used by the CONFIG_BLK_DEV_THROTTLING_LOW mechanism (in throtl_upgrade_check() and the downgrade logic) to timestamp the last upgrade/downgrade check. Commit bf20ab538c81 ("blk-throttle: remove CONFIG_BLK_DEV_THROTTLING_LOW") removed all five of its uses in blk-throttle.c and the surrounding LOW fields (latency_target, last_finish_time, checked_last_finish_time, avg_idletime, idletime_threshold, bio_cnt, bad_bio_cnt, bio_cnt_reset_time), but missed the field definition itself. It has been a dead field since then: zero references in blk-throttle.c and the whole tree. Remove it. Signed-off-by: Tao Cui Reviewed-by: Yu Kuai Reviewed-by: Tang Yizhou Link: https://patch.msgid.link/20260717054855.2018578-1-cui.tao@linux.dev Signed-off-by: Jens Axboe --- block/blk-throttle.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/block/blk-throttle.h b/block/blk-throttle.h index 9d7a42c039a1..1b5775771e07 100644 --- a/block/blk-throttle.h +++ b/block/blk-throttle.h @@ -120,8 +120,6 @@ struct throtl_grp { int64_t bytes_disp[2]; int io_disp[2]; - unsigned long last_check_time; - /* When did we start a new slice */ unsigned long slice_start[2]; unsigned long slice_end[2]; From cbe81d612038fa3fb986a1e31fe7b8f808079cf1 Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Mon, 3 Aug 2026 21:41:04 +0800 Subject: [PATCH 093/241] block/bfq-cgroup: use data_race() for online test bfqg_prfill_stat_recursive() and bfq_bio_bfqg() read blkg->online locklessly, same as blkg_rwstat_recursive_sum(). Annotate with data_race() to silence KCSAN. Signed-off-by: Tao Cui Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260803134104.138411-1-cui.tao@linux.dev Signed-off-by: Jens Axboe --- block/bfq-cgroup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/block/bfq-cgroup.c b/block/bfq-cgroup.c index e82ff03bda02..ef9d63e8542e 100644 --- a/block/bfq-cgroup.c +++ b/block/bfq-cgroup.c @@ -610,7 +610,7 @@ struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio) struct bfq_group *bfqg; while (blkg) { - if (!blkg->online) { + if (!data_race(blkg->online)) { blkg = blkg->parent; continue; } @@ -1168,7 +1168,7 @@ static u64 bfqg_prfill_stat_recursive(struct seq_file *sf, struct blkg_policy_data *pd; struct bfq_stat *stat; - if (!pos_blkg->online) + if (!data_race(pos_blkg->online)) continue; pd = blkg_to_pd(pos_blkg, &blkcg_policy_bfq); From 3831568792af75b6523fa93bb91560e29189cf55 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Wed, 29 Jul 2026 11:10:39 -0600 Subject: [PATCH 094/241] ublk: check import_ubuf() return value import_ubuf() can fail if the address range (provided by the userspace ublk server) is outside the allowed user address space. Return that 0 bytes were copied if import_ubuf() fails rather than passing an uninitialized struct iov_iter to ublk_copy_user_pages(). Fixes: 981f95a571e3 ("ublk: cleanup ublk_copy_user_pages") Reported-by: Ming Lei Signed-off-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260729171041.45061-2-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 4ca6ec738c93..098e046505ad 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1475,7 +1475,10 @@ static unsigned int ublk_map_io(const struct ublk_queue *ubq, struct iov_iter iter; const int dir = ITER_DEST; - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, &iter); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, + &iter) < 0) + return 0; + return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1496,7 +1499,10 @@ static unsigned int ublk_unmap_io(bool need_map, WARN_ON_ONCE(io->res > rq_bytes); - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, + &iter) < 0) + return 0; + return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; From 24fd3706178f1ae5501fd1ff9036e170ed0665ba Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Wed, 29 Jul 2026 11:10:40 -0600 Subject: [PATCH 095/241] ublk: check for ublk_unmap_io() returning 0 If the userspace ublk server passes an unmapped address as the data buffer for a completed ublk read, ublk_unmap_io() will return 0 indicating no bytes could be copied. Currently, this will result in calling blk_update_request() with nr_bytes=0, which doesn't seem supported. Fail the I/O with BLK_STS_IOERR in this case instead. Fixes: 71f28f3136af ("ublk_drv: add io_uring based userspace block driver") Signed-off-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260729171041.45061-3-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 098e046505ad..2cbc359dc196 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1590,8 +1590,14 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io, * * Re-read simply for this unlikely case. */ - if (unlikely(unmapped_bytes < io->res)) + if (unlikely(unmapped_bytes < io->res)) { + if (unlikely(!unmapped_bytes)) { + res = BLK_STS_IOERR; + goto exit; + } + io->res = unmapped_bytes; + } /* * Run bio->bi_end_io() with softirqs disabled. If the final fput From 15c1339ef44054cc79c49b40b24a26aa80a2253e Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Wed, 29 Jul 2026 11:10:41 -0600 Subject: [PATCH 096/241] ublk: remove WARN_ON_ONCE() in ublk_unmap_io() io->res is set from struct ublksrv_io_cmd's result field, which is controlled by the ublk server process, without any validation. It's thus possible for userspace to trigger the io->res > rq_bytes warning. ublk_copy_user_pages() already limits the copy length to the request data length, so drop the warning. Signed-off-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260729171041.45061-4-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 2cbc359dc196..8101c2a73efd 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1497,8 +1497,6 @@ static unsigned int ublk_unmap_io(bool need_map, struct iov_iter iter; const int dir = ITER_SOURCE; - WARN_ON_ONCE(io->res > rq_bytes); - if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter) < 0) return 0; From f510198855b6ddac0ffe62684dc407c56d3e0f24 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:33 -0600 Subject: [PATCH 097/241] ublk: consistently use u16 for queue and tag numbers The u16 nr_hw_queues and queue_depth fields of the ublk UAPI struct ublksrv_ctrl_dev_info constrain the number of queues and queue depth of each ublk device. However, the ublk driver is a bit inconsistent with the type it uses to represent these values, mixing u16 with int and unsigned int. Change all queue number, queue depth, q_id, and tag variables/fields to u16 to save some space. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei Link: https://patch.msgid.link/20260803211441.2538144-2-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 80 ++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 8101c2a73efd..0743b5ba241d 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -237,8 +237,8 @@ struct ublk_io { } ____cacheline_aligned_in_smp; struct ublk_queue { - int q_id; - int q_depth; + u16 q_id; + u16 q_depth; unsigned long flags; struct ublksrv_io_desc *io_cmd_buf; @@ -248,7 +248,7 @@ struct ublk_queue { bool fail_io; /* copy of dev->state == UBLK_S_DEV_FAIL_IO */ spinlock_t cancel_lock; struct ublk_device *dev; - u32 nr_io_ready; + u16 nr_io_ready; /* * For supporting UBLK_F_BATCH_IO only. @@ -327,7 +327,7 @@ struct ublk_device { struct ublk_params params; - u32 nr_queue_ready; + u16 nr_queue_ready; bool unprivileged_daemons; struct mutex cancel_mutex; bool canceling; @@ -403,7 +403,7 @@ static inline void ublk_io_evts_deinit(struct ublk_queue *q) } static inline struct ublksrv_io_desc * -ublk_get_iod(const struct ublk_queue *ubq, unsigned tag) +ublk_get_iod(const struct ublk_queue *ubq, u16 tag) { return &ubq->io_cmd_buf[tag]; } @@ -423,8 +423,7 @@ static inline bool ublk_support_shmem_zc(const struct ublk_queue *ubq) return ubq->flags & UBLK_F_SHMEM_ZC; } -static inline bool ublk_iod_is_shmem_zc(const struct ublk_queue *ubq, - unsigned int tag) +static inline bool ublk_iod_is_shmem_zc(const struct ublk_queue *ubq, u16 tag) { return ublk_get_iod(ubq, tag)->op_flags & UBLK_IO_F_SHMEM_ZC; } @@ -864,7 +863,7 @@ static unsigned int unprivileged_ublks_added; /* protected by ublk_ctl_mutex */ static struct miscdevice ublk_misc; -static inline unsigned ublk_pos_to_hwq(loff_t pos) +static inline u16 ublk_pos_to_hwq(loff_t pos) { return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_QID_OFF) & UBLK_QID_BITS_MASK; @@ -875,7 +874,7 @@ static inline unsigned ublk_pos_to_buf_off(loff_t pos) return (pos - UBLKSRV_IO_BUF_OFFSET) & UBLK_IO_BUF_BITS_MASK; } -static inline unsigned ublk_pos_to_tag(loff_t pos) +static inline u16 ublk_pos_to_tag(loff_t pos) { return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_TAG_OFF) & UBLK_TAG_BITS_MASK; @@ -1231,18 +1230,18 @@ static noinline void ublk_put_device(struct ublk_device *ub) } static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev, - int qid) + u16 qid) { return dev->queues[qid]; } static inline struct ublksrv_io_desc * -ublk_queue_cmd_buf(struct ublk_device *ub, int q_id) +ublk_queue_cmd_buf(struct ublk_device *ub, u16 q_id) { return ublk_get_queue(ub, q_id)->io_cmd_buf; } -static inline int __ublk_queue_cmd_buf_size(int depth) +static inline int __ublk_queue_cmd_buf_size(u16 depth) { return round_up(depth * sizeof(struct ublksrv_io_desc), PAGE_SIZE); } @@ -1667,7 +1666,7 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq, } static void -ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, unsigned tag) +ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, u16 tag) { struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag); @@ -1778,7 +1777,7 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req) { unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS; - int tag = req->tag; + u16 tag = req->tag; struct ublk_io *io = &ubq->ios[tag]; pr_devel("%s: complete: qid %d tag %d io_flags %x addr %llx\n", @@ -2368,7 +2367,7 @@ static const struct blk_mq_ops ublk_batch_mq_ops = { static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) { - int i; + u16 i; ubq->nr_io_ready = 0; @@ -2413,7 +2412,7 @@ static int ublk_ch_open(struct inode *inode, struct file *filp) static void ublk_reset_ch_dev(struct ublk_device *ub) { - int i; + u16 i; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { struct ublk_queue *ubq = ublk_get_queue(ub, i); @@ -2485,7 +2484,7 @@ out: static void ublk_set_canceling(struct ublk_device *ub, bool canceling) __must_hold(&ub->cancel_mutex) { - int i; + u16 i; ub->canceling = canceling; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) @@ -2494,7 +2493,7 @@ static void ublk_set_canceling(struct ublk_device *ub, bool canceling) static bool ublk_check_and_reset_active_ref(struct ublk_device *ub) { - int i, j; + u16 i, j; if (!ublk_dev_need_req_ref(ub)) return false; @@ -2527,7 +2526,7 @@ static void ublk_ch_release_work_fn(struct work_struct *work) struct ublk_device *ub = container_of(work, struct ublk_device, exit_work.work); struct gendisk *disk; - int i; + u16 i; /* * For zero-copy and auto buffer register modes, I/O references @@ -2646,7 +2645,8 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) size_t sz = vma->vm_end - vma->vm_start; unsigned max_sz = ublk_max_cmd_buf_size(); unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT; - int q_id, ret = 0; + int ret = 0; + u16 q_id; spin_lock(&ub->lock); if (!ub->mm) @@ -2719,7 +2719,7 @@ static void ublk_abort_batch_queue(struct ublk_device *ub, */ static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq) { - int i; + u16 i; for (i = 0; i < ubq->q_depth; i++) { struct ublk_io *io = &ubq->ios[i]; @@ -2762,7 +2762,7 @@ out: ublk_put_disk(disk); } -static void ublk_cancel_cmd(struct ublk_queue *ubq, unsigned tag, +static void ublk_cancel_cmd(struct ublk_queue *ubq, u16 tag, unsigned int issue_flags) { struct ublk_io *io = &ubq->ios[tag]; @@ -2913,7 +2913,7 @@ static inline bool ublk_dev_ready(const struct ublk_device *ub) static void ublk_cancel_queue(struct ublk_queue *ubq) { - int i; + u16 i; if (ublk_support_batch_io(ubq)) { ublk_batch_cancel_queue(ubq); @@ -2927,7 +2927,7 @@ static void ublk_cancel_queue(struct ublk_queue *ubq) /* Cancel all pending commands, must be called after del_gendisk() returns */ static void ublk_cancel_dev(struct ublk_device *ub) { - int i; + u16 i; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) ublk_cancel_queue(ublk_get_queue(ub, i)); @@ -2961,7 +2961,7 @@ static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub) static void ublk_force_abort_dev(struct ublk_device *ub) { - int i; + u16 i; pr_devel("%s: force abort ub: dev_id %d state %s\n", __func__, ub->dev_info.dev_id, @@ -3158,7 +3158,7 @@ ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io, static inline void ublk_prep_cancel(struct io_uring_cmd *cmd, unsigned int issue_flags, - struct ublk_queue *ubq, unsigned int tag) + struct ublk_queue *ubq, u16 tag) { struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); @@ -3962,8 +3962,8 @@ static int ublk_handle_non_batch_cmd(struct io_uring_cmd *cmd, const struct ublksrv_io_cmd *ub_cmd = io_uring_sqe_cmd(cmd->sqe, struct ublksrv_io_cmd); struct ublk_device *ub = cmd->file->private_data; - unsigned tag = READ_ONCE(ub_cmd->tag); - unsigned q_id = READ_ONCE(ub_cmd->q_id); + u16 tag = READ_ONCE(ub_cmd->tag); + u16 q_id = READ_ONCE(ub_cmd->q_id); unsigned index = READ_ONCE(ub_cmd->addr); struct ublk_queue *ubq; struct ublk_io *io; @@ -4169,7 +4169,8 @@ static const struct file_operations ublk_ch_batch_io_fops = { static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq) { - int size, i; + int size; + u16 i; size = ublk_queue_cmd_buf_size(ub); @@ -4190,7 +4191,7 @@ static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq) kvfree(ubq); } -static void ublk_deinit_queue(struct ublk_device *ub, int q_id) +static void ublk_deinit_queue(struct ublk_device *ub, u16 q_id) { struct ublk_queue *ubq = ub->queues[q_id]; @@ -4201,7 +4202,7 @@ static void ublk_deinit_queue(struct ublk_device *ub, int q_id) ub->queues[q_id] = NULL; } -static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id) +static int ublk_get_queue_numa_node(struct ublk_device *ub, u16 q_id) { unsigned int cpu; @@ -4214,14 +4215,15 @@ static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id) return NUMA_NO_NODE; } -static int ublk_init_queue(struct ublk_device *ub, int q_id) +static int ublk_init_queue(struct ublk_device *ub, u16 q_id) { - int depth = ub->dev_info.queue_depth; + u16 depth = ub->dev_info.queue_depth; gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO; struct ublk_queue *ubq; struct page *page; int numa_node; - int size, i, ret; + int size, ret; + u16 i; /* Determine NUMA node based on queue's CPU affinity */ numa_node = ublk_get_queue_numa_node(ub, q_id); @@ -4266,7 +4268,7 @@ fail: static void ublk_deinit_queues(struct ublk_device *ub) { - int i; + u16 i; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) ublk_deinit_queue(ub, i); @@ -4274,7 +4276,8 @@ static void ublk_deinit_queues(struct ublk_device *ub) static int ublk_init_queues(struct ublk_device *ub) { - int i, ret; + int ret; + u16 i; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { ret = ublk_init_queue(ub, i); @@ -5172,7 +5175,7 @@ out: struct count_busy { const struct ublk_queue *ubq; - unsigned int nr_busy; + u16 nr_busy; }; static bool ublk_count_busy_req(struct request *rq, void *data) @@ -5210,8 +5213,7 @@ static int ublk_wait_for_idle_io(struct ublk_device *ub, return 0; while (elapsed < timeout_ms && !signal_pending(current)) { - unsigned int queues_cancelable = 0; - int i; + u16 i, queues_cancelable = 0; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { struct ublk_queue *ubq = ublk_get_queue(ub, i); From 3a00b782a7b63b1876261ab883e1d8e1aa0e09fb Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:34 -0600 Subject: [PATCH 098/241] ublk: remove struct ublk_zoned_report_desc's operation field struct ublk_zoned_report_desc's operation field is only ever set to UBLK_IO_OP_REPORT_ZONES, so remove it. Replace its one load with the constant. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei Link: https://patch.msgid.link/20260803211441.2538144-3-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 0743b5ba241d..bbb3c9c5540f 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -528,7 +528,6 @@ static void ublk_init_iod(struct ublk_queue *ubq, struct request *req, struct ublk_zoned_report_desc { __u64 sector; - __u32 operation; __u32 nr_zones; }; @@ -658,7 +657,6 @@ static int ublk_report_zones(struct gendisk *disk, sector_t sector, goto out; } - desc.operation = UBLK_IO_OP_REPORT_ZONES; desc.sector = sector; desc.nr_zones = zones_in_request; ret = ublk_zoned_insert_report_desc(req, &desc); @@ -731,15 +729,9 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, desc = ublk_zoned_get_report_desc(req); if (!desc) return BLK_STS_IOERR; - ublk_op = desc->operation; - switch (ublk_op) { - case UBLK_IO_OP_REPORT_ZONES: - ublk_init_iod(ubq, req, ublk_op, desc->nr_zones, - desc->sector); - return BLK_STS_OK; - default: - return BLK_STS_IOERR; - } + ublk_init_iod(ubq, req, UBLK_IO_OP_REPORT_ZONES, desc->nr_zones, + desc->sector); + return BLK_STS_OK; case REQ_OP_DRV_OUT: /* We do not support drv_out */ return BLK_STS_NOTSUPP; From 8c76625ff9360cbe9ce0c47a624877aea14b3499 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:35 -0600 Subject: [PATCH 099/241] ublk: split request validation from io_desc init In preparation for moving the struct ublksrv_io_desc initialization from the thread submitting ublk requests to the daemon thread receiving them, split the fallible part of ublk_setup_iod{,_zoned}() into new helper ublk_validate_req{,_zoned}(). Only ublk_setup_iod{,_zoned}() accesses the io_desc and cannot error out. Return a bool value from ublk_validate_req{,_zoned}() as the existing error code ublk_setup_iod{,_zoned}() returns is only checked against BLK_STS_OK. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei Link: https://patch.msgid.link/20260803211441.2538144-4-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 70 +++++++++++++++++++++++++++------------- 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index bbb3c9c5540f..98be9a527a69 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -700,8 +700,24 @@ out: return ret; } -static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, - struct request *req) +static bool ublk_validate_req_zoned(const struct request *req) +{ + switch (req_op(req)) { + case REQ_OP_ZONE_OPEN: + case REQ_OP_ZONE_CLOSE: + case REQ_OP_ZONE_FINISH: + case REQ_OP_ZONE_RESET: + case REQ_OP_ZONE_APPEND: + case REQ_OP_ZONE_RESET_ALL: + return true; + case REQ_OP_DRV_IN: + return !!ublk_zoned_get_report_desc(req); + default: + return false; + } +} + +static void ublk_setup_iod_zoned(struct ublk_queue *ubq, struct request *req) { struct ublk_zoned_report_desc *desc; u32 ublk_op; @@ -727,20 +743,15 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, break; case REQ_OP_DRV_IN: desc = ublk_zoned_get_report_desc(req); - if (!desc) - return BLK_STS_IOERR; ublk_init_iod(ubq, req, UBLK_IO_OP_REPORT_ZONES, desc->nr_zones, desc->sector); - return BLK_STS_OK; - case REQ_OP_DRV_OUT: - /* We do not support drv_out */ - return BLK_STS_NOTSUPP; + return; default: - return BLK_STS_IOERR; + WARN_ON_ONCE(1); + return; } ublk_init_iod(ubq, req, ublk_op, blk_rq_sectors(req), blk_rq_pos(req)); - return BLK_STS_OK; } #else @@ -761,10 +772,14 @@ static int ublk_revalidate_disk_zones(struct ublk_device *ub) return 0; } -static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, - struct request *req) +static bool ublk_validate_req_zoned(const struct request *req) { - return BLK_STS_NOTSUPP; + return false; +} + +static void ublk_setup_iod_zoned(struct ublk_queue *ubq, struct request *req) +{ + WARN_ON_ONCE(1); } #endif @@ -1497,7 +1512,22 @@ static unsigned int ublk_unmap_io(bool need_map, return rq_bytes; } -static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) +static bool ublk_validate_req(const struct ublk_queue *ubq, + const struct request *req) +{ + switch (req_op(req)) { + case REQ_OP_READ: + case REQ_OP_WRITE: + case REQ_OP_FLUSH: + case REQ_OP_DISCARD: + case REQ_OP_WRITE_ZEROES: + return true; + default: + return ublk_queue_is_zoned(ubq) && ublk_validate_req_zoned(req); + } +} + +static void ublk_setup_iod(struct ublk_queue *ubq, struct request *req) { u32 ublk_op; @@ -1518,13 +1548,11 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) ublk_op = UBLK_IO_OP_WRITE_ZEROES; break; default: - if (ublk_queue_is_zoned(ubq)) - return ublk_setup_iod_zoned(ubq, req); - return BLK_STS_IOERR; + ublk_setup_iod_zoned(ubq, req); + return; } ublk_init_iod(ubq, req, ublk_op, blk_rq_sectors(req), blk_rq_pos(req)); - return BLK_STS_OK; } static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu( @@ -2138,8 +2166,6 @@ static enum blk_eh_timer_return ublk_timeout(struct request *rq) static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq, bool check_cancel) { - blk_status_t res; - if (unlikely(READ_ONCE(ubq->fail_io))) return BLK_STS_TARGET; @@ -2160,10 +2186,10 @@ static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq, return BLK_STS_IOERR; /* fill iod to slot in io cmd buffer */ - res = ublk_setup_iod(ubq, rq); - if (unlikely(res != BLK_STS_OK)) + if (unlikely(!ublk_validate_req(ubq, rq))) return BLK_STS_IOERR; + ublk_setup_iod(ubq, rq); blk_mq_start_request(rq); return BLK_STS_OK; } From 735409f58b3da45094f2dfd7c18fb9e1937431f1 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:36 -0600 Subject: [PATCH 100/241] ublk: initialize io_desc on daemon task ublk_setup_iod() is currently called to populate struct ublksrv_io_desc on the thread submitting I/O to a ublk device. However, only the ublk server threads read the io_descs. This basically guarantees a cache miss on both threads for each ublk I/O. There's really no need to initialize the io_descs on the submitting thread. Move the ublk_setup_iod() call to ublk_dispatch_req() (for non-UBLK_F_BATCH_IO) and __ublk_batch_prep_dispatch() (for UBLK_F_BATCH_IO), which runs on the ublk server daemon thread before dispatching the I/O to userspace. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei Link: https://patch.msgid.link/20260803211441.2538144-5-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 98be9a527a69..369757f5af08 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1800,6 +1800,7 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req) u16 tag = req->tag; struct ublk_io *io = &ubq->ios[tag]; + ublk_setup_iod(ubq, req); pr_devel("%s: complete: qid %d tag %d io_flags %x addr %llx\n", __func__, ubq->q_id, req->tag, io->flags, ublk_get_iod(ubq, req->tag)->addr); @@ -1853,6 +1854,7 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq, enum auto_buf_reg_res res = AUTO_BUF_REG_FALLBACK; struct io_uring_cmd *cmd = data->cmd; + ublk_setup_iod(ubq, req); if (!ublk_start_io(ubq, req, io)) return false; @@ -2189,7 +2191,6 @@ static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq, if (unlikely(!ublk_validate_req(ubq, rq))) return BLK_STS_IOERR; - ublk_setup_iod(ubq, rq); blk_mq_start_request(rq); return BLK_STS_OK; } From 5c0958d80190822c4614ca00478d9acd9671bba6 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:37 -0600 Subject: [PATCH 101/241] ublk: add UBLK_F_IO_DESC_SIZE ublk passes the parameters of incoming I/O in memory shared between the kernel ublk driver and userspace ublk server in struct ublksrv_io_desc. The size of this struct is currently fixed to 24 bytes, which has been an obstacle to extending it with additional fields [1]. Additionally, with multiple ublk server threads handling I/Os from the same ublk queue (possible with UBLK_F_PER_IO_DAEMON or UBLK_F_BATCH_IO), false sharing results from adjacent io_descs sharing the same cache line. Add a ublk feature UBLK_F_IO_DESC_SIZE to allow a ublk server to override the size of each io_desc. The size must be at least 24 and a multiple of 8 to store a properly-aligned struct ublksrv_io_desc. It's also limited to a maximum of 256, though this bound could be lifted in the future. The struct ublksrv_io_desc is located at the beginning of each io_desc and the remainder is padding. The mmap() performed for each queue must have a length of queue_depth * io_desc_size rounded up to the page size. The mmap() offset must be q_id * UBLK_MAX_QUEUE_DEPTH * io_desc_size, also rounded up to the page size. [1]: https://lore.kernel.org/linux-block/aV8QfvaNO5P6vOs6@fedora/ Suggested-by: Ming Lei Signed-off-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260803211441.2538144-6-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 38 ++++++++++++++++++++-------- include/uapi/linux/ublk_cmd.h | 5 +++- tools/testing/selftests/ublk/kublk.c | 1 + 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 369757f5af08..08d29c9ab898 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -89,7 +89,8 @@ | UBLK_F_SAFE_STOP_DEV \ | UBLK_F_BATCH_IO \ | UBLK_F_NO_AUTO_PART_SCAN \ - | UBLK_F_SHMEM_ZC) + | UBLK_F_SHMEM_ZC \ + | UBLK_F_IO_DESC_SIZE) #define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ | UBLK_F_USER_RECOVERY_REISSUE \ @@ -107,6 +108,8 @@ UBLK_BATCH_F_HAS_BUF_ADDR | \ UBLK_BATCH_F_AUTO_BUF_REG_FALLBACK) +#define UBLK_MAX_IO_DESC_SIZE 256 + /* ublk batch fetch uring_cmd */ struct ublk_batch_fetch_cmd { struct list_head node; @@ -239,6 +242,7 @@ struct ublk_io { struct ublk_queue { u16 q_id; u16 q_depth; + u16 io_desc_size; unsigned long flags; struct ublksrv_io_desc *io_cmd_buf; @@ -405,7 +409,7 @@ static inline void ublk_io_evts_deinit(struct ublk_queue *q) static inline struct ublksrv_io_desc * ublk_get_iod(const struct ublk_queue *ubq, u16 tag) { - return &ubq->io_cmd_buf[tag]; + return (void *)ubq->io_cmd_buf + tag * (size_t)ubq->io_desc_size; } static inline bool ublk_support_zero_copy(const struct ublk_queue *ubq) @@ -1248,19 +1252,20 @@ ublk_queue_cmd_buf(struct ublk_device *ub, u16 q_id) return ublk_get_queue(ub, q_id)->io_cmd_buf; } -static inline int __ublk_queue_cmd_buf_size(u16 depth) +static inline size_t __ublk_queue_cmd_buf_size(const struct ublk_device *ub, + u16 depth) { - return round_up(depth * sizeof(struct ublksrv_io_desc), PAGE_SIZE); + return round_up(depth * (size_t)ub->dev_info.io_desc_size, PAGE_SIZE); } -static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub) +static inline size_t ublk_queue_cmd_buf_size(const struct ublk_device *ub) { - return __ublk_queue_cmd_buf_size(ub->dev_info.queue_depth); + return __ublk_queue_cmd_buf_size(ub, ub->dev_info.queue_depth); } -static int ublk_max_cmd_buf_size(void) +static size_t ublk_max_cmd_buf_size(const struct ublk_device *ub) { - return __ublk_queue_cmd_buf_size(UBLK_MAX_QUEUE_DEPTH); + return __ublk_queue_cmd_buf_size(ub, UBLK_MAX_QUEUE_DEPTH); } /* @@ -2662,7 +2667,7 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) { struct ublk_device *ub = filp->private_data; size_t sz = vma->vm_end - vma->vm_start; - unsigned max_sz = ublk_max_cmd_buf_size(); + size_t max_sz = ublk_max_cmd_buf_size(ub); unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT; int ret = 0; u16 q_id; @@ -4188,7 +4193,7 @@ static const struct file_operations ublk_ch_batch_io_fops = { static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq) { - int size; + size_t size; u16 i; size = ublk_queue_cmd_buf_size(ub); @@ -4241,7 +4246,8 @@ static int ublk_init_queue(struct ublk_device *ub, u16 q_id) struct ublk_queue *ubq; struct page *page; int numa_node; - int size, ret; + size_t size; + int ret; u16 i; /* Determine NUMA node based on queue's CPU affinity */ @@ -4266,6 +4272,7 @@ static int ublk_init_queue(struct ublk_device *ub, u16 q_id) return -ENOMEM; } ubq->io_cmd_buf = page_address(page); + ubq->io_desc_size = ub->dev_info.io_desc_size; for (i = 0; i < ubq->q_depth; i++) spin_lock_init(&ubq->ios[i].lock); @@ -4750,6 +4757,15 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) if (info.flags & UBLK_F_INTEGRITY && !(info.flags & UBLK_F_USER_COPY)) return -EINVAL; + if (info.flags & UBLK_F_IO_DESC_SIZE) { + if (info.io_desc_size < sizeof(struct ublksrv_io_desc) || + info.io_desc_size % _Alignof(struct ublksrv_io_desc) || + info.io_desc_size > UBLK_MAX_IO_DESC_SIZE) + return -EINVAL; + } else { + info.io_desc_size = sizeof(struct ublksrv_io_desc); + } + /* the created device is always owned by current user */ ublk_store_owner_uid_gid(&info.owner_uid, &info.owner_gid); diff --git a/include/uapi/linux/ublk_cmd.h b/include/uapi/linux/ublk_cmd.h index 6991370a72ce..33b25dd13965 100644 --- a/include/uapi/linux/ublk_cmd.h +++ b/include/uapi/linux/ublk_cmd.h @@ -417,6 +417,9 @@ struct ublk_shmem_buf_reg { */ #define UBLK_F_SHMEM_ZC (1ULL << 19) +/* ublksrv_io_desc size is specified by ublksrv_ctrl_dev_info's io_desc_size */ +#define UBLK_F_IO_DESC_SIZE (1ULL << 20) + /* device state */ #define UBLK_S_DEV_DEAD 0 #define UBLK_S_DEV_LIVE 1 @@ -452,7 +455,7 @@ struct ublksrv_ctrl_dev_info { __u16 nr_hw_queues; __u16 queue_depth; __u16 state; - __u16 pad0; + __u16 io_desc_size; __u32 max_io_buf_bytes; __u32 dev_id; diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c index 0b23c09daea5..5c4a1f18d0a3 100644 --- a/tools/testing/selftests/ublk/kublk.c +++ b/tools/testing/selftests/ublk/kublk.c @@ -1970,6 +1970,7 @@ static int cmd_dev_get_features(void) FEAT_NAME(UBLK_F_BATCH_IO), FEAT_NAME(UBLK_F_NO_AUTO_PART_SCAN), FEAT_NAME(UBLK_F_SHMEM_ZC), + FEAT_NAME(UBLK_F_IO_DESC_SIZE), }; struct ublk_dev *dev; __u64 features = 0; From fc01b96d74b3a9eec2b558d49ec9f5176473766a Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:38 -0600 Subject: [PATCH 102/241] selftests: ublk: add support for --io_desc_size Add an optional --io_desc_size argument to the kublk add/recover commands to enable UBLK_F_IO_DESC on the ublk device. The mmap() arguments and ublk_get_iod() computation are adjusted accordingly. Display the configured io_desc_size in the kublk list output for ublk devices with UBLK_F_IO_DESC. Signed-off-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260803211441.2538144-7-csander@purestorage.com Signed-off-by: Jens Axboe --- tools/testing/selftests/ublk/kublk.c | 29 +++++++++++++++++++--------- tools/testing/selftests/ublk/kublk.h | 6 ++++-- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c index 5c4a1f18d0a3..0e3e2d74cc4d 100644 --- a/tools/testing/selftests/ublk/kublk.c +++ b/tools/testing/selftests/ublk/kublk.c @@ -352,6 +352,8 @@ static void ublk_ctrl_dump(struct ublk_dev *dev) ublk_log("\tmax rq size %d daemon pid %d flags 0x%llx state %s\n", info->max_io_buf_bytes, info->ublksrv_pid, info->flags, ublk_dev_state_desc(dev)); + if (info->flags & UBLK_F_IO_DESC_SIZE) + ublk_log("\tio_desc_size %u\n", info->io_desc_size); if (affinity) { char buf[512]; @@ -400,22 +402,22 @@ static struct ublk_dev *ublk_ctrl_init(void) return dev; } -static int __ublk_queue_cmd_buf_sz(unsigned depth) +static size_t __ublk_queue_cmd_buf_sz(const struct ublk_queue *q, __u16 depth) { - int size = depth * sizeof(struct ublksrv_io_desc); - unsigned int page_sz = getpagesize(); + size_t size = depth * (size_t)q->io_desc_size; + size_t page_sz = getpagesize(); return round_up(size, page_sz); } -static int ublk_queue_max_cmd_buf_sz(void) +static size_t ublk_queue_max_cmd_buf_sz(const struct ublk_queue *q) { - return __ublk_queue_cmd_buf_sz(UBLK_MAX_QUEUE_DEPTH); + return __ublk_queue_cmd_buf_sz(q, UBLK_MAX_QUEUE_DEPTH); } -static int ublk_queue_cmd_buf_sz(struct ublk_queue *q) +static size_t ublk_queue_cmd_buf_sz(const struct ublk_queue *q) { - return __ublk_queue_cmd_buf_sz(q->q_depth); + return __ublk_queue_cmd_buf_sz(q, q->q_depth); } static void ublk_queue_deinit(struct ublk_queue *q) @@ -453,7 +455,7 @@ static int ublk_queue_init(struct ublk_queue *q, unsigned long long extra_flags, struct ublk_dev *dev = q->dev; int depth = dev->dev_info.queue_depth; int i; - int cmd_buf_size, io_buf_size, integrity_size; + size_t cmd_buf_size, io_buf_size, integrity_size; unsigned long off; pthread_spin_init(&q->lock, PTHREAD_PROCESS_PRIVATE); @@ -463,12 +465,13 @@ static int ublk_queue_init(struct ublk_queue *q, unsigned long long extra_flags, q->flags = dev->dev_info.flags; q->flags |= extra_flags; q->metadata_size = metadata_size; + q->io_desc_size = dev->dev_info.io_desc_size; /* Cache fd in queue for fast path access */ q->ublk_fd = dev->fds[0]; cmd_buf_size = ublk_queue_cmd_buf_sz(q); - off = UBLKSRV_CMD_BUF_OFFSET + q->q_id * ublk_queue_max_cmd_buf_sz(); + off = UBLKSRV_CMD_BUF_OFFSET + q->q_id * ublk_queue_max_cmd_buf_sz(q); q->io_cmd_buf = mmap(0, cmd_buf_size, PROT_READ, MAP_SHARED | MAP_POPULATE, dev->fds[0], off); if (q->io_cmd_buf == MAP_FAILED) { @@ -1708,6 +1711,7 @@ static int __cmd_dev_add(const struct dev_ctx *ctx) info->dev_id = ctx->dev_id; info->nr_hw_queues = nr_queues; info->queue_depth = depth; + info->io_desc_size = ctx->io_desc_size; info->flags = ctx->flags; if ((features & UBLK_F_QUIESCE) && (info->flags & UBLK_F_USER_RECOVERY)) @@ -2069,6 +2073,7 @@ static void __cmd_create_help(char *exe, bool recovery) printf("\t[--integrity_capable] [--integrity_reftag] [--metadata_size SIZE] " "[--pi_offset OFFSET] [--csum_type ip|t10dif|nvme] [--tag_size SIZE]\n"); printf("\t[--batch|-b] [--no_auto_part_scan]\n"); + printf("\t[--io_desc_size SIZE]\n"); printf("\t[target options] [backfile1] [backfile2] ...\n"); printf("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n"); printf("\tdefault: nthreads=nr_queues"); @@ -2146,6 +2151,7 @@ int main(int argc, char *argv[]) { "shmem_zc", 0, NULL, 0 }, { "htlb", 1, NULL, 0 }, { "rdonly_shmem_buf", 0, NULL, 0 }, + { "io_desc_size", 1, NULL, 0 }, { 0, 0, 0, 0 } }; const struct ublk_tgt_ops *ops = NULL; @@ -2158,6 +2164,7 @@ int main(int argc, char *argv[]) .dev_id = -1, .tgt_type = "unknown", .csum_type = LBMD_PI_CSUM_NONE, + .io_desc_size = sizeof(struct ublksrv_io_desc), }; int ret = -EINVAL, i; int tgt_argc = 1; @@ -2267,6 +2274,10 @@ int main(int argc, char *argv[]) ctx.htlb_path = strdup(optarg); if (!strcmp(longopts[option_idx].name, "rdonly_shmem_buf")) ctx.rdonly_shmem_buf = 1; + if (!strcmp(longopts[option_idx].name, "io_desc_size")) { + ctx.flags |= UBLK_F_IO_DESC_SIZE; + ctx.io_desc_size = strtoul(optarg, NULL, 0); + } break; case '?': /* diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h index 742c41d77df1..15b56ff45bb6 100644 --- a/tools/testing/selftests/ublk/kublk.h +++ b/tools/testing/selftests/ublk/kublk.h @@ -87,6 +87,7 @@ struct dev_ctx { __u8 pi_offset; __u8 csum_type; __u8 tag_size; + __u16 io_desc_size; int _evtfd; int _shmid; @@ -187,6 +188,7 @@ struct ublk_queue { __u64 flags; int ublk_fd; /* cached ublk char device fd */ __u8 metadata_size; + __u16 io_desc_size; struct ublk_io ios[UBLK_QUEUE_DEPTH]; /* used for prep io commands */ @@ -461,9 +463,9 @@ static inline void ublk_mark_io_done(struct ublk_io *io, int res) io->result = res; } -static inline const struct ublksrv_io_desc *ublk_get_iod(const struct ublk_queue *q, int tag) +static inline const struct ublksrv_io_desc *ublk_get_iod(const struct ublk_queue *q, __u16 tag) { - return &q->io_cmd_buf[tag]; + return (void *)q->io_cmd_buf + tag * (size_t)q->io_desc_size; } static inline void ublk_set_sqe_cmd_op(struct io_uring_sqe *sqe, __u32 cmd_op) From d61d0f95e686be015cfaf193c0ae15156d1a0cc4 Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:39 -0600 Subject: [PATCH 103/241] selftests: ublk: add UBLK_F_IO_DESC_SIZE test Add test loop_08, which creates a ublk device with UBLK_F_IO_DESC_SIZE enabled and io_desc_size set to 64. The test issues verified I/O to the device using fio. Signed-off-by: Caleb Sander Mateos Link: https://patch.msgid.link/20260803211441.2538144-8-csander@purestorage.com Signed-off-by: Jens Axboe --- tools/testing/selftests/ublk/Makefile | 1 + tools/testing/selftests/ublk/test_loop_08.sh | 25 ++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100755 tools/testing/selftests/ublk/test_loop_08.sh diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile index 6e4fe8d1fed1..b00ef238a038 100644 --- a/tools/testing/selftests/ublk/Makefile +++ b/tools/testing/selftests/ublk/Makefile @@ -34,6 +34,7 @@ TEST_PROGS += test_loop_04.sh TEST_PROGS += test_loop_05.sh TEST_PROGS += test_loop_06.sh TEST_PROGS += test_loop_07.sh +TEST_PROGS += test_loop_08.sh TEST_PROGS += test_integrity_01.sh TEST_PROGS += test_integrity_02.sh diff --git a/tools/testing/selftests/ublk/test_loop_08.sh b/tools/testing/selftests/ublk/test_loop_08.sh new file mode 100755 index 000000000000..f7af2587482d --- /dev/null +++ b/tools/testing/selftests/ublk/test_loop_08.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh + +ERR_CODE=0 + +if ! _have_program fio; then + exit "$UBLK_SKIP_CODE" +fi + +_prep_test "loop" "write and verify with io_desc_size" + +_create_backfile 0 256M + +dev_id=$(_add_ublk_dev -t loop --io_desc_size 64 "${UBLK_BACKFILES[0]}") +_check_add_dev $TID $? + +# run fio over the ublk disk +_run_fio_verify_io --filename=/dev/ublkb"${dev_id}" --size=256M +ERR_CODE=$? + +_cleanup_test + +_show_result $TID $ERR_CODE From a8a79eba22dc4c11f2877bcf9e8557f6d95541ac Mon Sep 17 00:00:00 2001 From: Caleb Sander Mateos Date: Mon, 3 Aug 2026 15:14:40 -0600 Subject: [PATCH 104/241] ublk: lift checks out of ublk_{,un}map_io() ublk_map_io() and ublk_unmap_io() are no-ops for ublk devices that enable user copy or zero copy, as well as for requests without data to copy in the given direction. However, the implementation is a bit convoluted, returning the full request data length and relying on the caller to check the return value against the request length. UBLK_F_SHMEM_ZC recently added branches to skip the ublk_{,un}map_io() call for I/Os using a shared-memory buffer. This is a more logical place for the device need_map and the ublk_need_{,un}map_req() checks, so move them there from ublk_{,un}map_io(). Checking these conditions early also skips the expensive pointer-chasing for the ublk_iod_is_shmem_zc() check in __ublk_complete_rq() for the common case of a ublk device using user copy or zero copy. Drop the req_op() filter in __ublk_complete_rq(), as it's redundant with the ublk_need_unmap_req() check. Signed-off-by: Caleb Sander Mateos Reviewed-by: Ming Lei Link: https://patch.msgid.link/20260803211441.2538144-9-csander@purestorage.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 70 +++++++++++----------------------------- 1 file changed, 19 insertions(+), 51 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 08d29c9ab898..8ad61c2d434c 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1468,53 +1468,29 @@ static inline bool ublk_need_unmap_req(const struct request *req) (req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN); } -static unsigned int ublk_map_io(const struct ublk_queue *ubq, - const struct request *req, +static unsigned int ublk_map_io(const struct request *req, const struct ublk_io *io) { - const unsigned int rq_bytes = blk_rq_bytes(req); + struct iov_iter iter; + const int dir = ITER_DEST; - if (!ublk_need_map_io(ubq)) - return rq_bytes; + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), blk_rq_bytes(req), + &iter) < 0) + return 0; - /* - * no zero copy, we delay copy WRITE request data into ublksrv - * context and the big benefit is that pinning pages in current - * context is pretty fast, see ublk_pin_user_pages - */ - if (ublk_need_map_req(req)) { - struct iov_iter iter; - const int dir = ITER_DEST; - - if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, - &iter) < 0) - return 0; - - return ublk_copy_user_pages(req, 0, &iter, dir); - } - return rq_bytes; + return ublk_copy_user_pages(req, 0, &iter, dir); } -static unsigned int ublk_unmap_io(bool need_map, - const struct request *req, +static unsigned int ublk_unmap_io(const struct request *req, const struct ublk_io *io) { - const unsigned int rq_bytes = blk_rq_bytes(req); + struct iov_iter iter; + const int dir = ITER_SOURCE; - if (!need_map) - return rq_bytes; + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter) < 0) + return 0; - if (ublk_need_unmap_req(req)) { - struct iov_iter iter; - const int dir = ITER_SOURCE; - - if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, - &iter) < 0) - return 0; - - return ublk_copy_user_pages(req, 0, &iter, dir); - } - return rq_bytes; + return ublk_copy_user_pages(req, 0, &iter, dir); } static bool ublk_validate_req(const struct ublk_queue *ubq, @@ -1590,22 +1566,13 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io, goto exit; } - /* - * FLUSH, DISCARD or WRITE_ZEROES usually won't return bytes returned, so end them - * directly. - * - * Both the two needn't unmap. - */ - if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE && - req_op(req) != REQ_OP_DRV_IN) - goto exit; - /* shmem zero copy: no data to unmap, pages already shared */ - if (ublk_iod_is_shmem_zc(req->mq_hctx->driver_data, req->tag)) + if (!need_map || !ublk_need_unmap_req(req) || + ublk_iod_is_shmem_zc(req->mq_hctx->driver_data, req->tag)) goto exit; /* for READ request, writing data in iod->addr to rq buffers */ - unmapped_bytes = ublk_unmap_io(need_map, req, io); + unmapped_bytes = ublk_unmap_io(req, io); /* * Extremely impossible since we got data filled in just before @@ -1771,10 +1738,11 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, unsigned mapped_bytes; /* shmem zero copy: skip data copy, pages already shared */ - if (ublk_iod_is_shmem_zc(ubq, req->tag)) + if (!ublk_need_map_io(ubq) || !ublk_need_map_req(req) || + ublk_iod_is_shmem_zc(ubq, req->tag)) return true; - mapped_bytes = ublk_map_io(ubq, req, io); + mapped_bytes = ublk_map_io(req, io); /* partially mapped, update io descriptor */ if (unlikely(mapped_bytes != blk_rq_bytes(req))) { From ca5a01eee34c7cbe0f531a613b0292a3ad1a419b Mon Sep 17 00:00:00 2001 From: Yang Xiuwei Date: Thu, 30 Jul 2026 09:09:09 +0800 Subject: [PATCH 105/241] ublk: validate auto buf reg before taking uring_cmd With UBLK_F_AUTO_BUF_REG, invalid sqe->addr can fail after ublk_fill_io_cmd() has set UBLK_IO_FLAG_ACTIVE. The uring_cmd is completed while the tag stays active, which can hang teardown. Split validation from buffer apply so the check has no side effects, then take the uring_cmd and store the already-validated buffer. Apply the same order in FETCH so io->buf is not written before __ublk_fetch() state checks. Fixes: 52460dda3a77 ("ublk: move auto buffer register handling into one dedicated helper") Suggested-by: Caleb Sander Mateos Signed-off-by: Yang Xiuwei Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 74 +++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 8ad61c2d434c..a632fbcc03b5 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -3077,18 +3077,19 @@ static inline int ublk_check_cmd_op(u32 cmd_op) return 0; } -static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd) +/* Must run before ublk_fill_io_cmd() / __ublk_fetch(). */ +static inline int ublk_validate_io_buf(const struct ublk_device *ub, + struct io_uring_cmd *cmd, + struct ublk_auto_buf_reg *buf) { - struct ublk_auto_buf_reg buf; + if (!ublk_dev_support_auto_buf_reg(ub)) + return 0; - buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); - - if (buf.reserved0 || buf.reserved1) + *buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + if (buf->reserved0 || buf->reserved1) return -EINVAL; - - if (buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) + if (buf->flags & ~UBLK_AUTO_BUF_REG_F_MASK) return -EINVAL; - io->buf.auto_reg = buf; return 0; } @@ -3109,17 +3110,25 @@ static void ublk_clear_auto_buf_reg(struct ublk_io *io, * responsibility for unregistering the buffer, otherwise * this ublk request gets stuck. */ - if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) + if (buf_idx && + io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) *buf_idx = io->buf.auto_reg.index; } } -static int ublk_handle_auto_buf_reg(struct ublk_io *io, - struct io_uring_cmd *cmd, - u16 *buf_idx) +static inline void ublk_apply_io_buf(const struct ublk_device *ub, + struct ublk_io *io, + struct io_uring_cmd *cmd, + unsigned long buf_addr, + const struct ublk_auto_buf_reg *auto_buf, + u16 *buf_idx) { - ublk_clear_auto_buf_reg(io, cmd, buf_idx); - return ublk_set_auto_buf_reg(io, cmd); + if (ublk_dev_support_auto_buf_reg(ub)) { + ublk_clear_auto_buf_reg(io, cmd, buf_idx); + io->buf.auto_reg = *auto_buf; + } else { + io->buf.addr = buf_addr; + } } /* Once we return, `io->req` can't be used any more */ @@ -3136,18 +3145,6 @@ ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd) return req; } -static inline int -ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io, - struct io_uring_cmd *cmd, unsigned long buf_addr, - u16 *buf_idx) -{ - if (ublk_dev_support_auto_buf_reg(ub)) - return ublk_handle_auto_buf_reg(io, cmd, buf_idx); - - io->buf.addr = buf_addr; - return 0; -} - static inline void ublk_prep_cancel(struct io_uring_cmd *cmd, unsigned int issue_flags, struct ublk_queue *ubq, u16 tag) @@ -3288,6 +3285,7 @@ static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, struct ublk_io *io, __u64 buf_addr, u16 q_id) { + struct ublk_auto_buf_reg auto_buf; int ret; /* @@ -3296,11 +3294,13 @@ static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, * FETCH, so it is fine even for IO_URING_F_NONBLOCK. */ mutex_lock(&ub->mutex); - ret = __ublk_fetch(cmd, ub, io, q_id); - if (!ret) - ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL); + ret = ublk_validate_io_buf(ub, cmd, &auto_buf); if (!ret) + ret = __ublk_fetch(cmd, ub, io, q_id); + if (!ret) { + ublk_apply_io_buf(ub, io, cmd, buf_addr, &auto_buf, NULL); ublk_mark_io_ready(ub, q_id, io); + } mutex_unlock(&ub->mutex); return ret; } @@ -3443,13 +3443,18 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, case UBLK_IO_REGISTER_IO_BUF: return ublk_daemon_register_io_buf(cmd, ub, q_id, tag, io, addr, issue_flags); - case UBLK_IO_COMMIT_AND_FETCH_REQ: + case UBLK_IO_COMMIT_AND_FETCH_REQ: { + struct ublk_auto_buf_reg auto_buf; + ret = ublk_check_commit_and_fetch(ub, io, addr); + if (ret) + goto out; + ret = ublk_validate_io_buf(ub, cmd, &auto_buf); if (ret) goto out; io->res = result; req = ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); + ublk_apply_io_buf(ub, io, cmd, addr, &auto_buf, &buf_idx); if (buf_idx != UBLK_INVALID_BUF_IDX) io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); @@ -3458,10 +3463,8 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, req->__sector = addr; if (compl) __ublk_complete_rq(req, io, ublk_dev_need_map_io(ub), NULL); - - if (ret) - goto out; break; + } case UBLK_IO_NEED_GET_DATA: /* * ublk_get_data() may fail and fallback to requeue, so keep @@ -3469,8 +3472,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, * request */ req = ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, addr, NULL); - WARN_ON_ONCE(ret); + io->buf.addr = addr; if (likely(ublk_get_data(ubq, io, req))) { __ublk_prep_compl_io_cmd(io, req); return UBLK_IO_RES_OK; From d507d3cb19e71f10336e41deeac517cdc21c34ea Mon Sep 17 00:00:00 2001 From: Yang Xiuwei Date: Thu, 30 Jul 2026 10:40:50 +0800 Subject: [PATCH 106/241] selftests: ublk: add rotating auto_buf index regression test Batch AUTO_BUF_REG COMMIT must unregister the old auto_buf index before storing the next one. Fixed per-tag indexing (A == B) masks bugs that clear after overwriting io->buf. Add kublk --rotate_auto_buf so each tag alternates between two sparse buffer indices, and test_batch_04.sh to exercise that path. Without the driver fix, the request ref stays stuck and I/O hangs; the test uses a short timeout and kills the ublk daemon to recover. With the fix, a small write completes quickly. Signed-off-by: Yang Xiuwei Reviewed-by: Ming Lei Link: https://patch.msgid.link/20260730024050.1062354-1-yangxiuwei@kylinos.cn Signed-off-by: Jens Axboe --- tools/testing/selftests/ublk/Makefile | 1 + tools/testing/selftests/ublk/batch.c | 2 +- tools/testing/selftests/ublk/kublk.c | 19 +++++++- tools/testing/selftests/ublk/kublk.h | 19 +++++++- tools/testing/selftests/ublk/test_batch_04.sh | 44 +++++++++++++++++++ 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100755 tools/testing/selftests/ublk/test_batch_04.sh diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile index b00ef238a038..a3cec7b35db7 100644 --- a/tools/testing/selftests/ublk/Makefile +++ b/tools/testing/selftests/ublk/Makefile @@ -23,6 +23,7 @@ TEST_PROGS += test_generic_17.sh TEST_PROGS += test_batch_01.sh TEST_PROGS += test_batch_02.sh TEST_PROGS += test_batch_03.sh +TEST_PROGS += test_batch_04.sh TEST_PROGS += test_null_01.sh TEST_PROGS += test_null_02.sh diff --git a/tools/testing/selftests/ublk/batch.c b/tools/testing/selftests/ublk/batch.c index a54025b00917..d8d9ebed5979 100644 --- a/tools/testing/selftests/ublk/batch.c +++ b/tools/testing/selftests/ublk/batch.c @@ -535,7 +535,7 @@ void ublk_batch_complete_io(struct ublk_thread *t, struct ublk_queue *q, elem = (struct ublk_batch_elem *)(cb->elem + cb->done * t->commit_buf_elem_size); elem->tag = tag; - elem->buf_index = ublk_batch_io_buf_idx(t, q, tag); + elem->buf_index = ublk_batch_io_buf_idx_next(t, q, tag); elem->result = res; if (!ublk_queue_no_buf(q)) diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c index 0e3e2d74cc4d..be5a0d775952 100644 --- a/tools/testing/selftests/ublk/kublk.c +++ b/tools/testing/selftests/ublk/kublk.c @@ -543,9 +543,14 @@ static int ublk_thread_init(struct ublk_thread *t, unsigned long long extra_flag unsigned max_nr_ios_per_thread = nr_ios / dev->nthreads; max_nr_ios_per_thread += !!(nr_ios % dev->nthreads); + t->auto_buf_stride = max_nr_ios_per_thread; t->nr_bufs = max_nr_ios_per_thread; + if ((extra_flags & UBLKS_Q_ROTATE_AUTO_BUF) && + (dev->dev_info.flags & UBLK_F_AUTO_BUF_REG)) + t->nr_bufs *= 2; } else { t->nr_bufs = 0; + t->auto_buf_stride = 0; } if (ublk_dev_batch_io(dev)) @@ -1439,6 +1444,8 @@ static int ublk_start_daemon(const struct dev_ctx *ctx, struct ublk_dev *dev) extra_flags = UBLKS_Q_AUTO_BUF_REG_FALLBACK; if (ctx->no_ublk_fixed_fd) extra_flags |= UBLKS_Q_NO_UBLK_FIXED_FD; + if (ctx->rotate_auto_buf) + extra_flags |= UBLKS_Q_ROTATE_AUTO_BUF; for (i = 0; i < dinfo->nr_hw_queues; i++) { dev->q[i].dev = dev; @@ -2072,7 +2079,7 @@ static void __cmd_create_help(char *exe, bool recovery) printf("\t[--nthreads threads] [--per_io_tasks]\n"); printf("\t[--integrity_capable] [--integrity_reftag] [--metadata_size SIZE] " "[--pi_offset OFFSET] [--csum_type ip|t10dif|nvme] [--tag_size SIZE]\n"); - printf("\t[--batch|-b] [--no_auto_part_scan]\n"); + printf("\t[--batch|-b] [--rotate_auto_buf] [--no_auto_part_scan]\n"); printf("\t[--io_desc_size SIZE]\n"); printf("\t[target options] [backfile1] [backfile2] ...\n"); printf("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n"); @@ -2147,6 +2154,7 @@ int main(int argc, char *argv[]) { "tag_size", 1, NULL, 0 }, { "safe", 0, NULL, 0 }, { "batch", 0, NULL, 'b'}, + { "rotate_auto_buf", 0, NULL, 0 }, { "no_auto_part_scan", 0, NULL, 0 }, { "shmem_zc", 0, NULL, 0 }, { "htlb", 1, NULL, 0 }, @@ -2236,6 +2244,8 @@ int main(int argc, char *argv[]) ctx.flags |= UBLK_F_AUTO_BUF_REG; if (!strcmp(longopts[option_idx].name, "auto_zc_fallback")) ctx.auto_zc_fallback = 1; + if (!strcmp(longopts[option_idx].name, "rotate_auto_buf")) + ctx.rotate_auto_buf = 1; if (!strcmp(longopts[option_idx].name, "nthreads")) ctx.nthreads = strtol(optarg, NULL, 10); if (!strcmp(longopts[option_idx].name, "per_io_tasks")) @@ -2347,6 +2357,13 @@ int main(int argc, char *argv[]) return -EINVAL; } + if (ctx.rotate_auto_buf && + !((ctx.flags & UBLK_F_AUTO_BUF_REG) && + (ctx.flags & UBLK_F_BATCH_IO))) { + ublk_err("rotate_auto_buf requires --auto_zc and --batch\n"); + return -EINVAL; + } + i = optind; while (i < argc && ctx.nr_files < MAX_BACK_FILES) { ctx.files[ctx.nr_files++] = argv[i++]; diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h index 15b56ff45bb6..e27c154fc910 100644 --- a/tools/testing/selftests/ublk/kublk.h +++ b/tools/testing/selftests/ublk/kublk.h @@ -82,6 +82,7 @@ struct dev_ctx { unsigned int safe_stop:1; unsigned int no_auto_part_scan:1; unsigned int rdonly_shmem_buf:1; + unsigned int rotate_auto_buf:1; __u32 integrity_flags; __u8 metadata_size; __u8 pi_offset; @@ -135,6 +136,7 @@ struct ublk_io { unsigned short buf_index; unsigned short tgt_ios; + unsigned char auto_buf_phase; void *private_data; }; @@ -185,6 +187,7 @@ struct ublk_queue { #define UBLKS_Q_AUTO_BUF_REG_FALLBACK (1ULL << 63) #define UBLKS_Q_NO_UBLK_FIXED_FD (1ULL << 62) #define UBLKS_Q_PREPARED (1ULL << 61) +#define UBLKS_Q_ROTATE_AUTO_BUF (1ULL << 60) __u64 flags; int ublk_fd; /* cached ublk char device fd */ __u8 metadata_size; @@ -234,6 +237,7 @@ struct ublk_thread { unsigned int io_inflight; unsigned short nr_bufs; + unsigned short auto_buf_stride; /* followings are for BATCH_IO */ unsigned short commit_buf_start; @@ -552,7 +556,20 @@ static inline unsigned short ublk_batch_io_buf_idx( const struct ublk_thread *t, const struct ublk_queue *q, unsigned tag) { - return ublk_queue_idx_in_thread(t, q) * q->q_depth + tag; + unsigned short base = ublk_queue_idx_in_thread(t, q) * q->q_depth + tag; + + if (q->flags & UBLKS_Q_ROTATE_AUTO_BUF) + return base + q->ios[tag].auto_buf_phase * t->auto_buf_stride; + return base; +} + +static inline unsigned short ublk_batch_io_buf_idx_next( + const struct ublk_thread *t, struct ublk_queue *q, + unsigned tag) +{ + if (q->flags & UBLKS_Q_ROTATE_AUTO_BUF) + q->ios[tag].auto_buf_phase ^= 1; + return ublk_batch_io_buf_idx(t, q, tag); } /* Queue UBLK_U_IO_PREP_IO_CMDS for a specific queue with batch elements */ diff --git a/tools/testing/selftests/ublk/test_batch_04.sh b/tools/testing/selftests/ublk/test_batch_04.sh new file mode 100755 index 000000000000..cd5e1ff9d630 --- /dev/null +++ b/tools/testing/selftests/ublk/test_batch_04.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# --rotate_auto_buf: COMMIT must unregister old auto_buf index before store. + +. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh + +ERR_CODE=0 + +if ! _have_feature "BATCH_IO" || ! _have_feature "AUTO_BUF_REG"; then + exit "$UBLK_SKIP_CODE" +fi +if ! _have_program fio || ! _have_program timeout; then + exit "$UBLK_SKIP_CODE" +fi + +_prep_test "generic" "batch auto_buf unregister with rotating index" + +_create_backfile 0 64M + +dev_id=$(_add_ublk_dev_no_settle -t loop -q 1 --nthreads 1 -b --auto_zc \ + --rotate_auto_buf "${UBLK_BACKFILES[0]}") +_check_add_dev $TID $? + +for ((i = 0; i < 50; i++)); do + [ -b /dev/ublkb"${dev_id}" ] && break + sleep 0.1 +done +[ -b /dev/ublkb"${dev_id}" ] || { _cleanup_test; _show_result $TID 1; } + +timeout -k 2 5 fio --name=job1 --filename=/dev/ublkb"${dev_id}" \ + --ioengine=libaio --rw=write --direct=1 --bs=4k --iodepth=1 --size=64k \ + > /dev/null 2>&1 +ERR_CODE=$? + +if [ "$ERR_CODE" -ne 0 ]; then + kill -9 "$(_get_ublk_daemon_pid "$dev_id" 2>/dev/null)" 2>/dev/null || true + sleep 0.5 + pkill -9 fio 2>/dev/null || true + ERR_CODE=1 +fi + +_cleanup_test +_show_result $TID $ERR_CODE From 72e67c118642634c25465db0c8bcfa54c4ce086c Mon Sep 17 00:00:00 2001 From: Xu Rao Date: Tue, 4 Aug 2026 10:34:03 +0800 Subject: [PATCH 107/241] zloop: truncate finished zones to zone capacity The size of a sequential zone backing file records the amount of data written and is used to restore the zone state. A backing file whose size is equal to the zone capacity is restored as a full zone, while a file larger than the zone capacity is rejected as invalid. However, zloop_finish_zone() currently truncates the backing file to the zone size. For devices with a reduced zone capacity, finishing a zone therefore creates a backing file larger than the zone capacity. After the device is removed and later re-added, that zone file is rejected instead of being restored as a full zone. Truncate finished sequential zones to the zone capacity, matching the persistent representation accepted by zloop_update_seq_zone() for a full zone. Suggested-by: Damien Le Moal Fixes: eb0570c7df23 ("block: new zoned loop block device driver") Cc: stable@vger.kernel.org Signed-off-by: Xu Rao Reviewed-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/B39E5FD81D1A07F4+20260804023403.939767-1-raoxu@uniontech.com Signed-off-by: Jens Axboe --- Documentation/admin-guide/blockdev/zoned_loop.rst | 9 ++++----- drivers/block/zloop.c | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Documentation/admin-guide/blockdev/zoned_loop.rst b/Documentation/admin-guide/blockdev/zoned_loop.rst index f4f1f3121bf9..95974425ca05 100644 --- a/Documentation/admin-guide/blockdev/zoned_loop.rst +++ b/Documentation/admin-guide/blockdev/zoned_loop.rst @@ -30,11 +30,10 @@ indicates the position of the write pointer of the zone. When resetting a sequential zone, its backing file size is truncated to zero. Conversely, for a zone finish operation, the backing file is truncated to the -zone size. With this, the maximum capacity of a zloop zoned block device created -can be larger configured to be larger than the storage space available on the -backing file system. Of course, for such configuration, writing more data than -the storage space available on the backing file system will result in write -errors. +zone capacity. With this, a zloop zoned block device can be configured with a +larger capacity than the storage space available on the backing file system. Of +course, for such configuration, writing more data than the storage space +available on the backing file system will result in write errors. The zoned loop block device driver implements a complete zone transition state machine. That is, zones can be empty, implicitly opened, explicitly opened, diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index f97a20cfdb7c..58ec8161b7e2 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -479,7 +479,8 @@ static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) zone->cond == BLK_ZONE_COND_FULL) goto unlock; - if (vfs_truncate(&zone->file->f_path, zlo->zone_size << SECTOR_SHIFT)) { + if (vfs_truncate(&zone->file->f_path, + zlo->zone_capacity << SECTOR_SHIFT)) { set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); ret = -EIO; goto unlock; From 858d0abdb890006af3eb81e53c438d69debf51b1 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:50:17 -0700 Subject: [PATCH 108/241] block: remove a dead return statement in blk_zone_plug_bio The switch at the end of blk_zone_plug_bio always returns, so remove the dead extra return statement after it. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260804125038.740388-2-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-zoned.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/block/blk-zoned.c b/block/blk-zoned.c index ca30caec838e..034af9dfb5f9 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -1651,8 +1651,6 @@ bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) default: return false; } - - return false; } EXPORT_SYMBOL_GPL(blk_zone_plug_bio); From 9451934953bf1a4cc8e88179082785f3de5e127a Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:50:18 -0700 Subject: [PATCH 109/241] block: allow REQ_NOWAIT zone management commands Commit efae226c2ef1 ("block: handle zone management operations completions") moved all block layer tracking of zone management operations to the I/O completion handler. With that REQ_NOWAIT zone management operations are just fine, so allow them. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260804125038.740388-3-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-zoned.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 034af9dfb5f9..3b7a5f2bdf98 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -1586,15 +1586,6 @@ static bool blk_zone_wplug_handle_zone_mgmt(struct bio *bio) return true; } - /* - * No-wait zone management BIOs do not make much sense as the callers - * issue these as blocking operations in most cases. To avoid issues - * with the BIO execution potentially failing with BLK_STS_AGAIN, warn - * about REQ_NOWAIT being set and ignore that flag. - */ - if (WARN_ON_ONCE(bio->bi_opf & REQ_NOWAIT)) - bio->bi_opf &= ~REQ_NOWAIT; - return false; } From f3dfaf68ea3691a388dcd4c86c249f4544cfa898 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:50:19 -0700 Subject: [PATCH 110/241] block: remove blk_zone_wplug_handle_zone_mgmt blk_zone_wplug_handle_zone_mgmt now only checks that zone reset and zone finish operations are directed to a sequential write required zone. This has nothing to do with zone write plugs and is better handled with other bio validity checks in submit_bio_noacct. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260804125038.740388-4-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-core.c | 9 ++++++++- block/blk-zoned.c | 19 ------------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/block/blk-core.c b/block/blk-core.c index 365641266c9e..67cfd7bd8542 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -898,9 +898,16 @@ void submit_bio_noacct(struct bio *bio) goto not_supported; break; case REQ_OP_ZONE_RESET: + case REQ_OP_ZONE_FINISH: + /* + * Zone reset and zone finish operations do not apply to + * conventional zones. + */ + if (!bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector)) + goto end_io; + break; case REQ_OP_ZONE_OPEN: case REQ_OP_ZONE_CLOSE: - case REQ_OP_ZONE_FINISH: case REQ_OP_ZONE_RESET_ALL: if (!bdev_is_zoned(bio->bi_bdev)) goto not_supported; diff --git a/block/blk-zoned.c b/block/blk-zoned.c index 3b7a5f2bdf98..a5afb842bf35 100644 --- a/block/blk-zoned.c +++ b/block/blk-zoned.c @@ -1574,21 +1574,6 @@ static void blk_zone_wplug_handle_native_zone_append(struct bio *bio) disk_put_zone_wplug(zwplug); } -static bool blk_zone_wplug_handle_zone_mgmt(struct bio *bio) -{ - if (bio_op(bio) != REQ_OP_ZONE_RESET_ALL && - !bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector)) { - /* - * Zone reset and zone finish operations do not apply to - * conventional zones. - */ - bio_io_error(bio); - return true; - } - - return false; -} - /** * blk_zone_plug_bio - Handle a zone write BIO with zone write plugging * @bio: The BIO being submitted @@ -1635,10 +1620,6 @@ bool blk_zone_plug_bio(struct bio *bio, unsigned int nr_segs) case REQ_OP_WRITE: case REQ_OP_WRITE_ZEROES: return blk_zone_wplug_handle_write(bio, nr_segs); - case REQ_OP_ZONE_RESET: - case REQ_OP_ZONE_FINISH: - case REQ_OP_ZONE_RESET_ALL: - return blk_zone_wplug_handle_zone_mgmt(bio); default: return false; } From 13270876ce4eab47a218f31f0081885639011bb9 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:50:20 -0700 Subject: [PATCH 111/241] block: also reject zone open / close on conventional zones Just like zone reset / finish, these only apply to sequential zones. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260804125038.740388-5-hch@lst.de Signed-off-by: Jens Axboe --- block/blk-core.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/block/blk-core.c b/block/blk-core.c index 67cfd7bd8542..196bccf27f58 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -897,17 +897,14 @@ void submit_bio_noacct(struct bio *bio) if (!q->limits.max_write_zeroes_sectors) goto not_supported; break; + case REQ_OP_ZONE_OPEN: + case REQ_OP_ZONE_CLOSE: case REQ_OP_ZONE_RESET: case REQ_OP_ZONE_FINISH: - /* - * Zone reset and zone finish operations do not apply to - * conventional zones. - */ + /* Zone management operations require sequential zones. */ if (!bdev_zone_is_seq(bio->bi_bdev, bio->bi_iter.bi_sector)) goto end_io; break; - case REQ_OP_ZONE_OPEN: - case REQ_OP_ZONE_CLOSE: case REQ_OP_ZONE_RESET_ALL: if (!bdev_is_zoned(bio->bi_bdev)) goto not_supported; From b3c5f8d05e68963dea7c0cb9bc60e9e669b5c54e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:50:21 -0700 Subject: [PATCH 112/241] block: remove most blkdev_cmd_discard arguments All other arguments can be derived from cmd, so do that to simplify the calling convention. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260804125038.740388-6-hch@lst.de Signed-off-by: Jens Axboe --- block/ioctl.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/block/ioctl.c b/block/ioctl.c index 3d4ea1537457..af2a897f13f9 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -887,14 +887,13 @@ static void bio_cmd_bio_end_io(struct bio *bio) bio_put(bio); } -static int blkdev_cmd_discard(struct io_uring_cmd *cmd, - struct block_device *bdev, - uint64_t start, uint64_t len, bool nowait) +static int blkdev_cmd_discard(struct io_uring_cmd *cmd) { struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd); - gfp_t gfp = nowait ? GFP_NOWAIT : GFP_KERNEL; - sector_t sector = start >> SECTOR_SHIFT; - sector_t nr_sects = len >> SECTOR_SHIFT; + struct block_device *bdev = I_BDEV(cmd->file->f_mapping->host); + gfp_t gfp = bic->nowait ? GFP_NOWAIT : GFP_KERNEL; + sector_t sector = bic->start >> SECTOR_SHIFT; + sector_t nr_sects = bic->len >> SECTOR_SHIFT; struct bio *prev = NULL, *bio; int err; @@ -904,12 +903,12 @@ static int blkdev_cmd_discard(struct io_uring_cmd *cmd, return -EBADF; if (bdev_read_only(bdev)) return -EPERM; - err = blk_validate_byte_range(bdev, start, len); + err = blk_validate_byte_range(bdev, bic->start, bic->len); if (err) return err; - err = filemap_invalidate_pages(bdev->bd_mapping, start, - start + len - 1, nowait); + err = filemap_invalidate_pages(bdev->bd_mapping, bic->start, + bic->start + bic->len - 1, bic->nowait); if (err) return err; @@ -917,7 +916,7 @@ static int blkdev_cmd_discard(struct io_uring_cmd *cmd, bio = blk_alloc_discard_bio(bdev, §or, &nr_sects, gfp); if (!bio) break; - if (nowait) { + if (bic->nowait) { /* * Don't allow multi-bio non-blocking submissions as * subsequent bios may fail but we won't get a direct @@ -946,7 +945,6 @@ static int blkdev_cmd_discard(struct io_uring_cmd *cmd, int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { - struct block_device *bdev = I_BDEV(cmd->file->f_mapping->host); struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd); u32 cmd_op = cmd->cmd_op; @@ -967,8 +965,7 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) switch (cmd_op) { case BLOCK_URING_CMD_DISCARD: - return blkdev_cmd_discard(cmd, bdev, bic->start, bic->len, - bic->nowait); + return blkdev_cmd_discard(cmd); } return -EINVAL; } From ca8f6548e6f79761cba66ec8cb45bf026da8bf7d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:50:22 -0700 Subject: [PATCH 113/241] block: implement async io_uring zone reset all Add a new BLOCK_URING_CMD_ZONE_RESET_ALL uring cmd to reset all zones for a given block device. This can be used by storage systems or file system mkfs tools to initialize multiple devices in parallel. Signed-off-by: Christoph Hellwig Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260804125038.740388-7-hch@lst.de Signed-off-by: Jens Axboe --- block/ioctl.c | 37 +++++++++++++++++++++++++++++++++++++ include/uapi/linux/blkdev.h | 1 + 2 files changed, 38 insertions(+) diff --git a/block/ioctl.c b/block/ioctl.c index af2a897f13f9..64b4e6c0f696 100644 --- a/block/ioctl.c +++ b/block/ioctl.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "blk.h" #include "blk-crypto-internal.h" @@ -943,6 +944,40 @@ static int blkdev_cmd_discard(struct io_uring_cmd *cmd) return -EIOCBQUEUED; } +static int blkdev_cmd_zone_reset_all(struct io_uring_cmd *cmd) +{ + struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd); + struct block_device *bdev = I_BDEV(cmd->file->f_mapping->host); + struct bio *bio; + int err; + + if (!(file_to_blk_mode(cmd->file) & BLK_OPEN_WRITE)) + return -EBADF; + if (bdev_read_only(bdev)) + return -EPERM; + if (!bdev_is_zoned(bdev)) + return -EOPNOTSUPP; + if (bic->start || bic->len) + return -EINVAL; + + err = filemap_invalidate_pages(bdev->bd_mapping, 0, + bdev_nr_bytes(bdev) - 1, bic->nowait); + if (err) + return err; + + bio = bio_alloc(bdev, 0, REQ_OP_ZONE_RESET_ALL, + bic->nowait ? GFP_NOWAIT : GFP_KERNEL); + if (!bio) + return -EAGAIN; + if (bic->nowait) + bio->bi_opf |= REQ_NOWAIT; + trace_blkdev_zone_mgmt(bio, 0); + bio->bi_private = cmd; + bio->bi_end_io = bio_cmd_bio_end_io; + submit_bio(bio); + return -EIOCBQUEUED; +} + int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) { struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd); @@ -966,6 +1001,8 @@ int blkdev_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) switch (cmd_op) { case BLOCK_URING_CMD_DISCARD: return blkdev_cmd_discard(cmd); + case BLOCK_URING_CMD_ZONE_RESET_ALL: + return blkdev_cmd_zone_reset_all(cmd); } return -EINVAL; } diff --git a/include/uapi/linux/blkdev.h b/include/uapi/linux/blkdev.h index 66373cd1a83a..57b0bbd04e2b 100644 --- a/include/uapi/linux/blkdev.h +++ b/include/uapi/linux/blkdev.h @@ -10,5 +10,6 @@ * It's a different number space from ioctl(), reuse the block's code 0x12. */ #define BLOCK_URING_CMD_DISCARD _IO(0x12, 0) +#define BLOCK_URING_CMD_ZONE_RESET_ALL _IO(0x12, 1) #endif From 758b86f7bc8a2582d0783f3535854ea05d3ff97d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:55:10 -0700 Subject: [PATCH 114/241] block: split out a new blk_plug.h header blkdev.h gets included in various places outside the block layer just for struct blk_plug and related plugging functions. Split blk_plug into a separate helper to reduce the amount of code that needs to get rebuilt when blkdev.h changes and to slightly reduce compile times. In io_uring this requires pulling in a few other headers explicitly that previously were implicitly included through blkdev.h. Signed-off-by: Christoph Hellwig Reviewed-by: Christian Brauner (Amutable) Reviewed-by: Johannes Thumshirn Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260804125524.740996-1-hch@lst.de Signed-off-by: Jens Axboe --- fs/aio.c | 2 +- fs/fs-writeback.c | 2 +- include/linux/blk_plug.h | 95 ++++++++++++++++++++++++++++++++++ include/linux/blkdev.h | 86 +----------------------------- include/linux/io_uring_types.h | 2 +- io_uring/io_uring.h | 1 + io_uring/kbuf.c | 1 + io_uring/rsrc.h | 2 + io_uring/rw.h | 1 + kernel/exit.c | 1 - kernel/sched/core.c | 1 - mm/madvise.c | 2 +- mm/page-writeback.c | 1 - mm/readahead.c | 2 +- mm/swap_state.c | 2 +- mm/vmscan.c | 2 +- 16 files changed, 108 insertions(+), 95 deletions(-) create mode 100644 include/linux/blk_plug.h diff --git a/fs/aio.c b/fs/aio.c index f57fa21a2503..ebdb0e5b95fd 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -35,7 +35,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index fdb8766d275a..d064072284f4 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/linux/blk_plug.h b/include/linux/blk_plug.h new file mode 100644 index 000000000000..2ac1265662ad --- /dev/null +++ b/include/linux/blk_plug.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_BLK_PLUG_H +#define _LINUX_BLK_PLUG_H + +#include + +struct blk_plug_cb; +typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *cb, bool from_schedule); + +struct rq_list { + struct request *head; + struct request *tail; +}; + +#ifdef CONFIG_BLOCK +/* + * blk_plug permits building a queue of related requests by holding the I/O + * fragments for a short period. This allows merging of sequential requests + * into single larger request. As the requests are moved from a per-task list to + * the device's request_queue in a batch, this results in improved scalability + * as the lock contention for request_queue lock is reduced. + * + * It is ok not to disable preemption when adding the request to the plug list + * or when attempting a merge. For details, please see schedule() where + * blk_flush_plug() is called. + */ +struct blk_plug { + struct rq_list mq_list; /* blk-mq requests */ + + /* if ios_left is > 1, we can batch tag/rq allocations */ + struct rq_list cached_rqs; + u64 cur_ktime; + unsigned short nr_ios; + + unsigned short rq_count; + + bool multiple_queues; + bool has_elevator; + + struct list_head cb_list; /* md requires an unplug callback */ +}; + +void blk_start_plug(struct blk_plug *); +void blk_start_plug_nr_ios(struct blk_plug *, unsigned short); +void blk_finish_plug(struct blk_plug *); + +void __blk_flush_plug(struct blk_plug *plug, bool from_schedule); +static inline void blk_flush_plug(struct blk_plug *plug, bool async) +{ + if (plug) + __blk_flush_plug(plug, async); +} + +static __always_inline void blk_plug_invalidate_ts(void) +{ + if (unlikely(current->flags & PF_BLOCK_TS)) { + current->plug->cur_ktime = 0; + current->flags &= ~PF_BLOCK_TS; + } +} + +struct blk_plug_cb { + struct list_head list; + blk_plug_cb_fn callback; + void *data; +}; + +struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data, + int size); +#else /* CONFIG_BLOCK */ +struct blk_plug { +}; + +static inline void blk_start_plug(struct blk_plug *plug) +{ +} + +static inline void blk_start_plug_nr_ios(struct blk_plug *plug, + unsigned short nr_ios) +{ +} + +static inline void blk_finish_plug(struct blk_plug *plug) +{ +} + +static inline void blk_flush_plug(struct blk_plug *plug, bool async) +{ +} + +static inline void blk_plug_invalidate_ts(void) +{ +} +#endif /* CONFIG_BLOCK */ +#endif /* _LINUX_BLK_PLUG_H */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 9213a5716f95..20cb8ed7d987 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -21,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -1169,94 +1169,10 @@ extern void blk_put_queue(struct request_queue *); void blk_mark_disk_dead(struct gendisk *disk); -struct rq_list { - struct request *head; - struct request *tail; -}; - #ifdef CONFIG_BLOCK -/* - * blk_plug permits building a queue of related requests by holding the I/O - * fragments for a short period. This allows merging of sequential requests - * into single larger request. As the requests are moved from a per-task list to - * the device's request_queue in a batch, this results in improved scalability - * as the lock contention for request_queue lock is reduced. - * - * It is ok not to disable preemption when adding the request to the plug list - * or when attempting a merge. For details, please see schedule() where - * blk_flush_plug() is called. - */ -struct blk_plug { - struct rq_list mq_list; /* blk-mq requests */ - - /* if ios_left is > 1, we can batch tag/rq allocations */ - struct rq_list cached_rqs; - u64 cur_ktime; - unsigned short nr_ios; - - unsigned short rq_count; - - bool multiple_queues; - bool has_elevator; - - struct list_head cb_list; /* md requires an unplug callback */ -}; - -struct blk_plug_cb; -typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool); -struct blk_plug_cb { - struct list_head list; - blk_plug_cb_fn callback; - void *data; -}; -extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, - void *data, int size); -extern void blk_start_plug(struct blk_plug *); -extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short); -extern void blk_finish_plug(struct blk_plug *); - -void __blk_flush_plug(struct blk_plug *plug, bool from_schedule); -static inline void blk_flush_plug(struct blk_plug *plug, bool async) -{ - if (plug) - __blk_flush_plug(plug, async); -} - -static __always_inline void blk_plug_invalidate_ts(void) -{ - if (unlikely(current->flags & PF_BLOCK_TS)) { - current->plug->cur_ktime = 0; - current->flags &= ~PF_BLOCK_TS; - } -} - int blkdev_issue_flush(struct block_device *bdev); long nr_blockdev_pages(void); #else /* CONFIG_BLOCK */ -struct blk_plug { -}; - -static inline void blk_start_plug_nr_ios(struct blk_plug *plug, - unsigned short nr_ios) -{ -} - -static inline void blk_start_plug(struct blk_plug *plug) -{ -} - -static inline void blk_finish_plug(struct blk_plug *plug) -{ -} - -static inline void blk_flush_plug(struct blk_plug *plug, bool async) -{ -} - -static inline void blk_plug_invalidate_ts(void) -{ -} - static inline int blkdev_issue_flush(struct block_device *bdev) { return 0; diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 87151a5b62c1..954f34d6ca47 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -1,7 +1,7 @@ #ifndef IO_URING_TYPES_H #define IO_URING_TYPES_H -#include +#include #include #include #include diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h index cb736b815422..9771d4557ed3 100644 --- a/io_uring/io_uring.h +++ b/io_uring/io_uring.h @@ -3,6 +3,7 @@ #define IOU_CORE_H #include +#include #include #include #include diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index de0129bceaba..22b84129112e 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h index 98ae8ef51009..eacfdb70f203 100644 --- a/io_uring/rsrc.h +++ b/io_uring/rsrc.h @@ -2,8 +2,10 @@ #ifndef IOU_RSRC_H #define IOU_RSRC_H +#include #include #include +#include #define IO_VEC_CACHE_SOFT_CAP 256 diff --git a/io_uring/rw.h b/io_uring/rw.h index 9bd7fbf70ea9..1179506f929f 100644 --- a/io_uring/rw.h +++ b/io_uring/rw.h @@ -2,6 +2,7 @@ #include #include +#include struct io_meta_state { u32 seed; diff --git a/kernel/exit.c b/kernel/exit.c index 2c0b1c02920f..44cc17016572 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -48,7 +48,6 @@ #include /* for audit_free() */ #include #include -#include #include #include #include diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..616774777dea 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -40,7 +40,6 @@ #include #include -#include #include #include #include diff --git a/mm/madvise.c b/mm/madvise.c index 77552b03d318..7ec9b6cfb15e 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/mm/page-writeback.c b/mm/page-writeback.c index e98748112d1e..d1fd6ba58ae5 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include diff --git a/mm/readahead.c b/mm/readahead.c index 558c92957518..6e5563290287 100644 --- a/mm/readahead.c +++ b/mm/readahead.c @@ -113,7 +113,7 @@ * ->read_folio() which may be less efficient. */ -#include +#include #include #include #include diff --git a/mm/swap_state.c b/mm/swap_state.c index 9c3a5cf99778..727a17ee7821 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/mm/vmscan.c b/mm/vmscan.c index 35c3bb15ae96..b957664abb26 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include /* for buffer_heads_over_limit */ #include #include From af0955c8f26aa0f02e534084bc9948a69c9e7ce0 Mon Sep 17 00:00:00 2001 From: Yang Xiuwei Date: Thu, 30 Jul 2026 09:09:10 +0800 Subject: [PATCH 115/241] ublk: clear auto buf reg before updating io->buf in batch commit ublk_batch_commit_io() stored the new auto_buf into io->buf before calling ublk_clear_auto_buf_reg(). Clear takes the unregister index from io->buf.auto_reg, so it could drop the new slot and leave the old registered buffer behind. Fixes: 1e500e106d5a ("ublk: handle UBLK_U_IO_COMMIT_IO_CMDS") Signed-off-by: Yang Xiuwei Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index a632fbcc03b5..a67b9c26804b 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -3784,11 +3784,11 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq, ret = ublk_batch_commit_io_check(ubq, io, &buf); if (!ret) { io->res = elem->result; - io->buf = buf; req = ublk_fill_io_cmd(io, data->cmd); if (auto_reg) ublk_clear_auto_buf_reg(io, data->cmd, &buf_idx); + io->buf = buf; compl = ublk_need_complete_req(data->ub, io); } ublk_io_unlock(io); From d73b5b0690e36b2a9e6022f6712e9fa7fd338632 Mon Sep 17 00:00:00 2001 From: Xu Rao Date: Tue, 4 Aug 2026 13:29:41 +0800 Subject: [PATCH 116/241] Documentation: block: zloop: clarify capacity alignment zloop divides the requested capacity by the zone size to determine the number of zones. Since it uses one zone size for all zones, a smaller last zone is not supported and an unaligned capacity is rounded down. The capacity_mb description incorrectly states that the capacity is rounded up. Correct it to document the actual behavior. Fixes: 9e4f11c1228c ("Documentation: Document the new zoned loop block device driver") Suggested-by: Damien Le Moal Reviewed-by: Damien Le Moal Signed-off-by: Xu Rao Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/4659F8F0C6C328EA+20260804052942.1186727-1-raoxu@uniontech.com Signed-off-by: Jens Axboe --- Documentation/admin-guide/blockdev/zoned_loop.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Documentation/admin-guide/blockdev/zoned_loop.rst b/Documentation/admin-guide/blockdev/zoned_loop.rst index 95974425ca05..64277494fb36 100644 --- a/Documentation/admin-guide/blockdev/zoned_loop.rst +++ b/Documentation/admin-guide/blockdev/zoned_loop.rst @@ -70,8 +70,10 @@ follows. =================== ========================================================= id Device number (the X in /dev/zloopX). Default: automatically assigned. -capacity_mb Device total capacity in MiB. This is always rounded up - to the nearest higher multiple of the zone size. +capacity_mb Device total capacity in MiB. A smaller last zone is not + supported, so a capacity value that is not a multiple of + the zone size is rounded down to the closest multiple of + the zone size. Default: 16384 MiB (16 GiB). zone_size_mb Device zone size in MiB. Default: 256 MiB. zone_capacity_mb Device zone capacity (must always be equal to or lower From 6c13180dba60f835d6909e2a3b4f50862de156c6 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:39:23 -0700 Subject: [PATCH 117/241] block: remove bip_should_check There is no benefit in using this helper over the simple flags check. Signed-off-by: Christoph Hellwig Reviewed-by: Anuj Gupta Reviewed-by: Kanchan Joshi Link: https://patch.msgid.link/20260804123928.736596-2-hch@lst.de Signed-off-by: Jens Axboe --- block/bio-integrity-auto.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c index b1c733ecfd2e..43ac9f338183 100644 --- a/block/bio-integrity-auto.c +++ b/block/bio-integrity-auto.c @@ -45,10 +45,6 @@ static void bio_integrity_verify_fn(struct work_struct *work) } #define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) -static bool bip_should_check(struct bio_integrity_payload *bip) -{ - return bip->bip_flags & BIP_CHECK_FLAGS; -} /** * __bio_integrity_endio - Integrity I/O completion function @@ -66,7 +62,7 @@ bool __bio_integrity_endio(struct bio *bio) container_of(bip, struct bio_integrity_data, bip); if (bio_op(bio) == REQ_OP_READ && !bio->bi_status && - bip_should_check(bip)) { + (bip->bip_flags & BIP_CHECK_FLAGS)) { INIT_WORK(&bid->work, bio_integrity_verify_fn); queue_work(kintegrityd_wq, &bid->work); return false; @@ -99,7 +95,7 @@ void bio_integrity_prep(struct bio *bio, unsigned int action) bio_integrity_setup_default(bio); /* Auto-generate integrity metadata if this is a write */ - if (bio_data_dir(bio) == WRITE && bip_should_check(&bid->bip)) + if (bio_data_dir(bio) == WRITE && (bid->bip.bip_flags & BIP_CHECK_FLAGS)) bio_integrity_generate(bio); else bid->saved_bio_iter = bio->bi_iter; From 738f01912a1ad68c81a6aed06cac94e09b5f609d Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:39:24 -0700 Subject: [PATCH 118/241] block: lift BIP_CHECK_FLAGS to include/linux/bio-integrity.h To allow for users outside of bio-integrity-auto.c. Also add a little comment explaining it. Signed-off-by: Christoph Hellwig Reviewed-by: Anuj Gupta Reviewed-by: Kanchan Joshi Link: https://patch.msgid.link/20260804123928.736596-3-hch@lst.de Signed-off-by: Jens Axboe --- block/bio-integrity-auto.c | 2 -- include/linux/bio-integrity.h | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/block/bio-integrity-auto.c b/block/bio-integrity-auto.c index 43ac9f338183..9456dcffd17a 100644 --- a/block/bio-integrity-auto.c +++ b/block/bio-integrity-auto.c @@ -44,8 +44,6 @@ static void bio_integrity_verify_fn(struct work_struct *work) bio_endio(bio); } -#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) - /** * __bio_integrity_endio - Integrity I/O completion function * @bio: Protected bio diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h index c3dda32fd803..0ea2a8bf7efb 100644 --- a/include/linux/bio-integrity.h +++ b/include/linux/bio-integrity.h @@ -17,6 +17,9 @@ enum bip_flags { BIP_MEMPOOL = 1 << 15, /* buffer backed by mempool */ }; +/* flags that require generate/verify action. */ +#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) + struct bio_integrity_payload { struct bvec_iter bip_iter; From 3bf9a21e7bccfd8c35b440efd114c61cc9838a41 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Tue, 4 Aug 2026 05:39:25 -0700 Subject: [PATCH 119/241] block: handle nogenerate/noverify properly in fs-integrity Check the BIP_CHECK flags before generating or verifying PI information, otherwise this can be incorrectly called for non-PI metadata and cause generation of incorrect metadata and crashed in the verification handler. The new behavior matches that of the block layer auto-generated metadata. Fixes: 0bde8a12b554 ("block: add fs_bio_integrity helpers") Signed-off-by: Christoph Hellwig Reviewed-by: Kanchan Joshi Reviewed-by: Anuj Gupta Link: https://patch.msgid.link/20260804123928.736596-4-hch@lst.de Signed-off-by: Jens Axboe --- block/bio-integrity-fs.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c index 9c5fe5fa8f0d..692403dfa047 100644 --- a/block/bio-integrity-fs.c +++ b/block/bio-integrity-fs.c @@ -46,7 +46,8 @@ void fs_bio_integrity_free(struct bio *bio) void fs_bio_integrity_generate(struct bio *bio) { - if (fs_bio_integrity_alloc(bio)) + if (fs_bio_integrity_alloc(bio) && + (bio_integrity(bio)->bip_flags & BIP_CHECK_FLAGS)) bio_integrity_generate(bio); } EXPORT_SYMBOL_GPL(fs_bio_integrity_generate); @@ -60,6 +61,9 @@ int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size) .bi_size = size, }; + if (!bip || !(bip->bip_flags & BIP_CHECK_FLAGS)) + return 0; + /* * Reinitialize bip->bip_iter. * From 4d73bf0ca4fbe7f252154ce98d6693c6c198164c Mon Sep 17 00:00:00 2001 From: Tao Cui Date: Tue, 4 Aug 2026 13:41:20 +0800 Subject: [PATCH 120/241] block/blk-iocost: annotate ioc_pd_stat reads with data_race() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ioc_pd_stat() reads ioc->enabled, ioc->vtime_base_rate, and iocg->last_stat without holding ioc->lock, which trips KCSAN since ioc_adjust_base_vrate() and iocg_flush_stat_upward() write those fields under ioc->lock. Commit 35198e323001 fixed the same issue in ioc_qos_prfill() and ioc_cost_model_prfill() by adding spin_lock_irq(&ioc->lock). However, those functions read configuration parameters (qos/model) that need synchronized reads. In contrast, ioc_pd_stat() only reads stat values (vrate, usage) where stale reads are harmless, so data_race() is more appropriate — it silences the KCSAN warning without adding lock contention during high-frequency stat reads. Signed-off-by: Tao Cui Acked-by: Tejun Heo Link: https://patch.msgid.link/20260804054120.161933-1-cui.tao@linux.dev Signed-off-by: Jens Axboe --- block/blk-iocost.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/block/blk-iocost.c b/block/blk-iocost.c index b60625613e09..dd7749d59900 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -3093,23 +3093,23 @@ static void ioc_pd_stat(struct blkg_policy_data *pd, struct seq_file *s) struct ioc_gq *iocg = pd_to_iocg(pd); struct ioc *ioc = iocg->ioc; - if (!ioc->enabled) + if (!data_race(ioc->enabled)) return; if (iocg->level == 0) { unsigned vp10k = DIV64_U64_ROUND_CLOSEST( - ioc->vtime_base_rate * 10000, + data_race(ioc->vtime_base_rate) * 10000, VTIME_PER_USEC); seq_printf(s, " cost.vrate=%u.%02u", vp10k / 100, vp10k % 100); } - seq_printf(s, " cost.usage=%llu", iocg->last_stat.usage_us); + seq_printf(s, " cost.usage=%llu", data_race(iocg->last_stat.usage_us)); if (blkcg_debug_stats) seq_printf(s, " cost.wait=%llu cost.indebt=%llu cost.indelay=%llu", - iocg->last_stat.wait_us, - iocg->last_stat.indebt_us, - iocg->last_stat.indelay_us); + data_race(iocg->last_stat.wait_us), + data_race(iocg->last_stat.indebt_us), + data_race(iocg->last_stat.indelay_us)); } static u64 ioc_weight_prfill(struct seq_file *sf, struct blkg_policy_data *pd, From cddb447c62466f3076938ce120028d7b591f9f37 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:15:54 +0200 Subject: [PATCH 121/241] s390/dasd: Do not complete a failed ESE read as successful MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dasd_int_handler() completes an NRF read of an unallocated ESE track by calling ese_read() and unconditionally marking the request DASD_CQR_SUCCESS. dasd_eckd_ese_read() can return an error before it has zeroed the destination buffer: a failed sense-data parse or a current track outside the requested range both return early, leaving the destination pages untouched. The request is still completed successfully, so the block layer is handed stale / uninitialized memory instead of zeros. Check the ese_read() return value and fail the request through the normal error path instead of forcing DASD_CQR_SUCCESS. Fixes: 5e6bdd37c552 ("s390/dasd: fix data corruption for thin provisioned devices") Cc: stable@vger.kernel.org Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-2-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index 3181c06d91ce..dbe3caa1e0b4 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -1697,8 +1697,10 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm, return; } if (rq_data_dir(req) == READ) { - device->discipline->ese_read(cqr, irb); - cqr->status = DASD_CQR_SUCCESS; + if (device->discipline->ese_read(cqr, irb)) + cqr->status = DASD_CQR_ERROR; + else + cqr->status = DASD_CQR_SUCCESS; cqr->stopclk = now; dasd_device_clear_timer(device); dasd_schedule_device_bh(device); From 6fb5ba2e7e43173a3761e46f091070a8185efa14 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:15:55 +0200 Subject: [PATCH 122/241] s390/dasd: Propagate partial completion length across ERP recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dasd_default_erp_postaction() copies the timing and device state from the finished ERP request back to the original request but drops proc_bytes. A request that was partially completed, an ESE read of a not-yet-allocated track returns fewer bytes than requested, and then recovered through the ERP chain loses its partial-completion length. __dasd_cleanup_cqr() then sees proc_bytes == 0 and completes the whole request instead of requeueing the remainder, silently returning zeroed data for the part that was never read. Carry proc_bytes over to the original request like the other per-request state. Fixes: 5e6bdd37c552 ("s390/dasd: fix data corruption for thin provisioned devices") Cc: stable@vger.kernel.org Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-3-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_erp.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c index 89d7516b9ec8..468f0b2cc342 100644 --- a/drivers/s390/block/dasd_erp.c +++ b/drivers/s390/block/dasd_erp.c @@ -123,6 +123,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) int success; unsigned long startclk, stopclk; struct dasd_device *startdev; + unsigned int proc_bytes; BUG_ON(cqr->refers == NULL || cqr->function == NULL); @@ -130,6 +131,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) startclk = cqr->startclk; stopclk = cqr->stopclk; startdev = cqr->startdev; + proc_bytes = cqr->proc_bytes; /* free all ERPs - but NOT the original cqr */ while (cqr->refers != NULL) { @@ -147,6 +149,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) cqr->startclk = startclk; cqr->stopclk = stopclk; cqr->startdev = startdev; + cqr->proc_bytes = proc_bytes; if (success) cqr->status = DASD_CQR_DONE; else { From 2a1780f9fc2493bd34c418a0be6fc58943afcecf Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:15:56 +0200 Subject: [PATCH 123/241] s390/dasd: Guard sysfs discipline callbacks against unallocated private data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several sysfs show/store handlers call a discipline callback that dereferences device->private, either directly or through the DASD_DEFINE_ATTR() macro. During dasd_generic_set_online() the discipline is assigned before check_device() allocates device->private, so an unprivileged read of one of these world-readable attributes in that window dereferences a NULL pointer and panics. Guard the dereference inside each callback that actually touches device->private. Fixes: c729696bcf8b ("s390/dasd: Recognise data for ESE volumes") Cc: stable@vger.kernel.org Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-4-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 40 ++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 74fe73b5738a..6e2fd445688c 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1491,6 +1491,8 @@ static void dasd_eckd_reset_path(struct dasd_device *device, __u8 pm) struct dasd_eckd_private *private = device->private; unsigned long flags; + if (!private) + return; if (!private->fcx_max_data) private->fcx_max_data = get_fcx_max_data(device); spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); @@ -1646,6 +1648,9 @@ static int dasd_eckd_is_ese(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.vol_info.ese; } @@ -1653,6 +1658,9 @@ static int dasd_eckd_ext_pool_id(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.extent_pool_id; } @@ -1666,6 +1674,9 @@ static int dasd_eckd_space_configured(struct dasd_device *device) struct dasd_eckd_private *private = device->private; int rc; + if (!private) + return 0; + rc = dasd_eckd_read_vol_info(device); return rc ? : private->vsq.space_configured; @@ -1680,6 +1691,9 @@ static int dasd_eckd_space_allocated(struct dasd_device *device) struct dasd_eckd_private *private = device->private; int rc; + if (!private) + return 0; + rc = dasd_eckd_read_vol_info(device); return rc ? : private->vsq.space_allocated; @@ -1689,6 +1703,9 @@ static int dasd_eckd_logical_capacity(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->vsq.logical_capacity; } @@ -1831,7 +1848,11 @@ static int dasd_eckd_read_ext_pool_info(struct dasd_device *device) static int dasd_eckd_ext_size(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; - struct dasd_ext_pool_sum eps = private->eps; + struct dasd_ext_pool_sum eps; + + if (!private) + return 0; + eps = private->eps; if (!eps.flags.extent_size_valid) return 0; @@ -1847,6 +1868,9 @@ static int dasd_eckd_ext_pool_warn_thrshld(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.warn_thrshld; } @@ -1854,6 +1878,9 @@ static int dasd_eckd_ext_pool_cap_at_warnlevel(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.flags.capacity_at_warnlevel; } @@ -1864,6 +1891,9 @@ static int dasd_eckd_ext_pool_oos(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->eps.flags.pool_oos; } @@ -5935,8 +5965,11 @@ static int dasd_eckd_query_host_access(struct dasd_device *device, struct ccw1 *ccw; int rc; + if (!private) + return -ENODEV; + /* not available for HYPER PAV alias devices */ - if (!device->block && private->lcu->pav == HYPER_PAV) + if (!device->block && private->lcu && private->lcu->pav == HYPER_PAV) return -EOPNOTSUPP; /* may not be supported by the storage server */ @@ -6801,6 +6834,9 @@ static int dasd_eckd_hpf_enabled(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; + if (!private) + return 0; + return private->fcx_max_data ? 1 : 0; } From dc3e3f7306cb74066f231251602b1be5aaec7bd8 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:15:57 +0200 Subject: [PATCH 124/241] s390/dasd: Snapshot intrc before freeing the request block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit __dasd_cleanup_cqr() maps the completion result to a block status by reading cqr->intrc, but only after discipline->free_cp() has returned the request block to its memory pool (dasd_eckd_free_cp() ends in dasd_sfree_request()). On SMP another CPU can reallocate that block and overwrite cqr->intrc before it is read, completing the request with the wrong error. proc_bytes is already snapshotted before free_cp() for the same reason; do the same for intrc. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-5-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index dbe3caa1e0b4..f5585c549e09 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -2687,17 +2687,23 @@ static void __dasd_cleanup_cqr(struct dasd_ccw_req *cqr) struct request *req; blk_status_t error = BLK_STS_OK; unsigned int proc_bytes; - int status; + int status, intrc; req = (struct request *) cqr->callback_data; dasd_profile_end(cqr->block, cqr, req); + /* + * free_cp() returns the request block to its memory pool, so snapshot + * everything still needed from cqr before calling it - another CPU can + * reallocate and overwrite the block right after. + */ proc_bytes = cqr->proc_bytes; + intrc = cqr->intrc; status = cqr->block->base->discipline->free_cp(cqr, req); if (status < 0) error = errno_to_blk_status(status); else if (status == 0) { - switch (cqr->intrc) { + switch (intrc) { case -EPERM: /* * DASD doesn't implement SCSI/NVMe reservations, but it From e647f351da2c6601a3e280b5c3477421e981ca73 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:15:58 +0200 Subject: [PATCH 125/241] s390/dasd: Optimize max blocks per request for track alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With 4096-byte blocks a full ECKD track holds exactly 12 records. Lower DASD_ECKD_MAX_BLOCKS from 190 to 180 so requests align to track boundaries (15 full tracks); full-track I/O is more efficient than partial-track writes, and 190 had no alignment significance and could let a request cross a track boundary. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-6-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index f9299bd184ba..763733bcc4d2 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -131,7 +131,7 @@ /* * Maximum number of blocks to be chained */ -#define DASD_ECKD_MAX_BLOCKS 190 +#define DASD_ECKD_MAX_BLOCKS 180 #define DASD_ECKD_MAX_BLOCKS_RAW 256 /***************************************************************************** From feea12d1cd8110aac009df32d3c6cd1daf117f95 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:15:59 +0200 Subject: [PATCH 126/241] s390/dasd: Use GFP_KERNEL in dasd_alloc_device() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dasd_alloc_device() runs in process context (device set_online), so its pool allocations do not need GFP_ATOMIC. Use GFP_KERNEL instead, which is more reliable, especially for the larger DMA allocations that later ESE full-track work adds here. No functional change intended. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-7-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index f5585c549e09..d0f72d3d0f54 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -86,25 +86,25 @@ struct dasd_device *dasd_alloc_device(void) { struct dasd_device *device; - device = kzalloc_obj(struct dasd_device, GFP_ATOMIC); + device = kzalloc_obj(struct dasd_device, GFP_KERNEL); if (!device) return ERR_PTR(-ENOMEM); /* Get two pages for normal block device operations. */ - device->ccw_mem = (void *) __get_free_pages(GFP_ATOMIC | GFP_DMA, 1); + device->ccw_mem = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, 1); if (!device->ccw_mem) { kfree(device); return ERR_PTR(-ENOMEM); } /* Get one page for error recovery. */ - device->erp_mem = (void *) get_zeroed_page(GFP_ATOMIC | GFP_DMA); + device->erp_mem = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); if (!device->erp_mem) { free_pages((unsigned long) device->ccw_mem, 1); kfree(device); return ERR_PTR(-ENOMEM); } /* Get two pages for ese format. */ - device->ese_mem = (void *)__get_free_pages(GFP_ATOMIC | GFP_DMA, 1); + device->ese_mem = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, 1); if (!device->ese_mem) { free_page((unsigned long) device->erp_mem); free_pages((unsigned long) device->ccw_mem, 1); From 1edac73fa2e9e8cadf19d59a49d64c681e4e45a3 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:00 +0200 Subject: [PATCH 127/241] s390/dasd: Add defines for the Extended Address Volume track address MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The track address of an Extended Address Volume (more than 65520 cylinders) carries the high cylinder bits that do not fit the 16-bit cyl field in the upper part of the head field. set_ch_t() open-codes the corresponding shifts; name them so the encoding is explicit and can be reused. No functional change. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-8-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 4 ++-- drivers/s390/block/dasd_eckd.h | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 6e2fd445688c..785aba11066b 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -199,8 +199,8 @@ recs_per_track(struct dasd_eckd_characteristics * rdc, static void set_ch_t(struct ch_t *geo, __u32 cyl, __u8 head) { geo->cyl = (__u16) cyl; - geo->head = cyl >> 16; - geo->head <<= 4; + geo->head = cyl >> DASD_EAV_CYL_HI_SHIFT; + geo->head <<= DASD_EAV_HEAD_HI_SHIFT; geo->head |= head; } diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index 763733bcc4d2..bad7ba666370 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -146,6 +146,14 @@ struct eckd_count { __u16 dl; } __attribute__ ((packed)); +/* + * Extended Address Volume track address: the head field carries the actual + * head in its low-order 4 bits; the cylinder bits that do not fit the 16-bit + * cyl field are shifted in just above them. + */ +#define DASD_EAV_CYL_HI_SHIFT 16 /* cylinder bits beyond the 16-bit cyl field */ +#define DASD_EAV_HEAD_HI_SHIFT 4 /* head occupies the low-order 4 bits of head */ + struct ch_t { __u16 cyl; __u16 head; From 9cebfced13249fb02061dbee8a29cf91ba364519 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:01 +0200 Subject: [PATCH 128/241] s390/dasd: Add infrastructure for ESE full-track write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the driver internals to build WRITE_FULL_TRACK FCX channel programs in response to unformatted tracks on ESE devices. struct dasd_ccw_req: filldata, a pointer to the per-track metadata (an R0 record and the count records) that the WRITE_FULL_TRACK TIDAWs point at, and format/start_trk/end_trk/collision that link a request to its format-track guard entry so an overlapping format request can be detected. struct dasd_device: fill_mem/fill_chunks pool for those buffers and a zeroed nulldata page used as the data source for pad records. struct dasd_block: ese_staging/ese_lock, a hardirq-safe staging list. An ESE format CQR is created in the interrupt handler but has to be enqueued on ccw_queue under queue_lock; taking queue_lock while the ccwdev_lock is held there would invert the lock order, so the CQR is staged under ese_lock and dasd_block_tasklet splices it onto ccw_queue. Existing locking is unchanged. Add CQR states DASD_CQR_ABORT/ABORTED to retire the origin CQR of a replaced write without completing it to the block layer, and struct eckd_r0 for the track header record. The CCW and ESE format pools are enlarged (a full-track ITCW is roughly twice a plain track-mode one) to keep two maximum-size requests in flight. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-9-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 100 ++++++++++++++++++++++++++++----- drivers/s390/block/dasd_eckd.c | 11 ++++ drivers/s390/block/dasd_eckd.h | 5 ++ drivers/s390/block/dasd_int.h | 19 +++++++ 4 files changed, 120 insertions(+), 15 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index d0f72d3d0f54..79cd7e132418 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -90,31 +90,53 @@ struct dasd_device *dasd_alloc_device(void) if (!device) return ERR_PTR(-ENOMEM); - /* Get two pages for normal block device operations. */ - device->ccw_mem = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, 1); + /* + * Four pages: a full-track ITCW is roughly twice the size of a plain + * track-mode one, so this keeps two maximum-size requests in flight. + */ + device->ccw_mem = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, 2); if (!device->ccw_mem) { kfree(device); return ERR_PTR(-ENOMEM); } + /* per-request track-filler buffers (R0 + count records) */ + device->fill_mem = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, 1); + if (!device->fill_mem) { + free_pages((unsigned long)device->ccw_mem, 2); + kfree(device); + return ERR_PTR(-ENOMEM); + } /* Get one page for error recovery. */ device->erp_mem = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); if (!device->erp_mem) { - free_pages((unsigned long) device->ccw_mem, 1); + free_pages((unsigned long)device->fill_mem, 1); + free_pages((unsigned long)device->ccw_mem, 2); kfree(device); return ERR_PTR(-ENOMEM); } - /* Get two pages for ese format. */ - device->ese_mem = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, 1); + /* sized like ccw_chunks: two max-size NRF format requests in flight */ + device->ese_mem = (void *)__get_free_pages(GFP_KERNEL | GFP_DMA, 2); if (!device->ese_mem) { - free_page((unsigned long) device->erp_mem); - free_pages((unsigned long) device->ccw_mem, 1); + free_page((unsigned long)device->erp_mem); + free_pages((unsigned long)device->fill_mem, 1); + free_pages((unsigned long)device->ccw_mem, 2); + kfree(device); + return ERR_PTR(-ENOMEM); + } + device->nulldata = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA); + if (!device->nulldata) { + free_page((unsigned long)device->erp_mem); + free_pages((unsigned long)device->fill_mem, 1); + free_pages((unsigned long)device->ccw_mem, 2); + free_pages((unsigned long)device->ese_mem, 2); kfree(device); return ERR_PTR(-ENOMEM); } - dasd_init_chunklist(&device->ccw_chunks, device->ccw_mem, PAGE_SIZE*2); + dasd_init_chunklist(&device->ccw_chunks, device->ccw_mem, PAGE_SIZE * 4); + dasd_init_chunklist(&device->fill_chunks, device->fill_mem, PAGE_SIZE * 2); dasd_init_chunklist(&device->erp_chunks, device->erp_mem, PAGE_SIZE); - dasd_init_chunklist(&device->ese_chunks, device->ese_mem, PAGE_SIZE * 2); + dasd_init_chunklist(&device->ese_chunks, device->ese_mem, PAGE_SIZE * 4); spin_lock_init(&device->mem_lock); atomic_set(&device->tasklet_scheduled, 0); tasklet_init(&device->tasklet, dasd_device_tasklet, @@ -137,9 +159,11 @@ struct dasd_device *dasd_alloc_device(void) void dasd_free_device(struct dasd_device *device) { kfree(device->private); - free_pages((unsigned long) device->ese_mem, 1); - free_page((unsigned long) device->erp_mem); - free_pages((unsigned long) device->ccw_mem, 1); + free_pages((unsigned long)device->ese_mem, 2); + free_page((unsigned long)device->erp_mem); + free_pages((unsigned long)device->fill_mem, 1); + free_pages((unsigned long)device->ccw_mem, 2); + free_page((unsigned long)device->nulldata); kfree(device); } @@ -163,6 +187,8 @@ struct dasd_block *dasd_alloc_block(void) spin_lock_init(&block->queue_lock); INIT_LIST_HEAD(&block->format_list); spin_lock_init(&block->format_lock); + INIT_LIST_HEAD(&block->ese_staging); + spin_lock_init(&block->ese_lock); timer_setup(&block->timer, dasd_block_timeout, 0); spin_lock_init(&block->profile.lock); @@ -363,7 +389,8 @@ int _wait_for_empty_queues(struct dasd_device *device) { if (device->block) return list_empty(&device->ccw_queue) && - list_empty(&device->block->ccw_queue); + list_empty(&device->block->ccw_queue) && + list_empty(&device->block->ese_staging); else return list_empty(&device->ccw_queue); } @@ -1223,7 +1250,18 @@ void dasd_sfree_request(struct dasd_ccw_req *cqr, struct dasd_device *device) unsigned long flags; spin_lock_irqsave(&device->mem_lock, flags); - dasd_free_chunk(&device->ccw_chunks, cqr->mem_chunk); + /* + * Free the request block from the pool it came from: smalloc() sets + * mem_chunk (ccw_chunks), fmalloc() leaves it NULL (ese_chunks). A + * full-track request also frees its track-filler buffer. + */ + if (cqr->filldata) + dasd_free_chunk(&device->fill_chunks, cqr->filldata); + if (cqr->mem_chunk) + dasd_free_chunk(&device->ccw_chunks, cqr->mem_chunk); + else + dasd_free_chunk(&device->ese_chunks, cqr); + spin_unlock_irqrestore(&device->mem_lock, flags); dasd_put_device(device); } @@ -1234,6 +1272,8 @@ void dasd_ffree_request(struct dasd_ccw_req *cqr, struct dasd_device *device) unsigned long flags; spin_lock_irqsave(&device->mem_lock, flags); + if (cqr->filldata) + dasd_free_chunk(&device->fill_chunks, cqr->filldata); dasd_free_chunk(&device->ese_chunks, cqr); spin_unlock_irqrestore(&device->mem_lock, flags); dasd_put_device(device); @@ -1884,6 +1924,17 @@ static void __dasd_process_cqr(struct dasd_device *device, case DASD_CQR_CLEARED: cqr->status = DASD_CQR_TERMINATED; break; + case DASD_CQR_ABORT: + cqr->status = DASD_CQR_ABORTED; + /* + * ABORT is only set on the block-layer origin write that a + * full-track format replaces. Clear the callback so the request + * is not completed here - the replacement completes it. Internal + * requests never take this path, so no sleep_on waiter is left + * without its wakeup. + */ + cqr->callback = NULL; + break; default: dev_err(&device->cdev->dev, "Unexpected CQR status %02x", cqr->status); @@ -2211,6 +2262,7 @@ EXPORT_SYMBOL(dasd_add_request_tail); void dasd_wakeup_cb(struct dasd_ccw_req *cqr, void *data) { spin_lock_irq(get_ccwdev_lock(cqr->startdev->cdev)); + cqr->endclk = get_tod_clock(); cqr->callback_data = DASD_SLEEPON_END_TAG; spin_unlock_irq(get_ccwdev_lock(cqr->startdev->cdev)); wake_up(&generic_waitq); @@ -2767,7 +2819,8 @@ restart: if (cqr->status != DASD_CQR_DONE && cqr->status != DASD_CQR_FAILED && cqr->status != DASD_CQR_NEED_ERP && - cqr->status != DASD_CQR_TERMINATED) + cqr->status != DASD_CQR_TERMINATED && + cqr->status != DASD_CQR_ABORTED) continue; if (cqr->status == DASD_CQR_TERMINATED) { @@ -2878,6 +2931,14 @@ static void dasd_block_tasklet(unsigned long data) atomic_set(&block->tasklet_scheduled, 0); INIT_LIST_HEAD(&final_queue); spin_lock_irq(&block->queue_lock); + /* + * Splice the hardirq-staged ESE format CQRs onto ccw_queue. Splice to + * the tail so an aborted origin request (already on ccw_queue) is + * retired before its format-CQR replacement completes and requeues it. + */ + spin_lock(&block->ese_lock); + list_splice_tail_init(&block->ese_staging, &block->ccw_queue); + spin_unlock(&block->ese_lock); /* Finish off requests on ccw queue */ __dasd_process_block_ccw_queue(block, &final_queue); spin_unlock_irq(&block->queue_lock); @@ -2937,6 +2998,15 @@ static int _dasd_requests_to_flushqueue(struct dasd_block *block, int rc, i; spin_lock_irqsave(&block->queue_lock, flags); + /* + * Splice any hardirq-staged ESE format CQRs onto ccw_queue first so + * they are seen and canceled by the walk below instead of being + * orphaned across this flush / state transition. Mirrors the splice + * in dasd_block_tasklet(). + */ + spin_lock(&block->ese_lock); + list_splice_tail_init(&block->ese_staging, &block->ccw_queue); + spin_unlock(&block->ese_lock); rc = 0; restart: list_for_each_entry_safe(cqr, n, &block->ccw_queue, blocklist) { diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 785aba11066b..7184da554298 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -204,6 +204,17 @@ static void set_ch_t(struct ch_t *geo, __u32 cyl, __u8 head) geo->head |= head; } +static __maybe_unused void set_chr_t(void *addr, __u32 cyl, __u8 head, __u8 record) +{ + struct chr_t *geo = addr; + + geo->cyl = (__u16)cyl; + geo->head = cyl >> DASD_EAV_CYL_HI_SHIFT; + geo->head <<= DASD_EAV_HEAD_HI_SHIFT; + geo->head |= head; + geo->record = record; +} + /* * calculate failing track from sense data depending if * it is an EAV device or not diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index bad7ba666370..0fdb92fdddc8 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -146,6 +146,11 @@ struct eckd_count { __u16 dl; } __attribute__ ((packed)); +struct eckd_r0 { + struct eckd_count count; + __u8 data[8]; +} __packed; + /* * Extended Address Volume track address: the head field carries the actual * head in its low-order 4 bits; the cylinder bits that do not fit the 16-bit diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index 81cfb5c89681..cab16907ea5a 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -159,6 +159,11 @@ struct dasd_ccw_req { void *callback_data; unsigned int proc_bytes; /* bytes for partial completion */ unsigned int trkcount; /* count formatted tracks */ + void *filldata; /* address of filler data */ + struct dasd_format_entry *format; + sector_t start_trk; + sector_t end_trk; + bool collision; }; /* @@ -170,6 +175,7 @@ struct dasd_ccw_req { #define DASD_CQR_IN_ERP 0x03 /* request is in recovery */ #define DASD_CQR_FAILED 0x04 /* request is finally failed */ #define DASD_CQR_TERMINATED 0x05 /* request was stopped by driver */ +#define DASD_CQR_ABORTED 0x06 /* request was replaced and will be deleted */ #define DASD_CQR_QUEUED 0x80 /* request is queued to be processed */ #define DASD_CQR_IN_IO 0x81 /* request is currently in IO */ @@ -177,6 +183,7 @@ struct dasd_ccw_req { #define DASD_CQR_CLEAR_PENDING 0x83 /* request is clear pending */ #define DASD_CQR_CLEARED 0x84 /* request was cleared */ #define DASD_CQR_SUCCESS 0x85 /* request was successful */ +#define DASD_CQR_ABORT 0x86 /* request was replaced and will not be handled */ /* default expiration time*/ #define DASD_EXPIRES 300 @@ -573,9 +580,12 @@ struct dasd_device { struct list_head ccw_queue; spinlock_t mem_lock; void *ccw_mem; + void *fill_mem; void *erp_mem; void *ese_mem; + void *nulldata; struct list_head ccw_chunks; + struct list_head fill_chunks; struct list_head erp_chunks; struct list_head ese_chunks; @@ -640,6 +650,15 @@ struct dasd_block { struct list_head format_list; spinlock_t format_lock; atomic_t trkcount; + + /* + * ESE format CQRs staged from hardirq, spliced into + * ccw_queue in dasd_block_tasklet under queue_lock. Direct enqueue from + * the IRQ handler would invert the queue_lock / ccwdev_lock order. + */ + struct list_head ese_staging; + /* lock for ese_staging */ + spinlock_t ese_lock; }; struct dasd_attention_data { From 05697849292011dc828b3651c159cfe694d1b5ef Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:02 +0200 Subject: [PATCH 129/241] s390/dasd: Add range-based format-track collision detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the single per-device format_entry slot with an array of 16 slots so multiple format requests can be in flight at once, and extend struct dasd_format_entry with a start_trk/end_trk/cqr range (replacing the single track field). Rewrite test_and_set_format_track() to scan the array for range overlaps instead of a trkcount snapshot, honour the early-collision flag, and return the allocated slot to the caller. Add dasd_req_conflict() and extend dasd_return_cqr_cb() to mark in-flight data CQRs that overlap a just-completed format range, so the next test_and_set_format_track() detects the conflict early. Remove the now-obsolete trkcount snapshot in dasd_start_IO(). The detection added here only becomes active together with the WRITE_FULL_TRACK ESE format handler later in the series: that patch routes the format request through dasd_return_cqr_cb() (so completion runs the overlap hook with cqr->format set) and records each request's start_trk/end_trk range. Until then the array and the conflict check are in place but dormant. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-10-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 29 ++++++++++++---- drivers/s390/block/dasd_eckd.c | 62 +++++++++++++++++++++++----------- drivers/s390/block/dasd_int.h | 19 +++++++++-- 3 files changed, 81 insertions(+), 29 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index 79cd7e132418..a2a66e929884 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -1401,13 +1401,6 @@ int dasd_start_IO(struct dasd_ccw_req *cqr) if (!cqr->lpm) cqr->lpm = dasd_path_get_opm(device); } - /* - * remember the amount of formatted tracks to prevent double format on - * ESE devices - */ - if (cqr->block) - cqr->trkcount = atomic_read(&cqr->block->trkcount); - if (cqr->cpmode == 1) { rc = ccw_device_tm_start(device->cdev, cqr->cpaddr, (long) cqr, cqr->lpm); @@ -2868,6 +2861,28 @@ restart: static void dasd_return_cqr_cb(struct dasd_ccw_req *cqr, void *data) { + struct dasd_ccw_req *temp_cqr; + struct dasd_block *block; + + /* only format CQRs are candidates */ + if (!cqr->block || unlikely(!cqr->format)) + goto out; + + block = cqr->block; + /* + * Mark in-flight (IN_IO) CQRs that overlap this just-completed format + * range so they re-check in test_and_set_format on completion; FILLED + * or QUEUED CQRs re-check the format_list on their next round anyway. + */ + list_for_each_entry(temp_cqr, &block->ccw_queue, blocklist) { + if (temp_cqr != cqr && + temp_cqr->status != DASD_CQR_FILLED && + temp_cqr->status != DASD_CQR_QUEUED && + dasd_req_conflict(cqr, temp_cqr)) { + WRITE_ONCE(temp_cqr->collision, true); + } + } +out: dasd_schedule_block_bh(cqr->block); } diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 7184da554298..e9fea5f5c8a0 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -3150,32 +3150,44 @@ static int dasd_eckd_format_device(struct dasd_device *base, 0, NULL); } -static bool test_and_set_format_track(struct dasd_format_entry *to_format, - struct dasd_ccw_req *cqr) +static bool test_and_set_format_track(sector_t start, sector_t end, + struct dasd_ccw_req *cqr, + struct dasd_block *block, + struct dasd_device *device, + struct dasd_format_entry **entry) { - struct dasd_block *block = cqr->block; - struct dasd_format_entry *format; + struct dasd_format_entry *to_format, *format; unsigned long flags; bool rc = false; + int i = 0; + /* marked as a collision by dasd_return_cqr_cb last round: retry */ + if (cqr && READ_ONCE(cqr->collision)) { + WRITE_ONCE(cqr->collision, false); + return true; + } spin_lock_irqsave(&block->format_lock, flags); - if (cqr->trkcount != atomic_read(&block->trkcount)) { - /* - * The number of formatted tracks has changed after request - * start and we can not tell if the current track was involved. - * To avoid data corruption treat it as if the current track is - * involved - */ + while (i < DASD_NR_FORMAT_ENTRIES && + READ_ONCE(device->format_entry[i].cqr)) + i++; + + if (i >= DASD_NR_FORMAT_ENTRIES) { rc = true; goto out; } + list_for_each_entry(format, &block->format_list, list) { - if (format->track == to_format->track) { + if (!(end < format->start_trk || format->end_trk < start)) { rc = true; goto out; } } + to_format = &device->format_entry[i]; + to_format->start_trk = start; + to_format->end_trk = end; + to_format->cqr = cqr; list_add_tail(&to_format->list, &block->format_list); + *entry = to_format; out: spin_unlock_irqrestore(&block->format_lock, flags); @@ -3183,13 +3195,13 @@ out: } static void clear_format_track(struct dasd_format_entry *format, - struct dasd_block *block) + struct dasd_block *block) { unsigned long flags; spin_lock_irqsave(&block->format_lock, flags); - atomic_inc(&block->trkcount); list_del_init(&format->list); + format->cqr = NULL; spin_unlock_irqrestore(&block->format_lock, flags); } @@ -3211,8 +3223,8 @@ static struct dasd_ccw_req * dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, struct irb *irb) { + struct dasd_format_entry *format = NULL; struct dasd_eckd_private *private; - struct dasd_format_entry *format; struct format_data_t fdata; unsigned int recs_per_trk; struct dasd_ccw_req *fcqr; @@ -3231,7 +3243,6 @@ dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, private = base->private; blksize = block->bp_block; recs_per_trk = recs_per_track(&private->rdc_data, 0, blksize); - format = &startdev->format_entry; first_trk = blk_rq_pos(req) >> block->s2b_shift; sector_div(first_trk, recs_per_trk); @@ -3248,9 +3259,9 @@ dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, curr_trk, first_trk, last_trk); return ERR_PTR(-EINVAL); } - format->track = curr_trk; + /* test if track is already in formatting by another thread */ - if (test_and_set_format_track(format, cqr)) { + if (test_and_set_format_track(curr_trk, curr_trk, cqr, block, startdev, &format)) { /* this is no real error so do not count down retries */ cqr->retries++; return ERR_PTR(-EEXIST); @@ -3262,17 +3273,28 @@ dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, fdata.intensity = private->uses_cdl ? DASD_FMT_INT_COMPAT : 0; rc = dasd_eckd_format_sanity_checks(base, &fdata); - if (rc) + if (rc) { + if (format) + clear_format_track(format, block); return ERR_PTR(-EINVAL); + } /* * We're building the request with PAV disabled as we're reusing * the former startdev. */ fcqr = dasd_eckd_build_format(base, startdev, &fdata, 0); - if (IS_ERR(fcqr)) + if (IS_ERR(fcqr)) { + if (format) + clear_format_track(format, block); return fcqr; + } + if (format) { + /* occupancy marker; the free-slot scan reads it with READ_ONCE */ + WRITE_ONCE(format->cqr, fcqr); + fcqr->format = format; + } fcqr->callback = dasd_eckd_ese_format_cb; fcqr->callback_data = (void *) format; diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index cab16907ea5a..b342237b84de 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -545,9 +545,17 @@ struct dasd_profile { spinlock_t lock; }; +/* + * concurrent ESE format ranges in flight; also caps a WRITE_FULL_TRACK's + * track count, which the LRE track bitmask limits to 16 + */ +#define DASD_NR_FORMAT_ENTRIES 16 + struct dasd_format_entry { struct list_head list; - sector_t track; + struct dasd_ccw_req *cqr; + sector_t start_trk; + sector_t end_trk; }; struct dasd_device { @@ -617,7 +625,7 @@ struct dasd_device { struct dentry *debugfs_dentry; struct dentry *hosts_dentry; struct dasd_profile profile; - struct dasd_format_entry format_entry; + struct dasd_format_entry format_entry[DASD_NR_FORMAT_ENTRIES]; struct kset *paths_info; struct dasd_copy_relation *copy; unsigned long aq_mask; @@ -834,6 +842,13 @@ static inline void *dasd_get_callback_data(struct dasd_ccw_req *cqr) return cqr->callback_data; } +static inline bool dasd_req_conflict(struct dasd_ccw_req *cqr1, + struct dasd_ccw_req *cqr2) +{ + return !(cqr1->format->end_trk < cqr2->start_trk || + cqr2->end_trk < cqr1->format->start_trk); +} + /* externals in dasd.c */ #define DASD_PROFILE_OFF 0 #define DASD_PROFILE_ON 1 From 44f9bf47987ec25a90d46d171b8df847d773f033 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:03 +0200 Subject: [PATCH 130/241] s390/dasd: Extend prepare_itcw() to support WRITE_FULL_TRACK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prepare_itcw() builds the FCX prefix block (PFX + LRE) for track-mode I/O. Extend it to handle DASD_ECKD_CCW_WRITE_FULL_TRACK. WRITE_FULL_TRACK needs two extra bytes appended to the LRE for that bitmask. The prefix block is a scratch buffer copied into the TCCB by itcw_add_dcw(), so keep it on the stack (sized for the two extra bytes) rather than allocating it: this runs in the writeback path and must not depend on an allocation that can fail under memory pressure. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-11-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 64 +++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index e9fea5f5c8a0..6d948f099efb 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -4385,11 +4385,13 @@ static int prepare_itcw(struct itcw *itcw, unsigned int tlf, unsigned int blk_per_trk) { - struct PFX_eckd_data pfxdata; + u8 pfxbuf[sizeof(struct PFX_eckd_data) + 2] __aligned(8); + struct PFX_eckd_data *pfxdata = (struct PFX_eckd_data *)pfxbuf; struct dasd_eckd_private *basepriv, *startpriv; struct DE_eckd_data *dedata; struct LRE_eckd_data *lredata; struct dcw *dcw; + int pfxsize; u32 begcyl, endcyl; u16 heads, beghead, endhead; @@ -4399,26 +4401,31 @@ static int prepare_itcw(struct itcw *itcw, int sector = 0; int dn, d; + pfxsize = sizeof(struct PFX_eckd_data); + /* prefix + LRE extended data */ + if (cmd == DASD_ECKD_CCW_WRITE_FULL_TRACK) + pfxsize += 2; + + memset(pfxbuf, 0, pfxsize); /* setup prefix data */ basepriv = basedev->private; startpriv = startdev->private; - dedata = &pfxdata.define_extent; - lredata = &pfxdata.locate_record; + dedata = &pfxdata->define_extent; + lredata = &pfxdata->locate_record; - memset(&pfxdata, 0, sizeof(pfxdata)); - pfxdata.format = 1; /* PFX with LRE */ - pfxdata.base_address = basepriv->conf.ned->unit_addr; - pfxdata.base_lss = basepriv->conf.ned->ID; - pfxdata.validity.define_extent = 1; + pfxdata->format = 1; /* PFX with LRE */ + pfxdata->base_address = basepriv->conf.ned->unit_addr; + pfxdata->base_lss = basepriv->conf.ned->ID; + pfxdata->validity.define_extent = 1; /* private uid is kept up to date, conf_data may be outdated */ if (startpriv->uid.type == UA_BASE_PAV_ALIAS) - pfxdata.validity.verify_base = 1; + pfxdata->validity.verify_base = 1; if (startpriv->uid.type == UA_HYPER_PAV_ALIAS) { - pfxdata.validity.verify_base = 1; - pfxdata.validity.hyper_pav = 1; + pfxdata->validity.verify_base = 1; + pfxdata->validity.hyper_pav = 1; } switch (cmd) { @@ -4448,7 +4455,38 @@ static int prepare_itcw(struct itcw *itcw, * data as well. */ if (dedata->ga_extended & 0x08 && dedata->ga_extended & 0x02) - pfxdata.validity.time_stamp = 1; /* 'Time Stamp Valid' */ + pfxdata->validity.time_stamp = 1; /* 'Time Stamp Valid' */ + pfx_cmd = DASD_ECKD_CCW_PFX; + break; + case DASD_ECKD_CCW_WRITE_FULL_TRACK: + dedata->mask.perm = 0x3; + dedata->mask.auth = 0x00; + dedata->attributes.operation = basepriv->attrib.operation; + dedata->blk_size = blksize; + dedata->ga_extended |= 0x42; + rc = set_timestamp(NULL, dedata, basedev); + lredata->operation.orientation = 0x0; + lredata->operation.operation = 0x3F; + lredata->extended_operation = 0x11; + lredata->auxiliary.check_bytes = 0x2; + lredata->extended_parameter_length = 0x02; + if (count > 8) { + lredata->extended_parameter[0] = 0xFF; + lredata->extended_parameter[1] = 0xFF; + lredata->extended_parameter[1] <<= (16 - count); + } else { + lredata->extended_parameter[0] = 0xFF; + lredata->extended_parameter[0] <<= (8 - count); + lredata->extended_parameter[1] = 0x00; + } + sector = 0xFF; + /* + * If XRC is supported the System Time Stamp is set. The + * validity of the time stamp must be reflected in the prefix + * data as well. + */ + if (dedata->ga_extended & 0x08 && dedata->ga_extended & 0x02) + pfxdata->validity.time_stamp = 1; /* 'Time Stamp Valid' */ pfx_cmd = DASD_ECKD_CCW_PFX; break; case DASD_ECKD_CCW_READ_COUNT_MT: @@ -4527,7 +4565,7 @@ static int prepare_itcw(struct itcw *itcw, lredata->search_arg.record = rec_on_trk; dcw = itcw_add_dcw(itcw, pfx_cmd, 0, - &pfxdata, sizeof(pfxdata), total_data_size); + pfxdata, pfxsize, total_data_size); return PTR_ERR_OR_ZERO(dcw); } From 123ec1e9cb4200308ad1535c84a4cf2f81904517 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:04 +0200 Subject: [PATCH 131/241] s390/dasd: Add dasd_eckd_build_cp_tpm_writefulltrack() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the channel program builder for WRITE_FULL_TRACK requests, used by dasd_eckd_ese_format() (next patch) to format and write a set of tracks atomically and avoid the format cycle on ESE devices. The program is an ITCW with a TIDAW list. Per track it emits an eckd_r0 header, an eckd_count + data pair for every record (pad records before and after the caller's data window use device->nulldata, records in the window point into the bio payload), and a terminating 0xFF pseudo-count with TIDAW_FLAGS_INSERT_CBC. The descriptors come from the per-device fill_chunks pool so they can be freed in bulk in __dasd_cleanup_cqr(). Add inline helpers crosses_page() and reserve_nocross(), to keep each descriptor within one page since TIDAW addressing must not cross a page boundary. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-12-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 344 ++++++++++++++++++++++++++++++++- 1 file changed, 343 insertions(+), 1 deletion(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 6d948f099efb..04e2d46fc92d 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -123,6 +123,14 @@ static int prepare_itcw(struct itcw *, unsigned int, unsigned int, int, unsigned int, unsigned int); static int dasd_eckd_query_pprc_status(struct dasd_device *, struct dasd_pprc_data_sc4 *); +static struct dasd_ccw_req *dasd_eckd_build_cp_tpm_writefulltrack(struct dasd_device *, + struct dasd_block *, + struct request *, + sector_t, sector_t, + sector_t, sector_t, + unsigned int, unsigned int, + unsigned int, unsigned int, + struct dasd_ccw_req *); /* initial attempt at a probe function. this can be simplified once * the other detection code is gone */ @@ -204,7 +212,7 @@ static void set_ch_t(struct ch_t *geo, __u32 cyl, __u8 head) geo->head |= head; } -static __maybe_unused void set_chr_t(void *addr, __u32 cyl, __u8 head, __u8 record) +static void set_chr_t(void *addr, __u32 cyl, __u8 head, __u8 record) { struct chr_t *geo = addr; @@ -4742,6 +4750,340 @@ out_error: return ERR_PTR(ret); } +static __always_inline bool crosses_page(const void *addr, size_t len) +{ + return len && (offset_in_page(addr) + len > PAGE_SIZE); +} + +static __always_inline void *reserve_nocross(char **p, size_t *space, size_t len) +{ + size_t pad = crosses_page(*p, len) ? PAGE_SIZE - offset_in_page(*p) : 0; + void *ret; + + if (*space < pad + len) + return NULL; /* out of space */ + + *p += pad; + *space -= pad; + ret = *p; + *p += len; + *space -= len; + return ret; +} + +/* + * Helpers for dasd_eckd_build_cp_tpm_writefulltrack(): append the TIDAWs for + * one track-image element (R0 header, a count + data record, or the trailing + * pseudo track end count) to the itcw. Return the last TIDAW, or NULL on failure. + */ +static struct tidaw *add_track_r0(struct itcw *itcw, char **fill, + size_t *fillsize, u32 cyl, u16 head) +{ + struct tidaw *tidaw; + struct eckd_r0 *r0; + + r0 = reserve_nocross(fill, fillsize, sizeof(*r0)); + if (WARN_ON_ONCE(!r0)) + return NULL; + set_chr_t(r0, cyl, head, 0); + r0->count.dl = 8; + tidaw = itcw_add_tidaw(itcw, 0, r0, sizeof(*r0)); + return IS_ERR_OR_NULL(tidaw) ? NULL : tidaw; +} + +static struct tidaw *add_track_record(struct itcw *itcw, char **fill, + size_t *fillsize, u32 cyl, u16 head, + u8 rec, void *data, u32 dl) +{ + struct eckd_count *count; + struct tidaw *tidaw; + + count = reserve_nocross(fill, fillsize, sizeof(*count)); + if (WARN_ON_ONCE(!count)) + return NULL; + set_chr_t(count, cyl, head, rec); + count->dl = dl; + tidaw = itcw_add_tidaw(itcw, 0, count, sizeof(*count)); + if (IS_ERR_OR_NULL(tidaw)) + return NULL; + tidaw = itcw_add_tidaw(itcw, 0, data, dl); + return IS_ERR_OR_NULL(tidaw) ? NULL : tidaw; +} + +static struct tidaw *add_track_end(struct itcw *itcw, char **fill, + size_t *fillsize) +{ + struct eckd_count *count; + struct tidaw *tidaw; + + count = reserve_nocross(fill, fillsize, sizeof(*count)); + if (WARN_ON_ONCE(!count)) + return NULL; + count->cyl = 0xffff; + count->head = 0xffff; + count->dl = 0xffff; + count->record = 0xff; + count->kl = 0xff; + tidaw = itcw_add_tidaw(itcw, TIDAW_FLAGS_INSERT_CBC, count, sizeof(*count)); + return IS_ERR_OR_NULL(tidaw) ? NULL : tidaw; +} + +static __maybe_unused struct dasd_ccw_req * +dasd_eckd_build_cp_tpm_writefulltrack(struct dasd_device *startdev, + struct dasd_block *block, + struct request *req, + sector_t first_rec, + sector_t last_rec, + sector_t first_trk, + sector_t last_trk, + unsigned int first_offs, + unsigned int last_offs, + unsigned int blk_per_trk, + unsigned int blksize, + struct dasd_ccw_req *ocqr) +{ + struct dasd_eckd_private *private = block->base->private; + unsigned int seg_len, part_len, len_to_track_end; + unsigned int count, count_to_trk_end, offs; + unsigned int trkcount, ctidaw, tlf; + int itcw_op, rec_count, datasize; + struct tidaw *last_tidaw = NULL; + sector_t recid, trkid, curr_trk; + unsigned char cmd, new_track; + struct dasd_device *basedev; + size_t itcw_size, fillsize; + struct dasd_ccw_req *cqr; + struct req_iterator iter; + char *dst, *filldata; + unsigned long flags; + struct itcw *itcw; + struct bio_vec bv; + int ret = -EINVAL; + void *nullrecord; + u16 heads, head; + u32 cyl; + u8 rec; + + basedev = block->base; + cmd = DASD_ECKD_CCW_WRITE_FULL_TRACK; + itcw_op = ITCW_OP_WRITE; + + /* + * trackbased I/O needs address all memory via TIDAWs, + * not just for 64 bit addresses. This allows us to map + * each segment directly to one tidaw. + * In the case of write requests, additional tidaws may + * be needed when a segment crosses a track boundary. + * Per track we emit one R0 tidaw, two tidaws per record (count field + * plus data - a record never crosses a track or page boundary, as + * part_len is clamped to both blksize and the track end), and one track + * end tidaw: 2 * blk_per_trk + 2. + * Round the +2 up to blk_per_trk-independent headroom via 2 * (blk_per_trk + 2). + */ + trkcount = last_trk - first_trk + 1; + ctidaw = trkcount * 2 * (blk_per_trk + 2); + + /* + * build_cp (ocqr == NULL): the request owns its CCW program - block in + * the pdu, ITCW in ccw_chunks. ese_format (ocqr != NULL): the failing + * origin still owns its pdu, so take the replacement from ese_chunks. + */ + itcw_size = itcw_calc_size(0, ctidaw, 0); + if (ocqr) + cqr = dasd_fmalloc_request(DASD_ECKD_MAGIC, 0, itcw_size, startdev); + else + cqr = dasd_smalloc_request(DASD_ECKD_MAGIC, 0, itcw_size, startdev, + blk_mq_rq_to_pdu(req)); + if (IS_ERR(cqr)) + return cqr; + fillsize = trkcount * (sizeof(struct eckd_r0) + + (sizeof(struct eckd_count) * (blk_per_trk + 2))); + /* + * reserve_nocross() pads elements away from page boundaries and draws + * that padding from fillsize; budget one element per page the buffer + * may span so it never runs short. + */ + fillsize += (fillsize / PAGE_SIZE + 1) * sizeof(struct eckd_r0); + spin_lock_irqsave(&startdev->mem_lock, flags); + filldata = dasd_alloc_chunk(&startdev->fill_chunks, fillsize); + spin_unlock_irqrestore(&startdev->mem_lock, flags); + if (!filldata) { + ret = -ENOMEM; + goto out_error; + } + memset(filldata, 0, fillsize); + cqr->filldata = filldata; + + nullrecord = startdev->nulldata; + + /* count + data for each record, plus r0 and the pseudo count */ + tlf = blk_per_trk * (blksize + sizeof(struct eckd_count)); + tlf += sizeof(struct eckd_r0) + sizeof(struct eckd_count); + + itcw = itcw_init(cqr->data, itcw_size, itcw_op, 0, ctidaw, 0); + if (IS_ERR(itcw)) { + ret = -EINVAL; + goto out_error; + } + cqr->cpaddr = itcw_get_tcw(itcw); + datasize = trkcount * tlf; + if (prepare_itcw(itcw, first_trk, last_trk, + cmd, basedev, startdev, + 0, + trkcount, blksize, + datasize, + tlf, + blk_per_trk) == -EAGAIN) { + /* Clock not in sync and XRC is enabled. + * Try again later. + */ + ret = -EAGAIN; + goto out_error; + } + heads = private->rdc_data.trk_per_cyl; + /* + * A tidaw can address 4k of memory, but must not cross page boundaries + * We can let the block layer handle this by setting seg_boundary_mask + * to page boundaries and max_segment_size to page size when setting up + * the request queue. + */ + curr_trk = first_trk; + recid = first_rec; + trkid = recid; + offs = sector_div(trkid, blk_per_trk); + count = blk_per_trk; + len_to_track_end = count * blksize; + recid += count - first_offs; + new_track = 0; + + /* the R0 header of the first track */ + cyl = curr_trk / heads; + head = curr_trk % heads; + last_tidaw = add_track_r0(itcw, &filldata, &fillsize, cyl, head); + if (!last_tidaw) + goto out_error; + + /* empty records before the first data record */ + for (int i = 1; i <= first_offs; i++) { + len_to_track_end -= blksize; + last_tidaw = add_track_record(itcw, &filldata, &fillsize, + cyl, head, i, nullrecord, blksize); + if (!last_tidaw) + goto out_error; + } + + /* process data records */ + rec = first_offs + 1; + rec_count = 0; + rq_for_each_segment(bv, req, iter) { + dst = bvec_virt(&bv); + seg_len = bv.bv_len; + while (seg_len) { + if (new_track) { + trkid = recid; + offs = sector_div(trkid, blk_per_trk); + count_to_trk_end = blk_per_trk - offs; + count = min((last_rec - recid + 1), + (sector_t)count_to_trk_end); + /* + * Size to the physical track end: a short last + * track is padded in out_skip, so the track-end + * marker must not be emitted early here. + */ + len_to_track_end = count_to_trk_end * blksize; + recid += count; + new_track = 0; + /* the R0 header of the next track */ + cyl = curr_trk / heads; + head = curr_trk % heads; + last_tidaw = add_track_r0(itcw, &filldata, + &fillsize, cyl, head); + if (!last_tidaw) + goto out_error; + rec = 1; + } + /* + * One count + data record per block: a bvec segment can + * be up to a page, so clamp to blksize - otherwise the + * count field would describe one oversized record instead + * of several blksize ones for sub-page block sizes. + */ + part_len = min(seg_len, len_to_track_end); + part_len = min(part_len, blksize); + seg_len -= part_len; + len_to_track_end -= part_len; + /* + * This block ends the track; the next one starts a new + * track. The track-end marker emitted below carries the + * CBC flag. + */ + if (!len_to_track_end) + new_track = 1; + + last_tidaw = add_track_record(itcw, &filldata, &fillsize, + cyl, head, rec, dst, part_len); + if (!last_tidaw) + goto out_error; + + if (new_track) { + /* add track end marker */ + last_tidaw = add_track_end(itcw, &filldata, + &fillsize); + if (!last_tidaw) + goto out_error; + curr_trk++; + } + rec++; + dst += part_len; + rec_count++; + if (rec_count >= (last_rec - first_rec + 1)) + goto out_skip; + } + } + +out_skip: + new_track = 0; + /* empty records after the last data record */ + for (int i = last_offs + 2; i <= blk_per_trk; i++) { + len_to_track_end -= blksize; + last_tidaw = add_track_record(itcw, &filldata, &fillsize, + cyl, head, i, nullrecord, blksize); + if (!last_tidaw) + goto out_error; + new_track = 1; + } + + /* add track end marker */ + if (new_track) { + last_tidaw = add_track_end(itcw, &filldata, &fillsize); + if (!last_tidaw) + goto out_error; + } + + last_tidaw->flags |= TIDAW_FLAGS_LAST; + last_tidaw->flags &= ~TIDAW_FLAGS_INSERT_CBC; + itcw_finalize(itcw); + + if (blk_noretry_request(req) || + block->base->features & DASD_FEATURE_FAILFAST) + set_bit(DASD_CQR_FLAGS_FAILFAST, &cqr->flags); + cqr->cpmode = 1; + cqr->startdev = startdev; + cqr->memdev = startdev; + cqr->block = block; + cqr->expires = startdev->default_expires * HZ; /* default 5 minutes */ + cqr->lpm = dasd_path_get_ppm(startdev); + cqr->retries = startdev->default_retries; + cqr->buildclk = get_tod_clock(); + cqr->status = DASD_CQR_FILLED; + + return cqr; +out_error: + /* dasd_sfree_request frees from the right pool via cqr->mem_chunk */ + dasd_sfree_request(cqr, startdev); + return ERR_PTR(ret); +} + static struct dasd_ccw_req *dasd_eckd_build_cp(struct dasd_device *startdev, struct dasd_block *block, struct request *req) From 791d257a21ba5241ce723a6c17f395aa0d419982 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:05 +0200 Subject: [PATCH 132/241] s390/dasd: Use WRITE_FULL_TRACK in ESE format handler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire dasd_eckd_build_cp_tpm_writefulltrack() into the ESE unformated track handler. dasd_eckd_ese_format() now returns void (matching the revised discipline hook): it computes the failing track/record range, trims a partially covered last track when several tracks are involved (the block layer re-issues the remainder), claims the range with test_and_set_format_track(), builds a writefulltrack CQR, copies callback_data/proc_bytes from the origin, and stages it on block->ese_staging. The origin CQR is set to DASD_CQR_ABORT so __dasd_process_cqr() retires it without the normal completion. Drop dasd_eckd_ese_format_cb(); the format-entry slot is now released by dasd_eckd_free_alias_cp() via clear_format_track() when the CQR is freed. dasd_int_handler() calls the void hook directly and, for writefulltrack CQRs (cqr->filldata set), returns DASD_CQR_ERROR instead of looping on the NRF. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-13-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 59 ++++++--- drivers/s390/block/dasd_3990_erp.c | 1 + drivers/s390/block/dasd_eckd.c | 202 ++++++++++++++++++++--------- drivers/s390/block/dasd_erp.c | 8 +- drivers/s390/block/dasd_int.h | 3 +- 5 files changed, 193 insertions(+), 80 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index a2a66e929884..0e01b498b0e8 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -1617,7 +1617,7 @@ static int dasd_ese_oos_cond(u8 *sense) void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm, struct irb *irb) { - struct dasd_ccw_req *cqr, *next, *fcqr; + struct dasd_ccw_req *cqr, *next; struct dasd_device *device; unsigned long now; int nrf_suppressed = 0; @@ -1739,26 +1739,23 @@ void dasd_int_handler(struct ccw_device *cdev, unsigned long intparm, dasd_schedule_device_bh(device); return; } - fcqr = device->discipline->ese_format(device, cqr, irb); - if (IS_ERR(fcqr)) { - if (PTR_ERR(fcqr) == -EINVAL) { - cqr->status = DASD_CQR_ERROR; - return; - } + if (cqr->filldata) { /* - * If we can't format now, let the request go - * one extra round. Maybe we can format later. + * A WRITE_FULL_TRACK cqr carries the complete + * track image; INV_TRACK_FORMAT here means the + * generated image or the media itself is bad, not + * that the track still needs formatting - retrying + * via ese_format() would just resubmit the same + * write. Let it fail instead. */ - cqr->status = DASD_CQR_QUEUED; - dasd_schedule_device_bh(device); - return; - } else { - fcqr->status = DASD_CQR_QUEUED; - cqr->status = DASD_CQR_QUEUED; - list_add(&fcqr->devlist, &device->ccw_queue); + cqr->status = DASD_CQR_ERROR; + cqr->stopclk = now; + dasd_device_clear_timer(device); dasd_schedule_device_bh(device); return; } + device->discipline->ese_format(device, cqr, irb); + return; } /* Check for clear pending */ @@ -2721,6 +2718,13 @@ static void __dasd_process_erp(struct dasd_device *device, if (cqr->status == DASD_CQR_DONE) DBF_DEV_EVENT(DBF_NOTICE, device, "%s", "ERP successful"); + else if (cqr->status == DASD_CQR_ABORTED) + /* + * ESE format aborts the request and replaces it with a format + * CQR - this is not an ERP failure. + */ + DBF_DEV_EVENT(DBF_NOTICE, device, "%s", + "ERP request aborted, replaced by ESE format"); else dev_err(&device->cdev->dev, "ERP failed for the DASD\n"); erp_fn = device->discipline->erp_postaction(cqr); @@ -2767,6 +2771,9 @@ static void __dasd_cleanup_cqr(struct dasd_ccw_req *cqr) error = BLK_STS_IOERR; break; } + } else if (status == DASD_CQR_ABORTED) { + /* aborted requests are replaced with a new one so do not complete this */ + return; } /* @@ -3173,6 +3180,13 @@ static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx, } goto out; } + if (!cqr) { + /* build_cp may collapse a non-transient build error to NULL */ + DBF_DEV_EVENT(DBF_ERR, basedev, + "CCW creation returned NULL on request %p", req); + rc = BLK_STS_IOERR; + goto out; + } /* * Note: callback is set to dasd_return_cqr_cb in * __dasd_block_start_head to cover erp requests as well @@ -3966,6 +3980,19 @@ restart_cb: */ goto restart_cb; } + /* + * An aborted request was replaced by a full-track write and is + * retired by that replacement; do not requeue it, just release + * it (mirrors the DASD_CQR_ABORTED handling in + * __dasd_cleanup_cqr()). + */ + if (cqr->status == DASD_CQR_ABORTED) { + struct request *req = cqr->callback_data; + + list_del_init(&cqr->blocklist); + cqr->block->base->discipline->free_cp(cqr, req); + continue; + } _dasd_requeue_request(cqr); list_del_init(&cqr->blocklist); cqr->block->base->discipline->free_cp( diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c index d0aa267462c5..736459477c19 100644 --- a/drivers/s390/block/dasd_3990_erp.c +++ b/drivers/s390/block/dasd_3990_erp.c @@ -2400,6 +2400,7 @@ static struct dasd_ccw_req *dasd_3990_erp_add_erp(struct dasd_ccw_req *cqr) erp->startdev = device; erp->memdev = device; erp->block = cqr->block; + erp->filldata = cqr->filldata; erp->magic = cqr->magic; erp->expires = cqr->expires; erp->retries = device->default_retries; diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 04e2d46fc92d..7828f564a11b 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -3213,36 +3213,24 @@ static void clear_format_track(struct dasd_format_entry *format, spin_unlock_irqrestore(&block->format_lock, flags); } -/* - * Callback function to free ESE format requests. - */ -static void dasd_eckd_ese_format_cb(struct dasd_ccw_req *cqr, void *data) -{ - struct dasd_device *device = cqr->startdev; - struct dasd_eckd_private *private = device->private; - struct dasd_format_entry *format = data; - - clear_format_track(format, cqr->basedev->block); - private->count--; - dasd_ffree_request(cqr, device); -} - -static struct dasd_ccw_req * -dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, - struct irb *irb) +static void dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, + struct irb *irb) { struct dasd_format_entry *format = NULL; + unsigned int first_offs, last_offs; struct dasd_eckd_private *private; - struct format_data_t fdata; - unsigned int recs_per_trk; + struct dasd_ccw_req *base_cqr; + sector_t first_rec, last_rec; + sector_t first_trk, last_trk; + unsigned int proc_bytes = 0; struct dasd_ccw_req *fcqr; + unsigned int recs_per_trk; struct dasd_device *base; struct dasd_block *block; unsigned int blksize; struct request *req; - sector_t first_trk; - sector_t last_trk; sector_t curr_trk; + unsigned int diff; int rc; req = dasd_get_callback_data(cqr); @@ -3252,50 +3240,94 @@ dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, blksize = block->bp_block; recs_per_trk = recs_per_track(&private->rdc_data, 0, blksize); - first_trk = blk_rq_pos(req) >> block->s2b_shift; - sector_div(first_trk, recs_per_trk); - last_trk = - (blk_rq_pos(req) + blk_rq_sectors(req) - 1) >> block->s2b_shift; - sector_div(last_trk, recs_per_trk); - rc = dasd_eckd_track_from_irb(irb, base, &curr_trk); - if (rc) - return ERR_PTR(rc); + /* Calculate record id of first and last block. */ + first_rec = blk_rq_pos(req) >> block->s2b_shift; + first_trk = first_rec; + first_offs = sector_div(first_trk, recs_per_trk); + last_rec = (blk_rq_pos(req) + blk_rq_sectors(req) - 1) >> block->s2b_shift; + last_trk = last_rec; + last_offs = sector_div(last_trk, recs_per_trk); + /* + * detect if some data has already been processed and the unformatted track is + * within the request. + * If so, finish the request first with the already processed bytes and let the + * blocklayer only redrive unformatted part. + * With this we ensure that there is no overlap of existing data with unformatted + * zero blocks + */ + rc = dasd_eckd_track_from_irb(irb, base, &curr_trk); + if (rc) { + /* sense data could not be parsed - this will not resolve by retrying */ + cqr->status = DASD_CQR_ERROR; + goto out; + } + if (curr_trk >= (sector_t)private->real_cyl * private->rdc_data.trk_per_cyl) { + DBF_DEV_EVENT(DBF_WARNING, startdev, + "ESE error track %llu exceeds device geometry\n", + curr_trk); + cqr->status = DASD_CQR_ERROR; + goto out; + } if (curr_trk < first_trk || curr_trk > last_trk) { DBF_DEV_EVENT(DBF_WARNING, startdev, "ESE error track %llu not within range %llu - %llu\n", curr_trk, first_trk, last_trk); - return ERR_PTR(-EINVAL); + cqr->status = DASD_CQR_ERROR; + goto out; } - - /* test if track is already in formatting by another thread */ - if (test_and_set_format_track(curr_trk, curr_trk, cqr, block, startdev, &format)) { - /* this is no real error so do not count down retries */ - cqr->retries++; - return ERR_PTR(-EEXIST); - } - - fdata.start_unit = curr_trk; - fdata.stop_unit = curr_trk; - fdata.blksize = blksize; - fdata.intensity = private->uses_cdl ? DASD_FMT_INT_COMPAT : 0; - - rc = dasd_eckd_format_sanity_checks(base, &fdata); - if (rc) { - if (format) - clear_format_track(format, block); - return ERR_PTR(-EINVAL); + if (curr_trk != first_trk) { + proc_bytes = ((curr_trk - first_trk) * recs_per_trk - first_offs) * blksize; + cqr->proc_bytes = proc_bytes; + cqr->status = DASD_CQR_SUCCESS; + cqr->stopclk = get_tod_clock(); + goto out; } /* - * We're building the request with PAV disabled as we're reusing - * the former startdev. + * If there are multiple tracks to be format-written, we can not write + * the partial last track since we do not know if it is already formatted + * or not so skip the partial last track for now. Return the partial + * completion to blocklayer and let it redo the remainder */ - fcqr = dasd_eckd_build_format(base, startdev, &fdata, 0); + if (first_trk != last_trk && last_offs + 1 < recs_per_trk) { + diff = last_offs + 1; + last_rec = last_rec - diff; + last_trk = last_rec; + last_offs = sector_div(last_trk, recs_per_trk); + proc_bytes = (last_rec - first_rec + 1) * blksize; + } + if (first_offs > 0 || last_offs + 1 < recs_per_trk) { + /* test if tracks are already in formatting by another thread */ + if (test_and_set_format_track(first_trk, last_trk, cqr, + cqr->block, cqr->startdev, &format)) { + /* this is no real error so do not count down retries */ + cqr->retries++; + goto out_retry; + } + } + + fcqr = dasd_eckd_build_cp_tpm_writefulltrack(startdev, block, req, + first_rec, last_rec, + first_trk, last_trk, + first_offs, last_offs, + recs_per_trk, blksize, cqr); if (IS_ERR(fcqr)) { if (format) - clear_format_track(format, block); - return fcqr; + clear_format_track(format, cqr->block); + if (PTR_ERR(fcqr) == -EINVAL) { + /* permanent build failure - fail instead of retrying */ + cqr->status = DASD_CQR_ERROR; + goto out; + } + /* + * Transient conditions - the XRC clock is not in sync (-EAGAIN) + * or the format request pool is momentarily exhausted under load + * (-ENOMEM). Retry the origin without counting down its retries. + */ + if (PTR_ERR(fcqr) == -EAGAIN || PTR_ERR(fcqr) == -ENOMEM) + cqr->retries++; + goto out_retry; } if (format) { @@ -3303,10 +3335,44 @@ dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, WRITE_ONCE(format->cqr, fcqr); fcqr->format = format; } - fcqr->callback = dasd_eckd_ese_format_cb; - fcqr->callback_data = (void *) format; - return fcqr; + /* + * cqr may be an ERP request; dq and the owning request are only set on + * the base request at the end of the ERP chain, so copy from there. + */ + base_cqr = cqr; + while (base_cqr->refers) + base_cqr = base_cqr->refers; + fcqr->dq = base_cqr->dq; + fcqr->callback_data = base_cqr->callback_data; + if (proc_bytes) + fcqr->proc_bytes = proc_bytes; + fcqr->status = DASD_CQR_FILLED; + ((struct dasd_eckd_private *)fcqr->memdev->private)->count++; + /* + * stage under ese_lock; dasd_block_tasklet splices it into ccw_queue. + * Direct enqueue here would invert queue_lock / ccwdev_lock. + */ + spin_lock(&block->ese_lock); + list_add(&fcqr->blocklist, &block->ese_staging); + spin_unlock(&block->ese_lock); + /* mark origin CQR as aborted; ccwdev_lock is held by the IRQ handler */ + cqr->status = DASD_CQR_ABORT; + goto out; + +out_retry: + /* + * If we can't format now, let the request go + * one extra round. Maybe we can format later. + * re-queue at the end to let potential format collision finish first + */ + list_move_tail(&cqr->devlist, &cqr->startdev->ccw_queue); + cqr->status = DASD_CQR_QUEUED; +out: + dasd_device_clear_timer(startdev); + dasd_schedule_block_bh(block); + dasd_schedule_device_bh(startdev); + return; } /* @@ -4828,7 +4894,7 @@ static struct tidaw *add_track_end(struct itcw *itcw, char **fill, return IS_ERR_OR_NULL(tidaw) ? NULL : tidaw; } -static __maybe_unused struct dasd_ccw_req * +static struct dasd_ccw_req * dasd_eckd_build_cp_tpm_writefulltrack(struct dasd_device *startdev, struct dasd_block *block, struct request *req, @@ -5119,7 +5185,7 @@ static struct dasd_ccw_req *dasd_eckd_build_cp(struct dasd_device *startdev, fcx_multitrack = private->features.feature[40] & 0x20; data_size = blk_rq_bytes(req); - if (data_size % blksize) + if (data_size % blksize || data_size == 0) return ERR_PTR(-EINVAL); /* tpm write request add CBC data on each track boundary */ if (rq_data_dir(req) == WRITE) @@ -5161,6 +5227,11 @@ static struct dasd_ccw_req *dasd_eckd_build_cp(struct dasd_device *startdev, first_trk, last_trk, first_offs, last_offs, blk_per_trk, blksize); + + if (!IS_ERR(cqr)) { + cqr->start_trk = first_trk; + cqr->end_trk = last_trk; + } return cqr; } @@ -5328,7 +5399,17 @@ dasd_eckd_free_cp(struct dasd_ccw_req *cqr, struct request *req) sector_t recid; int status; - if (!dasd_page_cache) + /* + * A format-aborted request finished nothing - its replacement + * completes the block request - so report ABORTED instead of DONE, + * but still release its bounce buffers like any other request. + */ + if (cqr->status == DASD_CQR_ABORTED) + status = DASD_CQR_ABORTED; + else + status = cqr->status == DASD_CQR_DONE; + /* transport mode has no dasd_page_cache bounce buffers to release */ + if (!dasd_page_cache || cqr->cpmode) goto out; private = cqr->block->base->private; blksize = cqr->block->bp_block; @@ -5363,7 +5444,6 @@ dasd_eckd_free_cp(struct dasd_ccw_req *cqr, struct request *req) } } out: - status = cqr->status == DASD_CQR_DONE; dasd_sfree_request(cqr, cqr->memdev); return status; } @@ -5440,6 +5520,8 @@ static int dasd_eckd_free_alias_cp(struct dasd_ccw_req *cqr, private = cqr->memdev->private; private->count--; spin_unlock_irqrestore(get_ccwdev_lock(cqr->memdev->cdev), flags); + if (cqr->format) + clear_format_track(cqr->format, cqr->block); return dasd_eckd_free_cp(cqr, req); } diff --git a/drivers/s390/block/dasd_erp.c b/drivers/s390/block/dasd_erp.c index 468f0b2cc342..05d5366484d7 100644 --- a/drivers/s390/block/dasd_erp.c +++ b/drivers/s390/block/dasd_erp.c @@ -120,7 +120,7 @@ dasd_default_erp_action(struct dasd_ccw_req *cqr) */ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) { - int success; + int success, aborted; unsigned long startclk, stopclk; struct dasd_device *startdev; unsigned int proc_bytes; @@ -128,6 +128,7 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) BUG_ON(cqr->refers == NULL || cqr->function == NULL); success = cqr->status == DASD_CQR_DONE; + aborted = cqr->status == DASD_CQR_ABORTED; startclk = cqr->startclk; stopclk = cqr->stopclk; startdev = cqr->startdev; @@ -150,7 +151,10 @@ struct dasd_ccw_req *dasd_default_erp_postaction(struct dasd_ccw_req *cqr) cqr->stopclk = stopclk; cqr->startdev = startdev; cqr->proc_bytes = proc_bytes; - if (success) + if (aborted) + /* base request is owned by the ESE format replacement CQR */ + cqr->status = DASD_CQR_ABORTED; + else if (success) cqr->status = DASD_CQR_DONE; else { cqr->status = DASD_CQR_FAILED; diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index b342237b84de..e1ffa20db10a 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -413,8 +413,7 @@ struct dasd_discipline { int (*ext_pool_warn_thrshld)(struct dasd_device *); int (*ext_pool_oos)(struct dasd_device *); int (*ext_pool_exhaust)(struct dasd_device *, struct dasd_ccw_req *); - struct dasd_ccw_req *(*ese_format)(struct dasd_device *, - struct dasd_ccw_req *, struct irb *); + void (*ese_format)(struct dasd_device *, struct dasd_ccw_req *, struct irb *); int (*ese_read)(struct dasd_ccw_req *, struct irb *); int (*pprc_status)(struct dasd_device *, struct dasd_pprc_data_sc4 *); bool (*pprc_enabled)(struct dasd_device *); From 42849375e9280f2ccf492ced782e9591910d6b47 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:06 +0200 Subject: [PATCH 133/241] s390/dasd: Add full_track_bias to control fulltrack write mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a single per-device 'full_track_bias' sysfs attribute (0..100) that gates the full-track write path. 0 disables it, 100 routes every aligned, full-track write through dasd_eckd_build_cp_tpm_writefulltrack(). Values in between are reserved for the adaptive heuristic added in the next patch. For now any non-zero value simply enables full-track writes. Internally the value is kept in the per-device 'ft_bias' field. This will control the default IO path only. In case we get an unformatted track error it will always be used to format and write the track in one go. The WRITE_FULL_TRACK command has an advantage on sparse formatted ESE devices but it has an overall penalty for maximum throughput compared to usual track based IO. The attribute lives at /sys/bus/ccw/devices//full_track_bias and accepts 0..100. The default is DASD_FT_BIAS_DEFAULT; together with the adaptive heuristic added in the next patch it uses full-track writes only where they pay off, avoiding the ESE format penalty out of the box while keeping the throughput cost off already-formatted volumes. A 'full_track_bias' module parameter sets the initial value applied to every device at online time; individual volumes can still be re-tuned through their sysfs attribute afterwards. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-14-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_devmap.c | 39 ++++++++++++++++++++++++++++++++ drivers/s390/block/dasd_eckd.c | 36 +++++++++++++++++++++++++---- drivers/s390/block/dasd_int.h | 18 +++++++++++++++ 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c index 381d616ad433..035c022255b6 100644 --- a/drivers/s390/block/dasd_devmap.c +++ b/drivers/s390/block/dasd_devmap.c @@ -1630,6 +1630,44 @@ dasd_expires_store(struct device *dev, struct device_attribute *attr, static DEVICE_ATTR(expires, 0644, dasd_expires_show, dasd_expires_store); +/* ESE fulltrack write aggressiveness knob (0..100, see DASD_FT_BIAS_*) */ +static ssize_t +full_track_bias_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct dasd_device *device; + int len; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + len = sysfs_emit(buf, "%u\n", device->ft_bias); + dasd_put_device(device); + return len; +} + +static ssize_t full_track_bias_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct dasd_device *device; + unsigned int val; + + if (kstrtouint(buf, 0, &val) || val > DASD_FT_BIAS_MAX) + return -EINVAL; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + + device->ft_bias = val; + device->fulltrack = val ? 1 : 0; + + dasd_put_device(device); + return count; +} + +static DEVICE_ATTR_RW(full_track_bias); + static ssize_t dasd_retries_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2425,6 +2463,7 @@ static struct attribute * dasd_attrs[] = { &dev_attr_erplog.attr, &dev_attr_failfast.attr, &dev_attr_expires.attr, + &dev_attr_full_track_bias.attr, &dev_attr_retries.attr, &dev_attr_timeout.attr, &dev_attr_reservation_policy.attr, diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 7828f564a11b..c34c3afb55d0 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -47,6 +47,18 @@ MODULE_DESCRIPTION("S/390 DASD ECKD Disks device driver"); MODULE_LICENSE("GPL"); +/* + * Default full-track write bias applied to every ESE volume at online time; + * individual volumes can be re-tuned afterwards through their per-device + * full_track_bias sysfs attribute. 0 disables full-track writes, 100 always + * uses them, 50 (the default) enables the adaptive heuristic. Values above + * DASD_FT_BIAS_MAX are capped when applied. + */ +static unsigned int full_track_bias = DASD_FT_BIAS_DEFAULT; +module_param(full_track_bias, uint, 0644); +MODULE_PARM_DESC(full_track_bias, + "Default ESE full-track write bias 0..100 (0=off, 1..99=adaptive, 100=always)"); + static struct dasd_discipline dasd_eckd_discipline; /* The ccw bus type uses this table to find devices that it sends to @@ -2148,6 +2160,11 @@ dasd_eckd_check_characteristics(struct dasd_device *device) device->path_interval = DASD_ECKD_PATH_INTERVAL; device->aq_timeouts = DASD_RETRIES_MAX; + /* default ESE fulltrack write aggressiveness from the module parameter */ + device->ft_bias = min_t(unsigned int, full_track_bias, DASD_FT_BIAS_MAX); + /* only the "always" endpoint forces fulltrack unconditionally here */ + device->fulltrack = (device->ft_bias >= DASD_FT_BIAS_MAX) ? 1 : 0; + if (private->conf.gneq) { value = 1; for (i = 0; i < private->conf.gneq->timeout.value; i++) @@ -5201,11 +5218,20 @@ static struct dasd_ccw_req *dasd_eckd_build_cp(struct dasd_device *startdev, /* do nothing, just fall through to the cmd mode single case */ } else if ((data_size <= private->fcx_max_data) && (fcx_multitrack || (first_trk == last_trk))) { - cqr = dasd_eckd_build_cp_tpm_track(startdev, block, req, - first_rec, last_rec, - first_trk, last_trk, - first_offs, last_offs, - blk_per_trk, blksize); + if (!first_offs && (last_offs + 1 == blk_per_trk) && + rq_data_dir(req) == WRITE && basedev->fulltrack) { + cqr = dasd_eckd_build_cp_tpm_writefulltrack(startdev, block, req, + first_rec, last_rec, + first_trk, last_trk, + first_offs, last_offs, + blk_per_trk, blksize, NULL); + } else { + cqr = dasd_eckd_build_cp_tpm_track(startdev, block, req, + first_rec, last_rec, + first_trk, last_trk, + first_offs, last_offs, + blk_per_trk, blksize); + } if (IS_ERR(cqr) && (PTR_ERR(cqr) != -EAGAIN) && (PTR_ERR(cqr) != -ENOMEM)) cqr = NULL; diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index e1ffa20db10a..b8a3190c9c93 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -629,6 +629,10 @@ struct dasd_device { struct dasd_copy_relation *copy; unsigned long aq_mask; unsigned int aq_timeouts; + + /* ESE fulltrack write control (see full_track_bias sysfs attribute) */ + unsigned int ft_bias; /* aggressiveness 0..100: 0=off, 100=always */ + unsigned int fulltrack; /* internal: use WRITE_FULL_TRACK for aligned writes */ }; struct dasd_block { @@ -686,6 +690,20 @@ struct dasd_queue { #define DASD_STOPPED_PPRC 32 /* PPRC swap */ #define DASD_STOPPED_NOSPC 128 /* no space left */ +/* + * ESE fulltrack write aggressiveness (full_track_bias sysfs attribute), 0..100: + * 0 - never use proactively WRITE_FULL_TRACK + * 100 - always use proactively WRITE_FULL_TRACK, no probing + * 1..99 - adaptive; higher means switch to ft more eagerly + * WRITE_FULL_TRACK has an advantage on sparse formatted ESE devices + * but it has an overall penalty for maximum throughput for fully + * formatted devices. + * The default of 50 tries to balance both and do some probing in between + * to choose the best mode for default IO. + */ +#define DASD_FT_BIAS_MAX 100 +#define DASD_FT_BIAS_DEFAULT 50 + /* per device flags */ #define DASD_FLAG_OFFLINE 3 /* device is in offline processing */ #define DASD_FLAG_EER_SNSS 4 /* A SNSS is required */ From 4e304b2e56f09bce314ffeb4d4033ef7b8776c30 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:07 +0200 Subject: [PATCH 134/241] s390/dasd: Derive adaptive ESE fulltrack heuristic from ft_bias MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Turn the middle of the ft_bias range (1..99) into an adaptive heuristic that switches between fulltrack write (ft1) and plain write ft0 depending on how sparse the device still is. A sparse device benefits from fulltrack writes (it avoids the format/retry cycle); once enough tracks are formatted the per-write overhead of ft1 outweighs that. An state machine measures the NRF rate in short ft0 probe windows and flips back to ft1 when it is high (FT1_ACTIVE -> PROBING -> FT0_STABLE, with a backing-off reprobe interval). The four parameters are derived from ft_bias by linear interpolation, anchored so ft_bias == 50 derives the following values: ese_heu_start_interval - 2000 - IOs in ft1, before first ft0-Probe starts ese_heu_probe_window - 100 - IOs in probe window ese_heu_nrf_high - 10 ‰ (= 1 %) - TRACK_FORMAT rate that leads to ft1 ese_heu_max_interval - 500000 - Backoff-Cap: max. IOs between two probes Higher is more eager to use ft1, and 0/100 skips the heuristic. The NRF counter is bumped in dasd_eckd_ese_format() for both the classic NRF sense and the HPF INV_TRACK_FORMAT equivalent. The state machine resets to ft1 on check_characteristics, full format, and release-space. A read-only ese_heuristic_state sysfs attribute exposes the current mode. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-15-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_devmap.c | 47 +++++++++++++++- drivers/s390/block/dasd_eckd.c | 97 ++++++++++++++++++++++++++++++-- drivers/s390/block/dasd_int.h | 85 ++++++++++++++++++++++++++++ 3 files changed, 223 insertions(+), 6 deletions(-) diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c index 035c022255b6..f6aab94b7be6 100644 --- a/drivers/s390/block/dasd_devmap.c +++ b/drivers/s390/block/dasd_devmap.c @@ -1659,8 +1659,14 @@ static ssize_t full_track_bias_store(struct device *dev, if (IS_ERR(device)) return -ENODEV; + /* + * ft_bias is the tuning target; fulltrack is a best-effort mode hint + * that the per-IO heuristic also updates locklessly. A racing writer can + * at most leave a transient mismatch that self-corrects on the next IO, + * never corruption, so the update is left unlocked. + */ device->ft_bias = val; - device->fulltrack = val ? 1 : 0; + dasd_ft_bias_apply(device); dasd_put_device(device); return count; @@ -1668,6 +1674,44 @@ static ssize_t full_track_bias_store(struct device *dev, static DEVICE_ATTR_RW(full_track_bias); +static const char * const dasd_ese_heu_state_names[] = { + [DASD_ESE_HEU_FT1_ACTIVE] = "fulltrack active", + [DASD_ESE_HEU_PROBING] = "probing", + [DASD_ESE_HEU_FT0_STABLE] = "fulltrack inactive", +}; + +/* read-only: current full-track mode / adaptive FSM state, for observability */ +static ssize_t +ese_heuristic_state_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct dasd_device *device; + unsigned int state; + int len; + + device = dasd_device_from_cdev(to_ccwdev(dev)); + if (IS_ERR(device)) + return -ENODEV; + if (device->ft_bias == 0) { + len = sysfs_emit(buf, "fulltrack deactivated\n"); + } else if (device->ft_bias >= DASD_FT_BIAS_MAX) { + len = sysfs_emit(buf, "fulltrack permanent active\n"); + } else if (!dasd_ese_adaptive(device)) { + /* adaptive range but not ESE: the heuristic does not run */ + len = sysfs_emit(buf, "fulltrack deactivated\n"); + } else { + state = device->ese_probe_state; + if (state < ARRAY_SIZE(dasd_ese_heu_state_names)) + len = sysfs_emit(buf, "%s\n", dasd_ese_heu_state_names[state]); + else + len = sysfs_emit(buf, "unknown\n"); + } + dasd_put_device(device); + return len; +} + +static DEVICE_ATTR_RO(ese_heuristic_state); + static ssize_t dasd_retries_show(struct device *dev, struct device_attribute *attr, char *buf) { @@ -2464,6 +2508,7 @@ static struct attribute * dasd_attrs[] = { &dev_attr_failfast.attr, &dev_attr_expires.attr, &dev_attr_full_track_bias.attr, + &dev_attr_ese_heuristic_state.attr, &dev_attr_retries.attr, &dev_attr_timeout.attr, &dev_attr_reservation_policy.attr, diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index c34c3afb55d0..41cb44242204 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -2160,11 +2160,6 @@ dasd_eckd_check_characteristics(struct dasd_device *device) device->path_interval = DASD_ECKD_PATH_INTERVAL; device->aq_timeouts = DASD_RETRIES_MAX; - /* default ESE fulltrack write aggressiveness from the module parameter */ - device->ft_bias = min_t(unsigned int, full_track_bias, DASD_FT_BIAS_MAX); - /* only the "always" endpoint forces fulltrack unconditionally here */ - device->fulltrack = (device->ft_bias >= DASD_FT_BIAS_MAX) ? 1 : 0; - if (private->conf.gneq) { value = 1; for (i = 0; i < private->conf.gneq->timeout.value; i++) @@ -2219,6 +2214,13 @@ dasd_eckd_check_characteristics(struct dasd_device *device) /* Read Volume Information */ dasd_eckd_read_vol_info(device); + /* + * is_ese() now reflects the hardware ESE state, so derive the default + * fulltrack write bias from the module parameter. + */ + device->ft_bias = min_t(unsigned int, full_track_bias, DASD_FT_BIAS_MAX); + dasd_ft_bias_apply(device); + /* Read Extent Pool Information */ dasd_eckd_read_ext_pool_info(device); @@ -3171,6 +3173,13 @@ out: static int dasd_eckd_format_device(struct dasd_device *base, struct format_data_t *fdata, int enable_pav) { + /* + * A full format (start_unit == 0) returns the device to a fully sparse + * state, so restart the heuristic from ft1 without an offline cycle. + */ + if (fdata->start_unit == 0) + dasd_ft_bias_apply(base); + return dasd_eckd_format_process_data(base, fdata, enable_pav, 0, NULL, 0, NULL); } @@ -3230,6 +3239,69 @@ static void clear_format_track(struct dasd_format_entry *format, spin_unlock_irqrestore(&block->format_lock, flags); } +/* + * Adaptive ft_bias heuristic, called once per IO from dasd_eckd_build_cp(). + * Probes the device formatting state by briefly switching to ft0 and measuring + * the NRF rate; parameters are derived from ft_bias. + */ +static void dasd_ese_heuristic_tick(struct dasd_device *basedev) +{ + int ios, nrf, rate; + + if (atomic_inc_return(&basedev->ese_io_cnt) < (int)basedev->ese_probe_interval) + return; + + /* + * One wins the race to evaluate, the rest see ios == 0 after the + * xchg and return early, preventing redundant state transitions. + */ + ios = atomic_xchg(&basedev->ese_io_cnt, 0); + if (ios <= 0) + return; + + switch (basedev->ese_probe_state) { + case DASD_ESE_HEU_FT1_ACTIVE: + /* Start ft0 probe window, reset NRF counter for clean measurement */ + basedev->fulltrack = 0; + basedev->ese_probe_state = DASD_ESE_HEU_PROBING; + basedev->ese_probe_interval = basedev->ese_heu_probe_window; + atomic_set(&basedev->ese_nrf_window, 0); + break; + + case DASD_ESE_HEU_PROBING: + case DASD_ESE_HEU_FT0_STABLE: + nrf = atomic_xchg(&basedev->ese_nrf_window, 0); + rate = (int)((u64)nrf * 1000 / ios); + if (rate > (int)basedev->ese_heu_nrf_high) { + /* NRF rate high: device still sparse, ft1 is better */ + basedev->fulltrack = 1; + basedev->ese_probe_state = DASD_ESE_HEU_FT1_ACTIVE; + basedev->ese_probe_interval = basedev->ese_heu_start_interval; + } else if (basedev->ese_probe_state == DASD_ESE_HEU_PROBING) { + /* + * NRF rate low: device mostly formatted, ft0 is faster. + * Re-probe frequently at first, then back off below. + */ + basedev->fulltrack = 0; + basedev->ese_probe_state = DASD_ESE_HEU_FT0_STABLE; + basedev->ese_probe_interval = basedev->ese_heu_probe_window; + } else { + /* + * Still stable in ft0: re-assert plain-write mode so a + * fulltrack value left behind by a racing sysfs write + * self-corrects, and back off the re-probe interval + * (double it, capped at max_interval) so a long-lived + * formatted device is not probed more often than needed. + */ + basedev->fulltrack = 0; + basedev->ese_probe_interval = + min(basedev->ese_probe_interval * 2, + basedev->ese_heu_max_interval); + } + break; + } +} + static void dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_req *cqr, struct irb *irb) { @@ -3254,6 +3326,8 @@ static void dasd_eckd_ese_format(struct dasd_device *startdev, struct dasd_ccw_r block = cqr->block; base = block->base; private = base->private; + if (dasd_ese_adaptive(base)) + atomic_inc(&base->ese_nrf_window); blksize = block->bp_block; recs_per_trk = recs_per_track(&private->rdc_data, 0, blksize); @@ -4017,6 +4091,14 @@ static int dasd_eckd_release_space_full(struct dasd_device *device) rc = dasd_sleep_on_interruptible(cqr); + if (!rc) { + /* + * Releasing all space (RAS) wipes every track and the device is fully + * sparse again, so restart the heuristic from ft1. + */ + dasd_ft_bias_apply(device); + } + dasd_sfree_request(cqr, cqr->memdev); return rc; @@ -5185,6 +5267,11 @@ static struct dasd_ccw_req *dasd_eckd_build_cp(struct dasd_device *startdev, struct dasd_ccw_req *cqr; basedev = block->base; + if (dasd_ese_adaptive(basedev)) + dasd_ese_heuristic_tick(basedev); + else + /* re-assert the endpoint mode: a stale heuristic write cannot stick */ + basedev->fulltrack = (basedev->ft_bias >= DASD_FT_BIAS_MAX) ? 1 : 0; private = basedev->private; /* Calculate number of blocks/records per track. */ diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index b8a3190c9c93..59cce4e7dbc1 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -633,6 +633,15 @@ struct dasd_device { /* ESE fulltrack write control (see full_track_bias sysfs attribute) */ unsigned int ft_bias; /* aggressiveness 0..100: 0=off, 100=always */ unsigned int fulltrack; /* internal: use WRITE_FULL_TRACK for aligned writes */ + /* adaptive heuristic (active for ft_bias 1..99), derived from ft_bias */ + unsigned int ese_probe_state; /* heuristic FSM state */ + unsigned int ese_probe_interval; /* IOs between evaluations */ + atomic_t ese_io_cnt; /* IO counter for current window */ + atomic_t ese_nrf_window; /* NRF/INV_TRACK_FORMAT events in window */ + unsigned int ese_heu_start_interval; /* IOs before first probe */ + unsigned int ese_heu_probe_window; /* IOs in probe window */ + unsigned int ese_heu_max_interval; /* max IOs between probes (backoff cap) */ + unsigned int ese_heu_nrf_high; /* NRF per-mille threshold → activate ft1 */ }; struct dasd_block { @@ -704,6 +713,25 @@ struct dasd_queue { #define DASD_FT_BIAS_MAX 100 #define DASD_FT_BIAS_DEFAULT 50 +/* ESE fulltrack heuristic FSM states (adaptive range, ft_bias 1..99) */ +#define DASD_ESE_HEU_FT1_ACTIVE 0 /* fulltrack write active */ +#define DASD_ESE_HEU_PROBING 1 /* ft0 probe window, measuring NRF rate */ +#define DASD_ESE_HEU_FT0_STABLE 2 /* device formatted, ft0 active */ + +/* + * Heuristic parameters are derived from ft_bias by linear interpolation, + * anchored so that ft_bias == 50 reproduces the previously shipped defaults + * and ft_bias == 100 is the most aggressive end of the range. + * probe_window is constant. + */ +#define DASD_ESE_HEU_PROBE_WINDOW 100 +#define DASD_ESE_HEU_NRF_HIGH_A50 10 /* NRF per-mille threshold */ +#define DASD_ESE_HEU_NRF_HIGH_A100 1 +#define DASD_ESE_HEU_START_A50 2000 /* IOs before first probe */ +#define DASD_ESE_HEU_START_A100 500 +#define DASD_ESE_HEU_MAX_A50 500000 /* backoff cap */ +#define DASD_ESE_HEU_MAX_A100 20000 + /* per device flags */ #define DASD_FLAG_OFFLINE 3 /* device is in offline processing */ #define DASD_FLAG_EER_SNSS 4 /* A SNSS is required */ @@ -866,6 +894,63 @@ static inline bool dasd_req_conflict(struct dasd_ccw_req *cqr1, cqr2->end_trk < cqr1->format->start_trk); } +/* + * true when device is ese device and ft_bias selects the adaptive + * heuristic (neither hard endpoint) + */ +static inline bool dasd_ese_adaptive(struct dasd_device *device) +{ + return device->discipline && + device->discipline->is_ese && + device->discipline->is_ese(device) && + device->ft_bias > 0 && + device->ft_bias < DASD_FT_BIAS_MAX; +} + +/* + * Linear interpolation of a heuristic parameter between its value at aggr==50 + * (v50) and its value at aggr==100 (v100). + */ +static inline unsigned int dasd_ese_lerp(unsigned int v50, unsigned int v100, + unsigned int aggr) +{ + return (unsigned int)((int)v50 + + ((int)v100 - (int)v50) * ((int)aggr - 50) / 50); +} + +/* + * Apply the ft_bias knob. For the hard endpoints just pin the mode; for the + * adaptive range derive the heuristic parameters from ft_bias and (re)start + * the FSM in ft1 so a freshly sparse device avoids the NRF penalty right away. + */ +static inline void dasd_ft_bias_apply(struct dasd_device *device) +{ + unsigned int a = device->ft_bias; + + if (!dasd_ese_adaptive(device)) { + device->fulltrack = (a >= DASD_FT_BIAS_MAX) ? 1 : 0; + device->ese_probe_state = DASD_ESE_HEU_FT1_ACTIVE; + return; + } + + device->ese_heu_nrf_high = + dasd_ese_lerp(DASD_ESE_HEU_NRF_HIGH_A50, + DASD_ESE_HEU_NRF_HIGH_A100, a); + device->ese_heu_start_interval = + dasd_ese_lerp(DASD_ESE_HEU_START_A50, + DASD_ESE_HEU_START_A100, a); + device->ese_heu_max_interval = + dasd_ese_lerp(DASD_ESE_HEU_MAX_A50, + DASD_ESE_HEU_MAX_A100, a); + device->ese_heu_probe_window = DASD_ESE_HEU_PROBE_WINDOW; + + device->ese_probe_state = DASD_ESE_HEU_FT1_ACTIVE; + device->ese_probe_interval = device->ese_heu_start_interval; + device->fulltrack = 1; + atomic_set(&device->ese_io_cnt, 0); + atomic_set(&device->ese_nrf_window, 0); +} + /* externals in dasd.c */ #define DASD_PROFILE_OFF 0 #define DASD_PROFILE_ON 1 From 7d206efdc2f29240cfd815ffebfa958e2295a63e Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:08 +0200 Subject: [PATCH 135/241] s390/dasd: Stamp a format label into newly formatted volumes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a CDL volume is formatted, write a small on-disk label so the format can later be recognised by the kernel. The next patch will use this for ESE detection. The label records a magic, a version, whether the volume is ESE, and whether it was formatted quick (space released, thin) or full. It lives in track 0, head 0, record 4 (the first non-special CDL record). R4 is written by the same channel program that formats track 0 - its WRITE_CKD transfers count + the label data instead of count-only - so label and track format reach the disk atomically; a valid magic then marks a completed format without a separate, racy write. Quick vs full is derived from a full space release (RAS) preceding the format: dasd_eckd_release_space_full() sets a per-device flag the next format consumes. Non-ESE volumes and formats without a preceding full release are recorded as full. struct dasd_format_label is exactly 512 bytes (the smallest block size) so it fits one record; larger blocks zero-pad the rest. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-16-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 81 +++++++++++++++++++++++++++++++--- drivers/s390/block/dasd_eckd.h | 38 ++++++++++++++++ 2 files changed, 114 insertions(+), 5 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 41cb44242204..33f843a21fa0 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -2722,6 +2723,28 @@ dasd_eckd_build_check(struct dasd_device *base, struct format_data_t *fdata, return cqr; } +/* Fill the format label into a R4 record buffer, zero-padded to blksize. */ +static void dasd_eckd_fill_format_label(struct dasd_device *device, void *data, + unsigned int blksize) +{ + struct dasd_eckd_private *private = device->private; + struct dasd_format_label *label = data; + + memset(label, 0, blksize); + label->magic = DASD_ESE_LABEL_MAGIC; + label->version = DASD_ESE_LABEL_VERSION; + if (dasd_eckd_is_ese(device)) + label->flags |= DASD_ESE_LABEL_F_ESE; + if (private->ese_format_quick) + label->flags |= DASD_ESE_LABEL_F_QUICK; + else + label->flags |= DASD_ESE_LABEL_F_FULL; + label->blksize = blksize; + label->format_tod = get_tod_clock(); + strscpy(label->kernel_version, init_utsname()->release, + sizeof(label->kernel_version)); +} + static struct dasd_ccw_req * dasd_eckd_build_format(struct dasd_device *base, struct dasd_device *startdev, struct format_data_t *fdata, int enable_pav) @@ -2740,6 +2763,7 @@ dasd_eckd_build_format(struct dasd_device *base, struct dasd_device *startdev, int r0_perm; int nr_tracks; int use_prefix; + int write_label; if (enable_pav) startdev = dasd_alias_get_start_dev(base); @@ -2773,6 +2797,15 @@ dasd_eckd_build_format(struct dasd_device *base, struct dasd_device *startdev, use_prefix = base_priv->features.feature[8] & 0x01; + /* + * Stamp the format label into R4 of the very first track. Only for CDL + * (R4 is the first non-special record there), only when this request + * covers track 0, only for the record-writing format intensities (not + * track invalidation), and only if the track actually has an R4. + */ + write_label = (intensity & 0x08) && !((intensity & ~0x08) & 0x04) && + fdata->start_unit == 0 && rpt > 3; + switch (intensity) { case 0x00: /* Normal format */ case 0x08: /* Normal format, use cdl. */ @@ -2819,6 +2852,10 @@ dasd_eckd_build_format(struct dasd_device *base, struct dasd_device *startdev, return ERR_PTR(-EINVAL); } + /* room for the label data that R4 carries in addition to its count */ + if (write_label) + datasize += fdata->blksize; + fcp = dasd_fmalloc_request(DASD_ECKD_MAGIC, cplength, datasize, startdev); if (IS_ERR(fcp)) return fcp; @@ -2963,7 +3000,21 @@ dasd_eckd_build_format(struct dasd_device *base, struct dasd_device *startdev, ccw->cmd_code = DASD_ECKD_CCW_WRITE_CKD_MT; ccw->flags = CCW_FLAG_SLI; - ccw->count = 8; + if (write_label && address.cyl == 0 && + address.head == 0 && i == 3) { + /* + * R4 carries the label as its record + * data; it follows ect contiguously so + * the CCW transfers count + data. + */ + dasd_eckd_fill_format_label(base, + data, + fdata->blksize); + data += fdata->blksize; + ccw->count = 8 + fdata->blksize; + } else { + ccw->count = 8; + } ccw->cda = virt_to_dma32(ect); ccw++; } @@ -3173,6 +3224,9 @@ out: static int dasd_eckd_format_device(struct dasd_device *base, struct format_data_t *fdata, int enable_pav) { + struct dasd_eckd_private *private = base->private; + int rc; + /* * A full format (start_unit == 0) returns the device to a fully sparse * state, so restart the heuristic from ft1 without an offline cycle. @@ -3180,8 +3234,18 @@ static int dasd_eckd_format_device(struct dasd_device *base, if (fdata->start_unit == 0) dasd_ft_bias_apply(base); - return dasd_eckd_format_process_data(base, fdata, enable_pav, 0, NULL, - 0, NULL); + rc = dasd_eckd_format_process_data(base, fdata, enable_pav, 0, NULL, + 0, NULL); + + /* + * The quick-format indicator was consumed by the label stamped into + * track 0; clear it so a later format that is not preceded by a full + * space release is recorded as a full format. + */ + if (fdata->start_unit == 0) + private->ese_format_quick = 0; + + return rc; } static bool test_and_set_format_track(sector_t start, sector_t end, @@ -4082,6 +4146,7 @@ dasd_eckd_dso_ras(struct dasd_device *device, struct dasd_block *block, static int dasd_eckd_release_space_full(struct dasd_device *device) { + struct dasd_eckd_private *private; struct dasd_ccw_req *cqr; int rc; @@ -4093,10 +4158,16 @@ static int dasd_eckd_release_space_full(struct dasd_device *device) if (!rc) { /* - * Releasing all space (RAS) wipes every track and the device is fully - * sparse again, so restart the heuristic from ft1. + * Releasing all space (RAS) wipes every track and the device is + * fully sparse again, so restart the heuristic from ft1. */ dasd_ft_bias_apply(device); + /* + * A full release is what makes a subsequent format a quick + * (thin) one; remember it so the format label records that. + */ + private = device->private; + private->ese_format_quick = 1; } dasd_sfree_request(cqr, cqr->memdev); diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index 0fdb92fdddc8..92fd8ac92b79 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -159,6 +159,39 @@ struct eckd_r0 { #define DASD_EAV_CYL_HI_SHIFT 16 /* cylinder bits beyond the 16-bit cyl field */ #define DASD_EAV_HEAD_HI_SHIFT 4 /* head occupies the low-order 4 bits of head */ +/* + * On-disk DASD format label. + * + * Written into track 0, head 0, record 4 (R4 - the first non-special CDL + * record) as part of the same channel program that formats track 0, so it is + * stored atomically with the track: either both the track format and the label + * make it to disk or neither does. Its presence with a valid magic therefore + * marks a completed format and can be used for format detection. + * + * The structure is exactly the smallest supported block size (512 bytes) so it + * always fits into a single record. + * For larger block sizes the rest of the record is zero padded. + * The magic together with the version is used to recognise a valid label. + */ +#define DASD_ESE_LABEL_MAGIC 0xC4C1E2C4C6D4E3F1ULL /* EBCDIC "DASDFMT1" */ +#define DASD_ESE_LABEL_VERSION 1 + +/* dasd_format_label.flags */ +#define DASD_ESE_LABEL_F_ESE 0x00000001 /* volume is extent space efficient */ +#define DASD_ESE_LABEL_F_QUICK 0x00000002 /* quick (space released) format */ +#define DASD_ESE_LABEL_F_FULL 0x00000004 /* full format */ + +struct dasd_format_label { + __u64 magic; /* DASD_ESE_LABEL_MAGIC */ + __u32 version; /* DASD_ESE_LABEL_VERSION */ + __u32 flags; /* DASD_ESE_LABEL_F_* */ + __u32 blksize; /* block size the volume was formatted with */ + __u32 reserved0; + __u64 format_tod; /* TOD clock at format time */ + __u8 kernel_version[64]; /* NUL terminated kernel release (uname -r) */ + __u8 reserved[416]; /* pad the struct to 512 bytes */ +} __packed; + struct ch_t { __u16 cyl; __u16 head; @@ -709,6 +742,11 @@ struct dasd_eckd_private { u32 fcx_max_data; char suc_reason; + /* + * Set when the whole volume's space was released (full RAS); consumed by + * the next format to mark the on-disk label as a quick (vs full) format. + */ + int ese_format_quick; }; From 268e40548da2758851b6962a0a4ed2ab241ddbe9 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:09 +0200 Subject: [PATCH 136/241] s390/dasd: Detect ESE volumes from the on-disk format label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read the format label from track 0 record 4 at device bring-up and cache it. When a valid label is present, is_ese() is derived from it instead of the hardware volume field. A volume copied off ESE storage onto other hardware is thus still handled as thin. Without a label (older format) is_ese() falls back to the hardware field as before. The cache is refreshed after a format so is_ese() stays coherent without an offline/online cycle. The label F_ESE bit is stamped from the hardware capability rather than is_ese(), and space release (quick format) is gated on the hardware capability, so a copied label cannot enable it on non-ESE hardware. The ese sysfs attribute, and with this lsdasd, shows the hardware capability and not the internal handling. This is in line with the view from storage server interface. To reflect the specific internal handling an additional attribute on_demand_formatting is added to show that a device is handled like an ESE device internally based on the disk label. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-17-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_devmap.c | 6 +- drivers/s390/block/dasd_eckd.c | 174 ++++++++++++++++++++++++++++--- drivers/s390/block/dasd_eckd.h | 7 ++ drivers/s390/block/dasd_int.h | 3 + 4 files changed, 175 insertions(+), 15 deletions(-) diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c index f6aab94b7be6..d07d384a004f 100644 --- a/drivers/s390/block/dasd_devmap.c +++ b/drivers/s390/block/dasd_devmap.c @@ -2482,9 +2482,10 @@ static ssize_t dasd_##_name##_show(struct device *dev, \ \ return sysfs_emit(buf, "%d\n", val); \ } \ -static DEVICE_ATTR(_name, 0444, dasd_##_name##_show, NULL); \ +static DEVICE_ATTR(_name, 0444, dasd_##_name##_show, NULL); -DASD_DEFINE_ATTR(ese, device->discipline->is_ese); +DASD_DEFINE_ATTR(ese, device->discipline->ese_capable); +DASD_DEFINE_ATTR(on_demand_formatting, device->discipline->on_demand_format); DASD_DEFINE_ATTR(extent_size, device->discipline->ext_size); DASD_DEFINE_ATTR(pool_id, device->discipline->ext_pool_id); DASD_DEFINE_ATTR(space_configured, device->discipline->space_configured); @@ -2522,6 +2523,7 @@ static struct attribute * dasd_attrs[] = { &dev_attr_path_reset.attr, &dev_attr_hpf.attr, &dev_attr_ese.attr, + &dev_attr_on_demand_formatting.attr, &dev_attr_fc_security.attr, &dev_attr_copy_pair.attr, &dev_attr_copy_role.attr, diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 33f843a21fa0..d3e107bf63c4 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1676,7 +1676,8 @@ static int dasd_eckd_read_vol_info(struct dasd_device *device) return rc; } -static int dasd_eckd_is_ese(struct dasd_device *device) +/* Hardware/volume ESE capability, from the Volume Storage Query. */ +static int dasd_eckd_ese_capable(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; @@ -1686,6 +1687,53 @@ static int dasd_eckd_is_ese(struct dasd_device *device) return private->vsq.vol_info.ese; } +/* + * Whether the volume is to be handled as ESE (thin). This reflects the state + * of the data, not the hardware: a volume copied off ESE storage onto other + * hardware still needs ESE handling. The on-disk format label is authoritative + * when present; without it (e.g. a volume formatted by an older driver) fall + * back to the hardware ESE field. + * + * Only the F_ESE flag gates this. An ESE volume is thin regardless of whether + * it was quick- or full-formatted (tracks are allocated on write, and discard + * re-thins a full one). + */ +static int dasd_eckd_is_ese(struct dasd_device *device) +{ + struct dasd_eckd_private *private = device->private; + + /* sysfs may read this during set_online before private is allocated */ + if (!private) + return 0; + + if (private->ese_label_valid) + return !!(private->ese_label.flags & DASD_ESE_LABEL_F_ESE); + + return dasd_eckd_ese_capable(device); +} + +/* + * Whether the volume is formatted on demand (thin), as opposed to fully + * formatted. This is the format mode, not the hardware ESE capability. When a + * label is present it is authoritative (F_QUICK). Without a label the mode is + * unknown, but an ESE volume is still handled on demand (NRF triggers the + * format), so fall back to the ESE state to stay consistent with the driver's + * behavior on older, label-less volumes. + */ +static int dasd_eckd_on_demand_format(struct dasd_device *device) +{ + struct dasd_eckd_private *private = device->private; + + /* sysfs may read this during set_online before private is allocated */ + if (!private) + return 0; + + if (private->ese_label_valid) + return !!(private->ese_label.flags & DASD_ESE_LABEL_F_QUICK); + + return dasd_eckd_is_ese(device); +} + static int dasd_eckd_ext_pool_id(struct dasd_device *device) { struct dasd_eckd_private *private = device->private; @@ -2105,6 +2153,69 @@ static bool dasd_eckd_pprc_enabled(struct dasd_device *device) return private->rdc_data.facilities.PPRC_enabled; } +/* + * Read the on-disk format label from track 0, record 4. On a formatted volume + * R4 holds the label as its record data; on an unformatted (fresh ESE) or + * label-less volume the read returns No Record Found, which is expected and + * leaves the cache invalid so is_ese() falls back to the hardware field. + */ +static void dasd_eckd_read_format_label(struct dasd_device *device) +{ + struct dasd_eckd_private *private = device->private; + struct dasd_format_label *label; + struct DE_eckd_data *dedata; + struct LO_eckd_data *lodata; + struct dasd_ccw_req *cqr; + struct ccw1 *ccw; + + private->ese_label_valid = false; + + /* The label lives on the base volume; aliases have none of their own. */ + if (private->uid.type == UA_BASE_PAV_ALIAS || + private->uid.type == UA_HYPER_PAV_ALIAS) + return; + + cqr = dasd_smalloc_request(DASD_ECKD_MAGIC, 3 /* DE + LO + READ */, + sizeof(*dedata) + sizeof(*lodata) + + sizeof(*label), device, NULL); + if (IS_ERR(cqr)) + return; + + dedata = cqr->data; + lodata = (struct LO_eckd_data *)(dedata + 1); + label = (struct dasd_format_label *)(lodata + 1); + + ccw = cqr->cpaddr; + define_extent(ccw++, dedata, 0, 0, DASD_ECKD_CCW_READ, device, 0); + ccw[-1].flags |= CCW_FLAG_CC; + locate_record(ccw++, lodata, 0, 4, 1, DASD_ECKD_CCW_READ, device, + sizeof(*label)); + ccw[-1].flags |= CCW_FLAG_CC; + ccw->cmd_code = DASD_ECKD_CCW_READ; + ccw->count = sizeof(*label); + ccw->flags = CCW_FLAG_SLI; + ccw->cda = virt_to_dma32(label); + + cqr->startdev = device; + cqr->memdev = device; + cqr->block = NULL; + cqr->retries = 256; + cqr->expires = 10 * HZ; + cqr->buildclk = get_tod_clock(); + cqr->status = DASD_CQR_FILLED; + /* R4 may be absent (unformatted) or larger than the label. */ + set_bit(DASD_CQR_SUPPRESS_NRF, &cqr->flags); + set_bit(DASD_CQR_SUPPRESS_IL, &cqr->flags); + + if (!dasd_sleep_on(cqr) && + label->magic == DASD_ESE_LABEL_MAGIC && + label->version == DASD_ESE_LABEL_VERSION) { + private->ese_label = *label; + private->ese_label_valid = true; + } + dasd_sfree_request(cqr, device); +} + /* * Check device characteristics. * If the device is accessible using ECKD discipline, the device is enabled. @@ -2215,9 +2326,12 @@ dasd_eckd_check_characteristics(struct dasd_device *device) /* Read Volume Information */ dasd_eckd_read_vol_info(device); + /* Read the on-disk format label for ESE detection */ + dasd_eckd_read_format_label(device); + /* - * is_ese() now reflects the hardware ESE state, so derive the default - * fulltrack write bias from the module parameter. + * is_ese() now reflects the real ESE state (vsq + on-disk label), so + * the adaptive heuristic can be derived correctly for this device. */ device->ft_bias = min_t(unsigned int, full_track_bias, DASD_FT_BIAS_MAX); dasd_ft_bias_apply(device); @@ -2733,7 +2847,12 @@ static void dasd_eckd_fill_format_label(struct dasd_device *device, void *data, memset(label, 0, blksize); label->magic = DASD_ESE_LABEL_MAGIC; label->version = DASD_ESE_LABEL_VERSION; - if (dasd_eckd_is_ese(device)) + /* + * F_ESE records the hardware capability at format time, not is_ese(): + * is_ese() is derived from the label, so using it here would let the + * flag flip on repeated quick/full reformats. + */ + if (dasd_eckd_ese_capable(device)) label->flags |= DASD_ESE_LABEL_F_ESE; if (private->ese_format_quick) label->flags |= DASD_ESE_LABEL_F_QUICK; @@ -2743,6 +2862,13 @@ static void dasd_eckd_fill_format_label(struct dasd_device *device, void *data, label->format_tod = get_tod_clock(); strscpy(label->kernel_version, init_utsname()->release, sizeof(label->kernel_version)); + + /* + * Populate the cache directly from the bytes just computed instead of + * synchronously reading them back from disk after the write lands. + */ + private->ese_label = *label; + private->ese_label_valid = true; } static struct dasd_ccw_req * @@ -3227,23 +3353,35 @@ static int dasd_eckd_format_device(struct dasd_device *base, struct dasd_eckd_private *private = base->private; int rc; - /* - * A full format (start_unit == 0) returns the device to a fully sparse - * state, so restart the heuristic from ft1 without an offline cycle. - */ - if (fdata->start_unit == 0) - dasd_ft_bias_apply(base); - rc = dasd_eckd_format_process_data(base, fdata, enable_pav, 0, NULL, 0, NULL); + if (fdata->start_unit != 0) + return rc; + + if (rc) { + /* + * The format failed, so the label cached speculatively during + * CCW build may not match the disk; drop it so is_ese() falls + * back to the hardware field until the next successful format + * or bring-up. + */ + private->ese_label_valid = false; + return rc; + } /* * The quick-format indicator was consumed by the label stamped into * track 0; clear it so a later format that is not preceded by a full * space release is recorded as a full format. */ - if (fdata->start_unit == 0) - private->ese_format_quick = 0; + private->ese_format_quick = 0; + + /* + * A full format returns the device to a fully sparse state and has just + * committed a fresh label; restart the heuristic from ft1 on the now + * current is_ese state, without an offline cycle. + */ + dasd_ft_bias_apply(base); return rc; } @@ -4246,6 +4384,14 @@ out: static int dasd_eckd_release_space(struct dasd_device *device, struct format_data_t *rdata) { + /* + * Space release (and thus a quick format) requires real ESE hardware. + * is_ese() may be true from a copied label on non-ESE hardware, so gate + * on the hardware capability, not on is_ese(). + */ + if (!dasd_eckd_ese_capable(device)) + return -EOPNOTSUPP; + if (rdata->intensity & DASD_FMT_INT_ESE_FULL) return dasd_eckd_release_space_full(device); else if (rdata->intensity == 0) @@ -7619,6 +7765,8 @@ static struct dasd_discipline dasd_eckd_discipline = { .hpf_enabled = dasd_eckd_hpf_enabled, .reset_path = dasd_eckd_reset_path, .is_ese = dasd_eckd_is_ese, + .ese_capable = dasd_eckd_ese_capable, + .on_demand_format = dasd_eckd_on_demand_format, .space_allocated = dasd_eckd_space_allocated, .space_configured = dasd_eckd_space_configured, .logical_capacity = dasd_eckd_logical_capacity, diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index 92fd8ac92b79..30745f62402b 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -747,6 +747,13 @@ struct dasd_eckd_private { * the next format to mark the on-disk label as a quick (vs full) format. */ int ese_format_quick; + /* + * Cached on-disk format label (R4), read at online and refreshed on + * format. When valid, is_ese() is derived from it; otherwise it falls + * back to the hardware ESE field (vsq.vol_info.ese). + */ + struct dasd_format_label ese_label; + bool ese_label_valid; }; diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index 59cce4e7dbc1..8c73850f7947 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -401,6 +401,9 @@ struct dasd_discipline { * Extent Space Efficient (ESE) relevant functions */ int (*is_ese)(struct dasd_device *); + int (*ese_capable)(struct dasd_device *); + /* Whether the volume is formatted on demand (thin), from the label */ + int (*on_demand_format)(struct dasd_device *); /* Capacity */ int (*space_allocated)(struct dasd_device *); int (*space_configured)(struct dasd_device *); From 6c1be943fdb6f8557596f5164ea48ae65cd0c1df Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:10 +0200 Subject: [PATCH 137/241] s390/dasd: Report ESE capability and format mode at device online MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the device information line logged when a volume comes online with the ESE hardware capability and the on-disk format mode. The format mode (full or on demand) is derived from the on-disk format label alone, so a volume that is not backed by ESE hardware but was still formatted on demand is reported correctly. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-18-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index d3e107bf63c4..c2c60530fe0d 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -2522,6 +2522,7 @@ static int dasd_eckd_end_analysis(struct dasd_block *block) struct dasd_device *device = block->base; struct dasd_eckd_private *private = device->private; struct eckd_count *count_area; + const char *ese_str, *fmt_str; unsigned int sb, blk_per_trk; int status, i; struct dasd_ccw_req *init_cqr; @@ -2608,15 +2609,29 @@ raw: private->rdc_data.trk_per_cyl * blk_per_trk); + /* + * Report the ESE hardware capability and the format mode. The mode + * comes from dasd_eckd_on_demand_format() (the on-disk label, or the + * ESE state when no label is present), matching the on_demand_formatting + * sysfs attribute. + */ + ese_str = dasd_eckd_ese_capable(device) ? ", ESE" : ""; + fmt_str = ""; + if (dasd_eckd_on_demand_format(device)) + fmt_str = ", on-demand format"; + else if (dasd_eckd_ese_capable(device)) + fmt_str = ", full format"; + dev_info(&device->cdev->dev, - "DASD with %u KB/block, %lu KB total size, %u KB/track, " - "%s\n", (block->bp_block >> 10), + "DASD with %u KB/block, %lu KB total size, %u KB/track, %s%s%s\n", + (block->bp_block >> 10), (((unsigned long) private->real_cyl * private->rdc_data.trk_per_cyl * blk_per_trk * (block->bp_block >> 9)) >> 1), ((blk_per_trk * block->bp_block) >> 10), private->uses_cdl ? - "compatible disk layout" : "linux disk layout"); + "compatible disk layout" : "linux disk layout", + ese_str, fmt_str); return 0; } From 04ea1579bc7707366de1d642115ad3b65c6171e1 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:11 +0200 Subject: [PATCH 138/241] s390/dasd: Re-enable discard support for ESE volumes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-enable block-layer discard for ESE ECKD volumes, releasing thin space via release allocated space (RAS). This is based on commit 7e64db1597fe ("s390/dasd: Add discard support for ESE volumes") but adapted to the current code and fixed. REQ_OP_DISCARD is routed to a RAS release over the request's track range, and discard requests run on the base device only. Discard limits use extent granularity via the disc_limits discipline hook so the block layer only issues extent-aligned discards. Discard is gated on the DASD_FEATURE_DISCARD device feature rather than a per-discipline flag: the driver sets the feature when the volume is on ESE hardware (i.e. RAS is available), and the block-layer setup enables discard limits for a device that has it. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-19-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 34 +++++-- drivers/s390/block/dasd_eckd.c | 173 ++++++++++++++++++++++++++------- drivers/s390/block/dasd_int.h | 2 + 3 files changed, 166 insertions(+), 43 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index 0e01b498b0e8..da5e6813d391 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -353,17 +353,19 @@ static int dasd_state_basic_to_ready(struct dasd_device *device) */ lim.dma_alignment = lim.logical_block_size - 1; - if (device->discipline->has_discard) { + if (device->features & DASD_FEATURE_DISCARD) { unsigned int max_bytes; - lim.discard_granularity = block->bp_block; - - /* Calculate max_discard_sectors and make it PAGE aligned */ - max_bytes = USHRT_MAX * block->bp_block; - max_bytes = ALIGN_DOWN(max_bytes, PAGE_SIZE); - - lim.max_hw_discard_sectors = max_bytes / block->bp_block; - lim.max_write_zeroes_sectors = lim.max_hw_discard_sectors; + if (device->discipline->disc_limits) { + device->discipline->disc_limits(block, &lim); + } else { + lim.discard_granularity = block->bp_block; + /* Calculate max_discard_sectors and make it PAGE aligned */ + max_bytes = USHRT_MAX * block->bp_block; + max_bytes = ALIGN_DOWN(max_bytes, PAGE_SIZE); + lim.max_hw_discard_sectors = max_bytes / block->bp_block; + lim.max_write_zeroes_sectors = lim.max_hw_discard_sectors; + } } rc = queue_limits_commit_update(block->gdp->queue, &lim); if (rc) @@ -3124,6 +3126,7 @@ static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx, struct dasd_device *basedev; struct dasd_ccw_req *cqr; blk_status_t rc = BLK_STS_OK; + bool complete_noop = false; basedev = block->base; spin_lock_irq(&dq->lock); @@ -3172,6 +3175,17 @@ static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx, rc = BLK_STS_RESOURCE; } else if (PTR_ERR(cqr) == -EINVAL) { rc = BLK_STS_INVAL; + } else if (PTR_ERR(cqr) == -EOPNOTSUPP) { + /* + * A discard that covers no whole extent releases + * nothing. Discard is advisory, so complete it as a + * benign no-op: the device does support discard, this + * range just does not align to the large ESE extent + * granularity. + * Completed after the lock is dropped. + */ + rc = BLK_STS_OK; + complete_noop = true; } else { DBF_DEV_EVENT(DBF_ERR, basedev, "CCW creation failed (rc=%ld) on request %p", @@ -3205,6 +3219,8 @@ static blk_status_t do_dasd_request(struct blk_mq_hw_ctx *hctx, out: spin_unlock_irq(&dq->lock); + if (complete_noop) + blk_mq_end_request(req, BLK_STS_OK); return rc; } diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index c2c60530fe0d..6eb2879b479b 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -2326,6 +2326,18 @@ dasd_eckd_check_characteristics(struct dasd_device *device) /* Read Volume Information */ dasd_eckd_read_vol_info(device); + /* + * Advertise discard through the device feature so the block layer sets + * up discard limits. Discard releases allocated space, so require a thin + * (ESE) volume whose storage reports support for the space-release + * function. Raw-track access bypasses the normal block CCW path (discard + * would reach the raw builder, which has no record data), so exclude it. + */ + if (dasd_eckd_ese_capable(device) && + (private->features.feature[56] & 0x01) && + !(device->features & DASD_FEATURE_USERAW)) + device->features |= DASD_FEATURE_DISCARD; + /* Read the on-disk format label for ESE detection */ dasd_eckd_read_format_label(device); @@ -4133,37 +4145,13 @@ static int dasd_eckd_ras_sanity_checks(struct dasd_device *device, } /* - * Helper function to count the amount of involved extents within a given range - * with extent alignment in mind. + * Number of extents the track range [from, to] spans. Extent n covers tracks + * [n * trks_per_ext, (n + 1) * trks_per_ext - 1], so the range touches the + * extents from (from / trks_per_ext) to (to / trks_per_ext) inclusive. */ static int count_exts(unsigned int from, unsigned int to, int trks_per_ext) { - int cur_pos = 0; - int count = 0; - int tmp; - - if (from == to) - return 1; - - /* Count first partial extent */ - if (from % trks_per_ext != 0) { - tmp = from + trks_per_ext - (from % trks_per_ext) - 1; - if (tmp > to) - tmp = to; - cur_pos = tmp - from + 1; - count++; - } - /* Count full extents */ - if (to - (from + cur_pos) + 1 >= trks_per_ext) { - tmp = to - ((to - trks_per_ext + 1) % trks_per_ext); - count += (tmp - (from + cur_pos) + 1) / trks_per_ext; - cur_pos = tmp; - } - /* Count last partial extent */ - if (cur_pos < to) - count++; - - return count; + return to / trks_per_ext - from / trks_per_ext + 1; } static int dasd_in_copy_relation(struct dasd_device *device) @@ -4214,9 +4202,17 @@ dasd_eckd_dso_ras(struct dasd_device *device, struct dasd_block *block, if (dasd_eckd_ras_sanity_checks(device, first_trk, last_trk)) return ERR_PTR(-EINVAL); - copy_relation = dasd_in_copy_relation(device); - if (copy_relation < 0) - return ERR_PTR(copy_relation); + /* + * The block-layer discard path (req != NULL) runs in atomic context, so + * it must not issue the sleeping copy-relation (PPRC) query. It also + * leaves guarantee_init off - discard does not promise zeroing anyway. + */ + copy_relation = 0; + if (!req) { + copy_relation = dasd_in_copy_relation(device); + if (copy_relation < 0) + return ERR_PTR(copy_relation); + } rq = req ? blk_mq_rq_to_pdu(req) : NULL; @@ -4248,7 +4244,7 @@ dasd_eckd_dso_ras(struct dasd_device *device, struct dasd_block *block, * not fully specified, but is only supported with a certain feature * subset and for devices not in a copy relation. */ - if (features->feature[56] & 0x01 && !copy_relation) + if (!req && features->feature[56] & 0x01 && !copy_relation) ras_data->op_flags.guarantee_init = 1; ras_data->lss = private->conf.ned->ID; @@ -4344,6 +4340,9 @@ static int dasd_eckd_release_space_trks(struct dasd_device *device, INIT_LIST_HEAD(&ras_queue); + if (dasd_eckd_ext_size(device) == 0) + return -EINVAL; + device_exts = private->real_cyl / dasd_eckd_ext_size(device); trks_per_ext = dasd_eckd_ext_size(device) * private->rdc_data.trk_per_cyl; @@ -5481,6 +5480,58 @@ out_error: return ERR_PTR(ret); } +static struct dasd_ccw_req * +dasd_eckd_build_cp_discard(struct dasd_device *device, struct dasd_block *block, + struct request *req, sector_t first_trk, + sector_t last_trk, unsigned int first_offs, + unsigned int last_offs, unsigned int blk_per_trk) +{ + struct dasd_eckd_private *private = device->private; + sector_t first_ext_trk, last_ext_end, last_ext_trk; + unsigned int trks_per_ext; + + trks_per_ext = dasd_eckd_ext_size(device) * private->rdc_data.trk_per_cyl; + if (!trks_per_ext) + return ERR_PTR(-EOPNOTSUPP); + + /* + * A discard range is rarely track-aligned: fstrim is FS-block granular + * and discard_granularity is only a hint. If it starts or ends mid-track, + * that boundary track still holds live records outside the range, so drop + * it from the whole-track span first. Otherwise a partial boundary track + * that happens to sit on an extent boundary would be released together + * with its live records resulting in silent data loss + */ + if (first_offs) /* partial first track */ + first_trk++; + if (last_offs != blk_per_trk - 1) { /* partial last track */ + if (!last_trk) + return ERR_PTR(-EOPNOTSUPP); + last_trk--; + } + if (first_trk > last_trk) + return ERR_PTR(-EOPNOTSUPP); /* no whole track fully covered */ + + /* + * RAS releases whole extents. Only release extents that lie entirely + * within the (now whole-track) discard range by rounding inward to extent + * boundaries - an extent shared with a live allocation must never be + * released. If no whole extent is covered there is nothing to release + * safely (e.g. a sub-extent discard, unavoidable with large extents), so + * reject the request rather than release too much. + */ + first_ext_trk = roundup(first_trk, trks_per_ext); + /* one past the last whole extent inside the range (exclusive) */ + last_ext_end = rounddown(last_trk + 1, trks_per_ext); + if (first_ext_trk >= last_ext_end) + return ERR_PTR(-EOPNOTSUPP); + /* inclusive last track; the guard above keeps this from underflowing */ + last_ext_trk = last_ext_end - 1; + + return dasd_eckd_dso_ras(device, block, req, first_ext_trk, + last_ext_trk, 1); +} + static struct dasd_ccw_req *dasd_eckd_build_cp(struct dasd_device *startdev, struct dasd_block *block, struct request *req) @@ -5519,6 +5570,12 @@ static struct dasd_ccw_req *dasd_eckd_build_cp(struct dasd_device *startdev, last_offs = sector_div(last_trk, blk_per_trk); cdlspecial = (private->uses_cdl && first_rec < 2*blk_per_trk); + if (req_op(req) == REQ_OP_DISCARD) + return dasd_eckd_build_cp_discard(startdev, block, req, + first_trk, last_trk, + first_offs, last_offs, + blk_per_trk); + fcx_multitrack = private->features.feature[40] & 0x20; data_size = blk_rq_bytes(req); if (data_size % blksize || data_size == 0) @@ -5832,11 +5889,13 @@ static struct dasd_ccw_req *dasd_eckd_build_alias_cp(struct dasd_device *base, struct request *req) { struct dasd_eckd_private *private; - struct dasd_device *startdev; + struct dasd_device *startdev = NULL; unsigned long flags; struct dasd_ccw_req *cqr; - startdev = dasd_alias_get_start_dev(base); + /* Discard requests (space release) can only run on the base device. */ + if (req_op(req) != REQ_OP_DISCARD) + startdev = dasd_alias_get_start_dev(base); if (!startdev) startdev = base; private = startdev->private; @@ -7724,6 +7783,51 @@ static unsigned int dasd_eckd_max_sectors(struct dasd_block *block) return DASD_ECKD_MAX_BLOCKS << block->s2b_shift; } +/* + * Discard on ECKD releases space through RAS, which works on whole extents. + * Advertise extent granularity so the block layer only sends extent-aligned + * discards (avoiding partially specified extents), and only for volumes on ESE + * hardware. Non-ESE devices are left without discard limits. + */ +static void dasd_eckd_disc_limits(struct dasd_block *block, + struct queue_limits *lim) +{ + struct dasd_device *device = block->base; + struct dasd_eckd_private *private = device->private; + unsigned int logical_block_size = block->bp_block; + unsigned int max_discard_sectors, max_bytes, ext_bytes; + int recs_per_trk, trks_per_cyl, ext_limit, ext_size; + + if (!dasd_eckd_ese_capable(device) || dasd_eckd_ext_size(device) == 0) + return; + + trks_per_cyl = private->rdc_data.trk_per_cyl; + recs_per_trk = recs_per_track(&private->rdc_data, 0, logical_block_size); + + ext_size = dasd_eckd_ext_size(device); + ext_limit = min(private->real_cyl / ext_size, DASD_ECKD_RAS_EXTS_MAX); + ext_bytes = ext_size * trks_per_cyl * recs_per_trk * logical_block_size; + if (!ext_bytes) /* malformed RDC data - leave discard unset */ + return; + max_bytes = UINT_MAX - (UINT_MAX % ext_bytes); + if (max_bytes / ext_bytes > ext_limit) + max_bytes = ext_bytes * ext_limit; + + max_discard_sectors = max_bytes / 512; + + lim->max_hw_discard_sectors = max_discard_sectors; + /* + * ext_bytes is the hardware extent size and is not a power of two, so + * the block layer's power-of-two round_up()/round_down() alignment + * helpers compute it only approximately. That is a hint, not a + * correctness requirement: RAS safety is enforced in the CCW builder, + * which rounds the range inward to whole extents and rejects a request + * that covers no whole extent, so a misaligned range is never + * over-released. At worst a few sub-extent discards are declined. + */ + lim->discard_granularity = ext_bytes; +} + static struct ccw_driver dasd_eckd_driver = { .driver = { .name = "dasd-eckd", @@ -7746,6 +7850,7 @@ static struct dasd_discipline dasd_eckd_discipline = { .owner = THIS_MODULE, .name = "ECKD", .ebcname = "ECKD", + .disc_limits = dasd_eckd_disc_limits, .check_device = dasd_eckd_check_characteristics, .uncheck_device = dasd_eckd_uncheck_device, .do_analysis = dasd_eckd_do_analysis, diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h index 8c73850f7947..ef4930432c09 100644 --- a/drivers/s390/block/dasd_int.h +++ b/drivers/s390/block/dasd_int.h @@ -404,6 +404,8 @@ struct dasd_discipline { int (*ese_capable)(struct dasd_device *); /* Whether the volume is formatted on demand (thin), from the label */ int (*on_demand_format)(struct dasd_device *); + /* Fill discard queue limits */ + void (*disc_limits)(struct dasd_block *, struct queue_limits *); /* Capacity */ int (*space_allocated)(struct dasd_device *); int (*space_configured)(struct dasd_device *); From a600051da14b4cacc7b00685c967f40b0425ef5b Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Wed, 5 Aug 2026 13:16:12 +0200 Subject: [PATCH 139/241] s390/dasd: Read cached unit address and LSS in the CCW build path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CCW build path (prefix_LRE, the full-track prefix and dso_ras) read the base address and LSS straight from conf.ned. That buffer is freed and reallocated by the reload worker (do_reload_device - dasd_eckd_read_conf - dasd_eckd_clear_conf_data), so a configuration change concurrent with I/O can free conf.ned while a request is being built. Use-after-free reported by KASAN in prefix_LRE. Read the cached copies instead. The unit address is already kept in uid.real_unit_addr, and the LSS is now cached in ned_lss. Both are refreshed under the ccwdev lock in dasd_eckd_generate_uid whenever the configuration is (re)read. Also fix for prepare for read subsystem data (prssd) users. Reviewed-by: Jan Höppner Signed-off-by: Stefan Haberland Link: https://patch.msgid.link/20260805111612.1285190-20-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 26 ++++++++++++++++---------- drivers/s390/block/dasd_eckd.h | 8 ++++++++ 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 6eb2879b479b..bacf770c0e1f 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -588,8 +588,9 @@ static int prefix_LRE(struct ccw1 *ccw, struct PFX_eckd_data *pfxdata, return -EINVAL; } pfxdata->format = format; - pfxdata->base_address = basepriv->conf.ned->unit_addr; - pfxdata->base_lss = basepriv->conf.ned->ID; + /* cached copies - conf.ned may be freed under us by the reload worker */ + pfxdata->base_address = READ_ONCE(basepriv->ned_ua); + pfxdata->base_lss = READ_ONCE(basepriv->ned_lss); pfxdata->validity.define_extent = 1; /* private uid is kept up to date, conf_data may be outdated */ @@ -806,6 +807,9 @@ static int dasd_eckd_generate_uid(struct dasd_device *device) return -ENODEV; spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags); create_uid(&private->conf, &private->uid); + /* cache LSS and unit address for the lockless CCW-build path */ + WRITE_ONCE(private->ned_lss, private->conf.ned->ID); + WRITE_ONCE(private->ned_ua, private->conf.ned->unit_addr); spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags); return 0; } @@ -1631,8 +1635,8 @@ static int dasd_eckd_read_vol_info(struct dasd_device *device) prssdp = cqr->data; prssdp->order = PSF_ORDER_PRSSD; prssdp->suborder = PSF_SUBORDER_VSQ; /* Volume Storage Query */ - prssdp->lss = private->conf.ned->ID; - prssdp->volume = private->conf.ned->unit_addr; + prssdp->lss = READ_ONCE(private->ned_lss); + prssdp->volume = READ_ONCE(private->ned_ua); ccw = cqr->cpaddr; ccw->cmd_code = DASD_ECKD_CCW_PSF; @@ -4247,8 +4251,9 @@ dasd_eckd_dso_ras(struct dasd_device *device, struct dasd_block *block, if (!req && features->feature[56] & 0x01 && !copy_relation) ras_data->op_flags.guarantee_init = 1; - ras_data->lss = private->conf.ned->ID; - ras_data->dev_addr = private->conf.ned->unit_addr; + /* cached copies - conf.ned may be freed under us by the reload worker */ + ras_data->lss = READ_ONCE(private->ned_lss); + ras_data->dev_addr = READ_ONCE(private->ned_ua); ras_data->nr_exts = nr_exts; if (by_extent) { @@ -4819,8 +4824,9 @@ static int prepare_itcw(struct itcw *itcw, lredata = &pfxdata->locate_record; pfxdata->format = 1; /* PFX with LRE */ - pfxdata->base_address = basepriv->conf.ned->unit_addr; - pfxdata->base_lss = basepriv->conf.ned->ID; + /* cached copies - conf.ned may be freed under us by the reload worker */ + pfxdata->base_address = READ_ONCE(basepriv->ned_ua); + pfxdata->base_lss = READ_ONCE(basepriv->ned_lss); pfxdata->validity.define_extent = 1; /* private uid is kept up to date, conf_data may be outdated */ @@ -6902,8 +6908,8 @@ static int dasd_eckd_query_host_access(struct dasd_device *device, prssdp->order = PSF_ORDER_PRSSD; prssdp->suborder = PSF_SUBORDER_QHA; /* query host access */ /* LSS and Volume that will be queried */ - prssdp->lss = private->conf.ned->ID; - prssdp->volume = private->conf.ned->unit_addr; + prssdp->lss = READ_ONCE(private->ned_lss); + prssdp->volume = READ_ONCE(private->ned_ua); /* all other bytes of prssdp must be zero */ ccw = cqr->cpaddr; diff --git a/drivers/s390/block/dasd_eckd.h b/drivers/s390/block/dasd_eckd.h index 30745f62402b..8e6f09e9ca7e 100644 --- a/drivers/s390/block/dasd_eckd.h +++ b/drivers/s390/block/dasd_eckd.h @@ -736,6 +736,14 @@ struct dasd_eckd_private { /* alias management */ struct dasd_uid uid; + /* + * Cached copies of conf.ned->ID (the LSS) and conf.ned->unit_addr, + * refreshed under ccwdev_lock. Kept outside uid because create_uid() + * memsets uid before repopulating it, which would expose a transient + * zero to the lockless CCW-build readers. + */ + __u8 ned_lss; + __u8 ned_ua; struct alias_pav_group *pavgroup; struct alias_lcu *lcu; int count; From b539aeacf8cc5e9d8e5d94625d2d9c697a167add Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Thu, 6 Aug 2026 13:19:27 +0100 Subject: [PATCH 140/241] block: rename bi_bvec_done struct bvec_iter::bi_bvec_done is used an offset in the current bvec, let's rename it accordingly for better clarity. I also plan to use it for non-bvec based iteration in the future like dma-buf, so drop the "bvec" part. Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/4e4c21858705a200bd8848ffe4080522e3eb5c1c.1786018753.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- Documentation/block/biovecs.rst | 8 ++++---- block/bio.c | 4 ++-- block/blk-merge.c | 8 ++++---- block/blk-mq-dma.c | 2 +- block/blk.h | 2 +- drivers/block/loop.c | 2 +- drivers/block/zloop.c | 2 +- drivers/md/dm-io-rewind.c | 10 +++++----- drivers/md/dm-pcache/segment.c | 4 ++-- drivers/nvdimm/btt.c | 2 +- drivers/nvme/host/tcp.c | 2 +- fs/btrfs/misc.h | 2 +- include/linux/bvec.h | 14 +++++++------- io_uring/net.c | 4 ++-- lib/iov_iter.c | 2 +- net/ceph/messenger.c | 4 ++-- 16 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Documentation/block/biovecs.rst b/Documentation/block/biovecs.rst index 11126ed6f40f..49da2147b7d8 100644 --- a/Documentation/block/biovecs.rst +++ b/Documentation/block/biovecs.rst @@ -16,16 +16,16 @@ bv_len by the number of bytes completed in that biovec. In the new scheme of things, everything that must be mutated in order to partially complete a bio is segregated into struct bvec_iter: bi_sector, bi_size and bi_idx have been moved there; and instead of modifying bv_offset -and bv_len, struct bvec_iter has bi_bvec_done, which represents the number of +and bv_len, struct bvec_iter has bi_offset, which represents the number of bytes completed in the current bvec. There are a bunch of new helper macros for hiding the gory details - in particular, presenting the illusion of partially completed biovecs so that -normal code doesn't have to deal with bi_bvec_done. +normal code doesn't have to deal with bi_offset. * Driver code should no longer refer to biovecs directly; we now have bio_iovec() and bio_iter_iovec() macros that return literal struct biovecs, - constructed from the raw biovecs but taking into account bi_bvec_done and + constructed from the raw biovecs but taking into account bi_offset and bi_size. bio_for_each_segment() has been updated to take a bvec_iter argument @@ -101,7 +101,7 @@ Other implications: I.e. instead of using bio_iovec_idx() (or bio->bi_iovec[bio->bi_idx]), you now use bio_iter_iovec(), which takes a bvec_iter and returns a literal struct bio_vec - constructed on the fly from the raw biovec but - taking into account bi_bvec_done (and bi_size). + taking into account bi_offset (and bi_size). * bi_vcnt can't be trusted or relied upon by driver code - i.e. anything that doesn't actually own the bio. The reason is twofold: firstly, it's not diff --git a/block/bio.c b/block/bio.c index 500389f332d9..4074a0496b93 100644 --- a/block/bio.c +++ b/block/bio.c @@ -229,7 +229,7 @@ void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, bio->bi_iter.bi_sector = 0; bio->bi_iter.bi_size = 0; bio->bi_iter.bi_idx = 0; - bio->bi_iter.bi_bvec_done = 0; + bio->bi_iter.bi_offset = 0; bio->bi_end_io = NULL; bio->bi_private = NULL; #ifdef CONFIG_BLK_CGROUP @@ -1188,7 +1188,7 @@ void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter) bio->bi_io_vec = (struct bio_vec *)iter->bvec; bio->bi_iter.bi_idx = 0; - bio->bi_iter.bi_bvec_done = iter->iov_offset; + bio->bi_iter.bi_offset = iter->iov_offset; bio->bi_iter.bi_size = iov_iter_count(iter); bio_set_flag(bio, BIO_CLONED); } diff --git a/block/blk-merge.c b/block/blk-merge.c index ab1161ca69f1..258a726071d1 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -33,7 +33,7 @@ static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv) bio_advance_iter(bio, &iter, iter.bi_size); - if (!iter.bi_bvec_done) + if (!iter.bi_offset) idx = iter.bi_idx - 1; else /* in the middle of bvec */ idx = iter.bi_idx; @@ -41,11 +41,11 @@ static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv) *bv = bio->bi_io_vec[idx]; /* - * iter.bi_bvec_done records actual length of the last bvec + * iter.bi_offset records actual length of the last bvec * if this bio ends in the middle of one io vector */ - if (iter.bi_bvec_done) - bv->bv_len = iter.bi_bvec_done; + if (iter.bi_offset) + bv->bv_len = iter.bi_offset; } static inline bool bio_will_gap(struct request_queue *q, diff --git a/block/blk-mq-dma.c b/block/blk-mq-dma.c index bfdb9ed70741..88fd9cbc951f 100644 --- a/block/blk-mq-dma.c +++ b/block/blk-mq-dma.c @@ -44,7 +44,7 @@ static bool blk_map_iter_next(struct request *req, struct blk_map_iter *iter, * one could be merged into it. This typically happens when moving to * the next bio, but some callers also don't pack bvecs tight. */ - while (!iter->iter.bi_size || !iter->iter.bi_bvec_done) { + while (!iter->iter.bi_size || !iter->iter.bi_offset) { struct bio_vec next; if (!__blk_map_iter_next(iter)) diff --git a/block/blk.h b/block/blk.h index eaac05815cb0..50abfd932886 100644 --- a/block/blk.h +++ b/block/blk.h @@ -406,7 +406,7 @@ static inline bool bio_may_need_split(struct bio *bio, return true; bv = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); - if (bio->bi_iter.bi_size > bv->bv_len - bio->bi_iter.bi_bvec_done) + if (bio->bi_iter.bi_size > bv->bv_len - bio->bi_iter.bi_offset) return true; if ((bv->bv_offset | bv->bv_len) & lim->dma_alignment) return true; diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 26d7130c3f55..8639fa34b847 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -379,7 +379,7 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, iov_iter_bvec(&iter, rw, __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter), nr_bvec, blk_rq_bytes(rq)); - iter.iov_offset = rq->bio->bi_iter.bi_bvec_done; + iter.iov_offset = rq->bio->bi_iter.bi_offset; } atomic_set(&cmd->ref, 2); diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 58ec8161b7e2..4323ac108cae 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -555,7 +555,7 @@ static int zloop_do_rw(struct zloop_cmd *cmd) iov_iter_bvec(&iter, rw, __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter), nr_bvec, blk_rq_bytes(rq)); - iter.iov_offset = rq->bio->bi_iter.bi_bvec_done; + iter.iov_offset = rq->bio->bi_iter.bi_offset; } cmd->iocb.ki_pos = (cmd->sector - zone->start) << SECTOR_SHIFT; diff --git a/drivers/md/dm-io-rewind.c b/drivers/md/dm-io-rewind.c index 6155b0117c9d..04f3fc8aeb6f 100644 --- a/drivers/md/dm-io-rewind.c +++ b/drivers/md/dm-io-rewind.c @@ -16,12 +16,12 @@ static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv, int idx; iter->bi_size += bytes; - if (bytes <= iter->bi_bvec_done) { - iter->bi_bvec_done -= bytes; + if (bytes <= iter->bi_offset) { + iter->bi_offset -= bytes; return true; } - bytes -= iter->bi_bvec_done; + bytes -= iter->bi_offset; idx = iter->bi_idx - 1; while (idx >= 0 && bytes && bytes > bv[idx].bv_len) { @@ -32,13 +32,13 @@ static inline bool dm_bvec_iter_rewind(const struct bio_vec *bv, if (WARN_ONCE(idx < 0 && bytes, "Attempted to rewind iter beyond bvec's boundaries\n")) { iter->bi_size -= bytes; - iter->bi_bvec_done = 0; + iter->bi_offset = 0; iter->bi_idx = 0; return false; } iter->bi_idx = idx; - iter->bi_bvec_done = bv[idx].bv_len - bytes; + iter->bi_offset = bv[idx].bv_len - bytes; return true; } diff --git a/drivers/md/dm-pcache/segment.c b/drivers/md/dm-pcache/segment.c index 7e9818701445..8f8816e1c539 100644 --- a/drivers/md/dm-pcache/segment.c +++ b/drivers/md/dm-pcache/segment.c @@ -14,7 +14,7 @@ int segment_copy_to_bio(struct pcache_segment *segment, iov_iter_bvec(&iter, ITER_DEST, &bio->bi_io_vec[bio->bi_iter.bi_idx], bio_segments(bio), bio->bi_iter.bi_size); - iter.iov_offset = bio->bi_iter.bi_bvec_done; + iter.iov_offset = bio->bi_iter.bi_offset; if (bio_off) iov_iter_advance(&iter, bio_off); @@ -35,7 +35,7 @@ int segment_copy_from_bio(struct pcache_segment *segment, iov_iter_bvec(&iter, ITER_SOURCE, &bio->bi_io_vec[bio->bi_iter.bi_idx], bio_segments(bio), bio->bi_iter.bi_size); - iter.iov_offset = bio->bi_iter.bi_bvec_done; + iter.iov_offset = bio->bi_iter.bi_offset; if (bio_off) iov_iter_advance(&iter, bio_off); diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 7e1112960d7f..5d910a64503d 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1155,7 +1155,7 @@ static int btt_rw_integrity(struct btt *btt, struct bio_integrity_payload *bip, bv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter); /* * The 'bv' obtained from bvec_iter_bvec has its .bv_len and - * .bv_offset already adjusted for iter->bi_bvec_done, and we + * .bv_offset already adjusted for iter->bi_offset, and we * can use those directly */ diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index ba5c7b3e2a7c..ce03a0ea4ded 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -357,7 +357,7 @@ static void nvme_tcp_init_iter(struct nvme_tcp_request *req, iov_iter_bvec(&req->iter, dir, __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter), nr_bvec, bio->bi_iter.bi_size); - req->iter.iov_offset = bio->bi_iter.bi_bvec_done; + req->iter.iov_offset = bio->bi_iter.bi_offset; } } diff --git a/fs/btrfs/misc.h b/fs/btrfs/misc.h index 694be6d0562a..802060943180 100644 --- a/fs/btrfs/misc.h +++ b/fs/btrfs/misc.h @@ -74,7 +74,7 @@ static inline struct bvec_iter init_bvec_iter_for_bio(struct bio *bio) .bi_sector = 0, .bi_size = bio_size, .bi_idx = 0, - .bi_bvec_done = 0, + .bi_offset = 0, }; } diff --git a/include/linux/bvec.h b/include/linux/bvec.h index 92837e2743f1..fc566ee1c1ff 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -110,7 +110,7 @@ struct bvec_iter { /* * Current offset in the bvec entry pointed to by `bi_idx`. */ - unsigned int bi_bvec_done; + unsigned int bi_offset; } __packed __aligned(4); struct bvec_iter_all { @@ -135,14 +135,14 @@ mp_bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) static __always_inline unsigned int mp_bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) { - return min(__bvec_iter_bvec(bvecs, iter)->bv_len - iter.bi_bvec_done, + return min(__bvec_iter_bvec(bvecs, iter)->bv_len - iter.bi_offset, iter.bi_size); } static __always_inline unsigned int mp_bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) { - return __bvec_iter_bvec(bvecs, iter)->bv_offset + iter.bi_bvec_done; + return __bvec_iter_bvec(bvecs, iter)->bv_offset + iter.bi_offset; } static __always_inline unsigned int @@ -204,7 +204,7 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, } iter->bi_size -= bytes; - bytes += iter->bi_bvec_done; + bytes += iter->bi_offset; while (bytes && bytes >= bv[idx].bv_len) { bytes -= bv[idx].bv_len; @@ -212,7 +212,7 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, } iter->bi_idx = idx; - iter->bi_bvec_done = bytes; + iter->bi_offset = bytes; return true; } @@ -223,13 +223,13 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, static inline void bvec_iter_advance_single(const struct bio_vec *bv, struct bvec_iter *iter, unsigned int bytes) { - unsigned int done = iter->bi_bvec_done + bytes; + unsigned int done = iter->bi_offset + bytes; if (done == bv[iter->bi_idx].bv_len) { done = 0; iter->bi_idx++; } - iter->bi_bvec_done = done; + iter->bi_offset = done; iter->bi_size -= bytes; } diff --git a/io_uring/net.c b/io_uring/net.c index 00a7df803b99..7574008f97e1 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -1470,7 +1470,7 @@ static int io_sg_from_iter(struct sk_buff *skb, return zerocopy_fill_skb_from_iter(skb, from, length); bi.bi_size = min(from->count, length); - bi.bi_bvec_done = from->iov_offset; + bi.bi_offset = from->iov_offset; bi.bi_idx = 0; while (bi.bi_size && frag < MAX_SKB_FRAGS) { @@ -1489,7 +1489,7 @@ static int io_sg_from_iter(struct sk_buff *skb, from->bvec += bi.bi_idx; from->nr_segs -= bi.bi_idx; from->count -= copied; - from->iov_offset = bi.bi_bvec_done; + from->iov_offset = bi.bi_offset; skb->data_len += copied; skb->len += copied; diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 34a52e9ba9e1..81e5c5e5121f 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -1634,7 +1634,7 @@ static ssize_t iov_iter_extract_bvec_pages(struct iov_iter *i, } bi.bi_idx = 0; bi.bi_size = maxsize; - bi.bi_bvec_done = skip; + bi.bi_offset = skip; maxpages = want_pages_array(pages, maxsize, skip, maxpages); if (!maxpages) diff --git a/net/ceph/messenger.c b/net/ceph/messenger.c index 34b3097b4c7b..9c1b6cf8c36f 100644 --- a/net/ceph/messenger.c +++ b/net/ceph/messenger.c @@ -762,7 +762,7 @@ static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor, if (!cursor->resid) return false; /* no more data */ - if (!bytes || (it->iter.bi_size && it->iter.bi_bvec_done && + if (!bytes || (it->iter.bi_size && it->iter.bi_offset && page == bio_iter_page(it->bio, it->iter))) return false; /* more bytes to process in this segment */ @@ -817,7 +817,7 @@ static bool ceph_msg_data_bvecs_advance(struct ceph_msg_data_cursor *cursor, if (!cursor->resid) return false; /* no more data */ - if (!bytes || (cursor->bvec_iter.bi_bvec_done && + if (!bytes || (cursor->bvec_iter.bi_offset && page == bvec_iter_page(bvecs, cursor->bvec_iter))) return false; /* more bytes to process in this segment */ From 11d4f5e69fe25bc76e27eca75b4ef6f7aae53214 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 6 Aug 2026 15:00:49 +0200 Subject: [PATCH 141/241] s390/dasd: Add __context_unsafe() attribute to various functions Disable context analysis for various functions to get rid of context analysis compile time warnings using clang caused by conditional locking like e.g.: drivers/s390/block/dasd_eckd.c:1462:3: warning: releasing mutex 'dasd_pe_handler_mutex' that was not held [-Wthread-safety-analysis] 1462 | mutex_unlock(&dasd_pe_handler_mutex); | ^ Use __context_unsafe() to provide a short comment why context analysis is disabled for each function. It doesn't look like those functions can be easily reworked to get rid of conditional locking. Therefore disable context analysis for (only) those functions. Signed-off-by: Heiko Carstens Acked-by: Stefan Haberland Link: https://patch.msgid.link/20260806130050.2057443-2-hca@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_eckd.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index bacf770c0e1f..b4dcef06edaa 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -1468,6 +1468,7 @@ static void dasd_eckd_path_available_action(struct dasd_device *device, } static void do_pe_handler_work(struct work_struct *work) +__context_unsafe(/* Conditional locking */) { struct pe_handler_work_data *data; struct dasd_device *device; @@ -1501,6 +1502,7 @@ static void do_pe_handler_work(struct work_struct *work) static int dasd_eckd_pe_handler(struct dasd_device *device, __u8 tbvpm, __u8 fcsecpm) +__context_unsafe(/* Conditional locking */) { struct pe_handler_work_data *data; @@ -1601,6 +1603,7 @@ static int dasd_eckd_read_features(struct dasd_device *device) /* Read Volume Information - Volume Storage Query */ static int dasd_eckd_read_vol_info(struct dasd_device *device) +__context_unsafe(/* Conditional locking */) { struct dasd_eckd_private *private = device->private; struct dasd_psf_prssd_data *prssdp; @@ -5965,6 +5968,7 @@ dasd_eckd_fill_info(struct dasd_device * device, */ static int dasd_eckd_release(struct dasd_device *device) +__context_unsafe(/* Conditional locking */) { struct dasd_ccw_req *cqr; int rc; @@ -6020,6 +6024,7 @@ dasd_eckd_release(struct dasd_device *device) */ static int dasd_eckd_reserve(struct dasd_device *device) +__context_unsafe(/* Conditional locking */) { struct dasd_ccw_req *cqr; int rc; @@ -6074,6 +6079,7 @@ dasd_eckd_reserve(struct dasd_device *device) */ static int dasd_eckd_steal_lock(struct dasd_device *device) +__context_unsafe(/* Conditional locking */) { struct dasd_ccw_req *cqr; int rc; @@ -6129,6 +6135,7 @@ dasd_eckd_steal_lock(struct dasd_device *device) */ static int dasd_eckd_snid(struct dasd_device *device, void __user *argp) +__context_unsafe(/* Conditional locking */) { struct dasd_ccw_req *cqr; int rc; From 30df3ab3c92d0e7854c54df6ae66f4aa6eb5b3d2 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Thu, 6 Aug 2026 15:00:50 +0200 Subject: [PATCH 142/241] s390/block: Enable CONTEXT_ANALYSIS All drivers in drivers/s390/block pass clang's compile time context analysis. Therefore enable CONTEXT_ANALYSIS. Signed-off-by: Heiko Carstens Acked-by: Stefan Haberland Link: https://patch.msgid.link/20260806130050.2057443-3-hca@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/s390/block/Makefile b/drivers/s390/block/Makefile index a0a54d2f063f..3dd1a3ce7ed1 100644 --- a/drivers/s390/block/Makefile +++ b/drivers/s390/block/Makefile @@ -3,6 +3,8 @@ # S/390 block devices # +CONTEXT_ANALYSIS := y + dasd_eckd_mod-objs := dasd_eckd.o dasd_3990_erp.o dasd_alias.o dasd_fba_mod-objs := dasd_fba.o dasd_diag_mod-objs := dasd_diag.o From 2f6b2073ea631cb60d5176941b9815192b22a2b8 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:10 +0800 Subject: [PATCH 143/241] md/md-llbitmap: clear flush state after daemon flush llbitmap_flush() sets LLPageFlush on each bitmap page before it queues the daemon worker. The flag tells md_llbitmap_daemon_fn() to ignore the normal barrier_idle expiry check and clean the page immediately. The daemon only tested LLPageFlush. Once a page had been flushed explicitly, the flag stayed set, so later dirty bits on that page also bypassed barrier_idle and were cleaned the next time the daemon ran. That can make a new write look clean much earlier than the configured idle window. Consume LLPageFlush in md_llbitmap_daemon_fn() with test_and_clear_bit() and use the returned value for the current expiry check. The explicit flush still forces the current daemon pass, while later writes on the same page wait for barrier_idle again. This can be reproduced through normal sysfs operations: 1. Create a small RAID1 with --bitmap=lockless and --assume-clean. 2. Set llbitmap/daemon_sleep=1 and llbitmap/barrier_idle=10. 3. Toggle md/array_state from active to readonly and back to active to call llbitmap_flush() without destroying the in-memory bitmap. 4. Write one sector and read llbitmap/bits immediately, after 2 seconds, and after 12 seconds. On the bad kernel the dirty bit is already clean after 2 seconds. With this change it remains dirty until the barrier_idle window expires. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-2-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 2a2b38c663c3..71e9a21b98b2 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1066,14 +1066,14 @@ retry: for (idx = 0; idx < llbitmap->nr_pages; idx++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx]; + bool flush = test_and_clear_bit(LLPageFlush, &pctl->flags); if (idx > 0) { start = end + 1; end = min(end + PAGE_SIZE, llbitmap->chunks - 1); } - if (!test_bit(LLPageFlush, &pctl->flags) && - time_before(jiffies, pctl->expire)) { + if (!flush && time_before(jiffies, pctl->expire)) { restart = true; continue; } From 4b6cdc56c8412dc59904de2f915c72552b00bf6f Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:11 +0800 Subject: [PATCH 144/241] md/md-llbitmap: use GFP_NOIO for cache allocations llbitmap allocates its in-memory page cache and page-control structures from paths that can already be holding MD reconfiguration or bitmap state locks. For example, component_size_store() takes mddev_lock(), update_size() calls the personality resize method, and llbitmap_resize() can grow the page cache through llbitmap_prepare_resize(). Using GFP_KERNEL in those paths allows direct reclaim to enter filesystem or block I/O while MD resize state is locked. That can recurse back into the same array and wait on state that cannot make progress until the resize path finishes. Use GFP_NOIO for the llbitmap object, cached bitmap pages, page controls, page-control arrays, and percpu_ref initialization. Leave the explicit metadata zeroout path unchanged because it is intentional bitmap I/O rather than reclaim-driven allocation. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-3-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 71e9a21b98b2..3cd8373bc9b2 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -521,7 +521,7 @@ static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx) if (page) return page; - page = alloc_page(GFP_KERNEL | __GFP_ZERO); + page = alloc_page(GFP_NOIO | __GFP_ZERO); if (!page) return ERR_PTR(-ENOMEM); @@ -616,12 +616,12 @@ static int llbitmap_cache_pages(struct llbitmap *llbitmap) int i; llbitmap->pctl = kmalloc_array(nr_pages, sizeof(void *), - GFP_KERNEL | __GFP_ZERO); + GFP_NOIO | __GFP_ZERO); if (!llbitmap->pctl) return -ENOMEM; size = round_up(size, cache_line_size()); - pctl = kmalloc_array(nr_pages, size, GFP_KERNEL | __GFP_ZERO); + pctl = kmalloc_array(nr_pages, size, GFP_NOIO | __GFP_ZERO); if (!pctl) { kfree(llbitmap->pctl); return -ENOMEM; @@ -640,7 +640,7 @@ static int llbitmap_cache_pages(struct llbitmap *llbitmap) } if (percpu_ref_init(&pctl->active, active_release, - PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) { + PERCPU_REF_ALLOW_REINIT, GFP_NOIO)) { __free_page(page); llbitmap_free_pages(llbitmap); return -ENOMEM; @@ -1110,7 +1110,7 @@ static int llbitmap_create(struct mddev *mddev) if (ret) return ret; - llbitmap = kzalloc_obj(*llbitmap); + llbitmap = kzalloc_obj(*llbitmap, GFP_NOIO); if (!llbitmap) return -ENOMEM; From dbd21b489ac1afda73de401a446ecad2b53f413c Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:12 +0800 Subject: [PATCH 145/241] md/md-llbitmap: only end fully synced chunks llbitmap_cond_end_sync() is called with the sync thread's current sector. That value is an exclusive progress boundary: sectors below it have completed, but the llbitmap chunk containing it can still be in progress. The old code converted that sector directly to the last bit passed to BitmapActionEndsync. If resync had only advanced part-way into a large llbitmap chunk, the in-progress chunk was marked synced and flushed before the rest of the chunk was repaired. A later bitmap-assisted RAID1 resync could then skip the remainder of that chunk and leave stale mirror data behind. This can be reproduced without editing bitmap metadata by creating a large RAID1 with a lockless bitmap so llbitmap naturally selects a 524288-sector chunk (with the default 128 KiB bitmap area, an array just over 16 TiB is enough), making one mirror stale through the normal degraded write/re-add path, and throttling resync so the daemon checkpoint runs while resync is still inside the first chunk. On the bad kernel, bit 0 is ended early and a stale sector later in the same chunk is skipped. With this fix, bit 0 remains Syncing until resync reaches the next chunk boundary. Round the exclusive progress sector down to the nearest llbitmap chunk boundary and end only chunks strictly below that boundary. Also honor the force argument so callers that need an immediate checkpoint are not suppressed by daemon_sleep. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-4-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 3cd8373bc9b2..948bf64c5ad2 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1450,22 +1450,27 @@ static void llbitmap_cond_end_sync(struct mddev *mddev, sector_t sector, bool force) { struct llbitmap *llbitmap = mddev->bitmap; + sector_t complete; if (sector == 0) { llbitmap->last_end_sync = jiffies; return; } - if (time_before(jiffies, llbitmap->last_end_sync + - HZ * mddev->bitmap_info.daemon_sleep)) + if (!force && time_before(jiffies, llbitmap->last_end_sync + + HZ * mddev->bitmap_info.daemon_sleep)) return; wait_event(mddev->recovery_wait, !atomic_read(&mddev->recovery_active)); mddev->curr_resync_completed = sector; set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags); - llbitmap_state_machine(llbitmap, 0, sector >> llbitmap->chunkshift, - BitmapActionEndsync); + + complete = round_down(sector, llbitmap->chunksize); + if (complete) + llbitmap_state_machine(llbitmap, 0, + (complete >> llbitmap->chunkshift) - 1, + BitmapActionEndsync); __llbitmap_flush(mddev); llbitmap->last_end_sync = jiffies; From a41bb2ee1aca486853e84920565e50c15f86fa5d Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:13 +0800 Subject: [PATCH 146/241] md/raid5: reject zero-sector reshape chunks Sashiko reported that RAID5 can accept a reshape chunk size that becomes zero sectors. chunk_size_store() stores the sysfs byte value as n >> 9, so writing a value below 512 bytes sets mddev->new_chunk_sectors to zero. RAID5 then accepted that pending reshape geometry and raid5_start_reshape() installed it into conf->chunk_sectors, letting reshape code divide by zero. Reject zero-sector chunks both in check_reshape(), where normal sysfs requests are validated, and in raid5_start_reshape(), so assembly/resume paths also cannot install zero chunk geometry. Test script: in QEMU, create a plain three-disk RAID5 array with 64K chunks, write/read back a small pattern, write 1 to /sys/block/md0/md/chunk_size, add a fourth disk, and run mdadm --grow --raid-devices=4 --backup-file=... . The script scans dmesg for divide error/Oops/KASAN signatures. Bad kernel, eb29914412c3: echo 1 > /sys/block/md0/md/chunk_size mdadm --grow /dev/md0 --raid-devices=4 --backup-file=/root/md0-grow.bak Oops: divide error: 0000 [#1] SMP KASAN NOPTI RIP: raid5_get_active_stripe+0x863/0xc10 Call Trace: raid5_sync_request md_do_sync md_thread Kernel panic - not syncing: Fatal exception Fixed kernel: echo 1 > /sys/block/md0/md/chunk_size bash: echo: write error: Invalid argument chunk_write_rc=1 grow_rc=skipped RESULT: REJECTED_ZERO_CHUNK_NO_OOPS Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-5-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index e2c5a7072aca..d128d238e1da 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -8548,6 +8548,8 @@ static int check_reshape(struct mddev *mddev) return 0; /* nothing to do */ if (has_failed(conf)) return -EINVAL; + if (!mddev->new_chunk_sectors) + return -EINVAL; if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { /* We might be able to shrink, but the devices must * be made bigger first. @@ -8591,6 +8593,9 @@ static int raid5_start_reshape(struct mddev *mddev) if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) return -EBUSY; + if (!mddev->new_chunk_sectors) + return -EINVAL; + if (!check_stripe_cache(mddev)) return -ENOSPC; From 17ea021ae74987d6064c8195c4922fa025753892 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:14 +0800 Subject: [PATCH 147/241] md/raid5: round bitmap stripes with sector division raid5_bitmap_sector_map() aligns the array range to full RAID5 stripe widths before converting it to component sectors. That width is chunk_sectors multiplied by the number of data disks, and it is not always a power of two. Reproduce with a 4-disk RAID5, 1024-sector chunks, and three data disks. The full-stripe width is 3072 sectors. For a one-sector write at array sector 3072, correct rounding gives array range [3072, 6144), which maps to component range [1024, 2048). The old round_down()/round_up() logic instead gives [1024, 4096), which maps to [0, 1024). Use sector_div() based arithmetic so the rounded range is aligned to the actual RAID5 stripe width. The deterministic mapper test now reports the fixed component range as [1024, 2048), while the old mask-based range was [0, 1024). Fixes: 9c89f604476c ("md/raid5: implement pers->bitmap_sector()") Reported-by: Mykola Marzhan Link: https://lore.kernel.org/all/20260726185916.2223460-1-mykola@meshstor.io/ Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-6-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index d128d238e1da..2cc2546a29ae 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -6029,8 +6029,11 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sectors_per_chunk = conf->chunk_sectors * (conf->raid_disks - conf->max_degraded); - start = round_down(start, sectors_per_chunk); - end = round_up(end, sectors_per_chunk); + sector_div(start, sectors_per_chunk); + start *= sectors_per_chunk; + if (sector_div(end, sectors_per_chunk)) + end++; + end *= sectors_per_chunk; start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL); end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL); @@ -6048,8 +6051,10 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sectors_per_chunk = conf->prev_chunk_sectors * (conf->previous_raid_disks - conf->max_degraded); - prev_start = round_down(prev_start, sectors_per_chunk); - prev_end = round_down(prev_end, sectors_per_chunk); + sector_div(prev_start, sectors_per_chunk); + prev_start *= sectors_per_chunk; + sector_div(prev_end, sectors_per_chunk); + prev_end *= sectors_per_chunk; prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL); prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL); From 2a79365b2278f16e163e4024086105693b421601 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:15 +0800 Subject: [PATCH 148/241] md: wait for behind writes before destroying bitmap __md_stop() destroyed the bitmap before calling mddev_detach(). That made mddev_detach() skip bitmap_ops->wait_behind_writes(), because the bitmap was already disconnected from mddev. This was still safe for the legacy bitmap because bitmap_destroy() waits for behind writes itself. llbitmap keeps that wait in its ->wait_behind_writes() operation instead, while ->destroy() tears down the llbitmap storage. With the old ordering, RAID1 behind-write completions could still run after llbitmap storage had been freed. Call mddev_detach() before md_bitmap_destroy() so the common detach path can wait for behind writes while the bitmap is still alive. Only destroy the bitmap after those users are gone. Fixes: 5ab829f1971d ("md/md-llbitmap: introduce new lockless bitmap") Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-7-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 51b620edbef7..b61040315aef 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -7085,8 +7085,8 @@ static void __md_stop(struct mddev *mddev) { struct md_personality *pers = mddev->pers; - md_bitmap_destroy(mddev); mddev_detach(mddev); + md_bitmap_destroy(mddev); spin_lock(&mddev->lock); mddev->pers = NULL; spin_unlock(&mddev->lock); From 45102fc8330525d35675b1c193242bba101df5ee Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:16 +0800 Subject: [PATCH 149/241] md: avoid stale clone I/O accounting timestamps md_clone_bio() always allocates the clone from mddev->io_clone_set, even when queue I/O stats are disabled. In that case it does not call bio_start_io_acct(), but it also left md_io_clone->start_time untouched. The clone private data comes from a mempool and can contain data from a previous user. md_end_clone_io() checks start_time to decide whether it needs to call bio_end_io_acct(), so a stale non-zero value can make the completion path end accounting that was never started for this bio. Set start_time to 0 in the no-stats branch. This keeps the end path tied to whether bio_start_io_acct() actually ran. Fixes: c687297b8845 ("md: also clone new io if io accounting is disabled") Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-8-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/md/md.c b/drivers/md/md.c index b61040315aef..58fb5453a819 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9448,6 +9448,8 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio) md_io_clone->mddev = mddev; if (blk_queue_io_stat(bdev->bd_disk->queue)) md_io_clone->start_time = bio_start_io_acct(*bio); + else + md_io_clone->start_time = 0; if (bio_data_dir(*bio) == WRITE && md_bitmap_enabled(mddev, false)) { md_io_clone->offset = (*bio)->bi_iter.bi_sector; From 2116c2f0a0e547615886900e2ed8c529c016499b Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:17 +0800 Subject: [PATCH 150/241] md/md-llbitmap: prevent create failure bitmap UAF llbitmap_create() publishes mddev->bitmap before reading the bitmap superblock. This is needed because llbitmap_read_sb() can initialize a new bitmap and flush it through helpers that use mddev->bitmap. If llbitmap_read_sb() fails, the old cleanup dropped bitmap_info.mutex and freed llbitmap before clearing mddev->bitmap. Readers such as /proc/mdstat rely on bitmap_info.mutex to keep the bitmap pointer stable while collecting bitmap stats, so they could observe the stale pointer after the failed create path released the mutex. Clear mddev->bitmap while still holding bitmap_info.mutex, then free the failed llbitmap after dropping the mutex. This makes mutex-protected readers see either a live bitmap or no bitmap. Fixes: 5ab829f1971d ("md/md-llbitmap: introduce new lockless bitmap") Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-9-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 948bf64c5ad2..af80a630bd21 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1126,10 +1126,11 @@ static int llbitmap_create(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); mddev->bitmap = llbitmap; ret = llbitmap_read_sb(llbitmap); + if (ret) + mddev->bitmap = NULL; mutex_unlock(&mddev->bitmap_info.mutex); if (ret) { kfree(llbitmap); - mddev->bitmap = NULL; } return ret; From 5553d64e01d9a995be6c3de38501c6dd4ceede3b Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:18 +0800 Subject: [PATCH 151/241] md/md-llbitmap: stop daemon timer rearm on destroy llbitmap_destroy() deletes pending_timer before flushing md_llbitmap_io_wq. However, daemon_work can still be queued or running after the timer has been deleted, and the daemon path can arm pending_timer again when it finds dirty chunks that are not ready to flush yet. If that happens during teardown, pending_timer can remain armed after llbitmap is freed and later dereference freed memory. Add a BITMAP_SHUTDOWN bit to llbitmap->flags, set it before deleting the timer, and make the timer and daemon paths stop queueing or rearming work once teardown starts. Cancel daemon_work before flushing the shared workqueue so no already queued daemon instance can race with the free. Use timer_shutdown_sync() so a daemon instance that passed the shutdown check before teardown cannot rearm the timer afterward. BITMAP_SHUTDOWN is a runtime-only state. Mask it out when reading and updating the llbitmap superblock so the shutdown state is never loaded from disk or persisted to disk. Fixes: 5ab829f1971d ("md/md-llbitmap: introduce new lockless bitmap") Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-10-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-bitmap.h | 1 + drivers/md/md-llbitmap.c | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index 214f623c7e79..890276d9c66e 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -29,6 +29,7 @@ enum bitmap_state { BITMAP_FIRST_USE = 3, /* llbitmap is just created */ BITMAP_CLEAN = 4, /* llbitmap is created with assume_clean */ BITMAP_DAEMON_BUSY = 5, /* llbitmap daemon is not finished after daemon_sleep */ + BITMAP_SHUTDOWN = 6, /* llbitmap is being destroyed */ BITMAP_HOSTENDIAN =15, }; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index af80a630bd21..f5efecdb2cc4 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -789,6 +789,7 @@ write_bitmap: if (state == BitNeedSync || state == BitNeedSyncUnwritten) need_resync = !mddev->degraded; else if (state == BitDirty && + !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags) && !timer_pending(&llbitmap->pending_timer)) mod_timer(&llbitmap->pending_timer, jiffies + mddev->bitmap_info.daemon_sleep * HZ); @@ -981,7 +982,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) else mddev->bitmap_info.space = mddev->bitmap_info.default_space; } - llbitmap->flags = le32_to_cpu(sb->state); + llbitmap->flags = le32_to_cpu(sb->state) & ~BIT(BITMAP_SHUTDOWN); if (test_and_clear_bit(BITMAP_FIRST_USE, &llbitmap->flags)) { ret = llbitmap_init(llbitmap); goto out_put_page; @@ -1037,6 +1038,9 @@ static void llbitmap_pending_timer_fn(struct timer_list *pending_timer) struct llbitmap *llbitmap = container_of(pending_timer, struct llbitmap, pending_timer); + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (work_busy(&llbitmap->daemon_work)) { pr_warn("md/llbitmap: %s daemon_work not finished in %lu seconds\n", mdname(llbitmap->mddev), @@ -1057,6 +1061,9 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) bool restart; int idx; + if (test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) + return; + if (llbitmap->mddev->degraded) return; retry: @@ -1096,7 +1103,7 @@ retry: goto retry; /* If some page is dirty but not expired, setup timer again */ - if (restart) + if (restart && !test_bit(BITMAP_SHUTDOWN, &llbitmap->flags)) mod_timer(&llbitmap->pending_timer, jiffies + llbitmap->mddev->bitmap_info.daemon_sleep * HZ); } @@ -1179,7 +1186,9 @@ static void llbitmap_destroy(struct mddev *mddev) mutex_lock(&mddev->bitmap_info.mutex); - timer_delete_sync(&llbitmap->pending_timer); + set_bit(BITMAP_SHUTDOWN, &llbitmap->flags); + timer_shutdown_sync(&llbitmap->pending_timer); + cancel_work_sync(&llbitmap->daemon_work); flush_workqueue(md_llbitmap_io_wq); flush_workqueue(md_llbitmap_unplug_wq); @@ -1523,7 +1532,7 @@ static void llbitmap_update_sb(void *data) sb = kmap_local_page(sb_page); sb->events = cpu_to_le64(mddev->events); - sb->state = cpu_to_le32(llbitmap->flags); + sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN)); sb->chunksize = cpu_to_le32(llbitmap->chunksize); sb->sync_size = cpu_to_le64(mddev->resync_max_sectors); sb->events_cleared = cpu_to_le64(llbitmap->events_cleared); From 87c10252e3d6c28769d93bea9e6a9b592e5809d1 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:19 +0800 Subject: [PATCH 152/241] md: skip bitmap accounting for empty write ranges mkfs.ext4 can submit zero-sector flush/FUA bios. These bios are WRITE bios for md_write_start() purposes, but they do not cover any data sector and must not dirty bitmap bits. md bitmap accounting currently passes such bios to bitmap start_write(). For llbitmap this reaches llbitmap_start_write() with sectors == 0, which underflows the end chunk calculation. Personality bitmap mapping can also turn a non-empty bio into an empty bitmap range when the requested sectors are outside the active bitmap geometry. Treat both cases as not started, so the completion path will not call end_write() for an empty range. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-11-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 58fb5453a819..f88952371b9b 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9399,6 +9399,8 @@ static void md_bitmap_start(struct mddev *mddev, mddev->pers->bitmap_sector(mddev, &md_io_clone->offset, &md_io_clone->sectors); + if (!md_io_clone->sectors) + return; fn(mddev, md_io_clone->offset, md_io_clone->sectors); } @@ -9419,7 +9421,8 @@ static void md_end_clone_io(struct bio *bio) struct mddev *mddev = md_io_clone->mddev; struct completion *reshape_completion = bio->bi_private; - if (bio_data_dir(orig_bio) == WRITE && md_bitmap_enabled(mddev, false)) + if (bio_data_dir(orig_bio) == WRITE && md_io_clone->sectors && + md_bitmap_enabled(mddev, false)) md_bitmap_end(mddev, md_io_clone); if (bio->bi_status && !orig_bio->bi_status) @@ -9446,12 +9449,14 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio) md_io_clone = container_of(clone, struct md_io_clone, bio_clone); md_io_clone->orig_bio = *bio; md_io_clone->mddev = mddev; + md_io_clone->sectors = 0; if (blk_queue_io_stat(bdev->bd_disk->queue)) md_io_clone->start_time = bio_start_io_acct(*bio); else md_io_clone->start_time = 0; - if (bio_data_dir(*bio) == WRITE && md_bitmap_enabled(mddev, false)) { + if (bio_data_dir(*bio) == WRITE && bio_sectors(*bio) && + md_bitmap_enabled(mddev, false)) { md_io_clone->offset = (*bio)->bi_iter.bi_sector; md_io_clone->sectors = bio_sectors(*bio); md_io_clone->rw = op_stat_group(bio_op(*bio)); From ffd3e73d6e0c88f1cc7f5137978427a404fa5026 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:20 +0800 Subject: [PATCH 153/241] md: add helper to split bios at reshape offset Add mddev_bio_split_at_reshape_offset() so personalities can share reshape-offset bio splitting instead of open-coding the same boundary handling in multiple places. The helper first applies the optional max_sectors limit. If reshape is running and the bio crosses reshape_position, it further limits the front bio to the current reshape boundary so callers can account and submit one side of the reshape at a time. Snapshot reshape_position with READ_ONCE(). RAID5 and RAID10 update this field as reshape progresses, while the I/O path only needs one consistent decision point for the current bio. Using an explicit single load avoids a plain lockless access and prevents the compiler from refetching a different boundary while deciding whether and where to split. When a split is needed, bio_submit_split_bioset() submits the remainder and returns the front bio. Callers must therefore continue processing the returned bio, not the original pointer. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-12-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md.c | 39 +++++++++++++++++++++++++++++++++++++++ drivers/md/md.h | 4 ++++ 2 files changed, 43 insertions(+) diff --git a/drivers/md/md.c b/drivers/md/md.c index f88952371b9b..f0eecdfff1cc 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9388,6 +9388,45 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev, } EXPORT_SYMBOL_GPL(md_submit_discard_bio); +struct bio *mddev_bio_split_at_reshape_offset(struct mddev *mddev, + struct bio *bio, + unsigned int *max_sectors, + struct bio_set *bs) +{ + sector_t boundary; + sector_t start; + sector_t end; + unsigned int split_sectors; + + split_sectors = bio_sectors(bio); + if (max_sectors && *max_sectors && *max_sectors < split_sectors) + split_sectors = *max_sectors; + + if (!test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) + goto split; + + boundary = READ_ONCE(mddev->reshape_position); + start = bio->bi_iter.bi_sector; + end = bio_end_sector(bio); + if (start >= boundary || end <= boundary) + goto split; + + if (boundary - start < split_sectors) + split_sectors = boundary - start; + +split: + if (max_sectors) + *max_sectors = split_sectors; + if (split_sectors < bio_sectors(bio)) { + bio = bio_submit_split_bioset(bio, split_sectors, bs); + if (bio) + bio->bi_opf |= REQ_NOMERGE; + } + + return bio; +} +EXPORT_SYMBOL_GPL(mddev_bio_split_at_reshape_offset); + static void md_bitmap_start(struct mddev *mddev, struct md_io_clone *md_io_clone) { diff --git a/drivers/md/md.h b/drivers/md/md.h index bb2eb5f39914..8146a6f50a7d 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -920,6 +920,10 @@ extern void md_error(struct mddev *mddev, struct md_rdev *rdev); extern void md_finish_reshape(struct mddev *mddev); void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev, struct bio *bio, sector_t start, sector_t size); +struct bio *mddev_bio_split_at_reshape_offset(struct mddev *mddev, + struct bio *bio, + unsigned int *max_sectors, + struct bio_set *bs); void md_account_bio(struct mddev *mddev, struct bio **bio); extern bool __must_check md_flush_request(struct mddev *mddev, struct bio *bio); From 1ee6fef6e0dbff21fecfc1db05c79ede464e9500 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:21 +0800 Subject: [PATCH 154/241] md: add exact bitmap mapping and reshape hooks Add bitmap mapping and reshape hooks needed by llbitmap reshape support without teaching md core to account a single bio against multiple bitmap ranges. This also adds the old/new bitmap geometry helpers used by personalities to describe reshape mapping to llbitmap. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-13-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-bitmap.c | 8 ++++++++ drivers/md/md-bitmap.h | 8 ++++++++ drivers/md/md-llbitmap.c | 8 ++++++++ drivers/md/md.c | 11 ++++++++--- drivers/md/md.h | 4 ++++ 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 7e4fbca93ccb..b7f0d4acce04 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -1730,6 +1730,13 @@ static void bitmap_start_write(struct mddev *mddev, sector_t offset, } } +static void bitmap_prepare_range(struct mddev *mddev, sector_t *offset, + unsigned long *sectors) +{ + if (mddev->pers->bitmap_sector) + mddev->pers->bitmap_sector(mddev, offset, sectors); +} + static void bitmap_end_write(struct mddev *mddev, sector_t offset, unsigned long sectors) { @@ -3081,6 +3088,7 @@ static struct bitmap_operations bitmap_ops = { .flush = bitmap_flush, .write_all = bitmap_write_all, .dirty_bits = bitmap_dirty_bits, + .prepare_range = bitmap_prepare_range, .unplug = bitmap_unplug, .daemon_work = bitmap_daemon_work, diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index 890276d9c66e..6478cf9d8816 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -94,6 +94,14 @@ struct bitmap_operations { void (*write_all)(struct mddev *mddev); void (*dirty_bits)(struct mddev *mddev, unsigned long s, unsigned long e); + /* Prepare a range for this bitmap implementation. */ + void (*prepare_range)(struct mddev *mddev, + sector_t *offset, + unsigned long *sectors); + void (*reshape_finish)(struct mddev *mddev); + int (*reshape_can_start)(struct mddev *mddev); + void (*reshape_mark)(struct mddev *mddev, sector_t old_pos, + sector_t new_pos); void (*unplug)(struct mddev *mddev, bool sync); void (*daemon_work)(struct mddev *mddev); diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index f5efecdb2cc4..1c361f5a97f4 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1198,6 +1198,13 @@ static void llbitmap_destroy(struct mddev *mddev) mutex_unlock(&mddev->bitmap_info.mutex); } +static void llbitmap_prepare_range(struct mddev *mddev, sector_t *offset, + unsigned long *sectors) +{ + if (mddev->pers->bitmap_sector) + mddev->pers->bitmap_sector(mddev, offset, sectors); +} + static void llbitmap_start_write(struct mddev *mddev, sector_t offset, unsigned long sectors) { @@ -1789,6 +1796,7 @@ static struct bitmap_operations llbitmap_ops = { .update_sb = llbitmap_update_sb, .get_stats = llbitmap_get_stats, .dirty_bits = llbitmap_dirty_bits, + .prepare_range = llbitmap_prepare_range, .write_all = llbitmap_write_all, .groups = md_llbitmap_groups, diff --git a/drivers/md/md.c b/drivers/md/md.c index f0eecdfff1cc..538ba7bab060 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9427,6 +9427,12 @@ split: } EXPORT_SYMBOL_GPL(mddev_bio_split_at_reshape_offset); +static void md_bitmap_prepare_range(struct mddev *mddev, sector_t *offset, + unsigned long *sectors) +{ + mddev->bitmap_ops->prepare_range(mddev, offset, sectors); +} + static void md_bitmap_start(struct mddev *mddev, struct md_io_clone *md_io_clone) { @@ -9434,9 +9440,8 @@ static void md_bitmap_start(struct mddev *mddev, mddev->bitmap_ops->start_discard : mddev->bitmap_ops->start_write; - if (mddev->pers->bitmap_sector) - mddev->pers->bitmap_sector(mddev, &md_io_clone->offset, - &md_io_clone->sectors); + md_bitmap_prepare_range(mddev, &md_io_clone->offset, + &md_io_clone->sectors); if (!md_io_clone->sectors) return; diff --git a/drivers/md/md.h b/drivers/md/md.h index 8146a6f50a7d..b6d2e8929a0f 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -797,6 +797,10 @@ struct md_personality /* convert io ranges from array to bitmap */ void (*bitmap_sector)(struct mddev *mddev, sector_t *offset, unsigned long *sectors); + void (*bitmap_sector_map)(struct mddev *mddev, sector_t *offset, + unsigned long *sectors, bool previous); + sector_t (*bitmap_sync_size)(struct mddev *mddev, bool previous); + sector_t (*bitmap_array_sectors)(struct mddev *mddev, bool previous); }; struct md_sysfs_entry { From 35320d21e8b90198e94b95a17813415265d5742f Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:22 +0800 Subject: [PATCH 155/241] md/md-llbitmap: track bitmap sync_size explicitly Track llbitmap's own sync_size instead of always using mddev->resync_max_sectors directly. This is the minimal bookkeeping needed before llbitmap can track old and new reshape geometry independently. Reviewed-by: Su Yue Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-14-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 1c361f5a97f4..dfb1aff9f485 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -287,6 +287,8 @@ struct llbitmap { unsigned long chunksize; /* total number of chunks */ unsigned long chunks; + /* total number of sectors tracked by current bitmap geometry */ + sector_t sync_size; unsigned long last_end_sync; /* * time in seconds that dirty bits will be cleared if the page is not @@ -919,6 +921,7 @@ static int llbitmap_init(struct llbitmap *llbitmap) llbitmap->chunkshift = ffz(~chunksize); llbitmap->chunksize = chunksize; llbitmap->chunks = chunks; + llbitmap->sync_size = blocks; mddev->bitmap_info.daemon_sleep = DEFAULT_DAEMON_SLEEP; ret = llbitmap_cache_pages(llbitmap); @@ -939,6 +942,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) unsigned long daemon_sleep; unsigned long chunksize; unsigned long events; + sector_t sync_size; struct page *sb_page; bitmap_super_t *sb; int ret = -EINVAL; @@ -988,6 +992,14 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) goto out_put_page; } + sync_size = le64_to_cpu(sb->sync_size); + if (!sync_size) + sync_size = mddev->resync_max_sectors; + if (sync_size > mddev->resync_max_sectors) { + pr_err("md/llbitmap: %s: sync_size %llu exceeds array sync size %llu", + mdname(mddev), sync_size, mddev->resync_max_sectors); + goto out_put_page; + } chunksize = le32_to_cpu(sb->chunksize); if (!is_power_of_2(chunksize)) { pr_err("md/llbitmap: %s: chunksize not a power of 2", @@ -1023,8 +1035,9 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) llbitmap->barrier_idle = DEFAULT_BARRIER_IDLE; llbitmap->chunksize = chunksize; - llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(mddev->resync_max_sectors, chunksize); + llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(sync_size, chunksize); llbitmap->chunkshift = ffz(~chunksize); + llbitmap->sync_size = sync_size; ret = llbitmap_cache_pages(llbitmap); out_put_page: @@ -1161,6 +1174,7 @@ static int llbitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize) llbitmap->chunkshift = ffz(~chunksize); llbitmap->chunksize = chunksize; llbitmap->chunks = chunks; + llbitmap->sync_size = blocks; return 0; } @@ -1541,7 +1555,7 @@ static void llbitmap_update_sb(void *data) sb->events = cpu_to_le64(mddev->events); sb->state = cpu_to_le32(llbitmap->flags & ~BIT(BITMAP_SHUTDOWN)); sb->chunksize = cpu_to_le32(llbitmap->chunksize); - sb->sync_size = cpu_to_le64(mddev->resync_max_sectors); + sb->sync_size = cpu_to_le64(llbitmap->sync_size); sb->events_cleared = cpu_to_le64(llbitmap->events_cleared); sb->sectors_reserved = cpu_to_le32(mddev->bitmap_info.space); sb->daemon_sleep = cpu_to_le32(mddev->bitmap_info.daemon_sleep); @@ -1559,6 +1573,7 @@ static int llbitmap_get_stats(void *data, struct md_bitmap_stats *stats) stats->missing_pages = 0; stats->pages = llbitmap->nr_pages; stats->file_pages = llbitmap->nr_pages; + stats->sync_size = llbitmap->sync_size; stats->behind_writes = atomic_read(&llbitmap->behind_writes); stats->behind_wait = wq_has_sleeper(&llbitmap->behind_wait); From e9f0d66b5754888866b10fa0ff6c8642bb850028 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:23 +0800 Subject: [PATCH 156/241] md/md-llbitmap: allocate page controls independently Allocate one llbitmap page-control object at a time and free each object through the same model. Let llbitmap_read_page() return a zeroed page without reading disk when the page index is beyond the current bitmap size, so page-control allocation no longer needs a separate read_existing flag. This keeps the llbitmap page-control lifetime self-consistent and prepares the page-cache code for later in-place growth. Reviewed-by: Su Yue Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-15-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 99 +++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 37 deletions(-) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index dfb1aff9f485..5fec1db53436 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -512,13 +512,19 @@ static void llbitmap_write(struct llbitmap *llbitmap, enum llbitmap_state state, llbitmap_set_page_dirty(llbitmap, idx, bit, false); } +static unsigned int llbitmap_used_pages(struct llbitmap *llbitmap, + unsigned long chunks) +{ + return DIV_ROUND_UP(chunks + BITMAP_DATA_OFFSET, PAGE_SIZE); +} + static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx) { struct mddev *mddev = llbitmap->mddev; struct page *page = NULL; struct md_rdev *rdev; - if (llbitmap->pctl && llbitmap->pctl[idx]) + if (llbitmap->pctl && idx < llbitmap->nr_pages && llbitmap->pctl[idx]) page = llbitmap->pctl[idx]->page; if (page) return page; @@ -526,6 +532,8 @@ static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx) page = alloc_page(GFP_NOIO | __GFP_ZERO); if (!page) return ERR_PTR(-ENOMEM); + if (idx >= llbitmap_used_pages(llbitmap, llbitmap->chunks)) + return page; rdev_for_each(rdev, mddev) { sector_t sector; @@ -596,61 +604,78 @@ static void llbitmap_free_pages(struct llbitmap *llbitmap) for (i = 0; i < llbitmap->nr_pages; i++) { struct llbitmap_page_ctl *pctl = llbitmap->pctl[i]; - if (!pctl || !pctl->page) - break; - - __free_page(pctl->page); + if (!pctl) + continue; + if (pctl->page) + __free_page(pctl->page); percpu_ref_exit(&pctl->active); + kfree(pctl); } - kfree(llbitmap->pctl[0]); kfree(llbitmap->pctl); llbitmap->pctl = NULL; } -static int llbitmap_cache_pages(struct llbitmap *llbitmap) +static struct llbitmap_page_ctl * +llbitmap_alloc_page_ctl(struct llbitmap *llbitmap, int idx) { struct llbitmap_page_ctl *pctl; - unsigned int nr_pages = DIV_ROUND_UP(llbitmap->chunks + - BITMAP_DATA_OFFSET, PAGE_SIZE); + struct page *page; unsigned int size = struct_size(pctl, dirty, BITS_TO_LONGS( llbitmap->blocks_per_page)); + + size = round_up(size, cache_line_size()); + pctl = kzalloc(size, GFP_NOIO); + if (!pctl) + return ERR_PTR(-ENOMEM); + + page = llbitmap_read_page(llbitmap, idx); + + if (IS_ERR(page)) { + kfree(pctl); + return ERR_CAST(page); + } + + if (percpu_ref_init(&pctl->active, active_release, + PERCPU_REF_ALLOW_REINIT, GFP_NOIO)) { + __free_page(page); + kfree(pctl); + return ERR_PTR(-ENOMEM); + } + + pctl->page = page; + pctl->state = page_address(page); + init_waitqueue_head(&pctl->wait); + return pctl; +} + +static unsigned int llbitmap_reserved_pages(struct llbitmap *llbitmap) +{ + return DIV_ROUND_UP(llbitmap->mddev->bitmap_info.space << SECTOR_SHIFT, + PAGE_SIZE); +} + +static int llbitmap_alloc_pages(struct llbitmap *llbitmap) +{ + unsigned int used_pages = llbitmap_used_pages(llbitmap, llbitmap->chunks); + unsigned int nr_pages = max(used_pages, llbitmap_reserved_pages(llbitmap)); int i; - llbitmap->pctl = kmalloc_array(nr_pages, sizeof(void *), - GFP_NOIO | __GFP_ZERO); + llbitmap->pctl = kcalloc(nr_pages, sizeof(*llbitmap->pctl), GFP_NOIO); if (!llbitmap->pctl) return -ENOMEM; - size = round_up(size, cache_line_size()); - pctl = kmalloc_array(nr_pages, size, GFP_NOIO | __GFP_ZERO); - if (!pctl) { - kfree(llbitmap->pctl); - return -ENOMEM; - } - llbitmap->nr_pages = nr_pages; - for (i = 0; i < nr_pages; i++, pctl = (void *)pctl + size) { - struct page *page = llbitmap_read_page(llbitmap, i); + for (i = 0; i < nr_pages; i++) { + llbitmap->pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i); + if (IS_ERR(llbitmap->pctl[i])) { + int ret = PTR_ERR(llbitmap->pctl[i]); - llbitmap->pctl[i] = pctl; - - if (IS_ERR(page)) { + llbitmap->pctl[i] = NULL; llbitmap_free_pages(llbitmap); - return PTR_ERR(page); + return ret; } - - if (percpu_ref_init(&pctl->active, active_release, - PERCPU_REF_ALLOW_REINIT, GFP_NOIO)) { - __free_page(page); - llbitmap_free_pages(llbitmap); - return -ENOMEM; - } - - pctl->page = page; - pctl->state = page_address(page); - init_waitqueue_head(&pctl->wait); } return 0; @@ -924,7 +949,7 @@ static int llbitmap_init(struct llbitmap *llbitmap) llbitmap->sync_size = blocks; mddev->bitmap_info.daemon_sleep = DEFAULT_DAEMON_SLEEP; - ret = llbitmap_cache_pages(llbitmap); + ret = llbitmap_alloc_pages(llbitmap); if (ret) return ret; @@ -1038,7 +1063,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(sync_size, chunksize); llbitmap->chunkshift = ffz(~chunksize); llbitmap->sync_size = sync_size; - ret = llbitmap_cache_pages(llbitmap); + ret = llbitmap_alloc_pages(llbitmap); out_put_page: __free_page(sb_page); From 1cbc6ea5fa6527853803345cb6099917b77b0c61 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:24 +0800 Subject: [PATCH 157/241] md/md-llbitmap: grow the page cache in place for reshape Use the page-control helpers to grow llbitmap's cached pages in place for resize and later reshape preparation, instead of rebuilding the whole cache. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-16-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 143 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 132 insertions(+), 11 deletions(-) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 5fec1db53436..c34accb9233a 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -416,6 +416,19 @@ static char state_machine[BitStateCount][BitmapActionCount] = { }; static void __llbitmap_flush(struct mddev *mddev); +static void llbitmap_flush(struct mddev *mddev); +static void llbitmap_update_sb(void *data); + +static void llbitmap_calculate_chunks(struct mddev *mddev, sector_t blocks, + unsigned long *chunksize, + unsigned long *chunks) +{ + *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize); + while (*chunks > mddev->bitmap_info.space << SECTOR_SHIFT) { + *chunksize = *chunksize << 1; + *chunks = DIV_ROUND_UP_SECTOR_T(blocks, *chunksize); + } +} static enum llbitmap_state llbitmap_read(struct llbitmap *llbitmap, loff_t pos) { @@ -655,6 +668,48 @@ static unsigned int llbitmap_reserved_pages(struct llbitmap *llbitmap) PAGE_SIZE); } +static int llbitmap_expand_pages(struct llbitmap *llbitmap, + unsigned long chunks) +{ + struct llbitmap_page_ctl **pctl; + unsigned int old_nr_pages = llbitmap->nr_pages; + unsigned int nr_pages = llbitmap_used_pages(llbitmap, chunks); + unsigned int i; + int ret; + + if (nr_pages <= old_nr_pages) + return 0; + + pctl = kcalloc(nr_pages, sizeof(*pctl), GFP_NOIO); + if (!pctl) + return -ENOMEM; + + if (llbitmap->pctl) + memcpy(pctl, llbitmap->pctl, + array_size(old_nr_pages, sizeof(*pctl))); + + for (i = old_nr_pages; i < nr_pages; i++) { + pctl[i] = llbitmap_alloc_page_ctl(llbitmap, i); + if (IS_ERR(pctl[i])) + goto err_alloc_ptr; + } + + kfree(llbitmap->pctl); + llbitmap->pctl = pctl; + llbitmap->nr_pages = nr_pages; + return 0; + +err_alloc_ptr: + ret = PTR_ERR(pctl[i]); + while (i-- > old_nr_pages) { + __free_page(pctl[i]->page); + percpu_ref_exit(&pctl[i]->active); + kfree(pctl[i]); + } + kfree(pctl); + return ret; +} + static int llbitmap_alloc_pages(struct llbitmap *llbitmap) { unsigned int used_pages = llbitmap_used_pages(llbitmap, llbitmap->chunks); @@ -730,6 +785,34 @@ static bool llbitmap_zero_all_disks(struct llbitmap *llbitmap) return true; } +static void llbitmap_mark_range(struct llbitmap *llbitmap, + unsigned long start, + unsigned long end, + enum llbitmap_state state) +{ + while (start <= end) { + llbitmap_write(llbitmap, state, start); + start++; + } +} + +static int llbitmap_prepare_resize(struct llbitmap *llbitmap, + unsigned long old_chunks, + unsigned long new_chunks, + unsigned long cache_chunks) +{ + int ret; + + llbitmap_flush(llbitmap->mddev); + ret = llbitmap_expand_pages(llbitmap, cache_chunks); + if (ret) + return ret; + if (new_chunks > old_chunks) + llbitmap_mark_range(llbitmap, old_chunks, new_chunks - 1, + BitUnwritten); + return 0; +} + static void llbitmap_init_state(struct llbitmap *llbitmap) { struct mddev *mddev = llbitmap->mddev; @@ -1032,10 +1115,10 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) goto out_put_page; } - if (chunksize < DIV_ROUND_UP_SECTOR_T(mddev->resync_max_sectors, + if (chunksize < DIV_ROUND_UP_SECTOR_T(sync_size, mddev->bitmap_info.space << SECTOR_SHIFT)) { pr_err("md/llbitmap: %s: chunksize too small %lu < %llu / %lu", - mdname(mddev), chunksize, mddev->resync_max_sectors, + mdname(mddev), chunksize, sync_size, mddev->bitmap_info.space); goto out_put_page; } @@ -1184,24 +1267,62 @@ static int llbitmap_create(struct mddev *mddev) static int llbitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize) { struct llbitmap *llbitmap = mddev->bitmap; + sector_t old_blocks = llbitmap->sync_size; + unsigned long old_chunks = llbitmap->chunks; unsigned long chunks; + unsigned long cache_chunks; + int ret = 0; + unsigned long bitmap_chunksize; + bool reshape; + bool quiesced = false; if (chunksize == 0) chunksize = llbitmap->chunksize; - /* If there is enough space, leave the chunksize unchanged. */ - chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); - while (chunks > mddev->bitmap_info.space << SECTOR_SHIFT) { - chunksize = chunksize << 1; - chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); + bitmap_chunksize = chunksize; + llbitmap_calculate_chunks(mddev, blocks, &bitmap_chunksize, &chunks); + + reshape = mddev->delta_disks || mddev->new_level != mddev->level || + mddev->new_layout != mddev->layout || + mddev->new_chunk_sectors != mddev->chunk_sectors; + if (!reshape && bitmap_chunksize != llbitmap->chunksize) + return -EOPNOTSUPP; + if (blocks == old_blocks && chunks == llbitmap->chunks) + return 0; + + if (mddev->pers->quiesce) { + mddev->pers->quiesce(mddev, 1); + quiesced = true; } - llbitmap->chunkshift = ffz(~chunksize); - llbitmap->chunksize = chunksize; - llbitmap->chunks = chunks; - llbitmap->sync_size = blocks; + mutex_lock(&mddev->bitmap_info.mutex); + cache_chunks = reshape ? max(old_chunks, chunks) : chunks; + ret = llbitmap_prepare_resize(llbitmap, old_chunks, chunks, cache_chunks); + if (ret) + goto out; + if (reshape) { + llbitmap->chunks = max(old_chunks, chunks); + } else { + if (blocks < old_blocks && chunks < old_chunks) + llbitmap_mark_range(llbitmap, chunks, old_chunks - 1, + BitUnwritten); + mddev->bitmap_info.chunksize = bitmap_chunksize; + llbitmap->chunks = chunks; + llbitmap->sync_size = blocks; + llbitmap_update_sb(llbitmap); + } + __llbitmap_flush(mddev); + mutex_unlock(&mddev->bitmap_info.mutex); + if (quiesced) + mddev->pers->quiesce(mddev, 0); return 0; + +out: + mutex_unlock(&mddev->bitmap_info.mutex); + if (quiesced) + mddev->pers->quiesce(mddev, 0); + return ret; } static int llbitmap_load(struct mddev *mddev) From 3a92c67aef5aeb0ccf46cf291be87dac0485fd5f Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:25 +0800 Subject: [PATCH 158/241] md/md-llbitmap: track target reshape geometry fields Track llbitmap bookkeeping for the target reshape geometry while keeping a single live bitmap instance. Add the reshape geometry fields, refresh helper, and update the load and resize paths to keep the target geometry in sync. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-17-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index c34accb9233a..421a9a4ebbae 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -289,6 +289,9 @@ struct llbitmap { unsigned long chunks; /* total number of sectors tracked by current bitmap geometry */ sector_t sync_size; + unsigned long reshape_chunksize; + unsigned long reshape_chunks; + sector_t reshape_sync_size; unsigned long last_end_sync; /* * time in seconds that dirty bits will be cleared if the page is not @@ -430,6 +433,39 @@ static void llbitmap_calculate_chunks(struct mddev *mddev, sector_t blocks, } } +static bool llbitmap_reshaping(struct llbitmap *llbitmap) +{ + return llbitmap->mddev->reshape_position != MaxSector; +} + +static sector_t llbitmap_personality_sync_size(struct llbitmap *llbitmap, + bool previous) +{ + struct mddev *mddev = llbitmap->mddev; + + if (!llbitmap_reshaping(llbitmap) || !mddev->private || !mddev->pers || + !mddev->pers->bitmap_sync_size) + return llbitmap->sync_size; + return mddev->pers->bitmap_sync_size(mddev, previous); +} + +static void llbitmap_refresh_reshape(struct llbitmap *llbitmap) +{ + unsigned long old_chunks = DIV_ROUND_UP_SECTOR_T(llbitmap->sync_size, + llbitmap->chunksize); + sector_t blocks = llbitmap_personality_sync_size(llbitmap, false); + unsigned long chunksize = llbitmap->chunksize; + unsigned long chunks = DIV_ROUND_UP_SECTOR_T(blocks, chunksize); + + llbitmap->reshape_sync_size = blocks; + llbitmap->reshape_chunksize = chunksize; + llbitmap->reshape_chunks = chunks; + llbitmap_calculate_chunks(llbitmap->mddev, blocks, + &llbitmap->reshape_chunksize, + &llbitmap->reshape_chunks); + llbitmap->chunks = max(old_chunks, llbitmap->reshape_chunks); +} + static enum llbitmap_state llbitmap_read(struct llbitmap *llbitmap, loff_t pos) { unsigned int idx; @@ -1030,6 +1066,7 @@ static int llbitmap_init(struct llbitmap *llbitmap) llbitmap->chunksize = chunksize; llbitmap->chunks = chunks; llbitmap->sync_size = blocks; + llbitmap_refresh_reshape(llbitmap); mddev->bitmap_info.daemon_sleep = DEFAULT_DAEMON_SLEEP; ret = llbitmap_alloc_pages(llbitmap); @@ -1146,6 +1183,7 @@ static int llbitmap_read_sb(struct llbitmap *llbitmap) llbitmap->chunks = DIV_ROUND_UP_SECTOR_T(sync_size, chunksize); llbitmap->chunkshift = ffz(~chunksize); llbitmap->sync_size = sync_size; + llbitmap_refresh_reshape(llbitmap); ret = llbitmap_alloc_pages(llbitmap); out_put_page: @@ -1302,6 +1340,9 @@ static int llbitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize) goto out; if (reshape) { + llbitmap->reshape_sync_size = blocks; + llbitmap->reshape_chunksize = bitmap_chunksize; + llbitmap->reshape_chunks = chunks; llbitmap->chunks = max(old_chunks, chunks); } else { if (blocks < old_blocks && chunks < old_chunks) @@ -1310,6 +1351,7 @@ static int llbitmap_resize(struct mddev *mddev, sector_t blocks, int chunksize) mddev->bitmap_info.chunksize = bitmap_chunksize; llbitmap->chunks = chunks; llbitmap->sync_size = blocks; + llbitmap_refresh_reshape(llbitmap); llbitmap_update_sb(llbitmap); } __llbitmap_flush(mddev); From b094fa9d322302afe65ef67cdd89fada6578320b Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:26 +0800 Subject: [PATCH 159/241] md/md-llbitmap: finish reshape geometry Commit the staged llbitmap geometry when reshape finishes. When assembling a stopped reshape, md_run() creates the bitmap before publishing mddev->pers. llbitmap_read_sb() can therefore only initialize the reshape fields from the old on-disk sync size. Refresh the staged reshape geometry again from llbitmap_load(), after mddev->pers is available, and expand the in-memory page controls before replaying bitmap state. Reproduce on the old kernel by creating a RAID10 llbitmap with four active disks and two spares, growing it to six disks, then stopping and assembling while reshape is still running. The llbitmap chunk count was 32704 before grow, 49056 during reshape, then rolled back to 32704 after reassemble. The fixed kernel kept the target geometry across the same stop/reassemble flow: 65440 chunks before grow, 98160 during reshape, and 98160 after reassemble. Link: https://lore.kernel.org/all/20260726185916.2223460-1-mykola@meshstor.io/ Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-18-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 421a9a4ebbae..4e7070c14840 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1371,11 +1371,20 @@ static int llbitmap_load(struct mddev *mddev) { enum llbitmap_action action = BitmapActionReload; struct llbitmap *llbitmap = mddev->bitmap; + int ret; if (test_and_clear_bit(BITMAP_STALE, &llbitmap->flags)) action = BitmapActionStale; + mutex_lock(&mddev->bitmap_info.mutex); + llbitmap_refresh_reshape(llbitmap); + ret = llbitmap_expand_pages(llbitmap, llbitmap->chunks); + if (ret) { + mutex_unlock(&mddev->bitmap_info.mutex); + return ret; + } llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1, action); + mutex_unlock(&mddev->bitmap_info.mutex); return 0; } @@ -1709,6 +1718,30 @@ static void llbitmap_dirty_bits(struct mddev *mddev, unsigned long s, llbitmap_state_machine(mddev->bitmap, s, e, BitmapActionStartwrite); } +static void llbitmap_reshape_finish(struct mddev *mddev) +{ + struct llbitmap *llbitmap = mddev->bitmap; + + if (mddev->pers->quiesce) + mddev->pers->quiesce(mddev, 1); + + mutex_lock(&mddev->bitmap_info.mutex); + llbitmap_flush(mddev); + + llbitmap->chunksize = llbitmap->reshape_chunksize; + llbitmap->chunkshift = ffz(~llbitmap->chunksize); + llbitmap->chunks = llbitmap->reshape_chunks; + llbitmap->sync_size = llbitmap->reshape_sync_size; + llbitmap_refresh_reshape(llbitmap); + mddev->bitmap_info.chunksize = llbitmap->chunksize; + llbitmap_update_sb(llbitmap); + __llbitmap_flush(mddev); + mutex_unlock(&mddev->bitmap_info.mutex); + + if (mddev->pers->quiesce) + mddev->pers->quiesce(mddev, 0); +} + static void llbitmap_write_sb(struct llbitmap *llbitmap) { int nr_blocks = DIV_ROUND_UP(BITMAP_DATA_OFFSET, llbitmap->io_size); @@ -2000,6 +2033,7 @@ static struct bitmap_operations llbitmap_ops = { .get_stats = llbitmap_get_stats, .dirty_bits = llbitmap_dirty_bits, .prepare_range = llbitmap_prepare_range, + .reshape_finish = llbitmap_reshape_finish, .write_all = llbitmap_write_all, .groups = md_llbitmap_groups, From 807757d4c3d0e93e88ddad48f34f7075118917cf Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:27 +0800 Subject: [PATCH 160/241] md/md-llbitmap: refuse reshape while llbitmap still needs sync Reject reshape when llbitmap still contains NeedSync or Syncing bits. This keeps reshape from starting until the current llbitmap state has been reconciled. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-19-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 4e7070c14840..698162517ae9 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1718,6 +1718,29 @@ static void llbitmap_dirty_bits(struct mddev *mddev, unsigned long s, llbitmap_state_machine(mddev->bitmap, s, e, BitmapActionStartwrite); } +static int llbitmap_reshape_can_start(struct mddev *mddev) +{ + struct llbitmap *llbitmap = mddev->bitmap; + unsigned long chunk; + int ret = 0; + + if (!llbitmap) + return 0; + + mutex_lock(&mddev->bitmap_info.mutex); + for (chunk = 0; chunk < llbitmap->chunks; chunk++) { + enum llbitmap_state state = llbitmap_read(llbitmap, chunk); + + if (state == BitNeedSync || state == BitSyncing) { + ret = -EBUSY; + break; + } + } + mutex_unlock(&mddev->bitmap_info.mutex); + + return ret; +} + static void llbitmap_reshape_finish(struct mddev *mddev) { struct llbitmap *llbitmap = mddev->bitmap; @@ -2034,6 +2057,7 @@ static struct bitmap_operations llbitmap_ops = { .dirty_bits = llbitmap_dirty_bits, .prepare_range = llbitmap_prepare_range, .reshape_finish = llbitmap_reshape_finish, + .reshape_can_start = llbitmap_reshape_can_start, .write_all = llbitmap_write_all, .groups = md_llbitmap_groups, From 629c1659e90cdeb1b00c04c9e06028854d22cf2c Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:28 +0800 Subject: [PATCH 161/241] md/md-llbitmap: add reshape range mapping helpers Teach llbitmap to choose old versus new geometry during reshape and to encode exact bitmap ranges for the active geometry. This is the mapping groundwork for checkpoint remapping. Range preparation now distinguishes writes from discards. Normal writes must cover every touched bitmap chunk, while discards may only mark fully covered chunks unwritten. Without this distinction, a discard that starts or ends inside a chunk can make live data look unwritten after the range has been mapped and floored. Reproduce that with a RAID1 llbitmap using 128-sector chunks. A discard starting halfway into chunk 8 with a 128-sector length changed clean bits from 16352 to 16350 and unwritten bits from 0 to 2, even though no chunk was fully discarded. With discard-specific range encoding, both counts stay unchanged for the same test. Range preparation also clamps the pre-map range in the same coordinate space as the incoming IO. RAID5 receives array-sector offsets but tracks llbitmap sync size in component sectors, so steady-state RAID5 must use bitmap_array_sectors() before mapping and keep the existing sync-size clamp after mapping. Reproduce that with a 4-disk RAID5 llbitmap created --assume-clean. A write below dev_sectors changed dirty bits from 0 to 512, but a write at seek=2094080 left the count at 512. With the array-sector pre-map limit, writing at seek=component_size + 65536 increased dirty bits from 512 to 1024. Link: https://lore.kernel.org/all/20260726185916.2223460-1-mykola@meshstor.io/ Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-20-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-bitmap.c | 2 +- drivers/md/md-bitmap.h | 3 +- drivers/md/md-llbitmap.c | 139 +++++++++++++++++++++++++++++++++++---- drivers/md/md.c | 11 ++-- 4 files changed, 135 insertions(+), 20 deletions(-) diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index b7f0d4acce04..b8325cb09a37 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -1731,7 +1731,7 @@ static void bitmap_start_write(struct mddev *mddev, sector_t offset, } static void bitmap_prepare_range(struct mddev *mddev, sector_t *offset, - unsigned long *sectors) + unsigned long *sectors, bool discard) { if (mddev->pers->bitmap_sector) mddev->pers->bitmap_sector(mddev, offset, sectors); diff --git a/drivers/md/md-bitmap.h b/drivers/md/md-bitmap.h index 6478cf9d8816..b69c78174f02 100644 --- a/drivers/md/md-bitmap.h +++ b/drivers/md/md-bitmap.h @@ -97,7 +97,8 @@ struct bitmap_operations { /* Prepare a range for this bitmap implementation. */ void (*prepare_range)(struct mddev *mddev, sector_t *offset, - unsigned long *sectors); + unsigned long *sectors, + bool discard); void (*reshape_finish)(struct mddev *mddev); int (*reshape_can_start)(struct mddev *mddev); void (*reshape_mark)(struct mddev *mddev, sector_t old_pos, diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 698162517ae9..1283ad737692 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -433,22 +434,28 @@ static void llbitmap_calculate_chunks(struct mddev *mddev, sector_t blocks, } } -static bool llbitmap_reshaping(struct llbitmap *llbitmap) -{ - return llbitmap->mddev->reshape_position != MaxSector; -} - static sector_t llbitmap_personality_sync_size(struct llbitmap *llbitmap, bool previous) { struct mddev *mddev = llbitmap->mddev; - if (!llbitmap_reshaping(llbitmap) || !mddev->private || !mddev->pers || + if (READ_ONCE(mddev->reshape_position) == MaxSector || + !mddev->private || !mddev->pers || !mddev->pers->bitmap_sync_size) return llbitmap->sync_size; return mddev->pers->bitmap_sync_size(mddev, previous); } +static sector_t llbitmap_logical_size(struct llbitmap *llbitmap, bool previous) +{ + struct mddev *mddev = llbitmap->mddev; + + if (!mddev->private || !mddev->pers || + !mddev->pers->bitmap_array_sectors) + return llbitmap_personality_sync_size(llbitmap, previous); + return mddev->pers->bitmap_array_sectors(mddev, previous); +} + static void llbitmap_refresh_reshape(struct llbitmap *llbitmap) { unsigned long old_chunks = DIV_ROUND_UP_SECTOR_T(llbitmap->sync_size, @@ -466,6 +473,80 @@ static void llbitmap_refresh_reshape(struct llbitmap *llbitmap) llbitmap->chunks = max(old_chunks, llbitmap->reshape_chunks); } +static void llbitmap_map_layout(struct llbitmap *llbitmap, sector_t *offset, + unsigned long *sectors, bool previous) +{ + sector_t limit = llbitmap_logical_size(llbitmap, previous); + sector_t start = *offset; + sector_t end = start + *sectors; + + if (start >= limit) { + *sectors = 0; + return; + } + if (end > limit) + end = limit; + + *offset = start; + *sectors = end - start; + if (!*sectors) + return; + + if (llbitmap->mddev->pers->bitmap_sector_map) + llbitmap->mddev->pers->bitmap_sector_map(llbitmap->mddev, offset, + sectors, previous); + else if (!previous && llbitmap->mddev->pers->bitmap_sector) + llbitmap->mddev->pers->bitmap_sector(llbitmap->mddev, offset, + sectors); +} + +static void llbitmap_encode_range(struct llbitmap *llbitmap, sector_t *offset, + unsigned long *sectors, bool previous) +{ + unsigned long chunksize = previous ? llbitmap->chunksize : + llbitmap->reshape_chunksize; + u64 start; + u64 end; + + if (!*sectors) { + *offset = 0; + return; + } + + start = div64_u64(*offset, chunksize); + end = div64_u64(*offset + *sectors - 1, chunksize); + *offset = (sector_t)start << llbitmap->chunkshift; + *sectors = (end - start + 1) << llbitmap->chunkshift; +} + +static void llbitmap_encode_discard_range(struct llbitmap *llbitmap, + sector_t *offset, + unsigned long *sectors, + bool previous) +{ + unsigned long chunksize = previous ? llbitmap->chunksize : + llbitmap->reshape_chunksize; + sector_t end = *offset + *sectors; + u64 start; + u64 last; + + if (!*sectors) { + *offset = 0; + return; + } + + start = DIV_ROUND_UP_SECTOR_T(*offset, chunksize); + last = div64_u64(end, chunksize); + if (start >= last) { + *offset = 0; + *sectors = 0; + return; + } + + *offset = (sector_t)start << llbitmap->chunkshift; + *sectors = (last - start) << llbitmap->chunkshift; +} + static enum llbitmap_state llbitmap_read(struct llbitmap *llbitmap, loff_t pos) { unsigned int idx; @@ -1409,11 +1490,35 @@ static void llbitmap_destroy(struct mddev *mddev) mutex_unlock(&mddev->bitmap_info.mutex); } -static void llbitmap_prepare_range(struct mddev *mddev, sector_t *offset, - unsigned long *sectors) +static bool llbitmap_map_previous(struct llbitmap *llbitmap, sector_t offset, + unsigned long sectors) { - if (mddev->pers->bitmap_sector) - mddev->pers->bitmap_sector(mddev, offset, sectors); + struct mddev *mddev = llbitmap->mddev; + sector_t boundary = READ_ONCE(mddev->reshape_position); + + if (boundary == MaxSector) + return false; + + WARN_ON_ONCE(sectors && offset < boundary && offset + sectors > boundary); + + return mddev->reshape_backwards ? offset < boundary : offset >= boundary; +} + +static void llbitmap_prepare_range(struct mddev *mddev, sector_t *offset, + unsigned long *sectors, bool discard) +{ + struct llbitmap *llbitmap = mddev->bitmap; + bool previous; + + if (!llbitmap) + return; + + previous = llbitmap_map_previous(llbitmap, *offset, *sectors); + llbitmap_map_layout(llbitmap, offset, sectors, previous); + if (discard) + llbitmap_encode_discard_range(llbitmap, offset, sectors, previous); + else + llbitmap_encode_range(llbitmap, offset, sectors, previous); } static void llbitmap_start_write(struct mddev *mddev, sector_t offset, @@ -1582,7 +1687,11 @@ static bool llbitmap_blocks_synced(struct mddev *mddev, sector_t offset) { struct llbitmap *llbitmap = mddev->bitmap; unsigned long p = offset >> llbitmap->chunkshift; - enum llbitmap_state c = llbitmap_read(llbitmap, p); + enum llbitmap_state c; + + if (p >= llbitmap->chunks) + return false; + c = llbitmap_read(llbitmap, p); return c == BitClean || c == BitDirty || c == BitCleanUnwritten; } @@ -1592,7 +1701,11 @@ static sector_t llbitmap_skip_sync_blocks(struct mddev *mddev, sector_t offset) struct llbitmap *llbitmap = mddev->bitmap; unsigned long p = offset >> llbitmap->chunkshift; int blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); - enum llbitmap_state c = llbitmap_read(llbitmap, p); + enum llbitmap_state c; + + if (p >= llbitmap->chunks) + return 0; + c = llbitmap_read(llbitmap, p); /* always skip unwritten blocks */ if (c == BitUnwritten) @@ -1637,6 +1750,8 @@ static bool llbitmap_start_sync(struct mddev *mddev, sector_t offset, * if md_do_sync() loop more times. */ *blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1)); + if (p >= llbitmap->chunks) + return false; state = llbitmap_state_machine(llbitmap, p, p, BitmapActionStartsync); return state == BitSyncing || state == BitSyncingUnwritten; } diff --git a/drivers/md/md.c b/drivers/md/md.c index 538ba7bab060..e88381beb209 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9428,21 +9428,20 @@ split: EXPORT_SYMBOL_GPL(mddev_bio_split_at_reshape_offset); static void md_bitmap_prepare_range(struct mddev *mddev, sector_t *offset, - unsigned long *sectors) + unsigned long *sectors, bool discard) { - mddev->bitmap_ops->prepare_range(mddev, offset, sectors); + mddev->bitmap_ops->prepare_range(mddev, offset, sectors, discard); } static void md_bitmap_start(struct mddev *mddev, struct md_io_clone *md_io_clone) { - md_bitmap_fn *fn = unlikely(md_io_clone->rw == STAT_DISCARD) ? - mddev->bitmap_ops->start_discard : + bool discard = md_io_clone->rw == STAT_DISCARD; + md_bitmap_fn *fn = discard ? mddev->bitmap_ops->start_discard : mddev->bitmap_ops->start_write; md_bitmap_prepare_range(mddev, &md_io_clone->offset, - &md_io_clone->sectors); - + &md_io_clone->sectors, discard); if (!md_io_clone->sectors) return; fn(mddev, md_io_clone->offset, md_io_clone->sectors); From 44aa6154e17e8831a1168e600deaff7dcb052a18 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:29 +0800 Subject: [PATCH 162/241] md/md-llbitmap: don't skip reshape ranges from bitmap state Reshape progress is tracked by array metadata rather than llbitmap. Do not let llbitmap skip_sync_blocks() suppress reshape ranges based on stale bitmap state before the corresponding checkpoint is persisted. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-21-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 1283ad737692..a609e8d44901 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1707,6 +1707,14 @@ static sector_t llbitmap_skip_sync_blocks(struct mddev *mddev, sector_t offset) return 0; c = llbitmap_read(llbitmap, p); + /* + * Reshape progress is tracked by array metadata rather than llbitmap. + * Skipping reshape ranges from stale bitmap state can lose data after a + * restart before the corresponding bits are checkpointed to disk. + */ + if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) + return 0; + /* always skip unwritten blocks */ if (c == BitUnwritten) return blocks; From 9b2d455305f1f9f89ca41865fe625e31353e0bc8 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:30 +0800 Subject: [PATCH 163/241] md/md-llbitmap: remap checkpointed bits as reshape progresses Merge checkpointed old llbitmap state forward as reshape_position advances and record the checkpoint remap through reshape_mark(). Normal write accounting can run while the reshape thread checkpoints a new reshape position. llbitmap_reshape_mark() reads old state bytes, merges them into destination bits, and writes the result back. If llbitmap_start_write() or llbitmap_start_discard() updates the same state bytes at the same time, the two read/modify/write paths can overwrite each other and lose the state from one side. Serialize only this state-byte race with a rwlock. Normal I/O takes the read side around llbitmap_state_machine(), after page active references are raised, so concurrent normal I/O updates still run in parallel. Reshape checkpointing takes the write side only while merging the checkpointed range, avoiding page suspension and avoiding a sleeping mutex in the I/O accounting path. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-22-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 204 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 204 insertions(+) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index a609e8d44901..845b8cd10f83 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -302,6 +302,11 @@ struct llbitmap { /* fires on first BitDirty state */ struct timer_list pending_timer; struct work_struct daemon_work; + /* + * Serialize reshape checkpoint remapping against normal I/O bitmap + * updates without blocking concurrent I/O updates on each other. + */ + rwlock_t reshape_lock; unsigned long flags; __u64 events_cleared; @@ -498,6 +503,14 @@ static void llbitmap_map_layout(struct llbitmap *llbitmap, sector_t *offset, else if (!previous && llbitmap->mddev->pers->bitmap_sector) llbitmap->mddev->pers->bitmap_sector(llbitmap->mddev, offset, sectors); + + limit = llbitmap_personality_sync_size(llbitmap, previous); + start = *offset; + end = start + *sectors; + if (start >= limit) + *sectors = 0; + else if (end > limit) + *sectors = limit - start; } static void llbitmap_encode_range(struct llbitmap *llbitmap, sector_t *offset, @@ -930,6 +943,33 @@ static int llbitmap_prepare_resize(struct llbitmap *llbitmap, return 0; } +static enum llbitmap_state +llbitmap_rmerge_state(struct llbitmap *llbitmap, + enum llbitmap_state dst, + enum llbitmap_state src) +{ + bool level_456 = raid_is_456(llbitmap->mddev); + + if (dst == BitNeedSync || dst == BitSyncing || + src == BitNeedSync || src == BitSyncing) + return BitNeedSync; + + if (dst == BitDirty || src == BitDirty) + return BitDirty; + + /* + * Reshape generates valid target parity/data for both already-written + * and not-yet-written regions in the checkpointed range, so a mix of + * clean and unwritten still results in a clean destination bit. + */ + if (level_456 && ((dst == BitClean && src == BitUnwritten) || + (src == BitClean && dst == BitUnwritten))) + return BitClean; + if (dst == BitClean || src == BitClean) + return BitClean; + return BitUnwritten; +} + static void llbitmap_init_state(struct llbitmap *llbitmap) { struct mddev *mddev = llbitmap->mddev; @@ -1306,6 +1346,7 @@ static void md_llbitmap_daemon_fn(struct work_struct *work) if (llbitmap->mddev->degraded) return; + retry: start = 0; end = min(llbitmap->chunks, PAGE_SIZE - BITMAP_DATA_OFFSET) - 1; @@ -1367,6 +1408,7 @@ static int llbitmap_create(struct mddev *mddev) timer_setup(&llbitmap->pending_timer, llbitmap_pending_timer_fn, 0); INIT_WORK(&llbitmap->daemon_work, md_llbitmap_daemon_fn); + rwlock_init(&llbitmap->reshape_lock); atomic_set(&llbitmap->behind_writes, 0); init_waitqueue_head(&llbitmap->behind_wait); @@ -1535,7 +1577,9 @@ static void llbitmap_start_write(struct mddev *mddev, sector_t offset, page_start++; } + read_lock(&llbitmap->reshape_lock); llbitmap_state_machine(llbitmap, start, end, BitmapActionStartwrite); + read_unlock(&llbitmap->reshape_lock); } static void llbitmap_end_write(struct mddev *mddev, sector_t offset, @@ -1567,7 +1611,9 @@ static void llbitmap_start_discard(struct mddev *mddev, sector_t offset, page_start++; } + read_lock(&llbitmap->reshape_lock); llbitmap_state_machine(llbitmap, start, end, BitmapActionDiscard); + read_unlock(&llbitmap->reshape_lock); } static void llbitmap_end_discard(struct mddev *mddev, sector_t offset, @@ -1864,6 +1910,136 @@ static int llbitmap_reshape_can_start(struct mddev *mddev) return ret; } +struct llbitmap_reshape_range { + sector_t offset; + unsigned long sectors; + sector_t start; + sector_t end; +}; + +static enum llbitmap_state +llbitmap_reshape_init_dst(struct llbitmap *llbitmap, unsigned long dst, + const struct llbitmap_reshape_range *new) +{ + u64 bit_start = (u64)dst * llbitmap->reshape_chunksize; + u64 bit_end = bit_start + llbitmap->reshape_chunksize; + + if (!llbitmap->mddev->reshape_backwards) + return bit_start < new->offset ? llbitmap_read(llbitmap, dst) : + BitUnwritten; + return bit_end > new->end ? llbitmap_read(llbitmap, dst) : BitUnwritten; +} + +static void llbitmap_reshape_dst_range(struct llbitmap *llbitmap, + unsigned long dst, + const struct llbitmap_reshape_range *new, + struct llbitmap_reshape_range *dst_range) +{ + sector_t dst_bit_start = (sector_t)dst * llbitmap->reshape_chunksize; + + dst_range->start = max(dst_bit_start, new->offset); + dst_range->end = min(dst_bit_start + llbitmap->reshape_chunksize, + new->end); + dst_range->offset = dst_range->start; + dst_range->sectors = dst_range->end - dst_range->start; +} + +static void llbitmap_reshape_map_range(struct llbitmap *llbitmap, + sector_t lo, sector_t hi, + bool previous, + struct llbitmap_reshape_range *range) +{ + range->offset = lo; + range->sectors = hi - lo; + llbitmap_map_layout(llbitmap, &range->offset, &range->sectors, previous); + range->start = range->offset; + range->end = range->offset + range->sectors; +} + +static bool llbitmap_reshape_src_range(const struct llbitmap_reshape_range *old, + const struct llbitmap_reshape_range *new, + const struct llbitmap_reshape_range *dst, + struct llbitmap_reshape_range *src) +{ + if (!old->sectors) + return false; + + src->start = old->offset + + mul_u64_u64_div_u64(dst->start - new->offset, + old->sectors, new->sectors); + src->end = old->offset + + mul_u64_u64_div_u64_roundup(dst->end - new->offset, + old->sectors, new->sectors); + if (src->end > old->end) + src->end = old->end; + src->offset = src->start; + src->sectors = src->end - src->start; + + return src->sectors; +} + +static enum llbitmap_state llbitmap_rmerge_src(struct llbitmap *llbitmap, + enum llbitmap_state state, + const struct llbitmap_reshape_range *src) +{ + unsigned long bit = div64_u64(src->start, llbitmap->chunksize); + unsigned long end = div64_u64(src->end - 1, llbitmap->chunksize); + + while (bit <= end) { + enum llbitmap_state src_state = llbitmap_read(llbitmap, bit); + + state = llbitmap_rmerge_state(llbitmap, state, src_state); + bit++; + } + + return state; +} + +static void llbitmap_reshape_merge(struct llbitmap *llbitmap, + const struct llbitmap_reshape_range *old, + const struct llbitmap_reshape_range *new) +{ + unsigned long dst_start; + unsigned long dst_end; + unsigned long dst; + bool backwards = false; + + if (!new->sectors) + return; + + dst_start = div64_u64(new->offset, llbitmap->reshape_chunksize); + dst_end = div64_u64(new->end - 1, llbitmap->reshape_chunksize); + if (old->sectors) { + unsigned long src_start = div64_u64(old->offset, + llbitmap->chunksize); + unsigned long src_end = div64_u64(old->end - 1, + llbitmap->chunksize); + + backwards = src_start < dst_start && src_end >= dst_start; + } + + dst = backwards ? dst_end : dst_start; + while (true) { + struct llbitmap_reshape_range dst_range; + struct llbitmap_reshape_range src; + enum llbitmap_state state; + + llbitmap_reshape_dst_range(llbitmap, dst, new, &dst_range); + state = llbitmap_reshape_init_dst(llbitmap, dst, new); + if (llbitmap_reshape_src_range(old, new, &dst_range, &src)) + state = llbitmap_rmerge_src(llbitmap, state, &src); + else + state = llbitmap_rmerge_state(llbitmap, state, BitUnwritten); + llbitmap_write(llbitmap, state, dst); + if (dst == (backwards ? dst_start : dst_end)) + break; + if (backwards) + dst--; + else + dst++; + } +} + static void llbitmap_reshape_finish(struct mddev *mddev) { struct llbitmap *llbitmap = mddev->bitmap; @@ -1888,6 +2064,33 @@ static void llbitmap_reshape_finish(struct mddev *mddev) mddev->pers->quiesce(mddev, 0); } +static void llbitmap_reshape_mark(struct mddev *mddev, sector_t old_pos, + sector_t new_pos) +{ + struct llbitmap *llbitmap = mddev->bitmap; + sector_t lo; + sector_t hi; + struct llbitmap_reshape_range old; + struct llbitmap_reshape_range new; + + if (!llbitmap || old_pos == new_pos) + return; + + lo = min(old_pos, new_pos); + hi = max(old_pos, new_pos); + if (!hi) + return; + + llbitmap_reshape_map_range(llbitmap, lo, hi, true, &old); + llbitmap_reshape_map_range(llbitmap, lo, hi, false, &new); + if (!new.sectors) + return; + + write_lock(&llbitmap->reshape_lock); + llbitmap_reshape_merge(llbitmap, &old, &new); + write_unlock(&llbitmap->reshape_lock); +} + static void llbitmap_write_sb(struct llbitmap *llbitmap) { int nr_blocks = DIV_ROUND_UP(BITMAP_DATA_OFFSET, llbitmap->io_size); @@ -2181,6 +2384,7 @@ static struct bitmap_operations llbitmap_ops = { .prepare_range = llbitmap_prepare_range, .reshape_finish = llbitmap_reshape_finish, .reshape_can_start = llbitmap_reshape_can_start, + .reshape_mark = llbitmap_reshape_mark, .write_all = llbitmap_write_all, .groups = md_llbitmap_groups, From 3fa5499f32f16133e32c9fb696bc6c0a07483396 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:31 +0800 Subject: [PATCH 164/241] md/md-llbitmap: clamp state-machine walks to tracked bits llbitmap_state_machine() can be called with an end bit beyond llbitmap->chunks. In particular, llbitmap_cond_end_sync() passes sector >> chunkshift, and sector can reach the tracked boundary exactly. Clamp the state-machine range to llbitmap->chunks so it cannot walk past the tracked bitmap. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-23-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/md-llbitmap.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 845b8cd10f83..e1a783ee2032 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -1012,7 +1012,10 @@ static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap, llbitmap_init_state(llbitmap); return BitNone; } - + if (start >= llbitmap->chunks) + return BitNone; + if (end >= llbitmap->chunks) + end = llbitmap->chunks - 1; while (start <= end) { enum llbitmap_state c = llbitmap_read(llbitmap, start); From ecb66a97af620135f207cddabeb7f589510640fc Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:32 +0800 Subject: [PATCH 165/241] md/raid10: reject llbitmap reshape when md chunk shrinks llbitmap reshape keeps one live bitmap and cannot safely make an existing bitmap bit cover a smaller data range. The llbitmap chunksize itself will not shrink when mddev->chunk_sectors stays the same or grows. However, shrinking mddev->chunk_sectors can shrink the effective data range covered by each bit for the RAID10 reshape geometry. Reject that reshape while llbitmap is active. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-24-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index ed3c6fbe65f7..1c3393467667 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -4245,6 +4245,10 @@ static int raid10_check_reshape(struct mddev *mddev) if (conf->geo.far_copies != 1 && !conf->geo.far_offset) return -EINVAL; + if (mddev->bitmap_id == ID_LLBITMAP && + mddev->new_chunk_sectors && + mddev->new_chunk_sectors < mddev->chunk_sectors) + return -EOPNOTSUPP; if (setup_geo(&geo, mddev, geo_start) != conf->copies) /* mustn't change number of copies */ From b109d437dbc6741fbd6bc108626756c6e7f7ec20 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:33 +0800 Subject: [PATCH 166/241] md/raid10: wire llbitmap reshape lifecycle Prepare llbitmap before RAID10 starts growing, checkpoint the bitmap before advancing reshape_position, finish the llbitmap geometry update when reshape completes, and export the old and new tracked sizes. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-25-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 1c3393467667..bac9edd28c97 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -4356,6 +4356,12 @@ static int raid10_start_reshape(struct mddev *mddev) if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) return -EBUSY; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_can_start) { + ret = mddev->bitmap_ops->reshape_can_start(mddev); + if (ret) + return ret; + } if (setup_geo(&new, mddev, geo_start) != conf->copies) return -EINVAL; @@ -4679,6 +4685,13 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { /* Need to update reshape_position in metadata */ wait_barrier(conf); + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } mddev->reshape_position = conf->reshape_progress; if (mddev->reshape_backwards) mddev->curr_resync_completed = raid10_size(mddev, 0, 0) @@ -4877,9 +4890,19 @@ static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio) static void end_reshape(struct r10conf *conf) { + struct mddev *mddev = conf->mddev; + if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) return; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } + spin_lock_irq(&conf->device_lock); conf->prev = conf->geo; md_finish_reshape(conf->mddev); @@ -5011,10 +5034,15 @@ static void end_reshape_request(struct r10bio *r10_bio) static void raid10_finish_reshape(struct mddev *mddev) { struct r10conf *conf = mddev->private; + bool llbitmap = mddev->bitmap_id == ID_LLBITMAP && + md_bitmap_enabled(mddev, false); if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) return; + if (llbitmap && mddev->bitmap_ops->reshape_finish) + mddev->bitmap_ops->reshape_finish(mddev); + if (mddev->delta_disks > 0) { if (mddev->resync_offset > mddev->resync_max_sectors) { mddev->resync_offset = mddev->resync_max_sectors; @@ -5041,6 +5069,15 @@ static void raid10_finish_reshape(struct mddev *mddev) mddev->reshape_backwards = 0; } +static sector_t raid10_bitmap_sync_size(struct mddev *mddev, bool previous) +{ + struct r10conf *conf = mddev->private; + + if (previous) + return raid10_size(mddev, 0, 0); + return raid10_size(mddev, 0, conf->geo.raid_disks); +} + static struct md_personality raid10_personality = { .head = { @@ -5067,6 +5104,8 @@ static struct md_personality raid10_personality = .start_reshape = raid10_start_reshape, .finish_reshape = raid10_finish_reshape, .update_reshape_pos = raid10_update_reshape_pos, + .bitmap_sync_size = raid10_bitmap_sync_size, + .bitmap_array_sectors = raid10_bitmap_sync_size, }; static int __init raid10_init(void) From aa648f26a985d08f816740bb964a743406a5f40b Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:34 +0800 Subject: [PATCH 167/241] md/raid10: split reshape bios before bitmap accounting Use the shared mddev_bio_split_at_reshape_offset() helper so RAID10 submits only one-side bios to llbitmap during reshape. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-26-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index bac9edd28c97..562a325a7195 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1848,6 +1848,7 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio) { struct r10conf *conf = mddev->private; sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask); + const int rw = bio_data_dir(bio); int chunk_sects = chunk_mask + 1; int sectors = bio_sectors(bio); @@ -1873,6 +1874,15 @@ static bool raid10_make_request(struct mddev *mddev, struct bio *bio) sectors = chunk_sects - (bio->bi_iter.bi_sector & (chunk_sects - 1)); + + bio = mddev_bio_split_at_reshape_offset(mddev, bio, §ors, + &conf->bio_split); + if (!bio) { + if (rw == WRITE) + md_write_end(mddev); + return true; + } + if (!__make_request(mddev, bio, sectors)) md_write_end(mddev); From 9f59258d300494be3055f2e47f26b21f5c8ad710 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:35 +0800 Subject: [PATCH 168/241] md/raid5: add exact old and new llbitmap mapping helpers Teach RAID5 to export exact old and new llbitmap mappings and the corresponding sync and array sizes for reshape-aware bitmap users. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-27-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 81 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 2cc2546a29ae..88bf5a9ce573 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -6015,6 +6015,34 @@ static enum reshape_loc get_reshape_loc(struct mddev *mddev, return LOC_BEHIND_RESHAPE; } +static void raid5_bitmap_sector_map(struct mddev *mddev, sector_t *offset, + unsigned long *sectors, + bool previous) +{ + struct r5conf *conf = mddev->private; + sector_t start = *offset; + sector_t end = start + *sectors; + int sectors_per_chunk; + int dd_idx; + + if (previous) + sectors_per_chunk = conf->prev_chunk_sectors * + (conf->previous_raid_disks - conf->max_degraded); + else + sectors_per_chunk = conf->chunk_sectors * + (conf->raid_disks - conf->max_degraded); + sector_div(start, sectors_per_chunk); + start *= sectors_per_chunk; + if (sector_div(end, sectors_per_chunk)) + end++; + end *= sectors_per_chunk; + + start = raid5_compute_sector(conf, start, previous, &dd_idx, NULL); + end = raid5_compute_sector(conf, end, previous, &dd_idx, NULL); + *offset = start; + *sectors = end - start; +} + static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, unsigned long *sectors) { @@ -6022,21 +6050,11 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, sector_t start = *offset; sector_t end = start + *sectors; sector_t prev_start = start; - sector_t prev_end = end; - int sectors_per_chunk; + unsigned long prev_sectors = end - start; enum reshape_loc loc; - int dd_idx; - sectors_per_chunk = conf->chunk_sectors * - (conf->raid_disks - conf->max_degraded); - sector_div(start, sectors_per_chunk); - start *= sectors_per_chunk; - if (sector_div(end, sectors_per_chunk)) - end++; - end *= sectors_per_chunk; - - start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL); - end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL); + raid5_bitmap_sector_map(mddev, &start, sectors, false); + end = start + *sectors; /* * For LOC_INSIDE_RESHAPE, this IO will wait for reshape to make @@ -6045,19 +6063,10 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, loc = get_reshape_loc(mddev, conf, prev_start); if (likely(loc != LOC_AHEAD_OF_RESHAPE)) { *offset = start; - *sectors = end - start; return; } - sectors_per_chunk = conf->prev_chunk_sectors * - (conf->previous_raid_disks - conf->max_degraded); - sector_div(prev_start, sectors_per_chunk); - prev_start *= sectors_per_chunk; - sector_div(prev_end, sectors_per_chunk); - prev_end *= sectors_per_chunk; - - prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL); - prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL); + raid5_bitmap_sector_map(mddev, &prev_start, &prev_sectors, true); /* * for LOC_AHEAD_OF_RESHAPE, reshape can make progress before this IO @@ -6065,7 +6074,7 @@ static void raid5_bitmap_sector(struct mddev *mddev, sector_t *offset, * we set bits for both. */ *offset = min(start, prev_start); - *sectors = max(end, prev_end) - *offset; + *sectors = max(end, prev_start + prev_sectors) - *offset; } static enum stripe_result make_stripe_request(struct mddev *mddev, @@ -9131,6 +9140,21 @@ static void raid5_prepare_suspend(struct mddev *mddev) wake_up(&conf->wait_for_reshape); } +static sector_t raid5_bitmap_sync_size(struct mddev *mddev, bool previous) +{ + return mddev->dev_sectors; +} + +static sector_t raid5_bitmap_array_sectors(struct mddev *mddev, bool previous) +{ + struct r5conf *conf = mddev->private; + + if (previous) + return raid5_size(mddev, mddev->dev_sectors, + conf->previous_raid_disks); + return raid5_size(mddev, mddev->dev_sectors, conf->raid_disks); +} + static struct md_personality raid6_personality = { .head = { @@ -9160,6 +9184,9 @@ static struct md_personality raid6_personality = .change_consistency_policy = raid5_change_consistency_policy, .prepare_suspend = raid5_prepare_suspend, .bitmap_sector = raid5_bitmap_sector, + .bitmap_sector_map = raid5_bitmap_sector_map, + .bitmap_sync_size = raid5_bitmap_sync_size, + .bitmap_array_sectors = raid5_bitmap_array_sectors, }; static struct md_personality raid5_personality = { @@ -9190,6 +9217,9 @@ static struct md_personality raid5_personality = .change_consistency_policy = raid5_change_consistency_policy, .prepare_suspend = raid5_prepare_suspend, .bitmap_sector = raid5_bitmap_sector, + .bitmap_sector_map = raid5_bitmap_sector_map, + .bitmap_sync_size = raid5_bitmap_sync_size, + .bitmap_array_sectors = raid5_bitmap_array_sectors, }; static struct md_personality raid4_personality = @@ -9221,6 +9251,9 @@ static struct md_personality raid4_personality = .change_consistency_policy = raid5_change_consistency_policy, .prepare_suspend = raid5_prepare_suspend, .bitmap_sector = raid5_bitmap_sector, + .bitmap_sector_map = raid5_bitmap_sector_map, + .bitmap_sync_size = raid5_bitmap_sync_size, + .bitmap_array_sectors = raid5_bitmap_array_sectors, }; static int __init raid5_init(void) From 05a1b89689dfdc567a840dd06ba1327652cc9f58 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:36 +0800 Subject: [PATCH 169/241] md/raid5: reject llbitmap reshape when md chunk shrinks llbitmap reshape keeps one live bitmap and cannot safely make an existing bitmap bit cover a smaller data range. The llbitmap chunksize itself will not shrink when mddev->chunk_sectors stays the same or grows. However, shrinking mddev->chunk_sectors shrinks sectors_per_chunk used by raid5_bitmap_sector_map(). That can shrink the effective data range covered by each bit across the old and new RAID5 geometry. Reject that reshape while llbitmap is active. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-28-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 88bf5a9ce573..67d56c92c8a4 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -8580,6 +8580,9 @@ static int check_reshape(struct mddev *mddev) if (!check_stripe_cache(mddev)) return -ENOSPC; + if (mddev->bitmap_id == ID_LLBITMAP && + mddev->new_chunk_sectors < mddev->chunk_sectors) + return -EOPNOTSUPP; if (mddev->new_chunk_sectors > mddev->chunk_sectors || mddev->delta_disks > 0) if (resize_chunks(conf, From 816b25aca5b3664637c2b976d0e2a25b71dd88a9 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:37 +0800 Subject: [PATCH 170/241] md/raid5: wire llbitmap reshape lifecycle Prepare llbitmap before RAID5 reshape starts, checkpoint the bitmap before advancing reshape_position, and finish the llbitmap geometry update when reshape completes. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-29-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 67d56c92c8a4..5176de5b5956 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -6497,6 +6497,13 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *sk || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); if (atomic_read(&conf->reshape_stripes) != 0) return 0; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } mddev->reshape_position = conf->reshape_progress; mddev->curr_resync_completed = sector_nr; if (!mddev->reshape_backwards) @@ -6606,6 +6613,13 @@ finish: || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); if (atomic_read(&conf->reshape_stripes) != 0) goto ret; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } mddev->reshape_position = conf->reshape_progress; mddev->curr_resync_completed = sector_nr; if (!mddev->reshape_backwards) @@ -8648,6 +8662,12 @@ static int raid5_start_reshape(struct mddev *mddev) mdname(mddev)); return -EINVAL; } + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_id == ID_LLBITMAP) { + i = mddev->bitmap_ops->resize(mddev, mddev->dev_sectors, 0); + if (i) + return i; + } atomic_set(&conf->reshape_stripes, 0); spin_lock_irq(&conf->device_lock); @@ -8732,10 +8752,19 @@ static int raid5_start_reshape(struct mddev *mddev) */ static void end_reshape(struct r5conf *conf) { + struct mddev *mddev = conf->mddev; if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { struct md_rdev *rdev; + if (md_bitmap_enabled(mddev, false) && + mddev->bitmap_ops->reshape_mark && + conf->reshape_safe != conf->reshape_progress) { + mddev->bitmap_ops->reshape_mark(mddev, conf->reshape_safe, + conf->reshape_progress); + mddev->bitmap_ops->unplug(mddev, true); + } + spin_lock_irq(&conf->device_lock); conf->previous_raid_disks = conf->raid_disks; md_finish_reshape(conf->mddev); @@ -8762,8 +8791,16 @@ static void raid5_finish_reshape(struct mddev *mddev) { struct r5conf *conf = mddev->private; struct md_rdev *rdev; + bool llbitmap = mddev->bitmap_id == ID_LLBITMAP && + md_bitmap_enabled(mddev, false); if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { + if (llbitmap && mddev->bitmap_ops->reshape_finish) + mddev->bitmap_ops->reshape_finish(mddev); + if (llbitmap) { + mddev->resync_offset = 0; + mddev->resync_max_sectors = mddev->dev_sectors; + } if (mddev->delta_disks <= 0) { int d; From 661102bb87e43b7e476f3a2ff34e916b800b5627 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Mon, 3 Aug 2026 03:50:38 +0800 Subject: [PATCH 171/241] md/raid5: split reshape bios before bitmap accounting RAID5 maps array sectors through different geometries before and after the reshape position. During llbitmap reshape, md core cannot account one bio against both geometries as a single bitmap range, because the old and new bitmap mappings can cover different chunks. Split bios that cross reshape_position before md_account_bio(), so the bitmap only sees ranges that belong to one side of the reshape boundary. mddev_bio_split_at_reshape_offset() uses bio_submit_split_bioset(), which submits the remainder immediately and returns the front split bio. If that front bio later has to wait for reshape, md_handle_request() must not retry the original bio pointer, because after the split that pointer is the already-submitted remainder. Track whether the split happened, clear the temporary BLK_STS_RESOURCE status after the internal clone completion, and resubmit the front bio directly after the reshape wait. Keep the old return-false retry path for unsplit bios, where md_handle_request() still owns the same bio. Tested-by: Mykola Marzhan Link: https://patch.msgid.link/20260802195038.164272-30-yukuai@kernel.org Signed-off-by: Yu Kuai --- drivers/md/raid5.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 5176de5b5956..b91545ce090d 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -6221,9 +6221,11 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) struct r5conf *conf = mddev->private; const int rw = bio_data_dir(bi); struct stripe_request_ctx *ctx; + struct bio *front_bio; sector_t logical_sector; enum stripe_result res; int s, stripe_cnt; + bool split = false; bool on_wq; if (unlikely(bi->bi_opf & REQ_PREFLUSH)) { @@ -6257,6 +6259,18 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) return true; } + front_bio = bi; + bi = mddev_bio_split_at_reshape_offset(mddev, bi, NULL, + &conf->bio_split); + if (!bi) { + if (rw == WRITE) + md_write_end(mddev); + return true; + } + if (bi != front_bio) + split = true; + front_bio = bi; + logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1); bi->bi_next = NULL; @@ -6348,6 +6362,11 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi) bio_endio(bi); wait_for_completion(&done); + front_bio->bi_status = BLK_STS_OK; + if (split) { + submit_bio_noacct(front_bio); + return true; + } return false; } From 47f1441b281decde6954a2fa82b4131637d685ac Mon Sep 17 00:00:00 2001 From: Yunye Zhao Date: Thu, 23 Jul 2026 21:55:33 +0800 Subject: [PATCH 172/241] md/raid10: fix still_degraded being inverted in raid10_sync_request() Commit fe6a19d40ceb ("md/md-bitmap: merge md_bitmap_start_sync() into bitmap_operations") converted still_degraded from int to bool, but inverted the assignment in the loop that checks whether the array will still be degraded after the current device is recovered: "still_degraded = 1" became "still_degraded = false". As a result, recovering a device while another mirror is still missing calls md_bitmap_start_sync() with degraded == false, which clears bitmap bits that the still-missing device needs. When that device is re-added, its bitmap-based recovery finds the bits already cleared and skips every region written while the array was degraded, so it is marked In_sync while holding stale data: silent corruption. Reproducer (raid10 near=2, 4 disks, internal bitmap): - fail and remove one disk of each mirror pair - write to the degraded array - re-add both disks and let recovery finish - "check" reports mismatch_cnt=262272 after 256 MiB of degraded writes and file contents differ; the second disk's "recovery" completes in milliseconds because everything is skipped The same conversion in raid1 got it right (still_degraded = true). Restore the correct value. Fixes: fe6a19d40ceb ("md/md-bitmap: merge md_bitmap_start_sync() into bitmap_operations") Cc: stable@vger.kernel.org Signed-off-by: Yunye Zhao Reviewed-by: Mykola Marzhan Reviewed-by: Paul Menzel Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260723135535.101995-2-yunye.zhao@linux.alibaba.com Signed-off-by: Yu Kuai --- drivers/md/raid10.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 562a325a7195..1093c798d9dd 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -3337,7 +3337,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, struct md_rdev *rdev = conf->mirrors[j].rdev; if (rdev == NULL || test_bit(Faulty, &rdev->flags)) { - still_degraded = false; + still_degraded = true; break; } } From 162eb5ba791f6260fdaabae75fa2df20134f67d8 Mon Sep 17 00:00:00 2001 From: Yunye Zhao Date: Thu, 23 Jul 2026 21:55:34 +0800 Subject: [PATCH 173/241] md: add cond_resched() to md_do_sync()'s skip path When sync_request() reports a skipped region (*skipped == 1), md_do_sync()'s main loop advances the cursor and takes an early continue: j += sectors; ... if (last_check + window > io_sectors || j == max_sectors) continue; If the personality returns a small span per call (raid10 recovery returns only 128 sectors), syncing a large, mostly clean array iterates this branch an enormous number of times without ever yielding the CPU. On a non-preemptive kernel the resync thread then trips the soft-lockup watchdog: watchdog: BUG: soft lockup - CPU#149 stuck for 313s! [mdX_resync] md_bitmap_start_sync+0x6f/0xe0 raid10_sync_request+0x2c9/0x1530 [raid10] md_do_sync+0x810/0x1030 md_thread+0xa7/0x150 Add a cond_resched(). This does not reduce the wasted iterations; the excessive iteration count is a raid10 problem addressed separately. Signed-off-by: Yunye Zhao Link: https://patch.msgid.link/20260723135535.101995-3-yunye.zhao@linux.alibaba.com Signed-off-by: Yu Kuai --- drivers/md/md.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index e88381beb209..42fca0cec8e4 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9958,8 +9958,10 @@ update: */ md_new_event(); - if (last_check + window > io_sectors || j == max_sectors) + if (last_check + window > io_sectors || j == max_sectors) { + cond_resched(); continue; + } last_check = io_sectors; repeat: From efdffeb6d4915219a130607f5ae29613d2c5545a Mon Sep 17 00:00:00 2001 From: Wale Zhang Date: Fri, 31 Jul 2026 03:47:29 -0400 Subject: [PATCH 174/241] md: skip discard on unsupported member devices blk_stack_limits() uses min_not_zero() when stacking discard limits. Thus an array containing devices with different discard capabilities can expose discard support as long as at least one member has a non-zero discard limit. raid0 and raid10 use md_submit_discard_bio() to submit a discard bio to each member covered by the request. The helper currently also submits bios to members whose max_discard_sectors is zero. The block layer completes these bios with BLK_STS_NOTSUPP, and bio chaining propagates that status to the original discard request. Discard is optional, so skip members which do not support it. Members that do support discard continue to receive their portion of the request. Signed-off-by: Wale Zhang Link: https://patch.msgid.link/20260731074729.1885314-1-wale.zhang.ftd@gmail.com Signed-off-by: Yu Kuai --- drivers/md/md.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/md/md.c b/drivers/md/md.c index 42fca0cec8e4..680b34a63cb3 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -9377,6 +9377,10 @@ void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev, { struct bio *discard_bio = NULL; + /* Discard is optional, so silently skip members that do not support it. */ + if (unlikely(!bdev_max_discard_sectors(rdev->bdev))) + return; + __blkdev_issue_discard(rdev->bdev, start, size, GFP_NOIO, &discard_bio); if (!discard_bio) return; From dc386aa0ac0a3ec06c9a3ea9b064b073fb72a916 Mon Sep 17 00:00:00 2001 From: Bruce Johnston Date: Mon, 3 Aug 2026 14:02:39 -0400 Subject: [PATCH 175/241] md/raid1: don't set array_frozen in raid1_takeover() raid1_takeover() sets conf->array_frozen = 1 on the newly-allocated r1conf and nothing ever clears it, so every I/O to the array stalls permanently once _wait_barrier() sees it stuck at 1. This used to be harmless: level_store() called mddev_resume() right after pers->run(), which called raid1_quiesce(mddev, 0) and cleared array_frozen back to 0 regardless of what raid1_takeover() set. Commit b39f35ebe86d ("md: don't quiesce in mddev_suspend()") removed that quiesce(mddev, 0) call, so the pre-set now sticks. setup_conf() already zero-initializes the new r1conf via kzalloc, so just don't set array_frozen here. Same class of bug as commit 892da88d1cd9 ("md/raid10: fix a 'conf->barrier' leakage in raid10_takeover()"), also triggered by b39f35ebe86d. Fixes: b39f35ebe86d ("md: don't quiesce in mddev_suspend()") Link: https://issues.redhat.com/browse/RHEL-191802 Signed-off-by: Bruce Johnston Link: https://patch.msgid.link/20260803180240.1177104-1-bjohnsto@redhat.com Signed-off-by: Yu Kuai --- drivers/md/raid1.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index e9baba7b241f..f0646fb24371 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -3430,8 +3430,6 @@ static void *raid1_takeover(struct mddev *mddev) mddev->new_chunk_sectors = 0; conf = setup_conf(mddev); if (!IS_ERR(conf)) { - /* Array must appear to be quiesced */ - conf->array_frozen = 1; mddev_clear_unsupported_flags(mddev, UNSUPPORTED_MDDEV_FLAGS); } From 4a3f00262a044e8e15064b1a6860968bf0500bf4 Mon Sep 17 00:00:00 2001 From: Ibrahim Hashimov Date: Thu, 9 Jul 2026 15:25:33 +0200 Subject: [PATCH 176/241] nvmet-tcp: bound SGL data length before allocating command buffers nvmet_tcp_map_data() reads the host-controlled 32-bit sgl->length and, for the in-capsule offset descriptor (type 0x01), checks it against port->inline_data_size before use. Any other SGL descriptor type -- including the non-inline transport SGL data-block descriptor (type (NVME_TRANSPORT_SGL_DATA_DESC << 4) | NVME_SGL_FMT_TRANSPORT_A, the type a real host uses for out-of-capsule writes) skips that check entirely and falls straight through to: cmd->req.sg = sgl_alloc(len, GFP_KERNEL, &cmd->req.sg_cnt); with len taken directly from the wire, unbounded up to 4 GiB. nvmet_req_init() only parses the command and never inspects sgl->length, and nvmet_check_transfer_len() -- the only other place transfer_len is validated -- runs later, from req->execute(), after the allocation has already happened. For a write command the target responds with an R2T and parks the command waiting for the host to send the data; if the host (or an unauthenticated peer that simply never follows up) never does, the sgl_alloc() buffer stays resident for the life of the command. NVMe/TCP has no mandatory authentication in the default configuration, so any peer able to reach the target portal and complete a Fabrics connect can drive this with a single crafted command, repeatable across queues and connections for amplification. This is unbounded kernel memory allocation triggered by a remote, effectively unauthenticated peer. Validate len against the same NVMET_TCP_MAXH2CDATA ceiling this file already uses to bound per-PDU H2C data, for every SGL descriptor type, before doing any allocation. This closes the gap for the non-inline descriptor while leaving the existing, tighter inline_data_size check in place for the in-capsule case. Runtime-verified on a v6.19 KASAN stand: with this bound in place, a crafted write command carrying an oversized non-inline SGL length is rejected before sgl_alloc() runs, where the same request previously drove an unbounded ~256 MiB kernel allocation (up to 4 GiB) that stayed resident pending an R2T the host never satisfies. Fixes: 872d26a391da ("nvmet-tcp: add NVMe over TCP target driver") Cc: stable@vger.kernel.org Reviewed-by: Christoph Hellwig Signed-off-by: Ibrahim Hashimov Assisted-by: AuditCode-AI:2026.07 Signed-off-by: Keith Busch --- drivers/nvme/target/tcp.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index cb6d37798d74..e4f603b2ace7 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -422,6 +422,19 @@ static int nvmet_tcp_map_data(struct nvmet_tcp_cmd *cmd) if (!len) return 0; + /* + * inline_data_size only bounds the in-capsule (type 0x01) SGL + * descriptor below. A non-inline transport SGL data-block + * descriptor skips that check entirely and would otherwise reach + * sgl_alloc() with an attacker-controlled len of up to 4 GiB, + * pinning that much kernel memory for a command that may never + * complete. Bound every descriptor type here, before allocating + * anything, using the same ceiling this file already applies to + * per-PDU H2C data. + */ + if (len > NVMET_TCP_MAXH2CDATA) + return NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR; + if (sgl->type == ((NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET)) { if (!nvme_is_write(cmd->req.cmd)) From bc7f75eba50012ed447654d8ce169ebaba695193 Mon Sep 17 00:00:00 2001 From: Hari Mishal Date: Fri, 17 Jul 2026 16:43:53 +0200 Subject: [PATCH 177/241] nvmet: passthru: fix OOB reads when parsing ns id descriptor list nvmet_passthru_override_id_descs() walks a namespace identification descriptor list populated from the underlying passthru controller's Identify response, which is device reported. The loop advanced pos by device controlled amounts (sizeof(*cur) + nidl) without checking that the next descriptor header actually fits inside the buffer, so a malicious device could push pos to within a few bytes of the buffer end and cause cur->nidl, cur->nidt or the reserved field to be read past the allocation. Additionally, when a CSI descriptor lands exactly at the last valid header offset, cur + 1 points one byte past the end of the buffer. The unconditional memcpy(&csi, cur + 1, NVME_NIDT_CSI_LEN) could read that out-of-bounds byte and copy it back to the initiator via nvmet_copy_to_sgl(), leaking adjacent heap memory. Bounds check both the descriptor header and the CSI value before dereferencing them. Signed-off-by: Hari Mishal Signed-off-by: Keith Busch --- drivers/nvme/target/passthru.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index e27f84e3cf2b..fa6527c537e2 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -53,13 +53,22 @@ static u16 nvmet_passthru_override_id_descs(struct nvmet_req *req) for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) { struct nvme_ns_id_desc *cur = data + pos; + if (pos + sizeof(*cur) > NVME_IDENTIFY_DATA_SIZE) + break; + if (cur->nidl == 0) break; + if (cur->nidt == NVME_NIDT_CSI) { + if (pos + sizeof(*cur) + NVME_NIDT_CSI_LEN > + NVME_IDENTIFY_DATA_SIZE) + break; + memcpy(&csi, cur + 1, NVME_NIDT_CSI_LEN); csi_seen = true; break; } + len = sizeof(struct nvme_ns_id_desc) + cur->nidl; } From 58202950e39c127d593ec4f0624d8b8985a285b1 Mon Sep 17 00:00:00 2001 From: Geliang Tang Date: Sun, 26 Jul 2026 10:46:49 +0800 Subject: [PATCH 178/241] nvme-tcp: look up host_iface in the current netns nvme_tcp_alloc_ctrl() looks opts->host_iface up in &init_net, the boot-time netns. When called from any other netns - e.g. the selftest's ns2, where ns2eth1 actually lives - the lookup misses and the controller setup fails with "invalid interface passed": nvmet: adding nsid 1 to subsystem nqn.2014-08.org.nvmexpress.mptcpdev nvmet_tcp: enabling port 24660 (0.0.0.0:24099) # nvme discover -a 10.1.1.1 --tos=0x10 --host-iface=ns2eth1 nvme_tcp: invalid interface passed: ns2eth1 # failed to add controller, error invalid interface Look the device up in current->nsproxy->net_ns instead so the check sees the calling task's netns. Reviewed-by: Hannes Reinecke Signed-off-by: Geliang Tang Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 87d8067f3283..0b2ac150b675 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -2965,7 +2965,8 @@ static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev, } if (opts->mask & NVMF_OPT_HOST_IFACE) { - if (!__dev_get_by_name(&init_net, opts->host_iface)) { + if (!__dev_get_by_name(current->nsproxy->net_ns, + opts->host_iface)) { pr_err("invalid interface passed: %s\n", opts->host_iface); ret = -ENODEV; From ba98d6796d12258e837ece065d2ecb59d76ce4ff Mon Sep 17 00:00:00 2001 From: Jiang HongHui Date: Wed, 29 Jul 2026 19:02:06 +0800 Subject: [PATCH 179/241] nvmet-fc: fix invalid free in LS IOD error path nvmet_fc_alloc_ls_iodlist() advances iod while initializing the LS IOD array. If an rqstbuf allocation or response buffer DMA mapping fails, the unwind loop decrements iod past the start of the array. The final kfree(iod) therefore frees an address before the allocated object. This can be reproduced with nvme-fcloop and failslab by setting fail-nth to 6 before creating a target port. KASAN reports: BUG: KASAN: invalid-free in nvmet_fc_register_targetport Free of addr ffff88816cf8ff48 by task nvmet_fail_nth/9552 Free the original allocation base stored in tgtport->iod instead. With this fix applied, the same sysfs write with fail-nth=6 returns -ENOMEM without any KASAN report. Fixes: c53432030d86 ("nvme-fabrics: Add target support for FC transport") Cc: stable@vger.kernel.org Reviewed-by: Maurizio Lombardi Assisted-by: Codex:gpt-5 Signed-off-by: Jiang HongHui Signed-off-by: Keith Busch --- drivers/nvme/target/fc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index d161707559ce..1b557775e033 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -566,7 +566,7 @@ out_fail: list_del(&iod->ls_rcv_list); } - kfree(iod); + kfree(tgtport->iod); return -EFAULT; } From 0a96b9e440331bffbf049f80d7e5c96417d37e36 Mon Sep 17 00:00:00 2001 From: Zhengrong Li Date: Tue, 28 Jul 2026 16:26:01 +0800 Subject: [PATCH 180/241] nvmet: fix Reservation Register Replace for unregistered host with IEKEY When a host sends a Reservation Register command with RREGA=Replace and IEKEY=1 without being previously registered, nvmet returns Reservation Conflict. The NVMe specification states: "A host may replace its reservation key without regard to its registration status or current reservation key value by setting the Ignore Existing Key (IEKEY) bit to '1' in the Reservation Register command." Fix nvmet_pr_replace() to add a new registrant when the host is not found in the registrant list and IEKEY is set with a non-zero NRKEY. If IEKEY is set but NRKEY is zero, return Invalid Field since there is no valid reservation key to register. Tested with nvme-cli against nvmet-tcp: # no prior registration nvme resv-register /dev/nvmeXn1 -n 1 --rrega=2 --iekey --nrkey=0x9999 Before: RESERVATION_CONFLICT (0x4083) After: success, registrant created with rkey 0x9999 Fixes: 5a47c2080a73 ("nvmet: support reservation feature") Reviewed-by: Christoph Hellwig Reviewed-by: Guixin Liu Signed-off-by: Zhengrong Li Signed-off-by: Keith Busch --- drivers/nvme/target/pr.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/pr.c b/drivers/nvme/target/pr.c index 5dd2f3553d8c..0948a690a1c0 100644 --- a/drivers/nvme/target/pr.c +++ b/drivers/nvme/target/pr.c @@ -355,9 +355,15 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmet_pr *pr = &req->ns->pr; - struct nvmet_pr_registrant *reg; + struct nvmet_pr_registrant *reg, *new = NULL; u64 nrkey = le64_to_cpu(d->nrkey); + if (ignore_key && nrkey) { + new = kzalloc_obj(*new); + if (!new) + return NVME_SC_INTERNAL; + } + down(&pr->pr_sem); list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { if (uuid_equal(®->hostid, &ctrl->hostid)) { @@ -365,9 +371,26 @@ static u16 nvmet_pr_replace(struct nvmet_req *req, status = nvmet_pr_update_reg_attr(pr, reg, nvmet_pr_update_reg_rkey, &nrkey); - break; + goto free_data; } } + + if (ignore_key) { + if (!nrkey) { + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto free_data; + } + INIT_LIST_HEAD(&new->entry); + new->rkey = nrkey; + uuid_copy(&new->hostid, &ctrl->hostid); + list_add_tail_rcu(&new->entry, &pr->registrant_list); + status = NVME_SC_SUCCESS; + goto out; + } + +free_data: + kfree(new); +out: up(&pr->pr_sem); return status; } From bededeaaeff404978a5a8e2a605a6c3017cddd3e Mon Sep 17 00:00:00 2001 From: Yehyeong Lee Date: Thu, 30 Jul 2026 20:36:24 +0900 Subject: [PATCH 181/241] nvme: zero the discard fallback page nvme_setup_discard() always maps sizeof(struct nvme_dsm_range) * NVME_DSM_MAX_RANGES = 4096 bytes as the DSM payload however many ranges the command declares, because some devices ignore the 'Number of Ranges' field - the Fixes: commit records two that read past the declared ranges. A single-range discard fills only the first 16 bytes. Normally the buffer comes from kzalloc() and the other 4080 bytes are zero. When that allocation fails the code falls back to the per-controller ctrl->discard_page, which nvme_init_ctrl() obtains with alloc_page(GFP_KERNEL) and nothing ever zeroes, so those 4080 bytes are whatever the page last held and are handed to the controller. Reaching it requires the kzalloc(GFP_ATOMIC | __GFP_NOWARN) to fail, that is memory pressure; it is not remotely triggerable. Failing the allocation under KMSAN reproduces it, with the leaked tail full of vmemmap struct page pointers. The extent in the report is a partial transfer of the payload, not the whole 4096 bytes; the 16-byte boundary in it is the one declared range: [ 11.991601] BUG: KMSAN: uninit-value in dma_map_phys+0x14c8/0x1900 [ 11.991969] dma_map_phys+0x14c8/0x1900 [ 11.992220] dma_map_page_attrs+0xcf/0x130 [ 11.992485] e1000_xmit_frame+0x4099/0x6d10 [ 11.992768] dev_hard_start_xmit+0x22f/0xa80 [ 11.993068] sch_direct_xmit+0x35c/0xcb0 [ 11.993315] __dev_queue_xmit+0x1ee5/0x5eb0 [ 11.993608] ip_finish_output2+0x1903/0x1c30 [ 11.993881] ip_finish_output+0x288/0x870 [ 11.994125] ip_output+0x15e/0x400 [ 11.994365] __ip_queue_xmit+0x1e85/0x1fb0 [ 11.994639] ip_queue_xmit+0x60/0x80 [ 11.994899] __tcp_transmit_skb+0x4e71/0x5fa0 [ 11.995210] tcp_write_xmit+0x3a36/0x9160 [ 11.995533] __tcp_push_pending_frames+0xc5/0x3c0 [ 11.995854] tcp_push+0x7dc/0x840 [ 11.996076] tcp_sendmsg_locked+0x766c/0x8400 [ 11.996371] tcp_sendmsg+0x4b/0x90 [ 11.996572] inet_sendmsg+0x134/0x2a0 [ 11.996823] __sock_sendmsg+0x265/0x360 [ 11.997076] sock_sendmsg+0x100/0x1e0 [ 11.997293] nvme_tcp_try_send+0x196f/0x6370 [ 11.997605] nvme_tcp_queue_rq+0x1d54/0x20b0 [ 11.997882] blk_mq_dispatch_rq_list+0x5ee/0x2e50 [ 11.998175] __blk_mq_sched_dispatch_requests+0x16dc/0x24a0 [ 11.998539] blk_mq_sched_dispatch_requests+0x11b/0x2c0 [ 11.998865] blk_mq_run_work_fn+0x13b/0x280 [ 11.999146] process_scheduled_works+0x966/0x1ad0 [ 11.999465] worker_thread+0xe44/0x1480 [ 11.999709] kthread+0x53b/0x600 [ 11.999927] ret_from_fork+0x29f/0x7c0 [ 12.000191] ret_from_fork_asm+0x1a/0x30 [ 12.000460] [ 12.000558] Uninit was created at: [ 12.000788] __alloc_frozen_pages_noprof+0x8bf/0xd30 [ 12.001096] alloc_pages_mpol+0x1d0/0x5f0 [ 12.001326] alloc_pages_noprof+0x102/0x290 [ 12.001627] nvme_init_ctrl+0x5a3/0x9f0 [ 12.001891] nvme_tcp_create_ctrl+0xd75/0x19b0 [ 12.002170] nvmf_dev_write+0x4c68/0x4fd0 [ 12.002426] vfs_write+0x587/0x1a10 [ 12.002636] __x64_sys_write+0x207/0x4f0 [ 12.002874] x64_sys_call+0x2ff0/0x3ea0 [ 12.003123] do_syscall_64+0x147/0x3b0 [ 12.003400] entry_SYSCALL_64_after_hwframe+0x77/0x7f [ 12.003680] [ 12.003777] Bytes 16-2843 of 2844 are uninitialized [ 12.004068] Memory access of size 2844 starts at ffff888109f82000 [ 12.004412] [ 12.004530] CPU: 0 UID: 0 PID: 101 Comm: kworker/0:1H Not tainted 7.2.0-rc5-NVMECTL-gf5098b6bae76 #1 PREEMPT(lazy) [ 12.005127] Hardware name: QEMU Ubuntu 24.04 PC v2 (i440FX + PIIX, arch_caps fix, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014 [ 12.005762] Workqueue: kblockd blk_mq_run_work_fn [ 12.006073] ===================================================== Allocate the page with __GFP_ZERO. The single allocation site covers every use of it: bytes no discard has written stay zero, and bytes one did write hold that controller's own range list, which it has already been sent. Fixes: 530436c45ef2 ("nvme: Discard workaround for non-conformant devices") Cc: stable@vger.kernel.org Signed-off-by: Yehyeong Lee Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index cb93ada4376a..975181a74fae 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -5223,7 +5223,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev, BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) > PAGE_SIZE); - ctrl->discard_page = alloc_page(GFP_KERNEL); + ctrl->discard_page = alloc_page(GFP_KERNEL | __GFP_ZERO); if (!ctrl->discard_page) { ret = -ENOMEM; goto out; From 79aba4c9403419d822972d2851f2a96a2c0531cf Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 4 Aug 2026 10:18:57 +0800 Subject: [PATCH 182/241] nvmet: fix NULL pointer dereference in nvmet_execute_identify_nslist() When a host issues an Identify command with CNS 07h (Active Namespace ID List for a specific I/O Command Set), nvmet_execute_identify_nslist() is called with match_css set. The command-set filter dereferences req->ns, but this handler never calls nvmet_req_find_ns(), so req->ns is always NULL (nvmet_req_init() resets it to NULL). As soon as an enabled namespace with an NSID greater than the requested value exists, req->ns->csi dereferences a NULL pointer and oopses. Besides the crash, the comparison is logically wrong: to filter the list by command set it must test the command set of the namespace being iterated, not a single fixed value. Use the loop variable ns->csi. Fixes: 61c9967cd634 ("nvmet: implement active command set ns list") Signed-off-by: Guixin Liu Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Reviewed-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/target/admin-cmd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 0b24d31f966d..3fde09b4d78a 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -960,7 +960,7 @@ static void nvmet_execute_identify_nslist(struct nvmet_req *req, bool match_css) nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) { if (ns->nsid <= min_nsid) continue; - if (match_css && req->ns->csi != req->cmd->identify.csi) + if (match_css && ns->csi != req->cmd->identify.csi) continue; list[i++] = cpu_to_le32(ns->nsid); if (i == buf_size / sizeof(__le32)) From 751709592d2626eaa8dc17ef8137796758a3c37f Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 4 Aug 2026 10:18:58 +0800 Subject: [PATCH 183/241] nvmet: propagate percpu_ref_init() failure in nvmet_ns_enable() The return value of percpu_ref_init() is discarded. At this point ret is 0 from the preceding successful steps, so when the allocation inside percpu_ref_init() fails the code jumps to the out_pr_exit cleanup chain which ends with "return ret", i.e. reports success. The configfs enable store then tells userspace the namespace was enabled even though it was not and its backing device has already been torn down. Capture the return value so the failure is propagated. Fixes: 408232680707 ("nvmet: Fix crash when a namespace is disabled") Signed-off-by: Guixin Liu Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Reviewed-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/target/core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index a2403a808360..30a1eb77f60b 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -610,7 +610,8 @@ int nvmet_ns_enable(struct nvmet_ns *ns) goto out_dev_put; } - if (percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL)) + ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 0, GFP_KERNEL); + if (ret) goto out_pr_exit; nvmet_ns_changed(subsys, ns->nsid); From cb144c2f67128abfa5c7ba33318617d19f192156 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 4 Aug 2026 10:18:59 +0800 Subject: [PATCH 184/241] nvme-pci: release descriptor pools on probe failure The per-NUMA-node descriptor DMA pools are created lazily from nvme_init_hctx_common() once the admin tag set is allocated, but they are only destroyed in nvme_remove() via nvme_release_descriptor_pools(). Any probe failure after the admin tag set has been allocated unwinds through the out_disable label and nvme_pci_free_ctrl(), neither of which releases the pools, leaking the dma_pool objects. Release the descriptor pools in the out_disable error path. It must not be added to nvme_pci_free_ctrl(), as that would double-free against nvme_remove() on the normal teardown path. Fixes: d977506f8863 ("nvme-pci: make PRP list DMA pools per-NUMA-node") Signed-off-by: Guixin Liu Reviewed-by: Hannes Reinecke Reviewed-by: Christoph Hellwig Reviewed-by: Kanchan Joshi Reviewed-by: Nilay Shroff Signed-off-by: Keith Busch --- drivers/nvme/host/pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 375e7a1fc91d..ef06627b21ee 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -3854,6 +3854,7 @@ out_disable: nvme_dev_remove_admin(dev); nvme_dbbuf_dma_free(dev); nvme_free_queues(dev, 0); + nvme_release_descriptor_pools(dev); out_release_iod_mempool: mempool_destroy(dev->dmavec_mempool); out_dev_unmap: From 53cdaeab2e30e0cb849a74b94f93729ad98946b1 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 4 Aug 2026 10:19:01 +0800 Subject: [PATCH 185/241] nvme: raise FDP placement handle cap to U8_MAX and warn on overflow The RUH status buffer and the placement-handle clamp used S8_MAX - 1 (126) as the maximum descriptor count. That value was picked only so the io-mgmt-receive result fit in a page, not because of any protocol or driver restriction. The meaningful upper bound is U8_MAX: write hints (bio->bi_write_stream) are u8, so placement handles beyond U8_MAX can never be selected. Size the buffer and clamp nr_plids to U8_MAX. Suggested-by: Kanchan Joshi Signed-off-by: Guixin Liu Reviewed-by: Kanchan Joshi Reviewed-by: Nilay Shroff Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/host/core.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 975181a74fae..a59abd770aff 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -33,6 +33,13 @@ #define NVME_MINORS (1U << MINORBITS) +/* + * Write hints (bio->bi_write_stream) are u8, so FDP placement handles beyond + * U8_MAX can never be selected. Cap the handle count to bound both the RUH + * status buffer and the per-head plids array. + */ +#define NVME_MAX_PLIDS U8_MAX + struct nvme_ns_info { struct nvme_ns_ids ids; u32 nsid; @@ -2353,7 +2360,7 @@ static int nvme_query_fdp_info(struct nvme_ns *ns, struct nvme_ns_info *info) if (!info->runs) return ret; - size = struct_size(ruhs, ruhsd, S8_MAX - 1); + size = struct_size(ruhs, ruhsd, NVME_MAX_PLIDS); ruhs = kzalloc(size, GFP_KERNEL); if (!ruhs) return -ENOMEM; @@ -2368,7 +2375,7 @@ static int nvme_query_fdp_info(struct nvme_ns *ns, struct nvme_ns_info *info) goto free; } - head->nr_plids = min(le16_to_cpu(ruhs->nruhsd), S8_MAX - 1); + head->nr_plids = min(le16_to_cpu(ruhs->nruhsd), NVME_MAX_PLIDS); if (!head->nr_plids) goto free; From 5bb96cc218835769ab74ec7f3ea2bf81fbffe955 Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 4 Aug 2026 11:38:00 +0800 Subject: [PATCH 186/241] nvmet: fix heap out-of-bounds read in nvmet_auth_negotiate() nvmet_execute_auth_send() allocates the DH-HMAC-CHAP message buffer with the host-supplied transfer length (tl) and hands it to nvmet_auth_negotiate() without passing tl along. nvmet_auth_negotiate() then reads the negotiate header and, for each of the halen hash identifiers and dhlen DH group identifiers, indexes into the fixed idlist[60] array (hashes at idlist[0..halen), groups at idlist[30..]). Neither the transfer length nor halen/dhlen is validated. A malicious or non-conformant host can report a tl smaller than the negotiate structure, or a halen/dhlen larger than the array (both are u8, up to 255), making the loops read past the end of the allocated buffer (heap out-of-bounds read). The sibling nvmet_auth_reply() already validates tl against the structure size; the negotiate path did not. Pass tl into nvmet_auth_negotiate(), reject a tl that does not cover the negotiate data plus one full protocol descriptor, and reject halen/dhlen larger than NVME_AUTH_DHCHAP_MAX_DH_IDS. Fixes: db1312dd9548 ("nvmet: implement basic In-Band Authentication") Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- drivers/nvme/target/fabrics-cmd-auth.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/drivers/nvme/target/fabrics-cmd-auth.c b/drivers/nvme/target/fabrics-cmd-auth.c index d1b39e64d877..92f8a76f10ff 100644 --- a/drivers/nvme/target/fabrics-cmd-auth.c +++ b/drivers/nvme/target/fabrics-cmd-auth.c @@ -31,12 +31,16 @@ void nvmet_auth_sq_init(struct nvmet_sq *sq) sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; } -static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) +static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d, u32 tl) { struct nvmet_ctrl *ctrl = req->sq->ctrl; struct nvmf_auth_dhchap_negotiate_data *data = d; int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid; + if (tl < sizeof(*data) + + sizeof(struct nvmf_auth_dhchap_protocol_descriptor)) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n", __func__, ctrl->cntlid, req->sq->qid, data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid, @@ -72,6 +76,10 @@ static u8 nvmet_auth_negotiate(struct nvmet_req *req, void *d) NVME_AUTH_DHCHAP_AUTH_ID) return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + if (data->auth_protocol[0].dhchap.dhlen > NVME_AUTH_DHCHAP_MAX_DH_IDS || + data->auth_protocol[0].dhchap.halen > NVME_AUTH_DHCHAP_MAX_HASH_IDS) + return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; + for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) { u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i]; @@ -317,7 +325,7 @@ void nvmet_execute_auth_send(struct nvmet_req *req) } else if (data->auth_id != req->sq->dhchap_step) goto done_failure1; /* Validate negotiation parameters */ - dhchap_status = nvmet_auth_negotiate(req, d); + dhchap_status = nvmet_auth_negotiate(req, d, tl); if (dhchap_status == 0) req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE; From 87d5b9864c8118d26f54de4b66d2bddf2c659272 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Thu, 6 Aug 2026 17:27:33 +0200 Subject: [PATCH 187/241] nvme-apple: Destroy the admin queue on removal The admin queue is allocated with blk_mq_alloc_queue() but never destroyed. nvme_free_ctrl() only drops the last reference and blk_mq_exit_queue() and blk_sync_queue() never run: the hctx is never moved to q->unused_hctx_list and the timeout timer and work stay armed on a queue that is about to be freed which will eventually oops inside blk_mq_timeout_work(). This can only be triggered when the controller fails to come up and is then immediately torn down again which is why no one ever ran into this before. Let's just copy what the pcie driver does: unquiesce and destroy the admin queue before nvme_uninit_ctrl(). With this the following WARN followed by a panic no longer happens: WARNING: block/blk-mq.c:4390 at blk_mq_release+0x194/0x238, CPU#4: kworker/u34:4/119 CPU: 4 UID: 0 PID: 119 Comm: kworker/u34:4 Not tainted 7.2.0-rc1-dirty #248 PREEMPT Hardware name: Apple Mac mini (M1, 2020) (DT) Workqueue: nvme-wq apple_nvme_remove_dead_ctrl_work pstate: 61400005 (nZCv daif +PAN -UAO -TCO +DIT -SSBS BTYPE=--) pc : blk_mq_release+0x194/0x238 lr : blk_mq_release+0x58/0x238 sp : ffffc000833a3b50 x29: ffffc000833a3b50 x28: ffff80001d0450f8 x27: ffff800020c95200 x26: 0000000000000088 x25: 0000000000000000 x24: ffff800020f36805 x23: 0000000000000000 x22: ffffc00081a86878 x21: ffff800020be9c60 x20: 0000000000000000 x19: ffff800022501698 x18: 000000000000000a x17: 7365757165722066 x16: 666f7265776f7020 x15: 0000000000000000 x14: 0000000000000028 x13: 0000000000004def x12: 0000000000000003 x11: 0000000000000000 x10: 0000000000000000 x9 : ffffc000805b4fc8 x8 : ffffc00081915820 x7 : ffffc00081c4f3c8 x6 : 0000000000000001 x5 : 0000000000000004 x4 : ffff800022498d80 x3 : ffffc000833a3b14 x2 : 0000000000000000 x1 : 0000000000000000 x0 : ffff800022501698 Call trace: blk_mq_release+0x194/0x238 (P) blk_put_queue+0x8c/0xf0 nvme_free_ctrl+0x4c/0x260 device_release+0x44/0x128 kobject_put+0xa0/0x120 put_device+0x1c/0x40 nvme_uninit_ctrl+0x48/0x60 apple_nvme_remove+0x54/0xb0 platform_remove+0x28/0x40 device_remove+0x54/0x98 device_release_driver_internal+ device_release_driver+0x20/0x38 apple_nvme_remove_dead_ctrl_wor process_one_work+0x1f4/0x770 worker_thread+0x1b8/0x360 kthread+0x140/0x160 ret_from_fork+0x10/0x20 irq event stamp: 448 hardirqs last enabled at (447):in_unlock_irqrestore+0x74/0x80 hardirqs last disabled at (448): [] el1_brk64+0x20/0x60 softirqs last enabled at (0): [ess+0xb28/0x2698 softirqs last disabled at (0): [<0000000000000000>] 0x0 ---[ end trace 0000000000000000 Unable to handle kernel NULL pointer dereference at virtual address 0000000000000000 Mem abort info: ESR = 0x0000000096000005 EC = 0x25: DABT (current EL), SET = 0, FnV = 0 EA = 0, S1PTW = 0 FSC = 0x05: level 1 translation fault Data abort info: ISV = 0, ISS = 0x00000005, ISS2 = 0x00000000 CM = 0, WnR = 0, TnD = 0, TagA GCS = 0, Overlay = 0, DirtyBit = 0, Xs = 0 [0000000000000000] user address Internal error: Oops: 0000000096000005 [#1] SMP CPU: 7 UID: 0 PID: 54 Comm: kwor 7.2.0-rc1-dirty #248PREEMPT Tainted: [W]=WARN Hardware name: Apple Mac mini (M1, 2020) (DT) Workqueue: kblockd blk_mq_timeou pstate: 01400005 (nzcv daif +PAN -UAO -TCO +DIT -SSBS BTYPE=--) pc : percpu_ref_tryget_many.cons lr : percpu_ref_tryget_many.constprop.0+0xc0/0x168 sp : ffffc000829cbce0 x29: ffffc000829cbce0 x28: ffff800020be9f48 x27: ffff800013e503c0 x26: 0000000000000108 x25: 000009c05 x23: 0000000000000000 x22: ffffc000819f5000 x21: ffff800020be9f48 x20: ffff8001deda4808 x19: ffff8000a x17: 00000000580e1fac x16: ffffc00082bbbb7c x15: 0000000000000000 x14: 0000000000000028 x13: 000000001 x11: 0000000000000000 x10: 0000000000000000 x9 : ffffc000829cbc20 x8 : ffffc00081915820 x7 : ffffc0001 x5 : ffff80001ca77d08 x4 : 0000000000000000 x3 : ffff80001ca77cb8 x2 : 0000000000000000 x1 : 000000007 Call trace: percpu_ref_tryget_many.constpro blk_mq_timeout_work+0x48/0x298 process_one_work+0x1f4/0x770 worker_thread+0x1b8/0x360 kthread+0x140/0x160 ret_from_fork+0x10/0x20 Code: 91282000 97ed44b2 17ffffd2 ---[ end trace 0000000000000000 ]--- Fixes: 5bd2927aceba ("nvme-apple: Add initial Apple SoC NVMe driver") Tested-by: Joshua Peisach Tested-by: Janne Grunau Tested-by: Nick Chan Signed-off-by: Sven Peter --- drivers/nvme/host/apple.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 09eb2295ceee..7e6a83b30731 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1650,6 +1650,15 @@ static void apple_nvme_remove(struct platform_device *pdev) nvme_stop_ctrl(&anv->ctrl); nvme_remove_namespaces(&anv->ctrl); apple_nvme_disable(anv, true); + if (anv->ctrl.admin_q && !blk_queue_dying(anv->ctrl.admin_q)) { + /* + * If the controller was reset during removal, it's possible + * user requests may be waiting on a stopped queue. Start the + * queue to flush these to completion. + */ + nvme_unquiesce_admin_queue(&anv->ctrl); + blk_mq_destroy_queue(anv->ctrl.admin_q); + } nvme_uninit_ctrl(&anv->ctrl); if (apple_rtkit_is_running(anv->rtk)) { From 94dd5804938d6681dbf26f023b1356d511f4fc48 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Thu, 6 Aug 2026 17:27:34 +0200 Subject: [PATCH 188/241] nvme-apple: Don't set a DMA direction for commands without a data transfer Setting the DMA direction for commands that don't do any transfer likely triggered the PRP NULL check for which we needed a chicken bit. That bit has disappeared starting with macOS 15 so let's just do this correctly instead. Fixes: 5bd2927aceba ("nvme-apple: Add initial Apple SoC NVMe driver") Tested-by: Joshua Peisach Tested-by: Janne Grunau Tested-by: Nick Chan Signed-off-by: Sven Peter --- drivers/nvme/host/apple.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 7e6a83b30731..0eb31ab196cf 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -341,7 +341,9 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, tcb->length = cmd->rw.length; tcb->command_id = tag; - if (nvme_is_write(cmd)) + if (!cmd->common.dptr.prp1) + tcb->dma_flags = 0; + else if (nvme_is_write(cmd)) tcb->dma_flags = APPLE_ANS_TCB_DMA_TO_DEVICE; else tcb->dma_flags = APPLE_ANS_TCB_DMA_FROM_DEVICE; From cc0fec9b42cfbc69d70cb4c4b616408a7037b445 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Thu, 6 Aug 2026 17:27:35 +0200 Subject: [PATCH 189/241] nvme-apple: Never set the opcode in the NVMMU TCB macOS always sets this to zero and the firmware starting with macOS 15 has started to complain about what we're doing here. Fixes: 5bd2927aceba ("nvme-apple: Add initial Apple SoC NVMe driver") Tested-by: Joshua Peisach Tested-by: Janne Grunau Tested-by: Nick Chan Signed-off-by: Sven Peter --- drivers/nvme/host/apple.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 0eb31ab196cf..321d7f5ab902 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -335,7 +335,7 @@ static void apple_nvme_submit_cmd_t8103(struct apple_nvme_queue *q, u32 tag = nvme_tag_from_cid(cmd->common.command_id); struct apple_nvmmu_tcb *tcb = &q->tcbs[tag]; - tcb->opcode = cmd->common.opcode; + tcb->opcode = 0; tcb->prp1 = cmd->common.dptr.prp1; tcb->prp2 = cmd->common.dptr.prp2; tcb->length = cmd->rw.length; From 69d22a6b2f6984200d92dac689f8b00cc3d7d736 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Thu, 6 Aug 2026 17:27:36 +0200 Subject: [PATCH 190/241] nvme: Add a quirk for page aligned admin queue buffers Apple controllers seem to require any queue buffers on the admin queue to be aligned to the NVMe controller page size. Weirdly, this constraint does not apply to the i/o queue where any alignment is fine. This has always been required on pre-M1 controllers and is required starting with macOS 15 firmware or post-M4 controllers again. On M1/M2/M3 we only got away with this because there was a chicken bit to disable this requirement. Let's add a quirk that enforces this alignment. Tested-by: Joshua Peisach Tested-by: Janne Grunau Tested-by: Nick Chan Signed-off-by: Sven Peter --- drivers/nvme/host/core.c | 5 ++++- drivers/nvme/host/nvme.h | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a59abd770aff..1322c678f4eb 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2089,7 +2089,10 @@ static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl, lim->max_integrity_segments = ctrl->max_integrity_segments; lim->virt_boundary_mask = ctrl->ops->get_virt_boundary(ctrl, is_admin); lim->max_segment_size = UINT_MAX; - lim->dma_alignment = 3; + if (is_admin && (ctrl->quirks & NVME_QUIRK_ADMIN_PAGE_ALIGN)) + lim->dma_alignment = NVME_CTRL_PAGE_SIZE - 1; + else + lim->dma_alignment = 3; } static bool nvme_update_disk_info(struct nvme_ns *ns, struct nvme_id_ns *id, diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 862464301d01..28cec87e4427 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -178,6 +178,11 @@ enum nvme_quirks { * Align dma pool segment size to 512 bytes */ NVME_QUIRK_DMAPOOL_ALIGN_512 = (1 << 22), + + /* + * Admin queue DMA buffers must be page aligned + */ + NVME_QUIRK_ADMIN_PAGE_ALIGN = (1 << 23), }; static inline char *nvme_quirk_name(enum nvme_quirks q) @@ -229,6 +234,8 @@ static inline char *nvme_quirk_name(enum nvme_quirks q) return "broken_msi"; case NVME_QUIRK_DMAPOOL_ALIGN_512: return "dmapool_align_512"; + case NVME_QUIRK_ADMIN_PAGE_ALIGN: + return "admin_page_align"; } return "unknown"; From ea2160c7b78187ea9ab08c3190eef237c4ee99a7 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Thu, 6 Aug 2026 17:27:37 +0200 Subject: [PATCH 191/241] nvme-apple: Require page aligned buffers on the admin queue Now that we have a quick to align buffers on the admin queue to the NVMe controller page size use it for Apple controllers. This fixes pre-M1 controllers, which always rejected unaligned requests, and also makes this driver work for M4 SoCs and for M1/M2/M3 SoCs that have been updated to the firmware shipped with macOS 15. Fixes: 5bd2927aceba ("nvme-apple: Add initial Apple SoC NVMe driver") Tested-by: Joshua Peisach Tested-by: Janne Grunau Tested-by: Nick Chan Signed-off-by: Sven Peter --- drivers/nvme/host/apple.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 321d7f5ab902..806dd55d5518 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1597,7 +1597,8 @@ static struct apple_nvme *apple_nvme_alloc(struct platform_device *pdev) } ret = nvme_init_ctrl(&anv->ctrl, anv->dev, &nvme_ctrl_ops, - NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS); + NVME_QUIRK_SKIP_CID_GEN | NVME_QUIRK_IDENTIFY_CNS | + NVME_QUIRK_ADMIN_PAGE_ALIGN); if (ret) { dev_err_probe(dev, ret, "Failed to initialize nvme_ctrl"); goto put_dev; From 8ce883fd068b7ba9ab493cd3ecca3a7ea868c375 Mon Sep 17 00:00:00 2001 From: Sven Peter Date: Thu, 6 Aug 2026 17:27:38 +0200 Subject: [PATCH 192/241] nvme-apple: Drop the PRP null check chicken bit Now that we program the DMA direction correctly the NULL check that used to make commands fail passes. Another side effect of this bit was that non-align buffers on the admin queue were silently allowed and that's been fixed now as well and we this don't need this chicken bit anymore. More importantly, starting with the firmware installed with macOS 15, which is required for M4 but can also be installed on the previous SoCs, the controller no longer exposes this control register and any access SErrors instead. Just drop the write entirely. Fixes: 5bd2927aceba ("nvme-apple: Add initial Apple SoC NVMe driver") Tested-by: Joshua Peisach Tested-by: Janne Grunau Tested-by: Nick Chan Signed-off-by: Sven Peter --- drivers/nvme/host/apple.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index 806dd55d5518..c63e28c75766 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -47,9 +47,6 @@ #define APPLE_ANS_BOOT_STATUS 0x1300 #define APPLE_ANS_BOOT_STATUS_OK 0xde71ce55 -#define APPLE_ANS_UNKNOWN_CTRL 0x24008 -#define APPLE_ANS_PRP_NULL_CHECK BIT(11) - #define APPLE_ANS_LINEAR_SQ_CTRL 0x24908 #define APPLE_ANS_LINEAR_SQ_EN BIT(0) @@ -1143,17 +1140,6 @@ static void apple_nvme_reset_work(struct work_struct *work) /* Setup the NVMMU for the maximum admin and IO queue depth */ writel(anv->hw->max_queue_depth - 1, anv->mmio_nvme + APPLE_NVMMU_NUM_TCBS); - - /* - * This is probably a chicken bit: without it all commands - * where any PRP is set to zero (including those that don't use - * that field) fail and the co-processor complains about - * "completed with err BAD_CMD-" or a "NULL_PRP_PTR_ERR" in the - * syslog - */ - writel(readl(anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL) & - ~APPLE_ANS_PRP_NULL_CHECK, - anv->mmio_nvme + APPLE_ANS_UNKNOWN_CTRL); } /* Setup the admin queue */ From 659ae9d02cb5d72c76f74fff7441eb8fb64d8f5c Mon Sep 17 00:00:00 2001 From: Yifei Gao Date: Tue, 4 Aug 2026 21:36:25 +0000 Subject: [PATCH 193/241] nvmet: pci-epf: put CQ ref on create_cq mapping failure nvmet_pci_epf_create_cq() calls nvmet_cq_create(), which takes a reference on the controller and installs the completion queue. If the subsequent PCI address-space mapping fails or returns a too-small partial mapping, the function jumps to err_internal / err_unmap_queue without calling nvmet_cq_put(). The matching put in nvmet_pci_epf_delete_cq() is gated on NVMET_PCI_EPF_Q_LIVE, which is only set after the mapping succeeds, so teardown never releases these references. A remote PCI host that drives Create IO CQ commands with a failing PRP1/pci_addr therefore leaks the CQ and a controller reference on each attempt. Drop the CQ reference on the mapping-failure paths. The err_internal and err_unmap_queue labels are only reachable after nvmet_cq_create() has succeeded, so this pairs the create/put correctly. Fixes: 0faa0fe6f90e ("nvmet: New NVMe PCI endpoint function target driver") Cc: stable@vger.kernel.org Reviewed-by: Damien Le Moal Assisted-by: Claude:claude-opus-4-8 Signed-off-by: Yifei Gao Signed-off-by: Keith Busch --- drivers/nvme/target/pci-epf.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 4e9db96ebfec..794e88d1d9a5 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -1339,6 +1339,7 @@ err_unmap_queue: nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map); err_internal: status = NVME_SC_INTERNAL | NVME_STATUS_DNR; + nvmet_cq_put(&cq->nvme_cq); err: if (test_and_clear_bit(NVMET_PCI_EPF_Q_IRQ_ENABLED, &cq->flags)) nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector); From c9e9bb757971485b4e8414b1744507af186d72c9 Mon Sep 17 00:00:00 2001 From: Shin'ichiro Kawasaki Date: Thu, 30 Jul 2026 15:18:39 +0900 Subject: [PATCH 194/241] nvmet: pci-epf: fix use-after-free in nvmet_pci_epf_exec_iod_work() nvmet_pci_epf_exec_iod_work() submits an I/O command with req->execute() and then waits for the command to complete and transfers the data back to the host. This wait is not needed for commands that do not transfer data from the device to the host. To decide whether that wait is needed, it reads iod->data_len and iod->dma_dir after calling req->execute(). However, once req->execute() is called, the command may complete asynchronously on another CPU. For commands that do not require a device-to-host data transfer, nvmet_pci_epf_queue_response() calls nvmet_pci_epf_complete_iod() directly, which can free the iod before it reads iod->data_len and iod->dma_dir, resulting in the KFENCE use-after- free: BUG: KFENCE: use-after-free read in nvmet_pci_epf_exec_iod_work+0x288/0x798 [nvmet_pci_epf] Use-after-free read at 0x00000000fdfa6d03 (in kfence-#63): nvmet_pci_epf_exec_iod_work+0x288/0x798 [nvmet_pci_epf] process_one_work+0x15c/0x4f0 worker_thread+0x18c/0x30c kthread+0x130/0x140 ret_from_fork+0x10/0x20 kfence-#63: 0x00000000e3de0e71-0x00000000c938ad62, size=712, cache=kmalloc-1k allocated by task 10 on cpu 0 at 73.995480s (0.005122s ago): mempool_kmalloc+0x1c/0x28 mempool_alloc_noprof+0x40/0x9c nvmet_pci_epf_poll_sqs_work+0xd4/0x344 [nvmet_pci_epf] process_one_work+0x15c/0x4f0 worker_thread+0x18c/0x30c kthread+0x130/0x140 ret_from_fork+0x10/0x20 freed by task 131 on cpu 3 at 73.995521s (0.008385s ago): mempool_kfree+0x10/0x20 mempool_free+0x44/0x64 nvmet_pci_epf_free_iod+0x88/0x98 [nvmet_pci_epf] nvmet_pci_epf_cq_work+0xfc/0x280 [nvmet_pci_epf] process_one_work+0x15c/0x4f0 worker_thread+0x18c/0x30c kthread+0x130/0x140 ret_from_fork+0x10/0x20 Fix this by referring to iod->data_len and iod->dma_dir before calling req->execute(). The remaining iod accesses such as iod->status are only reached on the device-to-host read path. In this case, nvmet_pci_epf_queue_response() signals iod->done instead of freeing the iod, so the iod stays valid. Fixes: 0faa0fe6f90e ("nvmet: New NVMe PCI endpoint function target driver") Cc: stable@vger.kernel.org Reviewed-by: Damien Le Moal Reviewed-by: Christoph Hellwig Signed-off-by: Shin'ichiro Kawasaki Signed-off-by: Keith Busch --- drivers/nvme/target/pci-epf.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 794e88d1d9a5..346a4badd6b2 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -1595,6 +1595,7 @@ static void nvmet_pci_epf_exec_iod_work(struct work_struct *work) struct nvmet_pci_epf_iod *iod = container_of(work, struct nvmet_pci_epf_iod, work); struct nvmet_req *req = &iod->req; + bool no_wait; int ret; if (!iod->ctrl->link_up) { @@ -1639,14 +1640,16 @@ static void nvmet_pci_epf_exec_iod_work(struct work_struct *work) } } - req->execute(req); - /* * If we do not have data to transfer after the command execution * finishes, nvmet_pci_epf_queue_response() will complete the command * directly. No need to wait for the completion in this case. */ - if (!iod->data_len || iod->dma_dir != DMA_TO_DEVICE) + no_wait = !iod->data_len || iod->dma_dir != DMA_TO_DEVICE; + + req->execute(req); + + if (no_wait) return; wait_for_completion(&iod->done); From f594863967d87b7fcbff6e724d51135fd701a13d Mon Sep 17 00:00:00 2001 From: Guixin Liu Date: Tue, 4 Aug 2026 11:36:05 +0800 Subject: [PATCH 195/241] nvmet: fix NULL pointer dereference in nvmet_execute_identify_ns_zns() When a host issues an Identify command with CNS 05h (I/O Command Set specific Identify Namespace) and CSI 02h (ZNS) targeting a file-backed namespace, nvmet_execute_identify_ns_zns() calls bdev_is_zoned() on req->ns->bdev. A file-backed namespace has no block device, so req->ns->bdev is NULL and bdev_is_zoned() dereferences it, oopsing. The I/O command set is selected by the host-supplied CSI field and the command is routed here whenever CONFIG_BLK_DEV_ZONED is enabled, independent of the namespace backing type, so any file-backed namespace is exposed. Reject the command with Invalid Field when the namespace is not backed by a block device. Fixes: aaf2e048af27 ("nvmet: add ZBD over ZNS backend support") Reviewed-by: Damien Le Moal Reviewed-by: Christoph Hellwig Signed-off-by: Guixin Liu Signed-off-by: Keith Busch --- drivers/nvme/target/zns.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index f00921931eb6..a13befd5f3ad 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -116,7 +116,7 @@ void nvmet_execute_identify_ns_zns(struct nvmet_req *req) mutex_unlock(&req->ns->subsys->lock); } - if (!bdev_is_zoned(req->ns->bdev)) { + if (!req->ns->bdev || !bdev_is_zoned(req->ns->bdev)) { status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; req->error_loc = offsetof(struct nvme_identify, nsid); goto out; From 1161be71d1ecf7dc785382114c71afed0349531e Mon Sep 17 00:00:00 2001 From: Chao Shi Date: Wed, 15 Jul 2026 11:57:52 -0400 Subject: [PATCH 196/241] nvme: reject passthrough of driver-managed Set Features Since commit b58da2d270db ("nvme: update keep alive interval when kato is modified"), a Set Features (KATO) passthrough command lets userspace start keep-alive on any transport. nvme_keep_alive_work() allocates with BLK_MQ_REQ_RESERVED, but nvme_alloc_admin_tag_set() reserves admin tags only for fabrics, so on other transports the allocation trips WARN_ON_ONCE() in blk_mq_get_tag() and fails: nvme nvme0: keep-alive failed: -11 Several Set Features change controller state the driver manages itself and cannot react to when set behind its back. Reject these in nvme_admin_cmd_allowed(): - KATO on non-fabrics (keep-alive is only armed for fabrics; on PCIe it has no reserved tag and harms idle power states) - Host Behavior Support, Host Memory Buffer, Number of Queues, and Autonomous Power State Transition (all driver-managed) Keep Alive on fabrics is unchanged; I/O commands are unaffected as the check is confined to the admin path (ns == NULL). Link: https://lore.kernel.org/linux-nvme/20260523225629.3964037-1-coshi036@gmail.com/ Fixes: b58da2d270db ("nvme: update keep alive interval when kato is modified") Found by FuzzNvme. Acked-by: Sungwoo Kim Acked-by: Dave Tian Acked-by: Weidong Zhu Signed-off-by: Chao Shi Signed-off-by: Keith Busch --- drivers/nvme/host/ioctl.c | 111 ++++++++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 39 deletions(-) diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index f4ea52d11945..6539d4750098 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -14,45 +14,54 @@ enum { NVME_IOCTL_PARTITION = (1 << 1), }; -static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, - unsigned int flags, bool open_for_write) +static bool nvme_admin_cmd_allowed(struct nvme_ctrl *ctrl, + struct nvme_command *c) { - u32 effects; - - /* - * Do not allow unprivileged passthrough on partitions, as that allows an - * escape from the containment of the partition. - */ - if (flags & NVME_IOCTL_PARTITION) - goto admin; - - /* - * Do not allow unprivileged processes to send vendor specific or fabrics - * commands as we can't be sure about their effects. - */ - if (c->common.opcode >= nvme_cmd_vendor_start || - c->common.opcode == nvme_fabrics_command) - goto admin; - /* * Do not allow unprivileged passthrough of admin commands except * for a subset of identify commands that contain information required * to form proper I/O commands in userspace and do not expose any * potentially sensitive information. */ - if (!ns) { - if (c->common.opcode == nvme_admin_identify) { - switch (c->identify.cns) { - case NVME_ID_CNS_NS: - case NVME_ID_CNS_CS_NS: - case NVME_ID_CNS_NS_CS_INDEP: - case NVME_ID_CNS_CS_CTRL: - case NVME_ID_CNS_CTRL: - return true; - } + switch (c->common.opcode) { + case nvme_admin_identify: + switch (c->identify.cns) { + case NVME_ID_CNS_NS: + case NVME_ID_CNS_CS_NS: + case NVME_ID_CNS_NS_CS_INDEP: + case NVME_ID_CNS_CS_CTRL: + case NVME_ID_CNS_CTRL: + return true; } - goto admin; + break; + case nvme_admin_set_features: + /* + * Reject Set Features that change controller state the driver + * manages itself; setting them behind the driver's back from + * userspace leaves it unable to react correctly. Keep Alive is + * only armed for fabrics - on other transports it has no + * reserved tag and harms idle power states. + */ + switch (le32_to_cpu(c->features.fid) & 0xff) { + case NVME_FEAT_KATO: + if (ctrl->ops->flags & NVME_F_FABRICS) + break; + fallthrough; + case NVME_FEAT_HOST_BEHAVIOR: + case NVME_FEAT_HOST_MEM_BUF: + case NVME_FEAT_NUM_QUEUES: + case NVME_FEAT_AUTO_PST: + return false; + } + break; } + return capable(CAP_SYS_ADMIN); +} + +static bool nvme_ns_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, + bool open_for_write) +{ + u32 effects; /* * Check if the controller provides a Commands Supported and Effects log @@ -61,7 +70,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ effects = nvme_command_effects(ns->ctrl, ns, c->common.opcode); if (!(effects & NVME_CMD_EFFECTS_CSUPP)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Don't allow passthrough for command that have intrusive (or unknown) @@ -70,7 +79,7 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_UUID_SEL | NVME_CMD_EFFECTS_SCOPE_MASK)) - goto admin; + return capable(CAP_SYS_ADMIN); /* * Only allow I/O commands that transfer data to the controller or that @@ -79,11 +88,34 @@ static bool nvme_cmd_allowed(struct nvme_ns *ns, struct nvme_command *c, */ if ((nvme_is_write(c) || (effects & NVME_CMD_EFFECTS_LBCC)) && !open_for_write) - goto admin; + return capable(CAP_SYS_ADMIN); return true; -admin: - return capable(CAP_SYS_ADMIN); +} + +static bool nvme_cmd_allowed(struct nvme_ctrl *ctrl, struct nvme_ns *ns, + struct nvme_command *c, unsigned int flags, + bool open_for_write) +{ + /* + * Do not allow unprivileged passthrough on partitions, as that + * allows an escape from the containment of the partition. + */ + if (flags & NVME_IOCTL_PARTITION) + return capable(CAP_SYS_ADMIN); + + /* + * Do not allow unprivileged processes to send vendor specific or + * fabrics commands as we can't be sure about their effects. + */ + if (c->common.opcode >= nvme_cmd_vendor_start || + c->common.opcode == nvme_fabrics_command) + return capable(CAP_SYS_ADMIN); + + if (!ns) + return nvme_admin_cmd_allowed(ctrl, c); + + return nvme_ns_cmd_allowed(ns, c, open_for_write); } /* @@ -261,7 +293,7 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio, c.rw.lbat = cpu_to_le16(io.apptag); c.rw.lbatm = cpu_to_le16(io.appmask); - if (!nvme_cmd_allowed(ns, &c, flags, open_for_write)) + if (!nvme_cmd_allowed(ns->ctrl, ns, &c, flags, open_for_write)) return -EACCES; return nvme_submit_user_cmd(ns->queue, &c, io.addr, length, metadata, @@ -311,7 +343,7 @@ static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, 0, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -358,7 +390,7 @@ static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(cmd.cdw14); c.common.cdw15 = cpu_to_le32(cmd.cdw15); - if (!nvme_cmd_allowed(ns, &c, flags, open_for_write)) + if (!nvme_cmd_allowed(ctrl, ns, &c, flags, open_for_write)) return -EACCES; if (cmd.timeout_ms) @@ -453,6 +485,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, const struct nvme_uring_cmd *cmd = io_uring_sqe128_cmd(ioucmd->sqe, struct nvme_uring_cmd); struct request_queue *q = ns ? ns->queue : ctrl->admin_q; + bool open_for_write = ioucmd->file->f_mode & FMODE_WRITE; struct nvme_uring_data d; struct nvme_command c; struct iov_iter iter; @@ -483,7 +516,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14)); c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15)); - if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE)) + if (!nvme_cmd_allowed(ctrl, ns, &c, 0, open_for_write)) return -EACCES; d.metadata = READ_ONCE(cmd->metadata); From 36ac05f7cfd59d90c597071304b14e98090d5dd1 Mon Sep 17 00:00:00 2001 From: Dmitry Bogdanov Date: Thu, 16 Jul 2026 16:42:19 +0200 Subject: [PATCH 197/241] nvme-tcp: fix usage of page_frag_cache nvme uses page_frag_cache to preallocate PDU for each preallocated request of block device. Block devices are created in parallel threads, consequently page_frag_cache is used in not thread-safe manner. That leads to incorrect refcounting of backstore pages and premature free. That can be catched by !sendpage_ok inside network stack: WARNING: CPU: 7 PID: 467 at ../net/core/skbuff.c:6931 skb_splice_from_iter+0xfa/0x310. tcp_sendmsg_locked+0x782/0xce0 tcp_sendmsg+0x27/0x40 sock_sendmsg+0x8b/0xa0 nvme_tcp_try_send_cmd_pdu+0x149/0x2a0 Then random panic may occur. Fix that by serializing the usage of page_frag_cache. Fixes: 4e893ca81170 ("nvme_core: scan namespaces asynchronously") Signed-off-by: Dmitry Bogdanov Signed-off-by: Daniel Wagner Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 0b2ac150b675..1d303e54e13e 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -108,6 +108,7 @@ struct nvme_tcp_queue { struct mutex queue_lock; struct mutex send_mutex; + struct mutex pf_cache_lock; struct llist_head req_list; struct list_head send_list; @@ -552,9 +553,11 @@ static int nvme_tcp_init_request(struct blk_mq_tag_set *set, struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; u8 hdgst = nvme_tcp_hdgst_len(queue); + mutex_lock(&queue->pf_cache_lock); req->pdu = page_frag_alloc(&queue->pf_cache, sizeof(struct nvme_tcp_cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&queue->pf_cache_lock); if (!req->pdu) return -ENOMEM; @@ -1419,9 +1422,11 @@ static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) struct nvme_tcp_request *async = &ctrl->async_req; u8 hdgst = nvme_tcp_hdgst_len(queue); + mutex_lock(&queue->pf_cache_lock); async->pdu = page_frag_alloc(&queue->pf_cache, sizeof(struct nvme_tcp_cmd_pdu) + hdgst, GFP_KERNEL | __GFP_ZERO); + mutex_unlock(&queue->pf_cache_lock); if (!async->pdu) return -ENOMEM; @@ -1463,6 +1468,7 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) kfree(queue->pdu); mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); + mutex_destroy(&queue->pf_cache_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC lockdep_unregister_key(&queue->nvme_tcp_sk_key); @@ -1790,6 +1796,7 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid, INIT_LIST_HEAD(&queue->send_list); mutex_init(&queue->send_mutex); INIT_WORK(&queue->io_work, nvme_tcp_io_work); + mutex_init(&queue->pf_cache_lock); if (qid > 0) queue->cmnd_capsule_len = nctrl->ioccsz * 16; @@ -1930,6 +1937,7 @@ err_sock: err_destroy_mutex: mutex_destroy(&queue->send_mutex); mutex_destroy(&queue->queue_lock); + mutex_destroy(&queue->pf_cache_lock); return ret; } From 86985da12699360a2b20748c5a492ddd92db8c47 Mon Sep 17 00:00:00 2001 From: Xixin Liu Date: Mon, 13 Jul 2026 18:00:00 +0800 Subject: [PATCH 198/241] nvmet: zns: reject full zone report when buffer is too small Zone Management Receive uses the Partial Report (PR) bit in dword 13. On a partial report (PR bit set), the host accepts an incomplete listing and Number of Zones must not exceed the zone descriptors copied to the host buffer. On a full report (PR bit clear), Number of Zones is the total number of matching zones and every descriptor must fit in the buffer (ZNS Command Set Specification Rev 1.2, section 3.4.2). nvmet_bdev_zone_zmgmt_recv_work() already caps Number of Zones for partial reports, but on a full report it may still succeed when the buffer only holds part of the matching descriptors. Reject the command in that case. Signed-off-by: Xixin Liu Reviewed-by: Christoph Hellwig Signed-off-by: Keith Busch --- drivers/nvme/target/zns.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c index a13befd5f3ad..23a17c02abee 100644 --- a/drivers/nvme/target/zns.c +++ b/drivers/nvme/target/zns.c @@ -295,11 +295,18 @@ static void nvmet_bdev_zone_zmgmt_recv_work(struct work_struct *w) } /* - * When partial bit is set nr_zones must indicate the number of zone - * descriptors actually transferred. + * Partial report (PR bit set): the host accepts an incomplete listing, + * so cap Number of Zones to the descriptors that fit in the buffer. + * Full report (PR bit clear): Number of Zones is the match count; fail + * if the buffer cannot hold every matching zone descriptor. */ - if (req->cmd->zmr.pr) + if (req->cmd->zmr.pr) { rz_data.nr_zones = min(rz_data.nr_zones, rz_data.out_nr_zones); + } else if (rz_data.nr_zones > rz_data.out_nr_zones) { + req->error_loc = offsetof(struct nvme_zone_mgmt_recv_cmd, numd); + status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; + goto out; + } nr_zones = cpu_to_le64(rz_data.nr_zones); status = nvmet_copy_to_sgl(req, 0, &nr_zones, sizeof(nr_zones)); From 7fa3f73f6c8ddc5f0425b50fb2a626a782ef7d12 Mon Sep 17 00:00:00 2001 From: Yehyeong Lee Date: Sat, 1 Aug 2026 17:18:17 +0900 Subject: [PATCH 199/241] nvme-tcp: reject a read that transferred too few bytes nvme_tcp_recv_data() completes a request once the current C2HData PDU has been consumed. Nothing compares the total bytes received against the length the command asked for: struct nvme_tcp_request has no receive-side counter, queue->data_remaining is per queue, and blk_mq_end_request() completes for blk_rq_bytes(rq) unconditionally with no residual concept anywhere above. A controller can therefore answer a 4096-byte read with 512 bytes and have it reported as a complete read; user space then gets 4096 bytes of which 3584 are whatever was already in the page. I reproduced that with a test target. Count the bytes received and refuse to complete a successful read whose count does not match, at the two NVME_TCP_F_DATA_SUCCESS paths and in nvme_tcp_process_nvme_cqe(). The success test shifts req->status right by one, because the driver keeps the wire value there and shifts it on completion, so the check must see what the completion path will see. Only REQ_OP_READ is checked, because there the length comes from the sectors the request covers; a passthrough command is built by its submitter, which picks both command and buffer, so the kernel has nothing to compare against. Fixes: 3f2304f8c6d6 ("nvme-tcp: add NVMe over TCP host driver") Cc: stable@vger.kernel.org Signed-off-by: Yehyeong Lee Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 1d303e54e13e..3655f7607be0 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -80,6 +80,7 @@ struct nvme_tcp_request { struct bio *curr_bio; struct iov_iter iter; + u32 data_recvd; /* send state */ size_t offset; @@ -617,6 +618,29 @@ static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); } +/* + * NVMe has no short read: a read that completes successfully must + * have transferred everything it asked for. + */ +static bool nvme_tcp_data_in_short(struct nvme_tcp_queue *queue, + struct request *rq) +{ + struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + + if (le16_to_cpu(req->status) >> 1) + return false; + if (req_op(rq) != REQ_OP_READ || !req->data_len) + return false; + if (likely(req->data_recvd == req->data_len)) + return false; + + dev_err(queue->ctrl->ctrl.device, + "queue %d tag %#x short data-in: got %u of %u\n", + nvme_tcp_queue_id(queue), rq->tag, + req->data_recvd, req->data_len); + return true; +} + static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, struct nvme_completion *cqe) { @@ -636,6 +660,9 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) req->status = cqe->status; + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; + if (!nvme_try_complete_req(rq, req->status, cqe->result)) nvme_complete_rq(rq); queue->nr_cqe++; @@ -958,6 +985,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, *len -= recv_len; *offset += recv_len; queue->data_remaining -= recv_len; + req->data_recvd += recv_len; } if (!queue->data_remaining) { @@ -966,6 +994,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; } else { if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; nvme_tcp_end_request(rq, le16_to_cpu(req->status)); queue->nr_cqe++; @@ -1014,6 +1044,9 @@ static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, pdu->command_id); struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); + if (unlikely(nvme_tcp_data_in_short(queue, rq))) + return -EPROTO; + nvme_tcp_end_request(rq, le16_to_cpu(req->status)); queue->nr_cqe++; } @@ -2746,6 +2779,7 @@ static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, req->status = cpu_to_le16(NVME_SC_SUCCESS); req->offset = 0; req->data_sent = 0; + req->data_recvd = 0; req->pdu_len = 0; req->pdu_sent = 0; req->h2cdata_left = 0; From 3a4aa9e6ad3e35f8e24d5eaf38ee4d437075fb36 Mon Sep 17 00:00:00 2001 From: Yehyeong Lee Date: Sat, 1 Aug 2026 17:18:18 +0900 Subject: [PATCH 200/241] nvme-tcp: do not accept C2HData based on blk_rq_payload_bytes() alone Commit 25e5cb780e62 ("nvme-tcp: fix possible crash in write_zeroes processing") established that blk_rq_payload_bytes() must not be read without first checking blk_rq_nr_phys_segments(), and recorded the result in nvme_tcp_setup_cmd_pdu() as req->data_len. The receive side was left as it was. The two differ for REQ_OP_WRITE_ZEROES, which has no physical segments but a non-zero blk_rq_bytes(), so setup leaves req->iter untouched while the receive gate lets a C2HData through and nvme_tcp_recv_data() copies into whatever the previous command on that tag left there. The driver-private area is zeroed only when the tag set is allocated. Reproduced with a test target that leaves a residual iterator on a tag and then sends a C2HData for a WRITE_ZEROES command on the same tag: BUG: KASAN: wild-memory-access in _copy_to_iter+0x642/0x1330 Write of size 512 at addr ffe728c2175dfa81 by task kworker/0:1H/103 CPU: 0 UID: 0 PID: 103 Comm: kworker/0:1H Not tainted 7.2.0-rc5-NVMETCP-gf5098b6bae76 #1 PREEMPT(lazy) Hardware name: QEMU Ubuntu 24.04 PC v2 (i440FX + PIIX, arch_caps fix, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014 Workqueue: nvme_tcp_wq nvme_tcp_io_work Call Trace: dump_stack_lvl+0x53/0x70 kasan_report+0xce/0x100 ? _copy_to_iter+0x642/0x1330 kasan_check_range+0x105/0x1b0 __asan_memcpy+0x3c/0x60 _copy_to_iter+0x642/0x1330 ? __pfx_sock_has_perm+0x10/0x10 ? worker_thread+0x45b/0xd10 ? __pfx__copy_to_iter+0x10/0x10 ? _raw_spin_lock_bh+0x83/0xe0 ? __pfx__raw_spin_lock_bh+0x10/0x10 __skb_datagram_iter+0xf3/0x820 ? __pfx_simple_copy_to_iter+0x10/0x10 ? __asan_memcpy+0x3c/0x60 ? skb_copy_bits+0x58d/0x830 skb_copy_datagram_iter+0x37/0x120 nvme_tcp_recv_skb+0xa07/0x4320 ? __pfx_nvme_tcp_recv_skb+0x10/0x10 __tcp_read_sock+0x1ab/0x810 ? __pfx_nvme_tcp_recv_skb+0x10/0x10 ? __pfx_lock_sock_nested+0x10/0x10 ? __pfx___tcp_read_sock+0x10/0x10 nvme_tcp_try_recv+0x152/0x1e0 ? __pfx_nvme_tcp_try_recv+0x10/0x10 ? __pfx_mutex_unlock+0x10/0x10 nvme_tcp_io_work+0x1e4/0x6c0 ? __schedule+0x181a/0x49f0 ? __pfx_nvme_tcp_io_work+0x10/0x10 process_one_work+0x633/0x1030 Keep the blk_rq_payload_bytes() test and add req->data_len to it. The old test is what rejects a C2HData naming a tag that is no longer in flight, because blk_update_request() zeroes rq->__data_len on completion; req->data_len and req->curr_bio are driver-private and survive completion, so they cannot stand in for it. Setup initialises the iterator only when both req->curr_bio and req->data_len are set, so the gate now tests the same two. Fixes: 25e5cb780e62 ("nvme-tcp: fix possible crash in write_zeroes processing") Cc: stable@vger.kernel.org Signed-off-by: Yehyeong Lee Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 3655f7607be0..46c2acc6abe0 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -673,6 +673,7 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, struct nvme_tcp_data_pdu *pdu) { + struct nvme_tcp_request *req; struct request *rq; rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); @@ -683,7 +684,8 @@ static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, return -ENOENT; } - if (!blk_rq_payload_bytes(rq)) { + req = blk_mq_rq_to_pdu(rq); + if (!blk_rq_payload_bytes(rq) || !req->curr_bio || !req->data_len) { dev_err(queue->ctrl->ctrl.device, "queue %d tag %#x unexpected data\n", nvme_tcp_queue_id(queue), rq->tag); From 6efbc52237facda35d2d874fe1765bb4839275d8 Mon Sep 17 00:00:00 2001 From: Yehyeong Lee Date: Wed, 29 Jul 2026 14:46:02 +0900 Subject: [PATCH 201/241] nvme-tcp: fix host memory disclosure on R2T for a read command nvme_tcp_handle_r2t() does not check the direction of the request the R2T refers to. A malicious controller can send an R2T for a READ and the host will answer it: nvme_tcp_setup_h2c_data_pdu() builds the H2CData header and nvme_tcp_try_send_data() sends the request's data buffer. That buffer is the READ destination, so its contents go to the controller. The command then completes normally and nothing is logged. Against a test controller that answers every READ with an R2T, a 4096 byte buffered read returned all 4096 bytes, split over two R2Ts. The pages contained stale kernel data, including an array of struct page pointers. Reject an R2T for a request that is not a write. Fixes: 3f2304f8c6d6 ("nvme-tcp: add NVMe over TCP host driver") Cc: stable@vger.kernel.org Signed-off-by: Yehyeong Lee Signed-off-by: Keith Busch --- drivers/nvme/host/tcp.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 46c2acc6abe0..62c5e38f5207 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -779,6 +779,13 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, } req = blk_mq_rq_to_pdu(rq); + if (unlikely(rq_data_dir(rq) != WRITE)) { + dev_err(queue->ctrl->ctrl.device, + "req %d unexpected r2t for a non-write command\n", + rq->tag); + return -EPROTO; + } + if (unlikely(!r2t_length)) { dev_err(queue->ctrl->ctrl.device, "req %d r2t len is %u, probably a bug...\n", From c05c86681170e381e24498257db3a879deb1ae89 Mon Sep 17 00:00:00 2001 From: Chao Shi Date: Mon, 10 Aug 2026 18:02:58 -0400 Subject: [PATCH 202/241] nvme: ratelimit the completion-path messages driven by device data nvme_find_rq() and nvme_handle_cqe() print an unratelimited message for every completion queue entry whose command id does not resolve to an in-flight request. Both are reached from the completion interrupt path (nvme_irq() -> nvme_poll_cq() -> nvme_handle_cqe()) and the decision to print is made entirely from device-supplied data, so a controller that posts a stream of bogus command ids drives unbounded printk from hard interrupt context. This is not hypothetical. A single boot under an emulated controller that posts invalid completions produced 846 "could not locate request for tag 0x0", 846 "invalid id 0 completed on queue 2" and 123 "genctr mismatch" lines. Once the tag set has been torn down every subsequent completion resolves to nothing, so the print rate is bounded only by how fast the device can post entries. Ratelimit the three messages. The information they carry is diagnostic and repeats, so the suppression count printed by the ratelimit helpers is enough to tell that the condition persists. This matches how the other device-driven error prints in the driver are already handled, for example the status messages in nvme_log_error() and nvme_log_err_passthru(). nvme_find_rq() lives in nvme.h and is shared by pci, tcp, rdma, apple and target-loop, so all transports are covered. Found by FuzzNvme. Signed-off-by: Chao Shi Signed-off-by: Keith Busch --- drivers/nvme/host/nvme.h | 6 +++--- drivers/nvme/host/pci.c | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 28cec87e4427..75e5d5a8a77c 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -692,12 +692,12 @@ static inline struct request *nvme_find_rq(struct blk_mq_tags *tags, rq = blk_mq_tag_to_rq(tags, tag); if (unlikely(!rq)) { - pr_err("could not locate request for tag %#x\n", - tag); + pr_err_ratelimited("could not locate request for tag %#x\n", + tag); return NULL; } if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) { - dev_err(nvme_req(rq)->ctrl->device, + dev_err_ratelimited(nvme_req(rq)->ctrl->device, "request %#x genctr mismatch (got %#x expected %#x)\n", tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr)); return NULL; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index ef06627b21ee..c19b9c2ea89a 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1586,9 +1586,9 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, req = nvme_find_rq(nvme_queue_tagset(nvmeq), command_id); if (unlikely(!req)) { - dev_warn(nvmeq->dev->ctrl.device, - "invalid id %d completed on queue %d\n", - command_id, le16_to_cpu(cqe->sq_id)); + dev_warn_ratelimited(nvmeq->dev->ctrl.device, + "invalid id %d completed on queue %d\n", + command_id, le16_to_cpu(cqe->sq_id)); return; } From 22eb631bf86ee3246f47885e4fa94154a46863e4 Mon Sep 17 00:00:00 2001 From: "Ewan D. Milne" Date: Wed, 13 May 2026 15:25:51 -0400 Subject: [PATCH 203/241] nvme: nvme-fc: Fix nvme_fc_create_hw_io_queues() queue deletion in error path nvme_fc_create_hw_io_queues() will call __nvme_fc_delete_hw_queue() for the last queue on which __nvme_fc_create_hw_queue() reported an error when deleting all the io queues if they cannot all be created. This is incorrect since the last queue did not actually get created. The most recent change to this code was commit 17a1ec08ce70 ("nvme/fc: simplify error handling of nvme_fc_create_hw_io_queues") which moved the cleanup to the delete_queues: label and changed the loop bounds, however the code was not correct prior to this change in a different way. The original commit e399441de911 ("nvme-fabrics: Add host support for FC transport") had a different error which called __nvme_fc_delete_hw_queue() on queue index 0 which is used for the admin queue. Fix this by correcting the initial loop index when deleting the io queues. Fixes: 17a1ec08ce70 ("nvme/fc: simplify error handling of nvme_fc_create_hw_io_queues") Fixes: e399441de911 ("nvme-fabrics: Add host support for FC transport") Cc: stable@vger.kernel.org Assisted-by: Claude:claude-opus-4-6 Reviewed-by: Maurizio Lombardi Reviewed-by: Laurence Oberman Reviewed-by: Justin Tee Signed-off-by: Ewan D. Milne Signed-off-by: Keith Busch --- drivers/nvme/host/fc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 40f9da2833ff..023710e08e0d 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2324,7 +2324,7 @@ nvme_fc_create_hw_io_queues(struct nvme_fc_ctrl *ctrl, u16 qsize) return 0; delete_queues: - for (; i > 0; i--) + for (--i; i > 0; i--) __nvme_fc_delete_hw_queue(ctrl, &ctrl->queues[i], i); return ret; } From f1a8846e06388113dfdbb89dee005083fa9afdf9 Mon Sep 17 00:00:00 2001 From: Maurizio Lombardi Date: Thu, 13 Aug 2026 15:18:50 +0200 Subject: [PATCH 204/241] nvmet: fix max_qid race between configfs and controller allocation The function nvmet_subsys_attr_qid_max_store() can race against nvmet_alloc_ctrl() when a subsystem's max_qid limit is modified. Suppose max_qid is currently 64. If nvmet_alloc_ctrl() executes: ctrl->sqs = kzalloc_objs(struct nvmet_sq *, subsys->max_qid + 1); and at this exact point, a userspace process changes max_qid to 128, nvmet_subsys_attr_qid_max_store() will set the new max_qid value. It attempts to delete active controllers to force a reconnect, but the new controller won't be deleted because it hasn't been added to the subsys->ctrls list yet. nvmet_alloc_ctrl() then proceeds and adds the new controller to the subsys->ctrls list. Later, when nvmet_install_queue() is called, it will see max_qid set to 128, but the memory allocated for sqs is only sized for 64 entries. This results in a KASAN out-of-bounds warning and potential memory corruptions. Fix this by protecting the queue allocations and list insertion in nvmet_alloc_ctrl() with down_read(&nvmet_config_sem). Because nvmet_subsys_attr_qid_max_store() acquires down_write(&nvmet_config_sem) to modify the attribute, this safely prevents the configfs writer from modifying max_qid during controller creation. Copy the max_qid from the subsystem to the controller's structure during the allocation; ctrl->max_qid never changes as long as the controller remains in LIVE state, so this will prevent similar race conditions. Fixes: 3e980f5995e0 ("nvmet: expose max queues to configfs") Reported-by: syzbot+2626e846cd2585c9aa67@syzkaller.appspotmail.com Signed-off-by: Maurizio Lombardi Signed-off-by: Keith Busch --- drivers/nvme/target/admin-cmd.c | 8 ++--- drivers/nvme/target/core.c | 51 +++++++++++++++++-------------- drivers/nvme/target/fabrics-cmd.c | 2 +- drivers/nvme/target/nvmet.h | 6 ++++ drivers/nvme/target/pci-epf.c | 2 +- 5 files changed, 40 insertions(+), 29 deletions(-) diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c index 3fde09b4d78a..7764a3c0195c 100644 --- a/drivers/nvme/target/admin-cmd.c +++ b/drivers/nvme/target/admin-cmd.c @@ -1337,7 +1337,7 @@ static u16 nvmet_set_feat_arbitration(struct nvmet_req *req) void nvmet_execute_set_features(struct nvmet_req *req) { - struct nvmet_subsys *subsys = nvmet_req_subsys(req); + struct nvmet_ctrl *ctrl = nvmet_req_ctrl(req); u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11); u16 status = 0; @@ -1359,7 +1359,7 @@ void nvmet_execute_set_features(struct nvmet_req *req) break; } nvmet_set_result(req, - (subsys->max_qid - 1) | ((subsys->max_qid - 1) << 16)); + (ctrl->max_qid - 1) | ((ctrl->max_qid - 1) << 16)); break; case NVME_FEAT_IRQ_COALESCE: status = nvmet_set_feat_irq_coalesce(req); @@ -1496,7 +1496,7 @@ void nvmet_get_feat_async_event(struct nvmet_req *req) void nvmet_execute_get_features(struct nvmet_req *req) { - struct nvmet_subsys *subsys = nvmet_req_subsys(req); + struct nvmet_ctrl *ctrl = nvmet_req_ctrl(req); u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); u16 status = 0; @@ -1536,7 +1536,7 @@ void nvmet_execute_get_features(struct nvmet_req *req) break; case NVME_FEAT_NUM_QUEUES: nvmet_set_result(req, - (subsys->max_qid-1) | ((subsys->max_qid-1) << 16)); + (ctrl->max_qid-1) | ((ctrl->max_qid-1) << 16)); break; case NVME_FEAT_KATO: nvmet_get_feat_kato(req); diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 30a1eb77f60b..d74c01c98f19 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -878,7 +878,7 @@ u16 nvmet_check_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create) if (!ctrl->cqs) return NVME_SC_INTERNAL | NVME_STATUS_DNR; - if (cqid > ctrl->subsys->max_qid) + if (cqid > ctrl->max_qid) return NVME_SC_QID_INVALID | NVME_STATUS_DNR; if ((create && ctrl->cqs[cqid]) || (!create && !ctrl->cqs[cqid])) @@ -926,7 +926,7 @@ u16 nvmet_check_sqid(struct nvmet_ctrl *ctrl, u16 sqid, if (!ctrl->sqs) return NVME_SC_INTERNAL | NVME_STATUS_DNR; - if (sqid > ctrl->subsys->max_qid) + if (sqid > ctrl->max_qid) return NVME_SC_QID_INVALID | NVME_STATUS_DNR; if ((create && ctrl->sqs[sqid]) || @@ -1655,23 +1655,6 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) if (!ctrl->changed_ns_list) goto out_free_ctrl; - ctrl->sqs = kzalloc_objs(struct nvmet_sq *, subsys->max_qid + 1); - if (!ctrl->sqs) - goto out_free_changed_ns_list; - - ctrl->cqs = kzalloc_objs(struct nvmet_cq *, subsys->max_qid + 1); - if (!ctrl->cqs) - goto out_free_sqs; - - ret = ida_alloc_range(&cntlid_ida, - subsys->cntlid_min, subsys->cntlid_max, - GFP_KERNEL); - if (ret < 0) { - args->status = NVME_SC_CONNECT_CTRL_BUSY | NVME_STATUS_DNR; - goto out_free_cqs; - } - ctrl->cntlid = ret; - /* * Discovery controllers may use some arbitrary high value * in order to cleanup stale discovery sessions @@ -1685,9 +1668,28 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) ctrl->err_counter = 0; spin_lock_init(&ctrl->error_lock); - nvmet_start_keep_alive_timer(ctrl); - + down_read(&nvmet_config_sem); mutex_lock(&subsys->lock); + + ctrl->max_qid = subsys->max_qid; + + ctrl->sqs = kzalloc_objs(struct nvmet_sq *, ctrl->max_qid + 1); + if (!ctrl->sqs) + goto out_free_changed_ns_list; + + ctrl->cqs = kzalloc_objs(struct nvmet_cq *, ctrl->max_qid + 1); + if (!ctrl->cqs) + goto out_free_sqs; + + ret = ida_alloc_range(&cntlid_ida, + subsys->cntlid_min, subsys->cntlid_max, + GFP_KERNEL); + if (ret < 0) { + args->status = NVME_SC_CONNECT_CTRL_BUSY | NVME_STATUS_DNR; + goto out_free_cqs; + } + ctrl->cntlid = ret; + ret = nvmet_ctrl_init_pr(ctrl); if (ret) goto init_pr_fail; @@ -1695,6 +1697,9 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) nvmet_setup_p2p_ns_map(ctrl, args->p2p_client); nvmet_debugfs_ctrl_setup(ctrl); mutex_unlock(&subsys->lock); + up_read(&nvmet_config_sem); + + nvmet_start_keep_alive_timer(ctrl); if (args->hostid) uuid_copy(&ctrl->hostid, args->hostid); @@ -1724,14 +1729,14 @@ struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args) return ctrl; init_pr_fail: - mutex_unlock(&subsys->lock); - nvmet_stop_keep_alive_timer(ctrl); ida_free(&cntlid_ida, ctrl->cntlid); out_free_cqs: kfree(ctrl->cqs); out_free_sqs: kfree(ctrl->sqs); out_free_changed_ns_list: + mutex_unlock(&subsys->lock); + up_read(&nvmet_config_sem); kfree(ctrl->changed_ns_list); out_free_ctrl: kfree(ctrl); diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c index 7cadd1c9e44c..42d1d1811671 100644 --- a/drivers/nvme/target/fabrics-cmd.c +++ b/drivers/nvme/target/fabrics-cmd.c @@ -370,7 +370,7 @@ static void nvmet_execute_io_connect(struct nvmet_req *req) goto out; } - if (unlikely(qid > ctrl->subsys->max_qid)) { + if (unlikely(qid > ctrl->max_qid)) { pr_warn("invalid queue id (%d)\n", qid); status = NVME_SC_CONNECT_INVALID_PARAM | NVME_STATUS_DNR; req->cqe->result.u32 = IPO_IATTR_CONNECT_SQE(qid); diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index c672c9bf3053..e362d7913a38 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -268,6 +268,7 @@ struct nvmet_ctrl { uuid_t hostid; u16 cntlid; + u16 max_qid; u32 kato; struct nvmet_port *port; @@ -756,6 +757,11 @@ static inline struct nvmet_subsys *nvmet_req_subsys(struct nvmet_req *req) return req->sq->ctrl->subsys; } +static inline struct nvmet_ctrl *nvmet_req_ctrl(struct nvmet_req *req) +{ + return req->sq->ctrl; +} + static inline bool nvmet_is_disc_subsys(struct nvmet_subsys *subsys) { return subsys->type != NVME_NQN_NVME; diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 346a4badd6b2..803e85df50e5 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -2081,7 +2081,7 @@ static int nvmet_pci_epf_create_ctrl(struct nvmet_pci_epf *nvme_epf, } /* Allocate our queues, up to the maximum number. */ - ctrl->nr_queues = min(ctrl->tctrl->subsys->max_qid + 1, max_nr_queues); + ctrl->nr_queues = min(ctrl->tctrl->max_qid + 1, max_nr_queues); ret = nvmet_pci_epf_alloc_queues(ctrl); if (ret) goto out_put_ctrl; From 017dac7670909eaea3eb36e6b3b5a8be9ce0a14d Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:00 +0800 Subject: [PATCH 205/241] null_blk: use DEFINE_MUTEX for the file-scope mutex In null_init(), mutex_init(&lock) currently happens after configfs_register_subsystem(), which exposes the nullb subsystem to userspace. A racing mkdir() into /sys/kernel/config/nullb/ can reach null_find_dev_by_name() -> mutex_lock(&lock) before the mutex is initialized, trigger warning: [ 123.137788] DEBUG_LOCKS_WARN_ON(lock->magic != lock) [ 123.137796] WARNING: kernel/locking/mutex.c:159 at mutex_lock+0x171/0x1c0, CPU#13: mkdir/1301 [ 123.140090] Modules linked in: null_blk(+) nft_fib_inet nft_fib_ipv4 ...... [ 123.154926] Call Trace: [ 123.155172] [ 123.155419] ? __pfx_mutex_lock+0x10/0x10 [ 123.156181] ? __pfx__raw_spin_lock+0x10/0x10 [ 123.156571] nullb_group_make_group+0x20/0x100 [null_blk] [ 123.157011] configfs_mkdir+0x47b/0xc70 [ 123.157337] ? __pfx_configfs_mkdir+0x10/0x10 [ 123.157719] ? may_create_dentry+0x242/0x2e0 [ 123.158061] vfs_mkdir+0x2a9/0x6c0 [ 123.158352] filename_mkdirat+0x3dc/0x500 [ 123.158710] ? __pfx_filename_mkdirat+0x10/0x10 [ 123.159070] ? strncpy_from_user+0x3a/0x1d0 [ 123.159413] __x64_sys_mkdir+0x6b/0x90 [ 123.159760] do_syscall_64+0xea/0x600 Replace the runtime mutex_init(&lock) with a static DEFINE_MUTEX(lock) declaration to fix this issue. Fixes: 49c3b9266a71 ("block: null_blk: Improve device creation with configfs") Suggested-by: Bart Van Assche Signed-off-by: Zizhi Wo Reviewed-by: Bart Van Assche Reviewed-by: Damien Le Moal Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260725022509.714271-2-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index f8c0fd57e041..eba204b27785 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -66,7 +66,7 @@ struct nullb_page { #define NULLB_PAGE_FREE (MAP_SZ - 2) static LIST_HEAD(nullb_list); -static struct mutex lock; +static DEFINE_MUTEX(lock); static int null_major; static DEFINE_IDA(nullb_indexes); static struct blk_mq_tag_set tag_set; @@ -2166,8 +2166,6 @@ static int __init null_init(void) if (ret) return ret; - mutex_init(&lock); - null_major = register_blkdev(0, "nullb"); if (null_major < 0) { ret = null_major; From c9d293d6bb0575fcb1f3408129453187e2a28a4e Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:01 +0800 Subject: [PATCH 206/241] null_blk: register configfs subsystem after creating default devices In null_init(), configfs_register_subsystem() currently runs before register_blkdev(), so when null_blk is built as a module, a racing mkdir() + poweron from userspace can reach null_add_dev() while null_major is still 0. __add_disk() then hits WARN_ON(disk->minors) (major=0 with minors!=0) and fails: [root@fedora ~]# [ 2366.521436] WARNING: block/genhd.c:476 at __add_disk+0x8a7/0xde0, [ 2366.523552] Modules linked in: null_blk(+) nft_fib_inet nft_fib_ipv4 nft_fib_ipv6 nft_fib [ 2366.529081] CPU: 26 UID: 0 PID: 1600 Comm: sh Not tainted 7.2.0-rc1+ #66 PREEMPT(full) ...... [ 2366.547251] Call Trace: [ 2366.547575] [ 2366.547831] ? _raw_spin_lock+0x84/0xe0 [ 2366.548260] add_disk_fwnode+0x114/0x560 [ 2366.548739] null_add_dev+0x102d/0x1b80 [null_blk] [ 2366.549310] ? __pfx_null_add_dev+0x10/0x10 [null_blk] [ 2366.549906] ? mutex_lock+0xde/0x1c0 [ 2366.550361] ? __pfx_mutex_lock+0x10/0x10 [ 2366.550827] nullb_device_power_store+0x1e7/0x280 [null_blk] [ 2366.551499] ? __pfx_nullb_device_power_store+0x10/0x10 [null_blk] [ 2366.552177] ? __kmalloc_cache_noprof+0x1f5/0x470 [ 2366.552748] ? configfs_write_iter+0x35c/0x4e0 [ 2366.553242] configfs_write_iter+0x286/0x4e0 [ 2366.553787] vfs_write+0x52d/0xd00 [ 2366.554169] ? __pfx_vfs_write+0x10/0x10 [ 2366.554679] ? __pfx___css_rstat_updated+0x10/0x10 [ 2366.555196] ? fdget_pos+0x1cf/0x4c0 [ 2366.555649] ksys_write+0xfc/0x1d0 ...... Additionally, the err_dev path destroys all devices on nullb_list while configfs is still registered. If a racing mkdir() + poweron puts a user device on the list, null_destroy_dev()->null_free_dev() kfrees the user device's nullb_device but /sys/kernel/config/nullb/ is still reachable. Any userspace access to the item will trigger a UAF. For simplicity, move configfs_register_subsystem() to the end to solve the problems above. Fixes: 3bf2bd20734e ("nullb: add configfs interface") Signed-off-by: Zizhi Wo Reviewed-by: Damien Le Moal Reviewed-by: Bart Van Assche Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260725022509.714271-3-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index eba204b27785..4613035222cd 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -2162,15 +2162,9 @@ static int __init null_init(void) config_group_init(&nullb_subsys.su_group); mutex_init(&nullb_subsys.su_mutex); - ret = configfs_register_subsystem(&nullb_subsys); - if (ret) - return ret; - null_major = register_blkdev(0, "nullb"); - if (null_major < 0) { - ret = null_major; - goto err_conf; - } + if (null_major < 0) + return null_major; for (i = 0; i < nr_devices; i++) { ret = null_create_dev(); @@ -2178,6 +2172,10 @@ static int __init null_init(void) goto err_dev; } + ret = configfs_register_subsystem(&nullb_subsys); + if (ret) + goto err_dev; + pr_info("module loaded\n"); return 0; @@ -2187,8 +2185,6 @@ err_dev: null_destroy_dev(nullb); } unregister_blkdev(null_major, "nullb"); -err_conf: - configfs_unregister_subsystem(&nullb_subsys); return ret; } From 4ec26e8885161ce15b66957be638a08d786035cb Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:02 +0800 Subject: [PATCH 207/241] null_blk: move unregister_blkdev() after destroying dev in null_exit() In null_exit(), unregister_blkdev() is called before the null_blk instances are destroyed, which is inconsistent with the cleanup order in null_init(). Move it after null_destroy_dev() so that teardown happens in the reverse order of initialization. No functional change intended. Suggested-by: Bart Van Assche Signed-off-by: Zizhi Wo Reviewed-by: Damien Le Moal Reviewed-by: Bart Van Assche Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260725022509.714271-4-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 4613035222cd..6cb213779cc5 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -2194,8 +2194,6 @@ static void __exit null_exit(void) configfs_unregister_subsystem(&nullb_subsys); - unregister_blkdev(null_major, "nullb"); - mutex_lock(&lock); while (!list_empty(&nullb_list)) { nullb = list_entry(nullb_list.next, struct nullb, list); @@ -2203,6 +2201,8 @@ static void __exit null_exit(void) } mutex_unlock(&lock); + unregister_blkdev(null_major, "nullb"); + if (tag_set.ops) blk_mq_free_tag_set(&tag_set); From 5a1c5ff3a49ba93a1fd0b70537e7a0164071760d Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:03 +0800 Subject: [PATCH 208/241] null_blk: free global tag_set on init error path If shared_tags is enabled, null_setup_tagset() allocates the global tag_set via null_init_global_tag_set(). If device creation later fails, err_dev destroys the default devices and calls unregister_blkdev(), but never frees the global tag_set. Since module init failed, null_exit() is never invoked, so the global tag_set's tags and maps are permanently leaked. Free the global tag_set in err_dev, matching null_exit() which does if (tag_set.ops) blk_mq_free_tag_set(&tag_set). Fixes: 82f402fefa50 ("null_blk: add support for shared tags") Signed-off-by: Zizhi Wo Reviewed-by: Damien Le Moal Reviewed-by: Bart Van Assche Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260725022509.714271-5-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 6cb213779cc5..df85189f0b69 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -2185,6 +2185,8 @@ err_dev: null_destroy_dev(nullb); } unregister_blkdev(null_major, "nullb"); + if (tag_set.ops) + blk_mq_free_tag_set(&tag_set); return ret; } From 2a6357a9b935a34f5508618fee8a7fffbf7722a8 Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:04 +0800 Subject: [PATCH 209/241] null_blk: free zones array on device power-off null_init_zoned_dev() allocates dev->zones when a zoned device is powered on, but null_del_dev() never frees it on power-off; dev->zones is only freed later in null_free_dev(), when the configfs directory is removed. If the device is powered off and then on again, null_init_zoned_dev() allocates a new array and overwrites the dev->zones pointer, leaking the previous allocation each power cycle. Free dev->zones in null_del_dev() via null_free_zoned_dev() to solve it. And calling null_free_zoned_dev() in null_free_dev() is no longer necessary because every caller already invokes null_del_dev() first: via nullb_group_drop_item() before nullb_device_release(), in the null_add_dev() error path of null_create_dev(), and in null_destroy_dev(). Remove the redundant call. And take &lock around zone_cond_store() in the two store wrappers to serialize dev->zones check-and-deref against its alloc/free, which already run under &lock. The reason there was no problem before is that only nullb_device_release() or null_exit() frees the dev->zones, which guarantees that subsequent users won't access the configfs interface. Fixes: ca4b2a011948 ("null_blk: add zone support") Assisted-by: Claude-Code:GLM-5.2 Signed-off-by: Zizhi Wo Reviewed-by: Nilay Shroff Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20260725022509.714271-6-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index df85189f0b69..e063c931dfca 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -579,8 +579,13 @@ static ssize_t nullb_device_zone_readonly_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); + ssize_t ret; - return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); + mutex_lock(&lock); + ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); + mutex_unlock(&lock); + + return ret; } CONFIGFS_ATTR_WO(nullb_device_, zone_readonly); @@ -588,8 +593,13 @@ static ssize_t nullb_device_zone_offline_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); + ssize_t ret; - return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); + mutex_lock(&lock); + ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); + mutex_unlock(&lock); + + return ret; } CONFIGFS_ATTR_WO(nullb_device_, zone_offline); @@ -836,7 +846,6 @@ static void null_free_dev(struct nullb_device *dev) if (!dev) return; - null_free_zoned_dev(dev); badblocks_exit(&dev->badblocks); kfree(dev); } @@ -1777,6 +1786,7 @@ static void null_del_dev(struct nullb *nullb) } put_disk(nullb->disk); + null_free_zoned_dev(dev); if (nullb->tag_set == &nullb->__tag_set) blk_mq_free_tag_set(nullb->tag_set); kfree(nullb->queues); From 5bce98f9e76f62386a3264b9782afb4dac99f31f Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:05 +0800 Subject: [PATCH 210/241] null_blk: clean up null_del_dev() to use cached dev pointer Replace remaining nullb->dev dereferences with the already-cached local dev variable. No functional change. Signed-off-by: Zizhi Wo Reviewed-by: Nilay Shroff Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20260725022509.714271-7-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index e063c931dfca..249caaf6ce89 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1779,7 +1779,7 @@ static void null_del_dev(struct nullb *nullb) del_gendisk(nullb->disk); - if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) { + if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) { hrtimer_cancel(&nullb->bw_timer); atomic_long_set(&nullb->cur_bytes, LONG_MAX); blk_mq_start_stopped_hw_queues(nullb->q, true); @@ -1791,7 +1791,7 @@ static void null_del_dev(struct nullb *nullb) blk_mq_free_tag_set(nullb->tag_set); kfree(nullb->queues); if (null_cache_active(nullb)) - null_free_device_storage(nullb->dev, true); + null_free_device_storage(dev, true); kfree(nullb); dev->nullb = NULL; } From 1cdfe2fa62b48728a9b436fbbd3dbe4c11593e24 Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:06 +0800 Subject: [PATCH 211/241] null_blk: reject per-device queue resize for shared tag set When shared_tags is enabled, null_setup_tagset() makes the device use the global tag_set, whose driver_data stays NULL. null_map_queues() therefore falls back to the module-wide g_submit_queues/g_poll_queues instead of any per-device value. Resizing submit_queues or poll_queues via configfs on such a device calls blk_mq_update_nr_hw_queues() on the shared set, shrinking set->nr_hw_queues. __blk_mq_realloc_hw_ctxs() only grows the q->queue_hw_ctx[] allocation, so on shrink it merely exits and NULLs the now-excess hctx slots. null_map_queues(), however, keeps mapping CPUs with the unchanged g_submit_queues/g_poll_queues, so mq_map[] ends up pointing at those NULLed hctx slots. blk_mq_map_swqueue() then dereferences the NULL hctx (hctx->cpumask), crashing the kernel: [ 460.218374] KASAN: null-ptr-deref in range [0x0000000000000098-0x000000000000009f] [ 460.219003] CPU: 24 UID: 0 PID: 1492 Comm: sh Not tainted 7.2.0-rc2+ #67 PREEMPT(full) [ 460.219792] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.17.0-4.fc41 04/01/2014 [ 460.220452] RIP: 0010:blk_mq_map_swqueue+0x4db/0x1430 ...... [ 460.228977] Call Trace: [ 460.229175] [ 460.229354] blk_mq_update_nr_hw_queues+0xd49/0x11c0 [ 460.229779] ? __pfx_blk_mq_update_nr_hw_queues+0x10/0x10 [ 460.230200] nullb_update_nr_hw_queues+0x1a9/0x370 [null_blk] [ 460.230694] nullb_device_submit_queues_store+0xd9/0x170 [null_blk] [ 460.231190] ? __pfx_nullb_device_submit_queues_store+0x10/0x10 [null_blk] [ 460.231776] ? configfs_write_iter+0x35c/0x4e0 [ 460.232122] configfs_write_iter+0x286/0x4e0 [ 460.232460] vfs_write+0x52d/0xd00 [ 460.232779] ? __x64_sys_openat+0x108/0x1d0 [ 460.233106] ? __pfx_vfs_write+0x10/0x10 [ 460.233413] ? fdget_pos+0x1cf/0x4c0 [ 460.233745] ? fput_close+0x133/0x190 [ 460.234038] ? __pfx_expand_files+0x10/0x10 [ 460.234368] ksys_write+0xfc/0x1d0 Reproducer: modprobe null_blk shared_tags=1 submit_queues=64 poll_queues=1 mkdir /sys/kernel/config/nullb/dev echo 1 > /sys/kernel/config/nullb/dev/power echo 1 > /sys/kernel/config/nullb/dev/submit_queues A per-device resize of a shared tag set is meaningless anyway, so reject it with -EINVAL in nullb_update_nr_hw_queues() when the device is bound to the global tag_set. Fixes: 45919fbfe1c4 ("null_blk: Enable modifying 'submit_queues' after an instance has been configured") Suggested-by: Nilay Shroff Assisted-by: Claude-Code:GLM-5.2 Signed-off-by: Zizhi Wo Reviewed-by: Nilay Shroff Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20260725022509.714271-8-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 249caaf6ce89..ad6dfed12464 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -382,6 +382,15 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, if (!dev->nullb) return 0; + /* + * A shared tag_set is mapped via the module-wide queue counts, so a + * per-device resize is meaningless. On shrink it would also leave + * mq_map[] pointing at NULLed hctx slots, causing a NULL deref in + * blk_mq_map_swqueue(). Reject it. + */ + if (dev->shared_tags) + return -EINVAL; + /* * Make sure at least one submit queue exists. */ From e3ef4b1b76d6721067259577ab5df54a173d9341 Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:07 +0800 Subject: [PATCH 212/241] null_blk: convert file-scope mutex users to guard(mutex) Using guard()/scoped_guard() ties lock release to scope exit, removing the need for manual mutex_unlock() calls and preventing missed unlocks on error paths. The per-attribute apply wrappers are left untouched, as those are reworked separately by the configfs show/store serialization patches. Signed-off-by: Zizhi Wo Link: https://patch.msgid.link/20260725022509.714271-9-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 46 +++++++++++++---------------------- 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index ad6dfed12464..962b0e05ce50 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -502,15 +502,15 @@ static ssize_t nullb_device_power_store(struct config_item *item, return ret; ret = count; - mutex_lock(&lock); + guard(mutex)(&lock); if (!dev->power && newp) { if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags)) - goto out; + return ret; ret = null_add_dev(dev); if (ret) { clear_bit(NULLB_DEV_FL_UP, &dev->flags); - goto out; + return ret; } set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags); @@ -524,8 +524,6 @@ static ssize_t nullb_device_power_store(struct config_item *item, clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags); } -out: - mutex_unlock(&lock); return ret; } @@ -588,13 +586,9 @@ static ssize_t nullb_device_zone_readonly_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); - ssize_t ret; - mutex_lock(&lock); - ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); - mutex_unlock(&lock); - - return ret; + guard(mutex)(&lock); + return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); } CONFIGFS_ATTR_WO(nullb_device_, zone_readonly); @@ -602,13 +596,9 @@ static ssize_t nullb_device_zone_offline_store(struct config_item *item, const char *page, size_t count) { struct nullb_device *dev = to_nullb_device(item); - ssize_t ret; - mutex_lock(&lock); - ret = zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); - mutex_unlock(&lock); - - return ret; + guard(mutex)(&lock); + return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); } CONFIGFS_ATTR_WO(nullb_device_, zone_offline); @@ -726,10 +716,9 @@ nullb_group_drop_item(struct config_group *group, struct config_item *item) struct nullb_device *dev = to_nullb_device(item); if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) { - mutex_lock(&lock); + guard(mutex)(&lock); dev->power = false; null_del_dev(dev->nullb); - mutex_unlock(&lock); } nullb_del_fault_config(dev); config_item_put(item); @@ -2100,14 +2089,13 @@ static struct nullb *null_find_dev_by_name(const char *name) { struct nullb *nullb = NULL, *nb; - mutex_lock(&lock); + guard(mutex)(&lock); list_for_each_entry(nb, &nullb_list, list) { if (strcmp(nb->disk_name, name) == 0) { nullb = nb; break; } } - mutex_unlock(&lock); return nullb; } @@ -2121,9 +2109,9 @@ static int null_create_dev(void) if (!dev) return -ENOMEM; - mutex_lock(&lock); - ret = null_add_dev(dev); - mutex_unlock(&lock); + scoped_guard(mutex, &lock) { + ret = null_add_dev(dev); + } if (ret) { null_free_dev(dev); return ret; @@ -2215,12 +2203,12 @@ static void __exit null_exit(void) configfs_unregister_subsystem(&nullb_subsys); - mutex_lock(&lock); - while (!list_empty(&nullb_list)) { - nullb = list_entry(nullb_list.next, struct nullb, list); - null_destroy_dev(nullb); + scoped_guard(mutex, &lock) { + while (!list_empty(&nullb_list)) { + nullb = list_entry(nullb_list.next, struct nullb, list); + null_destroy_dev(nullb); + } } - mutex_unlock(&lock); unregister_blkdev(null_major, "nullb"); From 7e7fff51808237703a3a1df6dd5cae1dfd1db86d Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:08 +0800 Subject: [PATCH 213/241] null_blk: serialize configfs attribute stores with the lock The NULLB_DEVICE_ATTR _store takes no lock: apply_fn attributes (submit_queues, poll_queues) get dev->NAME written again after apply_fn returns, outside its lock; APPLY=NULL attributes are entirely lockless. configfs only serializes stores per-open-file, so concurrent stores on separate fds race. For apply_fn attributes, once one store's apply_fn has reconfigured the hardware, a second (losing) store can still overwrite dev->NAME afterwards. This leaves dev->submit_queues out of sync with the live queue count, which is later caught by the WARN_ON_ONCE() in null_map_queues(). For !apply_fn attributes, power_store()'s null_add_dev() validates and builds the device under "lock" but only sets CONFIGURED afterwards. A store slipping in during this window can change a field mid-setup -- for example, zone_nr_conv can be pushed above nr_zones after it has already been clamped, leading to an out-of-bounds dev->zones[] access. Take "lock" in the macro around the apply_fn call, the CONFIGURED test and the field write, and move it out of nullb_apply_submit_queues()/ nullb_apply_poll_queues() so both paths are covered once. This serializes stores with power_store's setup and with each other. Fixes: 45919fbfe1c4 ("null_blk: Enable modifying 'submit_queues' after an instance has been configured") Suggested-by: Bart Van Assche Signed-off-by: Zizhi Wo Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260725022509.714271-10-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 962b0e05ce50..c7dfbec3e7d6 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -360,6 +360,7 @@ nullb_device_##NAME##_store(struct config_item *item, const char *page, \ ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\ if (ret < 0) \ return ret; \ + guard(mutex)(&lock); \ if (apply_fn) \ ret = apply_fn(dev, new_value); \ else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \ @@ -430,25 +431,13 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, static int nullb_apply_submit_queues(struct nullb_device *dev, unsigned int submit_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); } static int nullb_apply_poll_queues(struct nullb_device *dev, unsigned int poll_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); } NULLB_DEVICE_ATTR(size, ulong, NULL); From 2cd9e14abece86d790c2117276ff49173b735b61 Mon Sep 17 00:00:00 2001 From: Zizhi Wo Date: Sat, 25 Jul 2026 10:25:09 +0800 Subject: [PATCH 214/241] null_blk: serialize configfs attribute shows with the lock The _show callback in the NULLB_DEVICE_ATTR macro reads dev->NAME and the _store path writes it. configfs does not serialize accesses across separate open file descriptions (buffer->mutex is per-fd), and _show takes no lock, so a concurrent read and write on the same attribute is a data race. The _show readers also race against writes to these fields that run after the configfs item becomes visible, e.g. in nullb_update_nr_hw_queues(). All of those writers now run under the file-scope lock: _store takes it unconditionally, and the setup-side writers run under power_store() which holds the same lock. The only remaining unsynchronized accesses are the plain reads in _show. Rather than annotating every field with READ_ONCE()/WRITE_ONCE() across files, simply take the file-scope lock in _show (and in power_show) as well. This closes the remaining _show-vs-write data races with a single lock and keeps the writers as plain assignments. configfs attribute access is not on the I/O hot path, so taking the mutex in _show is acceptable from a performance standpoint. The dev fields written in null_alloc_dev() and dev->power in nullb_group_drop_item() need no locking: the former runs from .make_group before the item is published, and the latter is serialized by configfs frag_sem/frag_dead against attribute show/store. Suggested-by: Nilay Shroff Suggested-by: Bart Van Assche Signed-off-by: Zizhi Wo Reviewed-by: Nilay Shroff Link: https://patch.msgid.link/20260725022509.714271-11-wozizhi@huaweicloud.com Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index c7dfbec3e7d6..1c20d773dd0c 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -345,6 +345,7 @@ static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, static ssize_t \ nullb_device_##NAME##_show(struct config_item *item, char *page) \ { \ + guard(mutex)(&lock); \ return nullb_device_##TYPE##_attr_show( \ to_nullb_device(item)->NAME, page); \ } \ @@ -476,6 +477,7 @@ NULLB_DEVICE_ATTR(badblocks_partial_io, bool, NULL); static ssize_t nullb_device_power_show(struct config_item *item, char *page) { + guard(mutex)(&lock); return nullb_device_bool_attr_show(to_nullb_device(item)->power, page); } From e228404b05c8293f6a1286856e49ba3e55da1933 Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 1 Aug 2026 16:46:17 +0100 Subject: [PATCH 215/241] block: move bvec init into __bio_clone Consolidate bi_io_vec assignment for cloning in __bio_clone to keep any further changes in one place. Suggested-by: Christoph Hellwig Reviewed-by: Christoph Hellwig Signed-off-by: Pavel Begunkov Link: https://patch.msgid.link/6ecfe8f9b1c6bfb8665fba7daf55d9ad7a8a3243.1785596451.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- block/bio.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/block/bio.c b/block/bio.c index 4074a0496b93..00f99d03ac91 100644 --- a/block/bio.c +++ b/block/bio.c @@ -860,6 +860,7 @@ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) bio->bi_write_hint = bio_src->bi_write_hint; bio->bi_write_stream = bio_src->bi_write_stream; bio->bi_iter = bio_src->bi_iter; + bio->bi_io_vec = bio_src->bi_io_vec; if (bio->bi_bdev) { if (bio->bi_bdev == bio_src->bi_bdev && @@ -902,8 +903,6 @@ struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src, bio_put(bio); return NULL; } - bio->bi_io_vec = bio_src->bi_io_vec; - return bio; } EXPORT_SYMBOL(bio_alloc_clone); @@ -923,7 +922,7 @@ int bio_init_clone(struct block_device *bdev, struct bio *bio, { int ret; - bio_init(bio, bdev, bio_src->bi_io_vec, 0, bio_src->bi_opf); + bio_init(bio, bdev, NULL, 0, bio_src->bi_opf); ret = __bio_clone(bio, bio_src, gfp); if (ret) bio_uninit(bio); From 8b8755e00847da1027fe6624277e43d289e2300e Mon Sep 17 00:00:00 2001 From: Pavel Begunkov Date: Sat, 1 Aug 2026 16:46:18 +0100 Subject: [PATCH 216/241] block: introduce bio_iov_iter_set() In preparation to supporting dma-buf backed iterators and bios, introduce bio_iov_iter_set() which attempts to set up the bio directly from the given iterator. For now, it only supports bvec and expects users to check the result and fall back to other means if fails, but later we'll add more types. Suggested-by: Christoph Hellwig Signed-off-by: Pavel Begunkov Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/4686a0e47fc14f3f888967a80d45a6f66044f1e0.1785596451.git.asml.silence@gmail.com Signed-off-by: Jens Axboe --- block/bio.c | 13 ++++++++----- block/blk-map.c | 2 +- block/fops.c | 16 +++++++--------- include/linux/bio.h | 2 +- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/block/bio.c b/block/bio.c index 00f99d03ac91..898b2f5ef8c8 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1181,8 +1181,11 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty) } EXPORT_SYMBOL_GPL(__bio_release_pages); -void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter) +bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter) { + if (!iov_iter_is_bvec(iter)) + return false; + WARN_ON_ONCE(bio->bi_max_vecs); bio->bi_io_vec = (struct bio_vec *)iter->bvec; @@ -1190,6 +1193,7 @@ void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter) bio->bi_iter.bi_offset = iter->iov_offset; bio->bi_iter.bi_size = iov_iter_count(iter); bio_set_flag(bio, BIO_CLONED); + return true; } /* @@ -1284,10 +1288,9 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return -EIO; - if (iov_iter_is_bvec(iter)) { - bio_iov_bvec_set(bio, iter); - - if (!bio_iov_bvec_aligned(bio, mem_align_mask)) + if (bio_iov_iter_set(bio, iter)) { + if (iov_iter_is_bvec(iter) && + !bio_iov_bvec_aligned(bio, mem_align_mask)) return -EINVAL; iov_iter_advance(iter, bio->bi_iter.bi_size); diff --git a/block/blk-map.c b/block/blk-map.c index 615d29bb840e..9cb9605d1f62 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -473,7 +473,7 @@ static int blk_rq_map_user_bvec(struct request *rq, const struct iov_iter *iter) bio = blk_rq_map_bio_alloc(rq, 0, GFP_KERNEL); if (!bio) return -ENOMEM; - bio_iov_bvec_set(bio, iter); + bio_iov_iter_set(bio, iter); ret = blk_rq_append_bio(rq, bio); if (ret) diff --git a/block/fops.c b/block/fops.c index 3c2099dfef1d..d11923053afe 100644 --- a/block/fops.c +++ b/block/fops.c @@ -342,15 +342,13 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, bio->bi_end_io = blkdev_bio_end_io_async; bio->bi_ioprio = iocb->ki_ioprio; - if (iov_iter_is_bvec(iter)) { - /* - * Users don't rely on the iterator being in any particular - * state for async I/O returning -EIOCBQUEUED, hence we can - * avoid expensive iov_iter_advance(). Bypass - * bio_iov_iter_get_pages() and set the bvec directly. - */ - bio_iov_bvec_set(bio, iter); - } else { + /* + * Users don't rely on the iterator being in any particular + * state for async I/O returning -EIOCBQUEUED, hence we can + * avoid expensive iov_iter_advance(). Bypass + * bio_iov_iter_get_pages() and set the bvec directly. + */ + if (!bio_iov_iter_set(bio, iter)) { ret = blkdev_iov_iter_get_pages(bio, iter, bdev); if (unlikely(ret)) goto out_bio_put; diff --git a/include/linux/bio.h b/include/linux/bio.h index 0445ecba3b24..bb3235497e67 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -518,7 +518,7 @@ int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, unsigned mem_align_mask, unsigned len_align_mask); -void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter); +bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter); void __bio_release_pages(struct bio *bio, bool mark_dirty); extern void bio_set_pages_dirty(struct bio *bio); extern void bio_check_pages_dirty(struct bio *bio); From 4e1f23f9c33c156be7e313b40695af5a3a834739 Mon Sep 17 00:00:00 2001 From: Niklas Cassel Date: Thu, 13 Aug 2026 16:14:56 +0200 Subject: [PATCH 217/241] null_blk: serialize configfs attribute updates with device setup The attribute store methods generated with NULLB_DEVICE_ATTR() refuse to change the configuration of a live device by testing NULLB_DEV_FL_CONFIGURED, but that flag is only set by nullb_device_power_store() after null_add_dev() has returned, and the store methods take no lock at all. configfs only serializes writes to the same open file (buffer->mutex), so a write to any attribute can run concurrently with null_add_dev() and change the device configuration while it is being used. null_add_dev() reads the configuration several times, e.g. dev->zoned is read once to set up the queue limits and once to initialize the zone resources: CPU0: echo 1 > nullb0/power CPU1: echo 1 > nullb0/zoned nullb_device_power_store() mutex_lock(&lock) null_add_dev() if (dev->zoned) -> false /* no BLK_FEAT_ZONED */ nullb_device_zoned_store() test_bit(FL_CONFIGURED) -> 0 dev->zoned = true blk_mq_alloc_disk() /* queue is not zoned */ if (nullb->dev->zoned) -> true null_register_zoned_dev() blk_revalidate_disk_zones() blk_revalidate_disk_zones() is then called for a queue that does not have BLK_FEAT_ZONED set, which triggers its WARN_ON_ONCE() and fails the device setup with -EIO: WARNING: CPU: 2 PID: 322 at block/blk-zoned.c:2357 blk_revalidate_disk_zones+0x4c/0x560 Clearing dev->zoned in the same window is worse: the queue is created with BLK_FEAT_ZONED but the zone resources are never initialized, so add_disk() succeeds for a zoned disk that has no zones. And a store that lands after the last dev->zoned test leaves dev->zoned set while dev->zones is still NULL, which null_process_zoned_cmd() dereferences on the first write. Fix this by taking the global lock, which nullb_device_power_store() already holds across null_add_dev() and null_del_dev(), around both the NULLB_DEV_FL_CONFIGURED test and the update of the device configuration. The submit_queues and poll_queues apply callbacks are now called with that lock held, so remove the locking they did themselves. Since the store methods can run as soon as configfs_register_subsystem() returns, that is, before null_init() gets to mutex_init(&lock), also initialize the lock statically with DEFINE_MUTEX(). Fixes: 3bf2bd20734e ("nullb: add configfs interface") Reported-by: syzbot+643a6dd130546afdf1fb@syzkaller.appspotmail.com Closes: https://lore.kernel.org/linux-block/6a7d0b3f.ac361c09.22ff0a.004c.GAE@google.com/ Signed-off-by: Niklas Cassel Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260813141456.1625857-2-cassel@kernel.org Signed-off-by: Jens Axboe --- drivers/block/null_blk/main.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 1c20d773dd0c..6beb1f5b71ac 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -340,7 +340,15 @@ static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, return count; } -/* The following macro should only be used with TYPE = {uint, ulong, bool}. */ +/* + * The following macro should only be used with TYPE = {uint, ulong, bool}. + * + * The device configuration is modified under the global lock to serialize + * attribute changes against null_add_dev() and null_del_dev(): without this, + * an attribute could be changed while null_add_dev() is running, that is, + * before NULLB_DEV_FL_CONFIGURED is set, which would let null_add_dev() + * observe inconsistent values for the device configuration. + */ #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \ static ssize_t \ nullb_device_##NAME##_show(struct config_item *item, char *page) \ @@ -381,6 +389,8 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, struct blk_mq_tag_set *set; int ret, nr_hw_queues; + lockdep_assert_held(&lock); + if (!dev->nullb) return 0; @@ -2205,8 +2215,6 @@ static void __exit null_exit(void) if (tag_set.ops) blk_mq_free_tag_set(&tag_set); - - mutex_destroy(&lock); } module_init(null_init); From 7fab47863aa2a1680415bd73b7e0a4212a2ebf0e Mon Sep 17 00:00:00 2001 From: Hongfu Li Date: Wed, 12 Aug 2026 12:07:29 +0800 Subject: [PATCH 218/241] block/mq-deadline: Drop unused dd parameters Commit c807ab520fc3 ("block/mq-deadline: Add I/O priority support") left the dd parameter unused in deadline_move_request(). Commit fde02699c242 ("block: mq-deadline: Remove support for zone write locking") left dd unused in deadline_fifo_request() and deadline_next_request(). Remove these unused function parameters. Signed-off-by: Hongfu Li Reviewed-by: Bart Van Assche Reviewed-by: Tao Cui Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/20260812040729.27551-1-hongfu.li@linux.dev Signed-off-by: Jens Axboe --- block/mq-deadline.c | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 824bfc17b2c6..5f643c0ce2a8 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -233,9 +233,8 @@ static void dd_merged_requests(struct request_queue *q, struct request *req, /* * move an entry to dispatch queue */ -static void -deadline_move_request(struct deadline_data *dd, struct dd_per_prio *per_prio, - struct request *rq) +static void deadline_move_request(struct dd_per_prio *per_prio, + struct request *rq) { /* * take it off the sort and fifo list @@ -269,9 +268,8 @@ static inline bool deadline_check_fifo(struct dd_per_prio *per_prio, * For the specified data direction, return the next request to * dispatch using arrival ordered lists. */ -static struct request * -deadline_fifo_request(struct deadline_data *dd, struct dd_per_prio *per_prio, - enum dd_data_dir data_dir) +static struct request *deadline_fifo_request(struct dd_per_prio *per_prio, + enum dd_data_dir data_dir) { if (list_empty(&per_prio->fifo_list[data_dir])) return NULL; @@ -283,9 +281,8 @@ deadline_fifo_request(struct deadline_data *dd, struct dd_per_prio *per_prio, * For the specified data direction, return the next request to * dispatch using sector position sorted lists. */ -static struct request * -deadline_next_request(struct deadline_data *dd, struct dd_per_prio *per_prio, - enum dd_data_dir data_dir) +static struct request *deadline_next_request(struct dd_per_prio *per_prio, + enum dd_data_dir data_dir) { return deadline_from_pos(per_prio, data_dir, per_prio->latest_pos[data_dir]); @@ -334,7 +331,7 @@ static struct request *__dd_dispatch_request(struct deadline_data *dd, /* * batches are currently reads XOR writes */ - rq = deadline_next_request(dd, per_prio, dd->last_dir); + rq = deadline_next_request(per_prio, dd->last_dir); if (rq && dd->batching < dd->fifo_batch) { /* we have a next request and are still entitled to batch */ data_dir = rq_data_dir(rq); @@ -349,7 +346,7 @@ static struct request *__dd_dispatch_request(struct deadline_data *dd, if (!list_empty(&per_prio->fifo_list[DD_READ])) { BUG_ON(RB_EMPTY_ROOT(&per_prio->sort_list[DD_READ])); - if (deadline_fifo_request(dd, per_prio, DD_WRITE) && + if (deadline_fifo_request(per_prio, DD_WRITE) && (dd->starved++ >= dd->writes_starved)) goto dispatch_writes; @@ -379,14 +376,14 @@ dispatch_find_request: /* * we are not running a batch, find best request for selected data_dir */ - next_rq = deadline_next_request(dd, per_prio, data_dir); + next_rq = deadline_next_request(per_prio, data_dir); if (deadline_check_fifo(per_prio, data_dir) || !next_rq) { /* * A deadline has expired, the last request was in the other * direction, or we have run out of higher-sectored requests. * Start again from the request with the earliest expiry time. */ - rq = deadline_fifo_request(dd, per_prio, data_dir); + rq = deadline_fifo_request(per_prio, data_dir); } else { /* * The last req was the same dir and we have a next request in @@ -409,7 +406,7 @@ dispatch_request: * rq is the selected appropriate request. */ dd->batching++; - deadline_move_request(dd, per_prio, rq); + deadline_move_request(per_prio, rq); return dd_start_request(dd, data_dir, rq); } From 8935bf22c0a0db517a7f72f7097300e05dd852f5 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 14 Aug 2026 09:56:37 -0700 Subject: [PATCH 219/241] blk-iolatency: clear delay state when freeing policy data io.latency can throttle a group which has no latency target of its own. When a sibling misses its target, check_scale_change() scales down its peers, and a peer that reaches queue depth one gets blkcg_use_delay() called on it on every further scale-down, even with min_lat_nsec == 0. iolatency_pd_offline() resets the target through iolatency_set_min_lat_nsec(), which clears the delay only on a nonzero to zero transition, so it never clears such a peer. Freeing the policy data then leaves blkg->use_delay set and blkcg->congestion_count elevated with nothing left that can drop it. blk_cgroup_congested() then returns true for every task in that cgroup and its descendants for as long as the cgroup lives: page_cache_sync_ra() cuts readahead to a single page, page_cache_async_ra() skips it altogether, and __folio_throttle_swaprate() takes swap_avail_lock and schedules a throttle on anonymous folio allocation. Clear the delay in iolatency_pd_free(). By then bio-held blkg references have drained, or the queue is frozen for policy deactivation, so check_scale_change() cannot re-arm it. The free callback can also see policy data which was never attached to a blkg, hence the pd->blkg check. Fixes: d70675121546 ("block: introduce blk-iolatency io controller") Signed-off-by: Usama Arif Acked-by: Tejun Heo Link: https://patch.msgid.link/20260814165712.510132-2-usama.arif@linux.dev Signed-off-by: Jens Axboe --- block/blk-iolatency.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c index 9eb69010c34e..2caa79a008ad 100644 --- a/block/blk-iolatency.c +++ b/block/blk-iolatency.c @@ -1043,6 +1043,15 @@ static void iolat_release(struct rcu_head *rcu) static void iolatency_pd_free(struct blkg_policy_data *pd) { + struct blkcg_gq *blkg = pd_to_blkg(pd); + + /* + * Groups throttled as collateral have min_lat_nsec == 0, so + * iolatency_pd_offline() leaves their delay set. Drop it here, where + * no in-flight bio can re-arm it via check_scale_change(). + */ + if (blkg) + blkcg_clear_delay(blkg); call_rcu(&pd->rcu_head, iolat_release); } From 97cb95d2148835ae86ff916b145aef332d40439d Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 14 Aug 2026 09:56:38 -0700 Subject: [PATCH 220/241] blk-iocost: clear delay state when freeing policy data iocg_kick_delay() turns sufficiently large debt into an explicit block-cgroup delay with blkcg_set_delay(), setting blkg->use_delay to -1 and incrementing blkcg->congestion_count. Clearing it again depends on iocg_kick_delay() running from the period timer, the waitq timer or the issue path. ioc_pd_free() removes the iocg from active_iocgs and cancels its waitq timer, and no further bios can arrive, so once it has run nothing is left which can reduce the debt and clear the delay. The blkcg stays marked congested for the rest of its life. blk_cgroup_congested() then returns true for every task in that cgroup and its descendants: page_cache_sync_ra() cuts readahead to a single page, page_cache_async_ra() skips it altogether, and __folio_throttle_swaprate() takes swap_avail_lock and schedules a throttle on anonymous folio allocation. Clear it explicitly, after the list removal and the synchronous hrtimer_cancel() so that neither timer processing nor an I/O path can re-arm it. The free callback can also see policy data which was never attached to a blkg, hence the pd->blkg check. Fixes: 7caa47151ab2 ("blkcg: implement blk-iocost") Signed-off-by: Usama Arif Acked-by: Tejun Heo Link: https://patch.msgid.link/20260814165712.510132-3-usama.arif@linux.dev Signed-off-by: Jens Axboe --- block/blk-iocost.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/block/blk-iocost.c b/block/blk-iocost.c index dd7749d59900..2745bffcd5ee 100644 --- a/block/blk-iocost.c +++ b/block/blk-iocost.c @@ -3063,6 +3063,7 @@ static void iocg_release(struct rcu_head *rcu) static void ioc_pd_free(struct blkg_policy_data *pd) { struct ioc_gq *iocg = pd_to_iocg(pd); + struct blkcg_gq *blkg = pd_to_blkg(pd); struct ioc *ioc = iocg->ioc; unsigned long flags; @@ -3085,6 +3086,12 @@ static void ioc_pd_free(struct blkg_policy_data *pd) hrtimer_cancel(&iocg->waitq_timer); } + /* off ->active_iocgs and timer gone, so nothing can re-arm the delay */ + iocg->delay = 0; + iocg->indelay_since = 0; + if (blkg) + blkcg_clear_delay(blkg); + call_rcu(&pd->rcu_head, iocg_release); } From 4febfe7d98948bf6693f5c6a0a7e198e8fb4e584 Mon Sep 17 00:00:00 2001 From: Usama Arif Date: Fri, 14 Aug 2026 09:56:39 -0700 Subject: [PATCH 221/241] block: skip blkcg walk in blk_cgroup_congested() when nothing throttled blk_cgroup_congested() walks the current task's blkcg ancestor chain on every readahead decision and, once swap is in use, on every anonymous and shmem folio allocation. The answer is almost always "no", but finding that out costs two loads per level on two cold cache lines, plus an out-of-line kthread_blkcg() and an RCU read-side pair. On a fleet profile of hosts running containers with 5-10 level hierarchies it costs about as much as all of mutex_lock(), 99.4% of it under __folio_throttle_swaprate(). Gate the walk on a global count of blkcgs with a non-zero congestion_count. The counter only moves on the 0 <-> 1 transitions of each blkcg's congestion_count, so the extra atomic stays in the throttle arm/disarm paths and never appears in steady state. When something is throttled the counter is non-zero and the walk runs as before. Signed-off-by: Usama Arif Acked-by: Tejun Heo Link: https://patch.msgid.link/20260814165712.510132-4-usama.arif@linux.dev Signed-off-by: Jens Axboe --- block/blk-cgroup.c | 15 ++++++++++++++- block/blk-cgroup.h | 25 +++++++++++++++++++++---- include/linux/blk-cgroup.h | 23 ++++++++++++++++++++++- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index 354637f3b158..2b5c29434e42 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -53,6 +53,9 @@ EXPORT_SYMBOL_GPL(blkcg_root); struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css; EXPORT_SYMBOL_GPL(blkcg_root_css); +/* number of blkcgs with a non-zero congestion_count */ +atomic_t blkcg_nr_congested __read_mostly = ATOMIC_INIT(0); + static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS]; static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */ @@ -1350,6 +1353,16 @@ static void blkcg_css_free(struct cgroup_subsys_state *css) struct blkcg *blkcg = css_to_blkcg(css); int i; + /* + * Every blkg holds a reference on this css and drops any delay it + * still has from pd_free_fn(), so this is expected to be zero. Should + * a policy ever leave one behind, drop it here rather than let it pin + * blkcg_nr_congested and disable the fast path for the rest of the + * boot. Nothing can race with us at this point. + */ + if (WARN_ON_ONCE(atomic_xchg(&blkcg->congestion_count, 0) > 0)) + atomic_dec(&blkcg_nr_congested); + mutex_lock(&blkcg_pol_mutex); list_del(&blkcg->all_blkcgs_node); @@ -2228,7 +2241,7 @@ void blk_cgroup_bio_start(struct bio *bio) put_cpu(); } -bool blk_cgroup_congested(void) +bool __blk_cgroup_congested(void) { struct blkcg *blkcg; bool ret = false; diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h index 615390f751aa..e67c69839129 100644 --- a/block/blk-cgroup.h +++ b/block/blk-cgroup.h @@ -375,12 +375,29 @@ static inline void blkg_put(struct blkcg_gq *blkg) if (((d_blkg) = blkg_lookup(css_to_blkcg(pos_css), \ (p_blkg)->q))) +/* + * blkcg_nr_congested gates the hierarchy walk in blk_cgroup_congested(). + * These two helpers keep it in step with each blkcg's congestion_count in + * normal operation; blkcg_css_free() drops a residual count as a backstop. + */ +static inline void blkcg_inc_congestion_count(struct blkcg *blkcg) +{ + if (atomic_inc_return(&blkcg->congestion_count) == 1) + atomic_inc(&blkcg_nr_congested); +} + +static inline void blkcg_dec_congestion_count(struct blkcg *blkcg) +{ + if (atomic_dec_and_test(&blkcg->congestion_count)) + atomic_dec(&blkcg_nr_congested); +} + static inline void blkcg_use_delay(struct blkcg_gq *blkg) { if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0)) return; if (atomic_add_return(1, &blkg->use_delay) == 1) - atomic_inc(&blkg->blkcg->congestion_count); + blkcg_inc_congestion_count(blkg->blkcg); } static inline int blkcg_unuse_delay(struct blkcg_gq *blkg) @@ -405,7 +422,7 @@ static inline int blkcg_unuse_delay(struct blkcg_gq *blkg) if (old == 0) return 0; if (old == 1) - atomic_dec(&blkg->blkcg->congestion_count); + blkcg_dec_congestion_count(blkg->blkcg); return 1; } @@ -424,7 +441,7 @@ static inline void blkcg_set_delay(struct blkcg_gq *blkg, u64 delay) /* We only want 1 person setting the congestion count for this blkg. */ if (!old && atomic_try_cmpxchg(&blkg->use_delay, &old, -1)) - atomic_inc(&blkg->blkcg->congestion_count); + blkcg_inc_congestion_count(blkg->blkcg); atomic64_set(&blkg->delay_nsec, delay); } @@ -441,7 +458,7 @@ static inline void blkcg_clear_delay(struct blkcg_gq *blkg) /* We only want 1 person clearing the congestion count for this blkg. */ if (old && atomic_try_cmpxchg(&blkg->use_delay, &old, 0)) - atomic_dec(&blkg->blkcg->congestion_count); + blkcg_dec_congestion_count(blkg->blkcg); } /** diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h index dd5841a42c33..58abde49f8c5 100644 --- a/include/linux/blk-cgroup.h +++ b/include/linux/blk-cgroup.h @@ -14,6 +14,8 @@ * Nauman Rafique */ +#include +#include #include struct bio; @@ -24,10 +26,29 @@ struct gendisk; #ifdef CONFIG_BLK_CGROUP extern struct cgroup_subsys_state * const blkcg_root_css; +extern atomic_t blkcg_nr_congested; void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay); void blkcg_maybe_throttle_current(void); -bool blk_cgroup_congested(void); +bool __blk_cgroup_congested(void); + +/** + * blk_cgroup_congested - is the current task in a throttled blkcg? + * + * Called from mm hot paths where the answer is almost always false, so keep + * that case to a load and a branch and only walk the hierarchy out of line + * when something in the system really is throttled. + * + * Return: %true if the current task's blkcg or any of its ancestors is + * throttled, %false otherwise. + */ +static inline bool blk_cgroup_congested(void) +{ + if (likely(!atomic_read(&blkcg_nr_congested))) + return false; + return __blk_cgroup_congested(); +} + void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css); void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css); struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css); From 2707acf1856da266139986c9398ad722fd4f48c0 Mon Sep 17 00:00:00 2001 From: Yao Sang Date: Fri, 14 Aug 2026 10:32:24 +0800 Subject: [PATCH 222/241] ublk: reject non-power-of-2 zone sizes in SET_PARAMS UBLK_F_ZONED uses params.basic.chunk_sectors as zone size. ublk uses ilog2(chunk_sectors) to get number of zones, so the value must be power of 2. If chunk_sectors is 96 and dev_sectors is 96 * 16, userspace asks for 16 zones. But the shift calculation gets 24 zones. Block layer rejects such zone size when the disk is started. But SET_PARAMS has already returned success, which is confusing for userspace. Reject it in SET_PARAMS with other zoned parameter checks. Fixes: 29802d7ca33b ("ublk: enable zoned storage support") Signed-off-by: Yao Sang Link: https://patch.msgid.link/20260814023226.354288-2-sangyao@kylinos.cn Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index a67b9c26804b..8a6f845285ce 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -980,7 +980,7 @@ static int ublk_validate_params(const struct ublk_device *ub) if (p->max_sectors < PAGE_SECTORS) return -EINVAL; - if (ublk_dev_is_zoned(ub) && !p->chunk_sectors) + if (ublk_dev_is_zoned(ub) && !is_power_of_2(p->chunk_sectors)) return -EINVAL; } else return -EINVAL; From 176f02a86c0384df4ba175e319321b07089a8016 Mon Sep 17 00:00:00 2001 From: Yao Sang Date: Fri, 14 Aug 2026 10:32:25 +0800 Subject: [PATCH 223/241] selftests: ublk: add helper for SET_PARAMS The normal kublk add command goes through device startup. It does not tell the shell test whether a bad parameter is rejected by SET_PARAMS or later by START_DEV. Add a set_params command. It creates a temporary ublk device, sends SET_PARAMS with the command line parameters, returns the ioctl result, and deletes the device before START_DEV. Signed-off-by: Yao Sang Link: https://patch.msgid.link/20260814023226.354288-3-sangyao@kylinos.cn Signed-off-by: Jens Axboe --- tools/testing/selftests/ublk/kublk.c | 161 ++++++++++++++++++++++++++- tools/testing/selftests/ublk/kublk.h | 18 +++ 2 files changed, 178 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c index be5a0d775952..2400b4615766 100644 --- a/tools/testing/selftests/ublk/kublk.c +++ b/tools/testing/selftests/ublk/kublk.c @@ -8,6 +8,13 @@ #include "kublk.h" #define MAX_NR_TGT_ARG 64 +#define KUBLK_PARAM_LOGICAL_BS_SHIFT 9 +#define KUBLK_PARAM_PHYSICAL_BS_SHIFT 12 +#define KUBLK_PARAM_ZONE_SECTORS 128 +#define KUBLK_PARAM_NR_ZONES 16 +#define KUBLK_PARAM_DEV_SECTORS \ + (KUBLK_PARAM_ZONE_SECTORS * KUBLK_PARAM_NR_ZONES) +#define KUBLK_PARAM_ZONE_APPEND_SECTORS 8 unsigned int ublk_dbg_mask = UBLK_LOG; static const struct ublk_tgt_ops *tgt_ops_list[] = { @@ -227,6 +234,55 @@ static int ublk_ctrl_get_features(struct ublk_dev *dev, return __ublk_ctrl_cmd(dev, &data); } +static int parse_param_types(const char *arg, __u32 *types) +{ + char buf[128], *save = NULL, *tok; + + if (strlen(arg) >= sizeof(buf)) + return -EINVAL; + + strcpy(buf, arg); + *types = 0; + tok = strtok_r(buf, ",", &save); + while (tok) { + if (!strcmp(tok, "none")) + ; + else if (!strcmp(tok, "basic")) + *types |= UBLK_PARAM_TYPE_BASIC; + else if (!strcmp(tok, "zoned")) + *types |= UBLK_PARAM_TYPE_ZONED; + else + return -EINVAL; + tok = strtok_r(NULL, ",", &save); + } + + return 0; +} + +static void ublk_init_params_from_ctx(const struct dev_ctx *ctx, + struct ublk_params *params) +{ + const struct params_ctx *p = &ctx->params; + + *params = (struct ublk_params) { + .types = p->types, + .basic = { + .logical_bs_shift = p->logical_bs_shift, + .physical_bs_shift = p->physical_bs_shift, + .io_min_shift = p->io_min_shift, + .io_opt_shift = p->io_opt_shift, + .max_sectors = p->max_sectors, + .chunk_sectors = p->chunk_sectors, + .dev_sectors = p->dev_sectors, + }, + .zoned = { + .max_open_zones = p->max_open_zones, + .max_active_zones = p->max_active_zones, + .max_zone_append_sectors = p->max_zone_append_sectors, + }, + }; +} + static int ublk_ctrl_update_size(struct ublk_dev *dev, __u64 nr_sects) { @@ -1772,6 +1828,51 @@ fail: static int __cmd_dev_list(struct dev_ctx *ctx); +static int cmd_dev_set_params(struct dev_ctx *ctx) +{ + struct ublksrv_ctrl_dev_info *info; + struct ublk_params params; + struct ublk_dev *dev; + __u64 features; + int ret, del_ret; + + dev = ublk_ctrl_init(); + if (!dev) + return -ENODEV; + + ret = ublk_ctrl_get_features(dev, &features); + if (ret < 0) + goto out; + + if (!(features & UBLK_F_CMD_IOCTL_ENCODE)) { + ret = -ENOTSUP; + goto out; + } + + info = &dev->dev_info; + info->dev_id = ctx->dev_id; + info->nr_hw_queues = ctx->nr_hw_queues; + info->queue_depth = ctx->queue_depth; + info->io_desc_size = ctx->io_desc_size; + info->flags = ctx->flags; + + ret = ublk_ctrl_add_dev(dev); + if (ret < 0) + goto out; + + ublk_init_params_from_ctx(ctx, ¶ms); + + ret = ublk_ctrl_set_params(dev, ¶ms); + printf("SET_PARAMS returned %d\n", ret); + + del_ret = ublk_ctrl_del_dev(dev); + if (del_ret < 0 && ret == 0) + ret = del_ret; +out: + ublk_ctrl_deinit(dev); + return ret < 0 ? ret : 0; +} + static int cmd_dev_add(struct dev_ctx *ctx) { int res; @@ -2117,6 +2218,9 @@ static int cmd_dev_help(char *exe) printf("\t --safe only stop if device has no active openers\n\n"); printf("%s list [-n dev_id] -a \n", exe); printf("\t -a list all devices, -n list specified device, default -a \n\n"); + printf("%s set_params [-n dev_id] [-q nr_queues] [-d depth] [-u] [--zoned]\n", exe); + printf("\t[--param_types basic[,zoned]|none]\n"); + printf("\t issue ADD_DEV, SET_PARAMS and DEL_DEV without START_DEV\n\n"); printf("%s features\n", exe); printf("%s update_size -n dev_id -s|--size size_in_bytes \n", exe); printf("%s quiesce -n dev_id\n", exe); @@ -2160,6 +2264,18 @@ int main(int argc, char *argv[]) { "htlb", 1, NULL, 0 }, { "rdonly_shmem_buf", 0, NULL, 0 }, { "io_desc_size", 1, NULL, 0 }, + { "zoned", 0, NULL, 0 }, + { "param_types", 1, NULL, 0 }, + { "logical_bs_shift", 1, NULL, 0 }, + { "physical_bs_shift", 1, NULL, 0 }, + { "io_min_shift", 1, NULL, 0 }, + { "io_opt_shift", 1, NULL, 0 }, + { "max_sectors", 1, NULL, 0 }, + { "chunk_sectors", 1, NULL, 0 }, + { "dev_sectors", 1, NULL, 0 }, + { "max_zone_append_sectors", 1, NULL, 0 }, + { "max_open_zones", 1, NULL, 0 }, + { "max_active_zones", 1, NULL, 0 }, { 0, 0, 0, 0 } }; const struct ublk_tgt_ops *ops = NULL; @@ -2173,6 +2289,19 @@ int main(int argc, char *argv[]) .tgt_type = "unknown", .csum_type = LBMD_PI_CSUM_NONE, .io_desc_size = sizeof(struct ublksrv_io_desc), + .params = { + .types = UBLK_PARAM_TYPE_BASIC, + .logical_bs_shift = KUBLK_PARAM_LOGICAL_BS_SHIFT, + .physical_bs_shift = KUBLK_PARAM_PHYSICAL_BS_SHIFT, + .io_min_shift = KUBLK_PARAM_LOGICAL_BS_SHIFT, + .io_opt_shift = KUBLK_PARAM_PHYSICAL_BS_SHIFT, + .max_sectors = + UBLK_IO_MAX_BYTES >> KUBLK_PARAM_LOGICAL_BS_SHIFT, + .chunk_sectors = KUBLK_PARAM_ZONE_SECTORS, + .dev_sectors = KUBLK_PARAM_DEV_SECTORS, + .max_zone_append_sectors = + KUBLK_PARAM_ZONE_APPEND_SECTORS, + }, }; int ret = -EINVAL, i; int tgt_argc = 1; @@ -2288,6 +2417,34 @@ int main(int argc, char *argv[]) ctx.flags |= UBLK_F_IO_DESC_SIZE; ctx.io_desc_size = strtoul(optarg, NULL, 0); } + if (!strcmp(longopts[option_idx].name, "zoned")) + ctx.flags |= UBLK_F_ZONED; + if (!strcmp(longopts[option_idx].name, "param_types")) { + ret = parse_param_types(optarg, &ctx.params.types); + if (ret) + return ret; + } + if (!strcmp(longopts[option_idx].name, "logical_bs_shift")) + ctx.params.logical_bs_shift = strtoul(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "physical_bs_shift")) + ctx.params.physical_bs_shift = strtoul(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "io_min_shift")) + ctx.params.io_min_shift = strtoul(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "io_opt_shift")) + ctx.params.io_opt_shift = strtoul(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "max_sectors")) + ctx.params.max_sectors = strtoul(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "chunk_sectors")) + ctx.params.chunk_sectors = strtoul(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "dev_sectors")) + ctx.params.dev_sectors = strtoull(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "max_zone_append_sectors")) + ctx.params.max_zone_append_sectors = + strtoul(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "max_open_zones")) + ctx.params.max_open_zones = strtoul(optarg, NULL, 0); + if (!strcmp(longopts[option_idx].name, "max_active_zones")) + ctx.params.max_active_zones = strtoul(optarg, NULL, 0); break; case '?': /* @@ -2377,7 +2534,9 @@ int main(int argc, char *argv[]) ops->parse_cmd_line(&ctx, tgt_argc, tgt_argv); } - if (!strcmp(cmd, "add")) + if (!strcmp(cmd, "set_params")) + ret = cmd_dev_set_params(&ctx); + else if (!strcmp(cmd, "add")) ret = cmd_dev_add(&ctx); else if (!strcmp(cmd, "recover")) { if (ctx.dev_id < 0) { diff --git a/tools/testing/selftests/ublk/kublk.h b/tools/testing/selftests/ublk/kublk.h index e27c154fc910..d98f3d612d88 100644 --- a/tools/testing/selftests/ublk/kublk.h +++ b/tools/testing/selftests/ublk/kublk.h @@ -63,6 +63,22 @@ struct fault_inject_ctx { bool die_during_fetch; }; +struct params_ctx { + __u32 types; + + __u32 logical_bs_shift; + __u32 physical_bs_shift; + __u32 io_min_shift; + __u32 io_opt_shift; + __u32 max_sectors; + __u32 chunk_sectors; + __u64 dev_sectors; + + __u32 max_open_zones; + __u32 max_active_zones; + __u32 max_zone_append_sectors; +}; + struct dev_ctx { char tgt_type[16]; unsigned long flags; @@ -99,6 +115,8 @@ struct dev_ctx { /* for 'update_size' command */ unsigned long long size; + struct params_ctx params; + char *htlb_path; union { From 007af5e5cc822aed3a0909534ffb810f7816ed1f Mon Sep 17 00:00:00 2001 From: Yao Sang Date: Fri, 14 Aug 2026 10:32:26 +0800 Subject: [PATCH 224/241] selftests: ublk: add SET_PARAMS validation test Add test_params_01.sh for SET_PARAMS. The test checks valid basic parameters and several invalid parameter cases. Also cover zoned parameters, including a non-power-of-2 zone size. This case must fail in SET_PARAMS instead of being accepted and rejected later when the device is started. Signed-off-by: Yao Sang Link: https://patch.msgid.link/20260814023226.354288-4-sangyao@kylinos.cn Signed-off-by: Jens Axboe --- tools/testing/selftests/ublk/Makefile | 2 + .../testing/selftests/ublk/test_params_01.sh | 114 ++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100755 tools/testing/selftests/ublk/test_params_01.sh diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile index a3cec7b35db7..5daf36c6c36c 100644 --- a/tools/testing/selftests/ublk/Makefile +++ b/tools/testing/selftests/ublk/Makefile @@ -55,6 +55,8 @@ TEST_PROGS += test_stripe_06.sh TEST_PROGS += test_part_01.sh TEST_PROGS += test_part_02.sh +TEST_PROGS += test_params_01.sh + TEST_PROGS += test_shmemzc_01.sh TEST_PROGS += test_shmemzc_02.sh TEST_PROGS += test_shmemzc_03.sh diff --git a/tools/testing/selftests/ublk/test_params_01.sh b/tools/testing/selftests/ublk/test_params_01.sh new file mode 100755 index 000000000000..928e72b1035d --- /dev/null +++ b/tools/testing/selftests/ublk/test_params_01.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh + +ERR_CODE=0 + +run_set_params_success() +{ + local name=$1 + + shift + + echo "$name" + if ! "$UBLK_PROG" set_params -q 1 -d 2 "$@"; then + echo "$name: SET_PARAMS check failed" + return 1 + fi +} + +run_set_params_failure() +{ + local name=$1 + + shift + + echo "$name" + if "$UBLK_PROG" set_params -q 1 -d 2 "$@"; then + echo "$name: SET_PARAMS succeeded unexpectedly" + return 1 + fi +} + +run_zoned_set_params_success() +{ + local name=$1 + + shift + + echo "$name" + if ! "$UBLK_PROG" set_params -q 1 -d 2 -u --zoned "$@"; then + echo "$name: SET_PARAMS check failed" + return 1 + fi +} + +run_zoned_set_params_failure() +{ + local name=$1 + + shift + + echo "$name" + if "$UBLK_PROG" set_params -q 1 -d 2 -u --zoned "$@"; then + echo "$name: SET_PARAMS succeeded unexpectedly" + return 1 + fi +} + +_prep_test "params" "SET_PARAMS validation" + +if [ ! -c /dev/ublk-control ]; then + _cleanup_test + _show_result $TID $UBLK_SKIP_CODE +fi + +run_set_params_success "valid basic params" || + ERR_CODE=1 + +run_set_params_failure "missing basic params" \ + --param_types none || + ERR_CODE=1 + +run_set_params_failure "logical block larger than physical block" \ + --logical_bs_shift 12 --physical_bs_shift 9 || + ERR_CODE=1 + +run_set_params_failure "too large max sectors" \ + --max_sectors 2049 || + ERR_CODE=1 + +if _have_feature "ZONED" && _have_feature "USER_COPY"; then + run_zoned_set_params_success "valid zoned params" \ + --param_types basic,zoned || + ERR_CODE=1 + + run_zoned_set_params_failure "missing zoned params" || + ERR_CODE=1 + + run_zoned_set_params_failure "non-power-of-2 zone size" \ + --param_types basic,zoned \ + --chunk_sectors 96 --dev_sectors $((96 * 16)) || + ERR_CODE=1 + + run_zoned_set_params_failure "zero max zone append" \ + --param_types basic,zoned \ + --max_zone_append_sectors 0 || + ERR_CODE=1 + + run_zoned_set_params_failure "too many open zones" \ + --param_types basic,zoned \ + --dev_sectors $((128 * 4)) --max_open_zones 5 || + ERR_CODE=1 + + run_zoned_set_params_failure "too many active zones" \ + --param_types basic,zoned \ + --dev_sectors $((128 * 4)) --max_active_zones 5 || + ERR_CODE=1 +else + echo "zoned ublk feature unavailable, skip zoned SET_PARAMS cases" +fi + +_cleanup_test +_show_result $TID $ERR_CODE From f9820056e40ccea2919fcac0d27d13eb59548274 Mon Sep 17 00:00:00 2001 From: Nathan Chancellor Date: Tue, 11 Aug 2026 14:03:20 -0700 Subject: [PATCH 225/241] swim3: Add missing MODULE_DESCRIPTION Since commit 6c6c1fc09de3 ("modpost: require a MODULE_DESCRIPTION()"), modpost complains that swim3.ko is missing a module description. WARNING: modpost: drivers/block/swim3.ko: missing MODULE_DESCRIPTION() Add one to clear up the warning. Signed-off-by: Nathan Chancellor Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/20260811-swim3-module-description-v1-1-28398c5a0e32@kernel.org Signed-off-by: Jens Axboe --- drivers/block/swim3.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c index 01f7aef3fcfb..d4f0d43e0a72 100644 --- a/drivers/block/swim3.c +++ b/drivers/block/swim3.c @@ -1290,4 +1290,5 @@ module_init(swim3_init) MODULE_LICENSE("GPL"); MODULE_AUTHOR("Paul Mackerras"); +MODULE_DESCRIPTION("SWIM3 floppy driver for PowerMacs"); MODULE_ALIAS_BLOCKDEV_MAJOR(FLOPPY_MAJOR); From 7e9a46004b471eaf69b082c473d865316a4158e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=83=A1=E8=BF=9E=E5=8B=A4?= Date: Wed, 12 Aug 2026 11:59:04 +0000 Subject: [PATCH 226/241] block: set QUEUE_FLAG_DYING unconditionally in blk_mark_disk_dead() Disks created via blk_mq_alloc_disk_for_queue() (e.g. SCSI SD disks) do not have GD_OWNS_QUEUE set. Currently __blk_mark_disk_dead() only sets QUEUE_FLAG_DYING when GD_OWNS_QUEUE is set, so for such disks blk_queue_enter() and __bio_queue_enter() cannot detect the dying state via blk_queue_dying() and remain blocked waiting for I/O that will never complete after surprise removal. blk_mark_disk_dead() is the explicit "surprise removal" API -- the caller has already decided the disk is dead. Setting QUEUE_FLAG_DYING unconditionally here is appropriate: any in-flight I/O from other threads should get -ENODEV immediately from blk_queue_enter() regardless of GD_OWNS_QUEUE ownership. For disks that already have GD_OWNS_QUEUE set, __blk_mark_disk_dead() will set the flag again which is harmless. Fixes: 6f8191fdf41d ("block: simplify disk shutdown") Cc: stable@vger.kernel.org Signed-off-by: Lianqin Hu Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/PUZPR06MB62247E82E66A3ED46CC3E6C7D2DC2@PUZPR06MB6224.apcprd06.prod.outlook.com Signed-off-by: Jens Axboe --- block/genhd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/block/genhd.c b/block/genhd.c index df2c3c69b467..7b089e2d52c4 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -681,6 +681,7 @@ static bool __blk_mark_disk_dead(struct gendisk *disk) */ void blk_mark_disk_dead(struct gendisk *disk) { + blk_queue_flag_set(QUEUE_FLAG_DYING, disk->queue); __blk_mark_disk_dead(disk); blk_report_disk_dead(disk, true); } From 06a2ff603f1f22dd314e24c50092717b4b8c3ade Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 13 Aug 2026 16:14:20 +0000 Subject: [PATCH 227/241] loop: Fix recently introduced lock inversion All block driver code except loop_set_dio() calls queue_limits_start_update() before it freezes the request queue. Make loop_set_dio() follow this convention. This patch fixes the following lockdep complaint: ====================================================== WARNING: possible circular locking dependency detected 7.2.0-rc5-dbg #11 Not tainted ------------------------------------------------------ losetup/2924 is trying to acquire lock: ffff88816c76da68 (&q->limits_lock){+.+.}-{4:4}, at: loop_set_dio+0x318/0x720 [loop] but task is already holding lock: ffff88816c76d430 (&q->q_usage_counter(io)#24){++++}-{0:0}, at: blk_mq_freeze_queue_nomemsave+0x1a/0x30 which lock already depends on the new lock. Cc: Keith Busch Fixes: 6c8dec275ccc ("loop: set dma_alignment from the backing file for direct I/O") Reported-by: syzbot+cc0de396bac84da51919@syzkaller.appspotmail.com Closes: https://lore.kernel.org/linux-block/6a7d5368.d5f0ebe7.22d851.0013.GAE@google.com/ Signed-off-by: Bart Van Assche Reviewed-by: Christoph Hellwig Reviewed-by: Keith Busch Link: https://patch.msgid.link/d919f5285d16afbec6c51ecdf201692a484566e5.1786637565.git.bvanassche@acm.org Signed-off-by: Jens Axboe --- drivers/block/loop.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 8639fa34b847..6f12976035b0 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1452,12 +1452,12 @@ static int loop_set_dio(struct loop_device *lo, unsigned long arg) vfs_fsync(lo->lo_backing_file, 0); } + lim = queue_limits_start_update(lo->lo_queue); memflags = blk_mq_freeze_queue(lo->lo_queue); if (use_dio) lo->lo_flags |= LO_FLAGS_DIRECT_IO; else lo->lo_flags &= ~LO_FLAGS_DIRECT_IO; - lim = queue_limits_start_update(lo->lo_queue); loop_set_dma_limit(lo, &lim); queue_limits_commit_update(lo->lo_queue, &lim); blk_mq_unfreeze_queue(lo->lo_queue, memflags); From 913dcbc5670cb772bddf9799d17f975e020e8326 Mon Sep 17 00:00:00 2001 From: Tal Zussman Date: Mon, 10 Aug 2026 12:39:19 -0400 Subject: [PATCH 228/241] xfs: avoid double deferrals for RWF_DONTCACHE writes XFS already defers some writes to a workqueue when transactions are needed to process the I/O completion. Disable the block layer bio task completion in this case to avoid a major performance drop. Fixes: efbde6f9f449 ("iomap: use BIO_COMPLETE_IN_TASK for dropbehind writeback") Link: https://lore.kernel.org/all/8124341f-3af2-4a16-897d-38db5ab5a9d4@columbia.edu/ Signed-off-by: Tal Zussman Reviewed-by: "Darrick J. Wong" Reviewed-by: Christoph Hellwig Link: https://patch.msgid.link/20260810-xfs-dontcache-double-defer-v1-1-aea7484b3e49@columbia.edu Signed-off-by: Jens Axboe --- fs/xfs/xfs_aops.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/fs/xfs/xfs_aops.c b/fs/xfs/xfs_aops.c index cd8de8c82d78..74a6089abadf 100644 --- a/fs/xfs/xfs_aops.c +++ b/fs/xfs/xfs_aops.c @@ -549,10 +549,14 @@ xfs_writeback_submit( } /* - * Send ioends that might require a transaction to the completion wq. + * Send ioends that might require a transaction to the completion wq, + * and disable the block layer task completion for them as there is no + * need to defer twice. */ - if (xfs_ioend_needs_wq_completion(ioend)) + if (xfs_ioend_needs_wq_completion(ioend)) { ioend->io_bio.bi_end_io = xfs_end_bio; + bio_clear_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK); + } return iomap_ioend_writeback_submit(wpc, error); } @@ -663,7 +667,14 @@ xfs_zoned_writeback_submit( { struct iomap_ioend *ioend = wpc->wb_ctx; + /* + * Defer all completions to our workqueue as all zoned writes require a + * transaction to be persisted. This also means we never need the block + * layer in-task completion for a task context. + */ ioend->io_bio.bi_end_io = xfs_end_bio; + bio_clear_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK); + if (error) { ioend->io_bio.bi_status = errno_to_blk_status(error); bio_endio(&ioend->io_bio); From 13c251b2b7fb150f22b55d652ee66c2a39c6635e Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 10 Aug 2026 07:45:59 -0700 Subject: [PATCH 229/241] block: don't include blk-integrity.h in bdev.c Not needed, probably since this file was split out. Signed-off-by: Christoph Hellwig Reviewed-by: Anuj Gupta Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20260810144603.453283-1-hch@lst.de Signed-off-by: Jens Axboe --- block/bdev.c | 1 - 1 file changed, 1 deletion(-) diff --git a/block/bdev.c b/block/bdev.c index 85ce57bd2ae4..90baef0bd174 100644 --- a/block/bdev.c +++ b/block/bdev.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include From 95491fb05105b61050cb623a5e0227eb26aa3525 Mon Sep 17 00:00:00 2001 From: Rik van Riel Date: Sat, 8 Aug 2026 11:42:39 -0400 Subject: [PATCH 230/241] null_blk: fix UBSAN shift-out-of-bounds when zone_size is 0 or overflows null_zone_no() does sect >> ilog2(dev->zone_size_sects). When zone_size_sects is 0, ilog2(0) returns -1, producing shift exponent -1 which UBSAN reports as shift-out-of-bounds. UBSAN: shift-out-of-bounds in drivers/block/null_blk/zoned.c:21:14 shift exponent -1 is negative Call Trace: null_zone_no drivers/block/null_blk/zoned.c:21 [inline] null_process_zoned_cmd+0xf76/0xf80 drivers/block/null_blk/zoned.c:728 null_handle_cmd drivers/block/null_blk/main.c:1455 [inline] null_queue_rq+0x8bc/0xe70 drivers/block/null_blk/main.c:1703 __blk_mq_issue_directly block/blk-mq.c:2694 [inline] blk_mq_try_issue_directly+0x3f4/0x880 block/blk-mq.c:2754 blk_mq_submit_bio+0x20c0/0x2a40 block/blk-mq.c:3208 submit_bio_noacct_nocheck+0x2f4/0xa40 block/blk-core.c:790 block_read_full_folio+0x7a6/0x810 fs/buffer.c:2463 filemap_read_folio+0x12c/0x3a0 mm/filemap.c:2510 read_part_sector+0xb6/0x2b0 block/partitions/core.c:724 adfspart_check_ICS+0xb1/0x960 block/partitions/acorn.c:357 check_partition block/partitions/core.c:143 [inline] blk_add_partitions block/partitions/core.c:591 [inline] bdev_disk_changed+0x851/0x17a0 block/partitions/core.c:695 blkdev_get_whole+0x372/0x510 block/bdev.c:751 add_disk_final block/genhd.c:412 [inline] add_disk_fwnode+0x24b/0x3a0 block/genhd.c:606 null_add_dev+0x130b/0x1d70 drivers/block/null_blk/main.c:2052 nullb_device_power_store+0x240/0x380 drivers/block/null_blk/main.c:501 configfs_write_iter+0x337/0x430 fs/configfs/file.c:229 Syzkaller triggers this by creating a zoned null_blk device via configfs. The Call Trace shows configfs_write_iter in configfs/file.c handling a write to power file, which calls nullb_device_power_store in main.c, which calls null_add_dev in main.c, which calls add_disk in genhd.c, which triggers partition scan via bdev_disk_changed in partitions/core.c. A zoned null_blk device with zone_size 0 should not be legal. Existing code tries to reject it via is_power_of_2() check in zoned.c and !zone_size check in main.c, but syzkaller can still reach null_zone_no() with zone_size_sects 0 via two paths: 1. Direct 0 via configfs: zone_size attribute store in main.c has NULLB_DEVICE_ATTR(zone_size, ulong, NULL) with no validation callback, so echo 0 > zone_size succeeds before power store. If zoned is false at power store time, the !zone_size check in main.c is skipped, and later zoned set true leaves zone_size 0. 2. Large value overflow: mb_to_sects() in zoned.c does (sector_t)mb * SZ_1M >> SECTOR_SHIFT which is mb * 2048. If mb is 1UL << 53 (9PB), mb * 2048 overflows 64-bit to 0. The value is power-of-two so is_power_of_2() passes, but mb_to_sects() returns 0. Check for zero zone_size explicitly in null_init_zoned_dev() in zoned.c, returning -EINVAL with "must be non-zero power-of-two". Check for zero zone_size_sects after mb_to_sects() conversion, returning -EINVAL for overflow case. Keep defensive check in null_zone_no() returning 0 for zero sectors to avoid shift out-of-bounds even if zero slips through. This change should be safe because zone_size is set once in null_init_zoned_dev() under device lock and never changes after, and 0 is never valid for a zoned device. Returning -EINVAL at init time fails device creation early with clear error, while defensive return 0 in null_zone_no() makes zoned command fail via offline zone check. No new locking is introduced. Reported-by: syzbot+abd6a8dca0f2b7726060@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=abd6a8dca0f2b7726060 Link: https://lore.kernel.org/all/6a75205c.01d0871a.3a0d52.0033.GAE@google.com/ Fixes: 8a3cf049af68 ("null_blk: add zoned block device emulation") Cc: stable@vger.kernel.org Assisted-by: Hermes:muse-spark-1.2 syzkaller Signed-off-by: Rik van Riel Reviewed-by: Damien Le Moal Link: https://patch.msgid.link/20260808114239.69167f68@fangorn Signed-off-by: Jens Axboe --- drivers/block/null_blk/zoned.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 384bdce6a9b7..7e9b5ffb9e84 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -18,6 +18,8 @@ static inline sector_t mb_to_sects(unsigned long mb) static inline unsigned int null_zone_no(struct nullb_device *dev, sector_t sect) { + if (WARN_ON_ONCE(!dev->zone_size_sects)) + return 0; return sect >> ilog2(dev->zone_size_sects); } @@ -56,8 +58,8 @@ int null_init_zoned_dev(struct nullb_device *dev, sector_t sector = 0; unsigned int i; - if (!is_power_of_2(dev->zone_size)) { - pr_err("zone_size must be power-of-two\n"); + if (!dev->zone_size || !is_power_of_2(dev->zone_size)) { + pr_err("zone_size must be non-zero power-of-two\n"); return -EINVAL; } if (dev->zone_size > dev->size) { @@ -88,6 +90,10 @@ int null_init_zoned_dev(struct nullb_device *dev, zone_capacity_sects = mb_to_sects(dev->zone_capacity); dev_capacity_sects = mb_to_sects(dev->size); dev->zone_size_sects = mb_to_sects(dev->zone_size); + if (!dev->zone_size_sects) { + pr_err("zone_size too large or too small, leads to zero sectors\n"); + return -EINVAL; + } dev->nr_zones = round_up(dev_capacity_sects, dev->zone_size_sects) >> ilog2(dev->zone_size_sects); From 4fd66a7f829f3f38f92a79081f0f2688aed644f0 Mon Sep 17 00:00:00 2001 From: Yao Sang Date: Tue, 4 Aug 2026 20:57:36 +0800 Subject: [PATCH 231/241] ublk: avoid teardown retry loop on xarray allocation failure __ublk_shmem_remove_ranges() removes matching maple tree ranges in batches, but first stores each range into a temporary xarray so that the pages can be unpinned after dropping the maple tree lock. That temporary xarray is filled under the maple tree lock with xa_store(..., GFP_ATOMIC). If the store fails before mas_erase(), the current range is left in the tree and the helper returns false. The outer ublk_shmem_remove_ranges() loop then immediately retries the same range. While the atomic allocation keeps failing, the teardown path has no forward progress. The issue can be reproduced with radix_tree_node failslab injection after a SHMEM_ZC buffer has already been registered: # Kernel config: # CONFIG_BLK_DEV_UBLK=y # CONFIG_DEBUG_FS=y # CONFIG_FAULT_INJECTION=y # CONFIG_FAULT_INJECTION_DEBUG_FS=y # CONFIG_FAILSLAB=y echo 10 > /proc/sys/vm/nr_hugepages mkdir -p /tmp/htlb mount -t hugetlbfs none /tmp/htlb fallocate -l 4M /tmp/htlb/ublk_buf dev_id=$(kublk add -t null --shmem_zc \ --htlb /tmp/htlb/ublk_buf | awk -F '[ :]' '/dev id/ {print $3}') echo 1 > /sys/kernel/slab/radix_tree_node/failslab echo Y > /sys/kernel/debug/failslab/cache-filter echo Y > /sys/kernel/debug/failslab/ignore-gfp-wait echo 1 > /sys/kernel/debug/failslab/interval echo -1 > /sys/kernel/debug/failslab/times echo 100 > /sys/kernel/debug/failslab/probability kublk del -n "$dev_id" On the unfixed kernel the delete command was still running after 3 seconds. Disabling failslab made it return. The fault-injection stack showed: should_failslab kmem_cache_alloc_lru_noprof __xas_nomem __xa_store xa_store __ublk_shmem_remove_ranges ublk_cdev_rel ublk_ctrl_del_dev Remove the allocation from the teardown loop. Keep the existing batch limit, but collect {base_pfn, nr_pages} pairs in a fixed-size stack array. Once a matching range is found, the range is erased from the maple tree before dropping the lock, so each successful scan makes progress without depending on any GFP_ATOMIC allocation. With the same failslab settings, the fixed kernel completed "kublk del -n $dev_id" successfully in about 45 ms. Fixes: 309e02dccf64 ("ublk: avoid unpinning pages under maple tree spinlock") Signed-off-by: Yao Sang Reviewed-by: Ming Lei Link: https://patch.msgid.link/20260804125736.2011774-1-sangyao@kylinos.cn Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 8a6f845285ce..f82b13a16192 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -5510,39 +5510,36 @@ static void ublk_unpin_range_pages(unsigned long base_pfn, /* * Inner loop: erase up to UBLK_REMOVE_BATCH matching ranges under - * mas_lock, collecting them into an xarray. Then drop the lock and - * unpin pages + free ranges outside spinlock context. + * mas_lock, collecting the page ranges in a fixed-size array. Then + * drop the lock and unpin pages + free ranges outside spinlock context. * * Returns true if the tree walk completed, false if more ranges remain. - * Xarray key is the base PFN, value encodes nr_pages via xa_mk_value(). */ #define UBLK_REMOVE_BATCH 64 +struct ublk_unpin_range { + unsigned long base_pfn; + unsigned long nr_pages; +}; + static bool __ublk_shmem_remove_ranges(struct ublk_device *ub, int buf_index, int *ret) { MA_STATE(mas, &ub->buf_tree, 0, ULONG_MAX); struct ublk_buf_range *range; - struct xarray to_unpin; - unsigned long idx; + struct ublk_unpin_range to_unpin[UBLK_REMOVE_BATCH]; unsigned int count = 0; + unsigned int i; bool done = false; - void *entry; - - xa_init(&to_unpin); mas_lock(&mas); mas_for_each(&mas, range, ULONG_MAX) { - unsigned long nr; - if (buf_index >= 0 && range->buf_index != buf_index) continue; *ret = 0; - nr = mas.last - mas.index + 1; - if (xa_err(xa_store(&to_unpin, mas.index, - xa_mk_value(nr), GFP_ATOMIC))) - goto unlock; + to_unpin[count].base_pfn = mas.index; + to_unpin[count].nr_pages = mas.last - mas.index + 1; mas_erase(&mas); kfree(range); if (++count >= UBLK_REMOVE_BATCH) @@ -5552,9 +5549,9 @@ static bool __ublk_shmem_remove_ranges(struct ublk_device *ub, unlock: mas_unlock(&mas); - xa_for_each(&to_unpin, idx, entry) - ublk_unpin_range_pages(idx, xa_to_value(entry)); - xa_destroy(&to_unpin); + for (i = 0; i < count; i++) + ublk_unpin_range_pages(to_unpin[i].base_pfn, + to_unpin[i].nr_pages); return done; } From 68940f841d013192086a0f6d7cfbac2cd079e228 Mon Sep 17 00:00:00 2001 From: Hongyan Xu Date: Thu, 6 Aug 2026 14:04:41 +0800 Subject: [PATCH 232/241] block: mtip32xx: synchronize ioctls with device removal The ioctl handlers only test REMOVE_PENDING before entering mtip_hw_ioctl(). Removal can set that bit immediately afterwards and free dd->port in mtip_hw_exit() while an ioctl still dereferences it. An already open block device can reach the handlers while del_gendisk() is in progress. Serialize both native and compat ioctls with removal. Set REMOVE_PENDING before taking the mutex so new callers fail after an in-flight ioctl has drained, and hold the mutex until the port has been torn down. Fixes: 88523a61558a ("block: Add driver for Micron RealSSD pcie flash cards") Signed-off-by: Hongyan Xu Link: https://patch.msgid.link/20260806060441.676-1-getshell@seu.edu.cn Signed-off-by: Jens Axboe --- drivers/block/mtip32xx/mtip32xx.c | 7 +++++++ drivers/block/mtip32xx/mtip32xx.h | 2 ++ 2 files changed, 9 insertions(+) diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index f214a616386c..113bdb868c46 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3048,6 +3048,8 @@ static int mtip_block_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3086,6 +3088,8 @@ static int mtip_block_compat_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3721,6 +3725,7 @@ static int mtip_pci_probe(struct pci_dev *pdev, dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); if (!dd) return -ENOMEM; + mutex_init(&dd->ioctl_mutex); /* Attach the private data to this PCI device. */ pci_set_drvdata(pdev, dd); @@ -3887,6 +3892,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) } set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); + mutex_lock(&dd->ioctl_mutex); if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) del_gendisk(dd->disk); @@ -3915,6 +3921,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) /* De-initialize the protocol layer. */ mtip_hw_exit(dd); + mutex_unlock(&dd->ioctl_mutex); if (dd->isr_workq) { destroy_workqueue(dd->isr_workq); diff --git a/drivers/block/mtip32xx/mtip32xx.h b/drivers/block/mtip32xx/mtip32xx.h index f7328f19ac5c..0963c07b5845 100644 --- a/drivers/block/mtip32xx/mtip32xx.h +++ b/drivers/block/mtip32xx/mtip32xx.h @@ -12,6 +12,7 @@ #define __MTIP32XX_H__ #include +#include #include #include #include @@ -432,6 +433,7 @@ struct driver_data { struct request_queue *queue; /* Our request queue. */ struct blk_mq_tag_set tags; /* blk_mq tags */ + struct mutex ioctl_mutex; struct mtip_port *port; /* Pointer to the port data structure. */ From 1207dbb91c111e2422e94b2e75091367369e6476 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Wed, 12 Aug 2026 08:05:08 +0200 Subject: [PATCH 233/241] blk-mq: add missing call to srcu_barrier() in blk_mq_free_tag_set() Commit 05c3e88488ed ("srcu: Queue sdp->work when the delay timer is successfully deleted") added a check in cleanup_srcu_struct() if the call to srcu_barrier() has been made before calling it, which revealed a missing call to srcu_barrier() before calling cleanup_srcu_struct(set->srcu). Fix this. Signed-off-by: Marek Szyprowski Reviewed-by: Paul E. McKenney Link: https://patch.msgid.link/20260812060510.3220294-1-m.szyprowski@samsung.com Signed-off-by: Jens Axboe --- block/blk-mq.c | 1 + 1 file changed, 1 insertion(+) diff --git a/block/blk-mq.c b/block/blk-mq.c index 2c850330a32b..a26a11c73ee3 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -4975,6 +4975,7 @@ void blk_mq_free_tag_set(struct blk_mq_tag_set *set) srcu_barrier(&set->tags_srcu); cleanup_srcu_struct(&set->tags_srcu); if (set->flags & BLK_MQ_F_BLOCKING) { + srcu_barrier(set->srcu); cleanup_srcu_struct(set->srcu); kfree(set->srcu); } From fe247030f1ce74e6bb917e219fe152270613073d Mon Sep 17 00:00:00 2001 From: Long Li Date: Wed, 5 Aug 2026 20:29:23 +0800 Subject: [PATCH 234/241] nbd: simplify find_fallback() by removing redundant logic The second conditional checking nsock->fallback_index validity is the logical inverse of the first, so drop it and let execution fall through naturally. Consolidate the two identical dev_err_ratelimited() + return paths into a single no_fallback label to reduce duplication. Reviewed-by: Yu Kuai Signed-off-by: Long Li Link: https://patch.msgid.link/20260805122930.57647-2-yangerkun@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 8f10762e90ef..b1a5acd57426 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1061,40 +1061,31 @@ static int find_fallback(struct nbd_device *nbd, int index) int new_index = -1; struct nbd_sock *nsock = config->socks[index]; int fallback = nsock->fallback_index; + int i; if (test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags)) return new_index; - if (config->num_connections <= 1) { - dev_err_ratelimited(disk_to_dev(nbd->disk), - "Dead connection, failed to find a fallback\n"); - return new_index; - } + if (config->num_connections <= 1) + goto no_fallback; if (fallback >= 0 && fallback < config->num_connections && !config->socks[fallback]->dead) return fallback; - if (nsock->fallback_index < 0 || - nsock->fallback_index >= config->num_connections || - config->socks[nsock->fallback_index]->dead) { - int i; - for (i = 0; i < config->num_connections; i++) { - if (i == index) - continue; - if (!config->socks[i]->dead) { - new_index = i; - break; - } - } - nsock->fallback_index = new_index; - if (new_index < 0) { - dev_err_ratelimited(disk_to_dev(nbd->disk), - "Dead connection, failed to find a fallback\n"); - return new_index; + for (i = 0; i < config->num_connections; i++) { + if (i != index && !config->socks[i]->dead) { + new_index = i; + break; } } - new_index = nsock->fallback_index; + nsock->fallback_index = new_index; + if (new_index >= 0) + return new_index; + +no_fallback: + dev_err_ratelimited(disk_to_dev(nbd->disk), + "Dead connection, failed to find a fallback\n"); return new_index; } From 04d8fb23e520419a283dd53c1d9cdfb7c5b1705e Mon Sep 17 00:00:00 2001 From: Yang Erkun Date: Wed, 5 Aug 2026 20:29:24 +0800 Subject: [PATCH 235/241] nbd: disallow NBD_SET_SOCK on an active device We cannot add a socket to an already running nbd device, the reconfigure for netlink can only active an inactive socket. But for ioctl path, we can call NBD_SET_SOCK after NBD_DO_IT, reject this using nbd->pid which has been setted when NBD_DO_IT. Besides, it is the root cause for commit b98e762e3d71 ("nbd: freeze the queue while we're adding connections"). Reviewed-by: Yu Kuai Signed-off-by: Yang Erkun Link: https://patch.msgid.link/20260805122930.57647-3-yangerkun@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index b1a5acd57426..7ec85f94f742 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1278,6 +1278,13 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg, /* Arg will be cast to int, check it to avoid overflow */ if (arg > INT_MAX) return -EINVAL; + + if (nbd->pid) { + dev_err(disk_to_dev(nbd->disk), + "Cannot add socket to a running device\n"); + return -EBUSY; + } + sock = nbd_get_socket(nbd, arg, &err); if (!sock) return err; From 0fdee7c5fa886554503001cecea049f053237381 Mon Sep 17 00:00:00 2001 From: Yang Erkun Date: Wed, 5 Aug 2026 20:29:25 +0800 Subject: [PATCH 236/241] nbd: clear queue limits on disconnect An inactive nbd device may refuse any I/O operations. The nbd_config_put function calls invalidate_disk, which sets the device capacity to zero to reject all read and write I/O. For zero-sector flush I/O requests from blkdev_issue_flush, if the write cache is disabled, the zero-sector flush I/O immediately returns 0 in submit_bio_noacct. However, since nbd_config_put does not clear the write cache state, an inactive nbd device might still have the write cache enabled. In this situation, zero-sector flush I/O will return -EIO because there is no active socket. Additionally, BLK_FEAT_FUA and BLK_FEAT_ROTATIONAL flags may also remain stale, resetting all of them ensures consistent behavior. The limits update uses queue_limits_commit_update() (the non-freezing variant) because config_refs == 0 here means every fd is closed and recv threads have drained, so no in-flight I/O can read q->limits concurrently. Reviewed-by: Yu Kuai Signed-off-by: Yang Erkun Link: https://patch.msgid.link/20260805122930.57647-4-yangerkun@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 7ec85f94f742..78df6f459da6 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -331,6 +331,26 @@ static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock, nsock->sent = 0; } +static void nbd_apply_limits(struct queue_limits *lim, u32 flags) +{ + lim->features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL); + lim->max_hw_discard_sectors = 0; + lim->max_write_zeroes_sectors = 0; + + if (flags & NBD_FLAG_SEND_TRIM) + lim->max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT; + if (flags & NBD_FLAG_SEND_FLUSH) { + lim->features |= BLK_FEAT_WRITE_CACHE; + if (flags & NBD_FLAG_SEND_FUA) + lim->features |= BLK_FEAT_FUA; + } + + if (flags & NBD_FLAG_ROTATIONAL) + lim->features |= BLK_FEAT_ROTATIONAL; + if (flags & NBD_FLAG_SEND_WRITE_ZEROES) + lim->max_write_zeroes_sectors = UINT_MAX >> SECTOR_SHIFT; +} + static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize) { struct queue_limits lim; @@ -352,23 +372,7 @@ static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize) return 0; lim = queue_limits_start_update(nbd->disk->queue); - if (nbd->config->flags & NBD_FLAG_SEND_TRIM) - lim.max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT; - else - lim.max_hw_discard_sectors = 0; - if (!(nbd->config->flags & NBD_FLAG_SEND_FLUSH)) { - lim.features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA); - } else if (nbd->config->flags & NBD_FLAG_SEND_FUA) { - lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA; - } else { - lim.features |= BLK_FEAT_WRITE_CACHE; - lim.features &= ~BLK_FEAT_FUA; - } - if (nbd->config->flags & NBD_FLAG_ROTATIONAL) - lim.features |= BLK_FEAT_ROTATIONAL; - if (nbd->config->flags & NBD_FLAG_SEND_WRITE_ZEROES) - lim.max_write_zeroes_sectors = UINT_MAX >> SECTOR_SHIFT; - + nbd_apply_limits(&lim, nbd->config->flags); lim.logical_block_size = blksize; lim.physical_block_size = blksize; error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim); @@ -1469,8 +1473,13 @@ static void nbd_config_put(struct nbd_device *nbd) if (refcount_dec_and_mutex_lock(&nbd->config_refs, &nbd->config_lock)) { struct nbd_config *config = nbd->config; + struct queue_limits lim; nbd_dev_dbg_close(nbd); invalidate_disk(nbd->disk); + /* reset queue limits to default */ + lim = queue_limits_start_update(nbd->disk->queue); + nbd_apply_limits(&lim, 0); + queue_limits_commit_update(nbd->disk->queue, &lim); if (nbd->config->bytesize) kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); if (test_and_clear_bit(NBD_RT_HAS_PID_FILE, From 285908f554fe32d9ca7cb7c3a03a05144faeb461 Mon Sep 17 00:00:00 2001 From: Yang Erkun Date: Wed, 5 Aug 2026 20:29:26 +0800 Subject: [PATCH 237/241] nbd: remove queue freeze in nbd_add_socket nbd_add_socket() kreallocs config->socks, which a concurrent reader in nbd_handle_cmd() could UAF; commit b98e762e3d71 ("nbd: freeze the queue while we're adding connections")froze the queue to block that. But the freeze costs an RCU grace period on every socket added, and setup adds them one by one. After the previous patch, nbd_add_socket() is rejected once nbd->pid is set, so it only runs during setup. There the capacity is 0 and the write cache is off (cleared on disconnect by the preceding patch, and re-enabled only later in nbd_set_size), so submit_bio_noacct() rejects every bio before it reaches the driver -- non-zero-sector ones via bio_check_eod(), and flush-only ones via the !bdev_write_cache() branch. No I/O is in flight, so the freeze is unnecessary. Reviewed-by: Yu Kuai Signed-off-by: Yang Erkun Link: https://patch.msgid.link/20260805122930.57647-5-yangerkun@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 78df6f459da6..cb7c1f8502f4 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1276,7 +1276,6 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg, struct socket *sock; struct nbd_sock **socks; struct nbd_sock *nsock; - unsigned int memflags; int err; /* Arg will be cast to int, check it to avoid overflow */ @@ -1294,12 +1293,6 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg, return err; nbd_reclassify_socket(sock); - /* - * We need to make sure we don't get any errant requests while we're - * reallocating the ->socks array. - */ - memflags = blk_mq_freeze_queue(nbd->disk->queue); - if (!netlink && !nbd->task_setup && !test_bit(NBD_RT_BOUND, &config->runtime_flags)) nbd->task_setup = current; @@ -1339,12 +1332,10 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg, INIT_WORK(&nsock->work, nbd_pending_cmd_work); socks[config->num_connections++] = nsock; atomic_inc(&config->live_connections); - blk_mq_unfreeze_queue(nbd->disk->queue, memflags); return 0; put_socket: - blk_mq_unfreeze_queue(nbd->disk->queue, memflags); sockfd_put(sock); return err; } From db285268df7948f6edbb7c8bb7cd02da3ceec6b4 Mon Sep 17 00:00:00 2001 From: Yang Erkun Date: Wed, 5 Aug 2026 20:29:27 +0800 Subject: [PATCH 238/241] nbd: skip queue freeze when setting size at device startup Commit 242a49e5c878 ("nbd: freeze the queue for queue limits updates") added the freeze to keep in-flight commands from seeing torn queue_limits. But at startup the capacity is still 0 (invalidate_disk cleared it) and the write cache is off (the previous patch cleared it on disconnect, and nbd_set_size sets it back only after the commit), so submit_bio_noacct() rejects any bio before it reaches the driver and no I/O is in flight. Drop the freeze by checking capacity and write cache state in nbd_set_size. Reviewed-by: Yu Kuai Signed-off-by: Yang Erkun Link: https://patch.msgid.link/20260805122930.57647-6-yangerkun@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index cb7c1f8502f4..cb662ae4b91d 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -375,7 +375,11 @@ static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize) nbd_apply_limits(&lim, nbd->config->flags); lim.logical_block_size = blksize; lim.physical_block_size = blksize; - error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim); + /* No need freeze with 0 capacity and write cache disabled */ + if (!get_capacity(nbd->disk) && !blk_queue_write_cache(nbd->disk->queue)) + error = queue_limits_commit_update(nbd->disk->queue, &lim); + else + error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim); if (error) return error; From f8d21c590e55d629abed675554b186291ac08915 Mon Sep 17 00:00:00 2001 From: Yang Erkun Date: Wed, 5 Aug 2026 20:29:28 +0800 Subject: [PATCH 239/241] nbd: factor out a nbd_genl_foreach_sock The NBD_ATTR_SOCKETS walk is duplicated in nbd_genl_connect (add sockets) and nbd_genl_reconfigure (reconnect). Factor out a single helper that walks the list and calls a callback per fd; with a NULL callback it is a pure counter, used by a later patch to learn nr_hw_queues before the device exists. Returns the number of fds walked (>= 0) or a negative errno; a callback >0 will stops early. Reviewed-by: Yu Kuai Signed-off-by: Yang Erkun Link: https://patch.msgid.link/20260805122930.57647-7-yangerkun@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 128 +++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 66 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index cb662ae4b91d..192c80aa738c 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1344,7 +1344,7 @@ put_socket: return err; } -static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg) +static int nbd_genl_reconnect_sock_cb(struct nbd_device *nbd, unsigned long arg) { struct nbd_config *config = nbd->config; struct socket *sock, *old; @@ -1399,11 +1399,12 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg) atomic_inc(&config->live_connections); wake_up(&config->conn_wait); + dev_info(nbd_to_dev(nbd), "reconnected socket\n"); return 0; } sockfd_put(sock); kfree(args); - return -ENOSPC; + return 1; } static void nbd_bdev_reset(struct nbd_device *nbd) @@ -2109,6 +2110,58 @@ static int nbd_genl_size_set(struct genl_info *info, struct nbd_device *nbd) return 0; } +/* + * Walk the NBD_ATTR_SOCKETS nested list can call @cb for each socket fd. + * + * Return the number of fds walked, or a negative errno. + */ +static int nbd_genl_foreach_sock(struct genl_info *info, + int (*cb)(struct nbd_device *nbd, unsigned long fd), + struct nbd_device *nbd) +{ + struct nlattr *attr; + int rem, count = 0; + + if (!info->attrs[NBD_ATTR_SOCKETS]) + return 0; + + nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS], rem) { + struct nlattr *socks[NBD_SOCK_MAX + 1]; + int ret; + + if (nla_type(attr) != NBD_SOCK_ITEM) { + pr_err("socks must be embedded in a SOCK_ITEM attr\n"); + return -EINVAL; + } + + if (nla_parse_nested_deprecated(socks, NBD_SOCK_MAX, + attr, + nbd_sock_policy, + info->extack)) { + pr_err("error processing sock list\n"); + return -EINVAL; + } + + if (!socks[NBD_SOCK_FD]) + continue; + + count++; + if (cb) { + ret = cb(nbd, (int)nla_get_u32(socks[NBD_SOCK_FD])); + if (ret > 0) + return count; + if (ret < 0) + return ret; + } + } + return count; +} + +static int nbd_genl_connect_sock_cb(struct nbd_device *nbd, unsigned long fd) +{ + return nbd_add_socket(nbd, fd, true); +} + static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info) { struct nbd_device *nbd; @@ -2228,36 +2281,9 @@ again: } } - if (info->attrs[NBD_ATTR_SOCKETS]) { - struct nlattr *attr; - int rem, fd; - - nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS], - rem) { - struct nlattr *socks[NBD_SOCK_MAX+1]; - - if (nla_type(attr) != NBD_SOCK_ITEM) { - pr_err("socks must be embedded in a SOCK_ITEM attr\n"); - ret = -EINVAL; - goto out; - } - ret = nla_parse_nested_deprecated(socks, NBD_SOCK_MAX, - attr, - nbd_sock_policy, - info->extack); - if (ret != 0) { - pr_err("error processing sock list\n"); - ret = -EINVAL; - goto out; - } - if (!socks[NBD_SOCK_FD]) - continue; - fd = (int)nla_get_u32(socks[NBD_SOCK_FD]); - ret = nbd_add_socket(nbd, fd, true); - if (ret) - goto out; - } - } + ret = nbd_genl_foreach_sock(info, nbd_genl_connect_sock_cb, nbd); + if (ret < 0) + goto out; if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) { nbd->backend = nla_strdup(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER], @@ -2442,40 +2468,10 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) } } - if (info->attrs[NBD_ATTR_SOCKETS]) { - struct nlattr *attr; - int rem, fd; - - nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS], - rem) { - struct nlattr *socks[NBD_SOCK_MAX+1]; - - if (nla_type(attr) != NBD_SOCK_ITEM) { - pr_err("socks must be embedded in a SOCK_ITEM attr\n"); - ret = -EINVAL; - goto out; - } - ret = nla_parse_nested_deprecated(socks, NBD_SOCK_MAX, - attr, - nbd_sock_policy, - info->extack); - if (ret != 0) { - pr_err("error processing sock list\n"); - ret = -EINVAL; - goto out; - } - if (!socks[NBD_SOCK_FD]) - continue; - fd = (int)nla_get_u32(socks[NBD_SOCK_FD]); - ret = nbd_reconnect_socket(nbd, fd); - if (ret) { - if (ret == -ENOSPC) - ret = 0; - goto out; - } - dev_info(nbd_to_dev(nbd), "reconnected socket\n"); - } - } + ret = nbd_genl_foreach_sock(info, nbd_genl_reconnect_sock_cb, nbd); + /* foreach_sock returns a positive count on success; doit must return 0 */ + if (ret >= 0) + ret = 0; out: mutex_unlock(&nbd->config_lock); nbd_config_put(nbd); From a9d414b4a15c69b5389da3ec08f23e9f2926c52a Mon Sep 17 00:00:00 2001 From: Yang Erkun Date: Wed, 5 Aug 2026 20:29:29 +0800 Subject: [PATCH 240/241] nbd: remove queue freeze for newly created nbd from netlink path Previous commits has removed the queue freeze in nbd_add_socket and nbd_set_size during nbd device setup. However, a queue freeze can still occur when nbd_start_device calls blk_mq_update_nr_hw_queues if the socket connection count does not match nbd->tag_set->nr_hw_queues. The nbd_start_device function can be invoked through either the ioctl or netlink paths. The ioctl path only allows reusing an existing inactivate nbd device, there is nothing more we can do to prevent the queue freeze since the old nbd->tag_set->nr_hw_queues may not match the new socket connection count. Similarly, the netlink path can reuse a preferred inactivate nbd device, and again, we cannot do more in this scenario. However, the netlink path can also add a new nbd device using nbd_dev_add. In this case, we can obtain the new number of socket connections, and by adding a new argument representing the expected nr_hw_queues in nbd_dev_add, we can ensure the queue freeze is avoided for this situation. Reviewed-by: Yu Kuai Signed-off-by: Yang Erkun Link: https://patch.msgid.link/20260805122930.57647-8-yangerkun@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 192c80aa738c..6c4aa19ea11a 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -1944,7 +1944,8 @@ static const struct blk_mq_ops nbd_mq_ops = { .timeout = nbd_xmit_timeout, }; -static struct nbd_device *nbd_dev_add(int index, unsigned int refs) +static struct nbd_device *nbd_dev_add(int index, unsigned int refs, + int nr_hw_queues) { struct queue_limits lim = { .max_hw_sectors = 65536, @@ -1961,7 +1962,7 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs) goto out; nbd->tag_set.ops = &nbd_mq_ops; - nbd->tag_set.nr_hw_queues = 1; + nbd->tag_set.nr_hw_queues = nr_hw_queues; nbd->tag_set.queue_depth = 128; nbd->tag_set.numa_node = NUMA_NO_NODE; nbd->tag_set.cmd_size = sizeof(struct nbd_cmd); @@ -2214,7 +2215,11 @@ again: mutex_unlock(&nbd_index_mutex); if (!nbd) { - nbd = nbd_dev_add(index, 2); + ret = nbd_genl_foreach_sock(info, NULL, NULL); + if (ret < 0) + return ret; + + nbd = nbd_dev_add(index, 2, ret > 0 ? ret : 1); if (IS_ERR(nbd)) { pr_err("failed to add new device\n"); return PTR_ERR(nbd); @@ -2724,7 +2729,7 @@ static int __init nbd_init(void) nbd_dbg_init(); for (i = 0; i < nbds_max; i++) - nbd_dev_add(i, 1); + nbd_dev_add(i, 1, 1); return 0; } From 326d49039c10b65522ac7277b19b9b5c42ed1aeb Mon Sep 17 00:00:00 2001 From: Yang Erkun Date: Wed, 5 Aug 2026 20:29:30 +0800 Subject: [PATCH 241/241] nbd: add pre_defined_connections module parameter for pre-created devices blk_mq_update_nr_hw_queues() in nbd_start_device() may cause a queue freeze. The previous commit addressed this for newly created nbd devices by setting the expected nr_hw_queues in nbd_dev_add(). However, when reusing an old inactive nbd device, the queue freeze can still occur if the old nbd->tag_set->nr_hw_queues does not match the new socket connection count. Inactive nbd devices can originate from two sources: loading the nbd module with nbds_max, which sets the default nr_hw_queues to 1, and the netlink method, which sets nr_hw_queues according to the expected number of socket connections. For the first case, add a module parameter so the default nr_hw_queues can be changed. Users who know their expected number of connections can then prevent queue freezes on pre-created devices via nbds_max. Before this patchset: real 0m2.195s user 0m0.005s sys 0m0.022s After this patchset: real 0m0.090s user 0m0.004s sys 0m0.018s Signed-off-by: Yang Erkun Reviewed-by: Yu Kuai Link: https://patch.msgid.link/20260805122930.57647-9-yangerkun@huawei.com Signed-off-by: Jens Axboe --- drivers/block/nbd.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index 6c4aa19ea11a..ffce519bf008 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -166,6 +166,7 @@ static struct dentry *nbd_dbg_dir; static unsigned int nbds_max = 16; static int max_part = 16; +static int pre_defined_connections = 1; static int part_shift; static int nbd_dev_dbg_init(struct nbd_device *nbd); @@ -2712,6 +2713,12 @@ static int __init nbd_init(void) if (nbds_max > 1UL << (MINORBITS - part_shift)) return -EINVAL; + /* An excessively large value will be adjusted in blk_mq_alloc_tag_set */ + if (pre_defined_connections < 1) { + pr_err("pre_defined_connections must be >= 1\n"); + return -EINVAL; + } + if (register_blkdev(NBD_MAJOR, "nbd")) return -EIO; @@ -2728,8 +2735,12 @@ static int __init nbd_init(void) } nbd_dbg_init(); + /* + * Set to the intended connection count so nbd_start_device() can skip + * the queue-freezing blk_mq_update_nr_hw_queues() call. + */ for (i = 0; i < nbds_max; i++) - nbd_dev_add(i, 1, 1); + nbd_dev_add(i, 1, pre_defined_connections); return 0; } @@ -2790,3 +2801,6 @@ module_param(nbds_max, int, 0444); MODULE_PARM_DESC(nbds_max, "number of network block devices to initialize (default: 16)"); module_param(max_part, int, 0444); MODULE_PARM_DESC(max_part, "number of partitions per device (default: 16)"); +module_param(pre_defined_connections, int, 0444); +MODULE_PARM_DESC(pre_defined_connections, +"number of connections for devices pre-created at module load (default: 1)");