Sandro/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-09-18 23:19:34 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

io_uring/zcrx: move RQ head/tail to separate cache lines

Browse Source 
RQ head and tail are currently put into the same cache line, which can
cause false sharing problems when refill is run on another CPU. Put them
into separate cache lines.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Reviewed-by: Mina Almasry <almasrymina@google.com>
Link: https://patch.msgid.link/f769116a3f5267f06cb9a9a819d482ddea2e0852.1786108672.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov
2026-08-15 17:29:56 -06:00
committed by Jens Axboe
parent 1afcef59fe
commit 433c57e9b1
3 changed files with 11 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
io_uring/query.c
+1 -1
View File
@@ -38,7 +38,7 @@ static ssize_t io_query_zcrx(union io_query_data *data)
e->register_flags = ZCRX_SUPPORTED_REG_FLAGS;
e->area_flags = IORING_ZCRX_AREA_DMABUF;
e->nr_ctrl_opcodes = __ZCRX_CTRL_LAST;
e->rq_hdr_size = sizeof(struct io_uring);
e->rq_hdr_size = sizeof(struct zcrx_rq_hdr);
e->rq_hdr_alignment = L1_CACHE_BYTES;
e->features = ZCRX_FEATURES;
e->__resv2 = 0;
io_uring/zcrx.c
+4 -4
View File
@@ -379,9 +379,9 @@ static void io_zcrx_get_niov_uref(struct net_iov *niov)
static void io_fill_zcrx_offsets(struct io_uring_zcrx_offsets *offsets)
{
offsets->head = offsetof(struct io_uring, head);
offsets->tail = offsetof(struct io_uring, tail);
offsets->rqes = ALIGN(sizeof(struct io_uring), L1_CACHE_BYTES);
offsets->head = offsetof(struct zcrx_rq_hdr, head);
offsets->tail = offsetof(struct zcrx_rq_hdr, tail);
offsets->rqes = ALIGN(sizeof(struct zcrx_rq_hdr), L1_CACHE_BYTES);
}
static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
@@ -409,7 +409,7 @@ static int io_allocate_rbuf_ring(struct io_ring_ctx *ctx,
return ret;
ptr = io_region_get_ptr(&ifq->rq_region);
ifq->rq.ring = (struct io_uring *)ptr;
ifq->rq.ring = (struct zcrx_rq_hdr *)ptr;
ifq->rq.rqes = (struct io_uring_zcrx_rqe *)(ptr + off);
memset(ifq->rq.ring, 0, sizeof(*ifq->rq.ring));
io_uring/zcrx.h
+6 -1
View File
@@ -43,9 +43,14 @@ struct io_zcrx_area {
struct io_zcrx_mem mem;
};
struct zcrx_rq_hdr {
u32 head ____cacheline_aligned_in_smp;
u32 tail ____cacheline_aligned_in_smp;
};
struct zcrx_rq {
spinlock_t lock;
struct io_uring *ring;
struct zcrx_rq_hdr *ring;
struct io_uring_zcrx_rqe *rqes;
u32 cached_head;
u32 nr_entries;