Sandro/linux
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-09-18 23:19:34 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

io_uring/zcrx: scale refilling with large pages

Browse Source 
io_zcrx_ring_refill() caps the loop by mixing the max number of
allocated netmems and the number of available RQEs together, which
caps the number of entries to process the pp cache size. As a result,
when niovs are heavily fragmented, the refilling logic allocates only a
small number of niovs per call on average and sometimes even none.

Keep a separate counter for the number of processed RQ entries, which is
capped by a roughly calculated from the page size value to keep the
cache full. And separately break if it allocates enough niovs.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://patch.msgid.link/143cf439299728759eb0a840c866363fe99293f0.1786108672.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov
2026-08-15 17:29:56 -06:00
committed by Jens Axboe
parent 297b5ccea4
commit 1afcef59fe
1 changed files with 14 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
io_uring/zcrx.c
+14 -7
View File
@@ -28,6 +28,13 @@
#include "zcrx.h"
#include "rsrc.h"
#define ZCRX_MAX_FRAGS_PER_PAGE MAX(PAGE_SIZE / 1024, 1)
/*
* We need a reasonable limit to be able to fill in 64 entries on average
* for 1500 byte MTU. Over-estimate it to keep it pow2.
*/
#define ZCRX_REFILL_CAP MIN(64 * ZCRX_MAX_FRAGS_PER_PAGE, 1024)
#define IO_ZCRX_AREA_SUPPORTED_FLAGS (IORING_ZCRX_AREA_DMABUF)
#define IO_DMA_ATTR (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING)
@@ -1124,17 +1131,15 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
{
struct zcrx_rq *rq = &ifq->rq;
unsigned int mask = rq->nr_entries - 1;
unsigned int entries;
unsigned int rqes_left;
unsigned allocated = 0;
guard(spinlock_bh)(&rq->lock);
entries = zcrx_rq_entries(rq);
entries = min_t(unsigned, entries, to_alloc);
if (unlikely(!entries))
return 0;
rqes_left = zcrx_rq_entries(rq);
rqes_left = min_t(unsigned, rqes_left, ZCRX_REFILL_CAP);
do {
for (; rqes_left; rqes_left--) {
struct io_uring_zcrx_rqe *rqe = zcrx_next_rqe(rq, mask);
struct net_iov *niov;
netmem_ref netmem;
@@ -1155,7 +1160,9 @@ static unsigned io_zcrx_ring_refill(struct page_pool *pp,
netmems[allocated] = netmem;
allocated++;
} while (--entries);
if (allocated >= to_alloc)
break;
}
smp_store_release(&rq->ring->head, rq->cached_head);
return allocated;