Merge tag 'for-7.2/io_uring-20260615' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux

Pull io_uring updates from Jens Axboe:

 - Rework the task_work infrastructure.

   Both the local (DEFER_TASKRUN) and the normal (tctx) task_work lists
   were llist based, which is LIFO ordered, and hence each run had to do
   an O(n) list reversal pass first to restore queue order.
   Additionally, to cap the amount of task_work run, each method needed
   a retry list as well.

   Add a lockless MPCS FIFO queue (based on Dmitry Vyukov's intrusive
   MPSC algorithm) and switch both task_work lists to it. It performs
   better than llists and we can then also ditch the retry lists as well
   as entries are popped one-at-the-time.

   On top of those changes, run the tctx fallback task_work directly and
   remove the now-unused per-ctx fallback machinery entirely.

 - zcrx user notifications.

   Add a mechanism for zcrx to communicate conditions back to userspace
   via a dedicated CQE, with the initial users being notification on
   running out of buffers and on a frag copy fallback, plus
   shared-memory notification statistics.

   Alongside that, a series of zcrx reliability and cleanup fixes: more
   reliable scrubbing, poisoning pointers on unregistration, dropping an
   extra ifq close, adding a ctx back-pointer, reordering fd allocation
   in the export path, and killing a dead 'sock' member.

 - Allow using io_uring registered buffers for plain SEND and RECV, not
   just for the zero-copy send path.

   This enables targets like ublk's NBD backend to push/pull IO data
   directly to/from a registered buffer over a plain send/recv on a TCP
   socket.

 - Registered buffer improvements: account huge pages correctly, bump
   the io_mapped_ubuf length field to size_t, and raise the previous 1GB
   registered buffer size limit.

 - Restrict the ctx access exposed to io_uring BPF struct_ops programs
   by handing them an opaque type rather than the full io_ring_ctx, and
   add a separate MAINTAINERS entry for the bpf-ops code.

 - Allow opcode filtering on IORING_OP_CONNECT.

 - Validate ring-provided buffer addresses with access_ok(), and align
   the legacy buffer add limit with MAX_BIDS_PER_BGID.

 - Various other cleanups and minor fixes, including avoiding msghdr
   async data on connect/bind, dropping async_size for OP_LISTEN, making
   the POLL_FIRST receive side checks consistent, re-checking
   IO_WQ_BIT_EXIT for each linked work item, and using
   trace_call__##name() at guarded tracepoint call sites.

* tag 'for-7.2/io_uring-20260615' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: (31 commits)
  io_uring/bpf-ops: add a separate maintainer entry
  io_uring/net: make POLL_FIRST receive side checks consistent
  io_uring: remove the per-ctx fallback task_work machinery
  io_uring: run the tctx task_work fallback directly
  io_uring: switch normal task_work to a mpscq
  io_uring: switch local task_work to a mpscq
  io_uring/mpscq: add lockless multi-producer, single-consumer FIFO queue
  io_uring: grab RCU read lock marking task run
  io_uring/zcrx: kill dead 'sock' member in struct io_zcrx_args
  io_uring/kbuf: validate ring provided buffer addresses with access_ok()
  io_uring/net: support registered buffer for plain send and recv
  io_uring/nop: Drop a wrong comment in struct io_nop
  io_uring/net: Remove async_size for OP_LISTEN
  io_uring/net: Avoid msghdr on op_connect/op_bind async data
  io_uring/bpf-ops: restrict ctx access to BPF
  io_uring/io-wq: re-check IO_WQ_BIT_EXIT for each linked work item
  io_uring/kbuf: align legacy buffer add limit with MAX_BIDS_PER_BGID
  io_uring/zcrx: add shared-memory notification statistics
  io_uring/zcrx: notify user on frag copy fallback
  io_uring/zcrx: notify user when out of buffers
  ...

Author: torvalds

MAINTAINERS

@@ -13542,6 +13542,14 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
 S:	Maintained
 F:	io_uring/zcrx.*
 
+IO_URING BPF-OPS
+M:	Pavel Begunkov <asml.silence@gmail.com>
+L:	io-uring@vger.kernel.org
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
+S:	Maintained
+F:	io_uring/bpf-ops.*
+F:	io_uring/loop.*
+
 IPMI SUBSYSTEM
 M:	Corey Minyard <corey@minyard.net>
 L:	openipmi-developer@lists.sourceforge.net (moderated for non-subscribers)

include/linux/io_uring_types.h

@@ -55,6 +55,18 @@ struct io_wq_work_list {
 	struct io_wq_work_node *last;
 };
 
+/*
+ * Lockless multi-producer, single-consumer FIFO queue, see
+ * io_uring/mpscq.h for the implementation and rules. Defined here so
+ * that it can be embedded in io_ring_ctx. This is the producer side
+ * only - the consumer cursor is kept separately, on a cacheline that
+ * isn't dirtied by the producers.
+ */
+struct mpscq {
+	struct llist_node	*tail;		/* producers */
+	struct llist_node	stub;
+};
+
 struct io_wq_work {
 	struct io_wq_work_node list;
 	atomic_t flags;
@@ -119,6 +131,11 @@ struct io_uring_task {
 	const struct io_ring_ctx 	*last;
 	struct task_struct		*task;
 	struct io_wq			*io_wq;
+	/*
+	 * Consumer cursor for ->task_list. Only popped by the task itself,
+	 * or by ->fallback_work once the task can no longer run task_work.
+	 */
+	struct llist_node		*task_head;
 	struct file			*registered_rings[IO_RINGFD_REG_MAX];
 
 	struct xarray			xa;
@@ -127,8 +144,13 @@ struct io_uring_task {
 	atomic_t			inflight_tracked;
 	struct percpu_counter		inflight;
 
+	/* drains ->task_list once the task can no longer run task_work */
+	struct work_struct		fallback_work;
+
 	struct { /* task_work */
-		struct llist_head	task_list;
+		struct mpscq		task_list;
+		/* BIT(0) guards adding tw only once */
+		unsigned long		tw_pending;
 		struct callback_head	task_work;
 	} ____cacheline_aligned_in_smp;
 };
@@ -290,6 +312,8 @@ enum {
 	IO_RING_F_IOWQ_LIMITS_SET	= BIT(12),
 };
 
+struct iou_ctx {};
+
 struct io_ring_ctx {
 	/* const or read-mostly hot data */
 	struct {
@@ -346,6 +370,14 @@ struct io_ring_ctx {
 		bool			poll_multi_queue;
 		struct list_head	iopoll_list;
 
+		/*
+		 * Consumer cursor for ->work_list, protected by ->uring_lock.
+		 * Deliberately kept away from the producer side of the queue,
+		 * as it's written for every popped entry, and the producer
+		 * cacheline is contended enough as it is.
+		 */
+		struct llist_node	*work_head;
+
 		struct io_file_table	file_table;
 		struct io_rsrc_data	buf_table;
 		struct io_alloc_cache	node_cache;
@@ -366,7 +398,7 @@ struct io_ring_ctx {
 		struct io_alloc_cache	rw_cache;
 		struct io_alloc_cache	cmd_cache;
 
-		int (*loop_step)(struct io_ring_ctx *ctx,
+		int (*loop_step)(struct iou_ctx *,
 				 struct iou_loop_params *);
 
 		/*
@@ -403,8 +435,7 @@ struct io_ring_ctx {
 	 */
 	struct {
 		struct io_rings	__rcu	*rings_rcu;
-		struct llist_head	work_llist;
-		struct llist_head	retry_llist;
+		struct mpscq		work_list;
 		unsigned long		check_cq;
 		atomic_t		cq_wait_nr;
 		atomic_t		cq_timeouts;
@@ -446,6 +477,9 @@ struct io_ring_ctx {
 	/* Stores zcrx object pointers of type struct io_zcrx_ifq */
 	struct xarray			zcrx_ctxs;
 
+	/* Used for accounting references on pages in registered buffers */
+	struct xarray		hpage_acct;
+
 	u32			pers_next;
 	struct xarray		personalities;
 
@@ -464,8 +498,6 @@ struct io_ring_ctx {
 	struct mutex			tctx_lock;
 
 	/* ctx exit and cancelation */
-	struct llist_head		fallback_llist;
-	struct delayed_work		fallback_work;
 	struct work_struct		exit_work;
 	struct completion		ref_comp;
 
@@ -725,8 +757,6 @@ struct io_kiocb {
 	 */
 	u16				buf_index;
 
-	unsigned			nr_tw;
-
 	/* REQ_F_* flags */
 	io_req_flags_t			flags;
 

include/uapi/linux/io_uring/bpf_filter.h

@@ -27,6 +27,22 @@ struct io_uring_bpf_ctx {
 			__u64	mode;
 			__u64	resolve;
 		} open;
+		/*
+		 * For CONNECT: fields are populated only when addr_len covers
+		 * them; unpopulated fields are zero from the caller-side memset
+		 * in io_uring_populate_bpf_ctx(). port and v4_addr are network
+		 * byte order. Filters may only issue BPF_LD|BPF_W|BPF_ABS at
+		 * 4-byte aligned offsets; load + mask for sub-word fields.
+		 */
+		struct {
+			__u32	family;	/* sa_family_t zero-extended */
+			__be16	port;
+			__u8	pad[2];
+			union {
+				__be32	v4_addr;
+				__u8	v6_addr[16];
+			};
+		} connect;
 	};
 };
 

include/uapi/linux/io_uring/query.h

@@ -23,6 +23,7 @@ enum {
 	IO_URING_QUERY_OPCODES			= 0,
 	IO_URING_QUERY_ZCRX			= 1,
 	IO_URING_QUERY_SCQ			= 2,
+	IO_URING_QUERY_ZCRX_NOTIF		= 3,
 
 	__IO_URING_QUERY_MAX,
 };
@@ -62,6 +63,17 @@ struct io_uring_query_zcrx {
 	__u64 __resv2;
 };
 
+struct io_uring_query_zcrx_notif {
+	/* Bitmask of supported ZCRX_NOTIF_* flags */
+	__u32 notif_flags;
+	/* Size of io_uring_zcrx_notif_stats */
+	__u32 notif_stats_size;
+	/* Required alignment for the stats struct within the region (ie stats_offset) */
+	__u32 notif_stats_off_alignment;
+	__u32 __resv1;
+	__u64 __resv2[4];
+};
+
 struct io_uring_query_scq {
 	/* The SQ/CQ rings header size */
 	__u64 hdr_size;

include/uapi/linux/io_uring/zcrx.h

@@ -65,6 +65,32 @@ enum zcrx_features {
 	 * value in struct io_uring_zcrx_ifq_reg::rx_buf_len.
 	 */
 	ZCRX_FEATURE_RX_PAGE_SIZE	= 1 << 0,
+	ZCRX_FEATURE_NOTIFICATION	= 1 << 1,
+};
+
+enum zcrx_notification_type {
+	ZCRX_NOTIF_NO_BUFFERS,
+	ZCRX_NOTIF_COPY,
+
+	__ZCRX_NOTIF_TYPE_LAST,
+};
+
+enum zcrx_notification_desc_flags {
+	/* If set, stats_offset holds a valid offset to a notif_stats struct */
+	ZCRX_NOTIF_DESC_FLAG_STATS = 1 << 0,
+};
+
+struct zcrx_notif_stats {
+	__u64	copy_count;	/* cumulative copy-fallback CQEs */
+	__u64	copy_bytes;	/* cumulative bytes copied */
+};
+
+struct zcrx_notification_desc {
+	__u64	user_data;
+	__u32	type_mask;
+	__u32	flags; /* see enum zcrx_notification_desc_flags */
+	__u64	stats_offset; /* offset from the beginning of refill ring region for stats */
+	__u64	__resv2[9];
 };
 
 /*
@@ -82,12 +108,14 @@ struct io_uring_zcrx_ifq_reg {
 	struct io_uring_zcrx_offsets offsets;
 	__u32	zcrx_id;
 	__u32	rx_buf_len;
-	__u64	__resv[3];
+	__u64	notif_desc; /* see struct zcrx_notification_desc */
+	__u64	__resv[2];
 };
 
 enum zcrx_ctrl_op {
 	ZCRX_CTRL_FLUSH_RQ,
 	ZCRX_CTRL_EXPORT,
+	ZCRX_CTRL_ARM_NOTIFICATION,
 
 	__ZCRX_CTRL_LAST,
 };
@@ -101,6 +129,11 @@ struct zcrx_ctrl_export {
 	__u32 		__resv1[11];
 };
 
+struct zcrx_ctrl_arm_notif {
+	__u32		notif_type;
+	__u32		__resv[11];
+};
+
 struct zcrx_ctrl {
 	__u32	zcrx_id;
 	__u32	op; /* see enum zcrx_ctrl_op */
@@ -109,6 +142,7 @@ struct zcrx_ctrl {
 	union {
 		struct zcrx_ctrl_export		zc_export;
 		struct zcrx_ctrl_flush_rq	zc_flush;
+		struct zcrx_ctrl_arm_notif	zc_arm_notif;
 	};
 };
 

io_uring/bpf-ops.c

@@ -14,15 +14,18 @@ static const struct btf_type *loop_params_type;
 
 __bpf_kfunc_start_defs();
 
-__bpf_kfunc int bpf_io_uring_submit_sqes(struct io_ring_ctx *ctx, u32 nr)
+__bpf_kfunc int bpf_io_uring_submit_sqes(struct iou_ctx *loop_ctx, u32 nr)
 {
+	struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx);
+
 	return io_submit_sqes(ctx, nr);
 }
 
 __bpf_kfunc
-__u8 *bpf_io_uring_get_region(struct io_ring_ctx *ctx, __u32 region_id,
+__u8 *bpf_io_uring_get_region(struct iou_ctx *loop_ctx, __u32 region_id,
 			      const size_t rdwr_buf_size)
 {
+	struct io_ring_ctx *ctx = io_loop_demangle_ctx(loop_ctx);
 	struct io_mapped_region *r;
 
 	lockdep_assert_held(&ctx->uring_lock);
@@ -58,7 +61,7 @@ static const struct btf_kfunc_id_set bpf_io_uring_kfunc_set = {
 	.set = &io_uring_kfunc_set,
 };
 
-static int io_bpf_ops__loop_step(struct io_ring_ctx *ctx,
+static int io_bpf_ops__loop_step(struct iou_ctx *ctx,
 				 struct iou_loop_params *lp)
 {
 	return IOU_LOOP_STOP;

io_uring/bpf-ops.h

@@ -11,7 +11,7 @@ enum {
 };
 
 struct io_uring_bpf_ops {
-	int (*loop_step)(struct io_ring_ctx *ctx, struct iou_loop_params *lp);
+	int (*loop_step)(struct iou_ctx *, struct iou_loop_params *lp);
 
 	__u32 ring_fd;
 	void *priv;

io_uring/cancel.c

@@ -565,8 +565,6 @@ __cold bool io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
 	mutex_unlock(&ctx->uring_lock);
 	if (tctx)
 		ret |= io_run_task_work() > 0;
-	else
-		ret |= flush_delayed_work(&ctx->fallback_work);
 	return ret;
 }
 

io_uring/fdinfo.c

@@ -224,7 +224,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
 		if (ctx->buf_table.nodes[i])
 			buf = ctx->buf_table.nodes[i]->buf;
 		if (buf)
-			seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len);
+			seq_printf(m, "%5u: 0x%llx/%zu\n", i, buf->ubuf, buf->len);
 		else
 			seq_printf(m, "%5u: <none>\n", i);
 	}

io_uring/io-wq.c

@@ -602,7 +602,6 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
 	struct io_wq *wq = worker->wq;
 
 	do {
-		bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
 		struct io_wq_work *work;
 
 		/*
@@ -638,6 +637,7 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
 
 		/* handle a whole dependent link */
 		do {
+			bool do_kill = test_bit(IO_WQ_BIT_EXIT, &wq->state);
 			struct io_wq_work *next_hashed, *linked;
 			unsigned int work_flags = atomic_read(&work->flags);
 			unsigned int hash = __io_wq_is_hashed(work_flags)