1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | #ifndef _LINUX_IO_URING_H |
3 | #define _LINUX_IO_URING_H |
4 | |
5 | #include <linux/sched.h> |
6 | #include <linux/xarray.h> |
7 | #include <uapi/linux/io_uring.h> |
8 | |
9 | enum io_uring_cmd_flags { |
10 | IO_URING_F_COMPLETE_DEFER = 1, |
11 | IO_URING_F_UNLOCKED = 2, |
12 | /* the request is executed from poll, it should not be freed */ |
13 | IO_URING_F_MULTISHOT = 4, |
14 | /* executed by io-wq */ |
15 | IO_URING_F_IOWQ = 8, |
16 | /* int's last bit, sign checks are usually faster than a bit test */ |
17 | IO_URING_F_NONBLOCK = INT_MIN, |
18 | |
19 | /* ctx state flags, for URING_CMD */ |
20 | IO_URING_F_SQE128 = (1 << 8), |
21 | IO_URING_F_CQE32 = (1 << 9), |
22 | IO_URING_F_IOPOLL = (1 << 10), |
23 | |
24 | /* set when uring wants to cancel a previously issued command */ |
25 | IO_URING_F_CANCEL = (1 << 11), |
26 | IO_URING_F_COMPAT = (1 << 12), |
27 | }; |
28 | |
29 | /* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */ |
30 | #define IORING_URING_CMD_CANCELABLE (1U << 30) |
31 | #define IORING_URING_CMD_POLLED (1U << 31) |
32 | |
33 | struct io_uring_cmd { |
34 | struct file *file; |
35 | const struct io_uring_sqe *sqe; |
36 | union { |
37 | /* callback to defer completions to task context */ |
38 | void (*task_work_cb)(struct io_uring_cmd *cmd, unsigned); |
39 | /* used for polled completion */ |
40 | void *cookie; |
41 | }; |
42 | u32 cmd_op; |
43 | u32 flags; |
44 | u8 pdu[32]; /* available inline for free use */ |
45 | }; |
46 | |
47 | static inline const void *io_uring_sqe_cmd(const struct io_uring_sqe *sqe) |
48 | { |
49 | return sqe->cmd; |
50 | } |
51 | |
52 | #if defined(CONFIG_IO_URING) |
53 | int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, |
54 | struct iov_iter *iter, void *ioucmd); |
55 | void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, ssize_t res2, |
56 | unsigned issue_flags); |
57 | struct sock *io_uring_get_socket(struct file *file); |
58 | void __io_uring_cancel(bool cancel_all); |
59 | void __io_uring_free(struct task_struct *tsk); |
60 | void io_uring_unreg_ringfd(void); |
61 | const char *io_uring_get_opcode(u8 opcode); |
62 | void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, |
63 | void (*task_work_cb)(struct io_uring_cmd *, unsigned), |
64 | unsigned flags); |
65 | /* users should follow semantics of IOU_F_TWQ_LAZY_WAKE */ |
66 | void io_uring_cmd_do_in_task_lazy(struct io_uring_cmd *ioucmd, |
67 | void (*task_work_cb)(struct io_uring_cmd *, unsigned)); |
68 | |
69 | static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, |
70 | void (*task_work_cb)(struct io_uring_cmd *, unsigned)) |
71 | { |
72 | __io_uring_cmd_do_in_task(ioucmd, task_work_cb, flags: 0); |
73 | } |
74 | |
75 | static inline void io_uring_files_cancel(void) |
76 | { |
77 | if (current->io_uring) { |
78 | io_uring_unreg_ringfd(); |
79 | __io_uring_cancel(cancel_all: false); |
80 | } |
81 | } |
82 | static inline void io_uring_task_cancel(void) |
83 | { |
84 | if (current->io_uring) |
85 | __io_uring_cancel(cancel_all: true); |
86 | } |
87 | static inline void io_uring_free(struct task_struct *tsk) |
88 | { |
89 | if (tsk->io_uring) |
90 | __io_uring_free(tsk); |
91 | } |
92 | int io_uring_cmd_sock(struct io_uring_cmd *cmd, unsigned int issue_flags); |
93 | void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, |
94 | unsigned int issue_flags); |
95 | struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd); |
96 | #else |
97 | static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, |
98 | struct iov_iter *iter, void *ioucmd) |
99 | { |
100 | return -EOPNOTSUPP; |
101 | } |
102 | static inline void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, |
103 | ssize_t ret2, unsigned issue_flags) |
104 | { |
105 | } |
106 | static inline void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd, |
107 | void (*task_work_cb)(struct io_uring_cmd *, unsigned)) |
108 | { |
109 | } |
110 | static inline void io_uring_cmd_do_in_task_lazy(struct io_uring_cmd *ioucmd, |
111 | void (*task_work_cb)(struct io_uring_cmd *, unsigned)) |
112 | { |
113 | } |
114 | static inline struct sock *io_uring_get_socket(struct file *file) |
115 | { |
116 | return NULL; |
117 | } |
118 | static inline void io_uring_task_cancel(void) |
119 | { |
120 | } |
121 | static inline void io_uring_files_cancel(void) |
122 | { |
123 | } |
124 | static inline void io_uring_free(struct task_struct *tsk) |
125 | { |
126 | } |
127 | static inline const char *io_uring_get_opcode(u8 opcode) |
128 | { |
129 | return "" ; |
130 | } |
131 | static inline int io_uring_cmd_sock(struct io_uring_cmd *cmd, |
132 | unsigned int issue_flags) |
133 | { |
134 | return -EOPNOTSUPP; |
135 | } |
136 | static inline void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd, |
137 | unsigned int issue_flags) |
138 | { |
139 | } |
140 | static inline struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd) |
141 | { |
142 | return NULL; |
143 | } |
144 | #endif |
145 | |
146 | #endif |
147 | |