1 | /* SPDX-License-Identifier: GPL-2.0 */ |
2 | #ifndef _ELEVATOR_H |
3 | #define _ELEVATOR_H |
4 | |
5 | #include <linux/percpu.h> |
6 | #include <linux/hashtable.h> |
7 | #include "blk-mq.h" |
8 | |
9 | struct io_cq; |
10 | struct elevator_type; |
11 | struct blk_mq_debugfs_attr; |
12 | |
13 | /* |
14 | * Return values from elevator merger |
15 | */ |
16 | enum elv_merge { |
17 | ELEVATOR_NO_MERGE = 0, |
18 | ELEVATOR_FRONT_MERGE = 1, |
19 | ELEVATOR_BACK_MERGE = 2, |
20 | ELEVATOR_DISCARD_MERGE = 3, |
21 | }; |
22 | |
23 | struct blk_mq_alloc_data; |
24 | struct blk_mq_hw_ctx; |
25 | |
26 | struct elevator_mq_ops { |
27 | int (*init_sched)(struct request_queue *, struct elevator_type *); |
28 | void (*exit_sched)(struct elevator_queue *); |
29 | int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); |
30 | void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); |
31 | void (*depth_updated)(struct blk_mq_hw_ctx *); |
32 | |
33 | bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); |
34 | bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); |
35 | int (*request_merge)(struct request_queue *q, struct request **, struct bio *); |
36 | void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); |
37 | void (*requests_merged)(struct request_queue *, struct request *, struct request *); |
38 | void (*limit_depth)(blk_opf_t, struct blk_mq_alloc_data *); |
39 | void (*prepare_request)(struct request *); |
40 | void (*finish_request)(struct request *); |
41 | void (*insert_requests)(struct blk_mq_hw_ctx *hctx, struct list_head *list, |
42 | blk_insert_t flags); |
43 | struct request *(*dispatch_request)(struct blk_mq_hw_ctx *); |
44 | bool (*has_work)(struct blk_mq_hw_ctx *); |
45 | void (*completed_request)(struct request *, u64); |
46 | void (*requeue_request)(struct request *); |
47 | struct request *(*former_request)(struct request_queue *, struct request *); |
48 | struct request *(*next_request)(struct request_queue *, struct request *); |
49 | void (*init_icq)(struct io_cq *); |
50 | void (*exit_icq)(struct io_cq *); |
51 | }; |
52 | |
53 | #define ELV_NAME_MAX (16) |
54 | |
55 | struct elv_fs_entry { |
56 | struct attribute attr; |
57 | ssize_t (*show)(struct elevator_queue *, char *); |
58 | ssize_t (*store)(struct elevator_queue *, const char *, size_t); |
59 | }; |
60 | |
61 | /* |
62 | * identifies an elevator type, such as AS or deadline |
63 | */ |
64 | struct elevator_type |
65 | { |
66 | /* managed by elevator core */ |
67 | struct kmem_cache *icq_cache; |
68 | |
69 | /* fields provided by elevator implementation */ |
70 | struct elevator_mq_ops ops; |
71 | |
72 | size_t icq_size; /* see iocontext.h */ |
73 | size_t icq_align; /* ditto */ |
74 | struct elv_fs_entry *elevator_attrs; |
75 | const char *elevator_name; |
76 | const char *elevator_alias; |
77 | const unsigned int elevator_features; |
78 | struct module *elevator_owner; |
79 | #ifdef CONFIG_BLK_DEBUG_FS |
80 | const struct blk_mq_debugfs_attr *queue_debugfs_attrs; |
81 | const struct blk_mq_debugfs_attr *hctx_debugfs_attrs; |
82 | #endif |
83 | |
84 | /* managed by elevator core */ |
85 | char icq_cache_name[ELV_NAME_MAX + 6]; /* elvname + "_io_cq" */ |
86 | struct list_head list; |
87 | }; |
88 | |
89 | static inline bool elevator_tryget(struct elevator_type *e) |
90 | { |
91 | return try_module_get(module: e->elevator_owner); |
92 | } |
93 | |
94 | static inline void __elevator_get(struct elevator_type *e) |
95 | { |
96 | __module_get(module: e->elevator_owner); |
97 | } |
98 | |
99 | static inline void elevator_put(struct elevator_type *e) |
100 | { |
101 | module_put(module: e->elevator_owner); |
102 | } |
103 | |
104 | #define ELV_HASH_BITS 6 |
105 | |
106 | void elv_rqhash_del(struct request_queue *q, struct request *rq); |
107 | void elv_rqhash_add(struct request_queue *q, struct request *rq); |
108 | void elv_rqhash_reposition(struct request_queue *q, struct request *rq); |
109 | struct request *elv_rqhash_find(struct request_queue *q, sector_t offset); |
110 | |
111 | /* |
112 | * each queue has an elevator_queue associated with it |
113 | */ |
114 | struct elevator_queue |
115 | { |
116 | struct elevator_type *type; |
117 | void *elevator_data; |
118 | struct kobject kobj; |
119 | struct mutex sysfs_lock; |
120 | unsigned long flags; |
121 | DECLARE_HASHTABLE(hash, ELV_HASH_BITS); |
122 | }; |
123 | |
124 | #define ELEVATOR_FLAG_REGISTERED 0 |
125 | #define ELEVATOR_FLAG_DISABLE_WBT 1 |
126 | |
127 | /* |
128 | * block elevator interface |
129 | */ |
130 | extern enum elv_merge elv_merge(struct request_queue *, struct request **, |
131 | struct bio *); |
132 | extern void elv_merge_requests(struct request_queue *, struct request *, |
133 | struct request *); |
134 | extern void elv_merged_request(struct request_queue *, struct request *, |
135 | enum elv_merge); |
136 | extern bool elv_attempt_insert_merge(struct request_queue *, struct request *, |
137 | struct list_head *); |
138 | extern struct request *elv_former_request(struct request_queue *, struct request *); |
139 | extern struct request *elv_latter_request(struct request_queue *, struct request *); |
140 | void elevator_init_mq(struct request_queue *q); |
141 | |
142 | /* |
143 | * io scheduler registration |
144 | */ |
145 | extern int elv_register(struct elevator_type *); |
146 | extern void elv_unregister(struct elevator_type *); |
147 | |
148 | /* |
149 | * io scheduler sysfs switching |
150 | */ |
151 | extern ssize_t elv_iosched_show(struct request_queue *, char *); |
152 | extern ssize_t elv_iosched_store(struct request_queue *, const char *, size_t); |
153 | |
154 | extern bool elv_bio_merge_ok(struct request *, struct bio *); |
155 | extern struct elevator_queue *elevator_alloc(struct request_queue *, |
156 | struct elevator_type *); |
157 | |
158 | /* |
159 | * Helper functions. |
160 | */ |
161 | extern struct request *elv_rb_former_request(struct request_queue *, struct request *); |
162 | extern struct request *elv_rb_latter_request(struct request_queue *, struct request *); |
163 | |
164 | /* |
165 | * rb support functions. |
166 | */ |
167 | extern void elv_rb_add(struct rb_root *, struct request *); |
168 | extern void elv_rb_del(struct rb_root *, struct request *); |
169 | extern struct request *elv_rb_find(struct rb_root *, sector_t); |
170 | |
171 | /* |
172 | * Insertion selection |
173 | */ |
174 | #define ELEVATOR_INSERT_FRONT 1 |
175 | #define ELEVATOR_INSERT_BACK 2 |
176 | #define ELEVATOR_INSERT_SORT 3 |
177 | #define ELEVATOR_INSERT_REQUEUE 4 |
178 | #define ELEVATOR_INSERT_FLUSH 5 |
179 | #define ELEVATOR_INSERT_SORT_MERGE 6 |
180 | |
181 | #define rb_entry_rq(node) rb_entry((node), struct request, rb_node) |
182 | |
183 | #define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist) |
184 | #define rq_fifo_clear(rq) list_del_init(&(rq)->queuelist) |
185 | |
186 | #endif /* _ELEVATOR_H */ |
187 | |