1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_BLKDEV_H
3#define _LINUX_BLKDEV_H
4
5#include <linux/sched.h>
6#include <linux/sched/clock.h>
7
8#ifdef CONFIG_BLOCK
9
10#include <linux/major.h>
11#include <linux/genhd.h>
12#include <linux/list.h>
13#include <linux/llist.h>
14#include <linux/timer.h>
15#include <linux/workqueue.h>
16#include <linux/pagemap.h>
17#include <linux/backing-dev-defs.h>
18#include <linux/wait.h>
19#include <linux/mempool.h>
20#include <linux/pfn.h>
21#include <linux/bio.h>
22#include <linux/stringify.h>
23#include <linux/gfp.h>
24#include <linux/bsg.h>
25#include <linux/smp.h>
26#include <linux/rcupdate.h>
27#include <linux/percpu-refcount.h>
28#include <linux/scatterlist.h>
29#include <linux/blkzoned.h>
30
31struct module;
32struct scsi_ioctl_command;
33
34struct request_queue;
35struct elevator_queue;
36struct blk_trace;
37struct request;
38struct sg_io_hdr;
39struct bsg_job;
40struct blkcg_gq;
41struct blk_flush_queue;
42struct pr_ops;
43struct rq_qos;
44struct blk_queue_stats;
45struct blk_stat_callback;
46
47#define BLKDEV_MIN_RQ 4
48#define BLKDEV_MAX_RQ 128 /* Default maximum */
49
50/* Must be consistent with blk_mq_poll_stats_bkt() */
51#define BLK_MQ_POLL_STATS_BKTS 16
52
53/* Doing classic polling */
54#define BLK_MQ_POLL_CLASSIC -1
55
56/*
57 * Maximum number of blkcg policies allowed to be registered concurrently.
58 * Defined here to simplify include dependency.
59 */
60#define BLKCG_MAX_POLS 5
61
62typedef void (rq_end_io_fn)(struct request *, blk_status_t);
63
64/*
65 * request flags */
66typedef __u32 __bitwise req_flags_t;
67
68/* elevator knows about this request */
69#define RQF_SORTED ((__force req_flags_t)(1 << 0))
70/* drive already may have started this one */
71#define RQF_STARTED ((__force req_flags_t)(1 << 1))
72/* may not be passed by ioscheduler */
73#define RQF_SOFTBARRIER ((__force req_flags_t)(1 << 3))
74/* request for flush sequence */
75#define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << 4))
76/* merge of different types, fail separately */
77#define RQF_MIXED_MERGE ((__force req_flags_t)(1 << 5))
78/* track inflight for MQ */
79#define RQF_MQ_INFLIGHT ((__force req_flags_t)(1 << 6))
80/* don't call prep for this one */
81#define RQF_DONTPREP ((__force req_flags_t)(1 << 7))
82/* set for "ide_preempt" requests and also for requests for which the SCSI
83 "quiesce" state must be ignored. */
84#define RQF_PREEMPT ((__force req_flags_t)(1 << 8))
85/* contains copies of user pages */
86#define RQF_COPY_USER ((__force req_flags_t)(1 << 9))
87/* vaguely specified driver internal error. Ignored by the block layer */
88#define RQF_FAILED ((__force req_flags_t)(1 << 10))
89/* don't warn about errors */
90#define RQF_QUIET ((__force req_flags_t)(1 << 11))
91/* elevator private data attached */
92#define RQF_ELVPRIV ((__force req_flags_t)(1 << 12))
93/* account into disk and partition IO statistics */
94#define RQF_IO_STAT ((__force req_flags_t)(1 << 13))
95/* request came from our alloc pool */
96#define RQF_ALLOCED ((__force req_flags_t)(1 << 14))
97/* runtime pm request */
98#define RQF_PM ((__force req_flags_t)(1 << 15))
99/* on IO scheduler merge hash */
100#define RQF_HASHED ((__force req_flags_t)(1 << 16))
101/* track IO completion time */
102#define RQF_STATS ((__force req_flags_t)(1 << 17))
103/* Look at ->special_vec for the actual data payload instead of the
104 bio chain. */
105#define RQF_SPECIAL_PAYLOAD ((__force req_flags_t)(1 << 18))
106/* The per-zone write lock is held for this request */
107#define RQF_ZONE_WRITE_LOCKED ((__force req_flags_t)(1 << 19))
108/* already slept for hybrid poll */
109#define RQF_MQ_POLL_SLEPT ((__force req_flags_t)(1 << 20))
110/* ->timeout has been called, don't expire again */
111#define RQF_TIMED_OUT ((__force req_flags_t)(1 << 21))
112
113/* flags that prevent us from merging requests: */
114#define RQF_NOMERGE_FLAGS \
115 (RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
116
117/*
118 * Request state for blk-mq.
119 */
120enum mq_rq_state {
121 MQ_RQ_IDLE = 0,
122 MQ_RQ_IN_FLIGHT = 1,
123 MQ_RQ_COMPLETE = 2,
124};
125
126/*
127 * Try to put the fields that are referenced together in the same cacheline.
128 *
129 * If you modify this structure, make sure to update blk_rq_init() and
130 * especially blk_mq_rq_ctx_init() to take care of the added fields.
131 */
132struct request {
133 struct request_queue *q;
134 struct blk_mq_ctx *mq_ctx;
135 struct blk_mq_hw_ctx *mq_hctx;
136
137 unsigned int cmd_flags; /* op and common flags */
138 req_flags_t rq_flags;
139
140 int internal_tag;
141
142 /* the following two fields are internal, NEVER access directly */
143 unsigned int __data_len; /* total data len */
144 int tag;
145 sector_t __sector; /* sector cursor */
146
147 struct bio *bio;
148 struct bio *biotail;
149
150 struct list_head queuelist;
151
152 /*
153 * The hash is used inside the scheduler, and killed once the
154 * request reaches the dispatch list. The ipi_list is only used
155 * to queue the request for softirq completion, which is long
156 * after the request has been unhashed (and even removed from
157 * the dispatch list).
158 */
159 union {
160 struct hlist_node hash; /* merge hash */
161 struct list_head ipi_list;
162 };
163
164 /*
165 * The rb_node is only used inside the io scheduler, requests
166 * are pruned when moved to the dispatch queue. So let the
167 * completion_data share space with the rb_node.
168 */
169 union {
170 struct rb_node rb_node; /* sort/lookup */
171 struct bio_vec special_vec;
172 void *completion_data;
173 int error_count; /* for legacy drivers, don't use */
174 };
175
176 /*
177 * Three pointers are available for the IO schedulers, if they need
178 * more they have to dynamically allocate it. Flush requests are
179 * never put on the IO scheduler. So let the flush fields share
180 * space with the elevator data.
181 */
182 union {
183 struct {
184 struct io_cq *icq;
185 void *priv[2];
186 } elv;
187
188 struct {
189 unsigned int seq;
190 struct list_head list;
191 rq_end_io_fn *saved_end_io;
192 } flush;
193 };
194
195 struct gendisk *rq_disk;
196 struct hd_struct *part;
197 /* Time that I/O was submitted to the kernel. */
198 u64 start_time_ns;
199 /* Time that I/O was submitted to the device. */
200 u64 io_start_time_ns;
201
202#ifdef CONFIG_BLK_WBT
203 unsigned short wbt_flags;
204#endif
205#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
206 unsigned short throtl_size;
207#endif
208
209 /*
210 * Number of scatter-gather DMA addr+len pairs after
211 * physical address coalescing is performed.
212 */
213 unsigned short nr_phys_segments;
214
215#if defined(CONFIG_BLK_DEV_INTEGRITY)
216 unsigned short nr_integrity_segments;
217#endif
218
219 unsigned short write_hint;
220 unsigned short ioprio;
221
222 unsigned int extra_len; /* length of alignment and padding */
223
224 enum mq_rq_state state;
225 refcount_t ref;
226
227 unsigned int timeout;
228 unsigned long deadline;
229
230 union {
231 struct __call_single_data csd;
232 u64 fifo_time;
233 };
234
235 /*
236 * completion callback.
237 */
238 rq_end_io_fn *end_io;
239 void *end_io_data;
240};
241
242static inline bool blk_op_is_scsi(unsigned int op)
243{
244 return op == REQ_OP_SCSI_IN || op == REQ_OP_SCSI_OUT;
245}
246
247static inline bool blk_op_is_private(unsigned int op)
248{
249 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
250}
251
252static inline bool blk_rq_is_scsi(struct request *rq)
253{
254 return blk_op_is_scsi(req_op(rq));
255}
256
257static inline bool blk_rq_is_private(struct request *rq)
258{
259 return blk_op_is_private(req_op(rq));
260}
261
262static inline bool blk_rq_is_passthrough(struct request *rq)
263{
264 return blk_rq_is_scsi(rq) || blk_rq_is_private(rq);
265}
266
267static inline bool bio_is_passthrough(struct bio *bio)
268{
269 unsigned op = bio_op(bio);
270
271 return blk_op_is_scsi(op) || blk_op_is_private(op);
272}
273
274static inline unsigned short req_get_ioprio(struct request *req)
275{
276 return req->ioprio;
277}
278
279#include <linux/elevator.h>
280
281struct blk_queue_ctx;
282
283typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
284
285struct bio_vec;
286typedef int (dma_drain_needed_fn)(struct request *);
287
288enum blk_eh_timer_return {
289 BLK_EH_DONE, /* drivers has completed the command */
290 BLK_EH_RESET_TIMER, /* reset timer and try again */
291};
292
293enum blk_queue_state {
294 Queue_down,
295 Queue_up,
296};
297
298#define BLK_TAG_ALLOC_FIFO 0 /* allocate starting from 0 */
299#define BLK_TAG_ALLOC_RR 1 /* allocate starting from last allocated tag */
300
301#define BLK_SCSI_MAX_CMDS (256)
302#define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
303
304/*
305 * Zoned block device models (zoned limit).
306 */
307enum blk_zoned_model {
308 BLK_ZONED_NONE, /* Regular block device */
309 BLK_ZONED_HA, /* Host-aware zoned block device */
310 BLK_ZONED_HM, /* Host-managed zoned block device */
311};
312
313struct queue_limits {
314 unsigned long bounce_pfn;
315 unsigned long seg_boundary_mask;
316 unsigned long virt_boundary_mask;
317
318 unsigned int max_hw_sectors;
319 unsigned int max_dev_sectors;
320 unsigned int chunk_sectors;
321 unsigned int max_sectors;
322 unsigned int max_segment_size;
323 unsigned int physical_block_size;
324 unsigned int alignment_offset;
325 unsigned int io_min;
326 unsigned int io_opt;
327 unsigned int max_discard_sectors;
328 unsigned int max_hw_discard_sectors;
329 unsigned int max_write_same_sectors;
330 unsigned int max_write_zeroes_sectors;
331 unsigned int discard_granularity;
332 unsigned int discard_alignment;
333
334 unsigned short logical_block_size;
335 unsigned short max_segments;
336 unsigned short max_integrity_segments;
337 unsigned short max_discard_segments;
338
339 unsigned char misaligned;
340 unsigned char discard_misaligned;
341 unsigned char raid_partial_stripes_expensive;
342 enum blk_zoned_model zoned;
343};
344
345#ifdef CONFIG_BLK_DEV_ZONED
346
347extern unsigned int blkdev_nr_zones(struct block_device *bdev);
348extern int blkdev_report_zones(struct block_device *bdev,
349 sector_t sector, struct blk_zone *zones,
350 unsigned int *nr_zones, gfp_t gfp_mask);
351extern int blkdev_reset_zones(struct block_device *bdev, sector_t sectors,
352 sector_t nr_sectors, gfp_t gfp_mask);
353extern int blk_revalidate_disk_zones(struct gendisk *disk);
354
355extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
356 unsigned int cmd, unsigned long arg);
357extern int blkdev_reset_zones_ioctl(struct block_device *bdev, fmode_t mode,
358 unsigned int cmd, unsigned long arg);
359
360#else /* CONFIG_BLK_DEV_ZONED */
361
362static inline unsigned int blkdev_nr_zones(struct block_device *bdev)
363{
364 return 0;
365}
366
367static inline int blk_revalidate_disk_zones(struct gendisk *disk)
368{
369 return 0;
370}
371
372static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
373 fmode_t mode, unsigned int cmd,
374 unsigned long arg)
375{
376 return -ENOTTY;
377}
378
379static inline int blkdev_reset_zones_ioctl(struct block_device *bdev,
380 fmode_t mode, unsigned int cmd,
381 unsigned long arg)
382{
383 return -ENOTTY;
384}
385
386#endif /* CONFIG_BLK_DEV_ZONED */
387
388struct request_queue {
389 /*
390 * Together with queue_head for cacheline sharing
391 */
392 struct list_head queue_head;
393 struct request *last_merge;
394 struct elevator_queue *elevator;
395
396 struct blk_queue_stats *stats;
397 struct rq_qos *rq_qos;
398
399 make_request_fn *make_request_fn;
400 dma_drain_needed_fn *dma_drain_needed;
401
402 const struct blk_mq_ops *mq_ops;
403
404 /* sw queues */
405 struct blk_mq_ctx __percpu *queue_ctx;
406 unsigned int nr_queues;
407
408 unsigned int queue_depth;
409
410 /* hw dispatch queues */
411 struct blk_mq_hw_ctx **queue_hw_ctx;
412 unsigned int nr_hw_queues;
413
414 struct backing_dev_info *backing_dev_info;
415
416 /*
417 * The queue owner gets to use this for whatever they like.
418 * ll_rw_blk doesn't touch it.
419 */
420 void *queuedata;
421
422 /*
423 * various queue flags, see QUEUE_* below
424 */
425 unsigned long queue_flags;
426 /*
427 * Number of contexts that have called blk_set_pm_only(). If this
428 * counter is above zero then only RQF_PM and RQF_PREEMPT requests are
429 * processed.
430 */
431 atomic_t pm_only;
432
433 /*
434 * ida allocated id for this queue. Used to index queues from
435 * ioctx.
436 */
437 int id;
438
439 /*
440 * queue needs bounce pages for pages above this limit
441 */
442 gfp_t bounce_gfp;
443
444 spinlock_t queue_lock;
445
446 /*
447 * queue kobject
448 */
449 struct kobject kobj;
450
451 /*
452 * mq queue kobject
453 */
454 struct kobject *mq_kobj;
455
456#ifdef CONFIG_BLK_DEV_INTEGRITY
457 struct blk_integrity integrity;
458#endif /* CONFIG_BLK_DEV_INTEGRITY */
459
460#ifdef CONFIG_PM
461 struct device *dev;
462 int rpm_status;
463 unsigned int nr_pending;
464#endif
465
466 /*
467 * queue settings
468 */
469 unsigned long nr_requests; /* Max # of requests */
470
471 unsigned int dma_drain_size;
472 void *dma_drain_buffer;
473 unsigned int dma_pad_mask;
474 unsigned int dma_alignment;
475
476 unsigned int rq_timeout;
477 int poll_nsec;
478
479 struct blk_stat_callback *poll_cb;
480 struct blk_rq_stat poll_stat[BLK_MQ_POLL_STATS_BKTS];
481
482 struct timer_list timeout;
483 struct work_struct timeout_work;
484
485 struct list_head icq_list;
486#ifdef CONFIG_BLK_CGROUP
487 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
488 struct blkcg_gq *root_blkg;
489 struct list_head blkg_list;
490#endif
491
492 struct queue_limits limits;
493
494#ifdef CONFIG_BLK_DEV_ZONED
495 /*
496 * Zoned block device information for request dispatch control.
497 * nr_zones is the total number of zones of the device. This is always
498 * 0 for regular block devices. seq_zones_bitmap is a bitmap of nr_zones
499 * bits which indicates if a zone is conventional (bit clear) or
500 * sequential (bit set). seq_zones_wlock is a bitmap of nr_zones
501 * bits which indicates if a zone is write locked, that is, if a write
502 * request targeting the zone was dispatched. All three fields are
503 * initialized by the low level device driver (e.g. scsi/sd.c).
504 * Stacking drivers (device mappers) may or may not initialize
505 * these fields.
506 *
507 * Reads of this information must be protected with blk_queue_enter() /
508 * blk_queue_exit(). Modifying this information is only allowed while
509 * no requests are being processed. See also blk_mq_freeze_queue() and
510 * blk_mq_unfreeze_queue().
511 */
512 unsigned int nr_zones;
513 unsigned long *seq_zones_bitmap;
514 unsigned long *seq_zones_wlock;
515#endif /* CONFIG_BLK_DEV_ZONED */
516
517 /*
518 * sg stuff
519 */
520 unsigned int sg_timeout;
521 unsigned int sg_reserved_size;
522 int node;
523#ifdef CONFIG_BLK_DEV_IO_TRACE
524 struct blk_trace *blk_trace;
525 struct mutex blk_trace_mutex;
526#endif
527 /*
528 * for flush operations
529 */
530 struct blk_flush_queue *fq;
531
532 struct list_head requeue_list;
533 spinlock_t requeue_lock;
534 struct delayed_work requeue_work;
535
536 struct mutex sysfs_lock;
537
538 atomic_t mq_freeze_depth;
539
540#if defined(CONFIG_BLK_DEV_BSG)
541 struct bsg_class_device bsg_dev;
542#endif
543
544#ifdef CONFIG_BLK_DEV_THROTTLING
545 /* Throttle data */
546 struct throtl_data *td;
547#endif
548 struct rcu_head rcu_head;
549 wait_queue_head_t mq_freeze_wq;
550 struct percpu_ref q_usage_counter;
551 struct list_head all_q_node;
552
553 struct blk_mq_tag_set *tag_set;
554 struct list_head tag_set_list;
555 struct bio_set bio_split;
556
557#ifdef CONFIG_BLK_DEBUG_FS
558 struct dentry *debugfs_dir;
559 struct dentry *sched_debugfs_dir;
560 struct dentry *rqos_debugfs_dir;
561#endif
562
563 bool mq_sysfs_init_done;
564
565 size_t cmd_size;
566
567 struct work_struct release_work;
568
569#define BLK_MAX_WRITE_HINTS 5
570 u64 write_hints[BLK_MAX_WRITE_HINTS];
571};
572
573#define QUEUE_FLAG_STOPPED 0 /* queue is stopped */
574#define QUEUE_FLAG_DYING 1 /* queue being torn down */
575#define QUEUE_FLAG_NOMERGES 3 /* disable merge attempts */
576#define QUEUE_FLAG_SAME_COMP 4 /* complete on same CPU-group */
577#define QUEUE_FLAG_FAIL_IO 5 /* fake timeout */
578#define QUEUE_FLAG_NONROT 6 /* non-rotational device (SSD) */
579#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
580#define QUEUE_FLAG_IO_STAT 7 /* do disk/partitions IO accounting */
581#define QUEUE_FLAG_DISCARD 8 /* supports DISCARD */
582#define QUEUE_FLAG_NOXMERGES 9 /* No extended merges */
583#define QUEUE_FLAG_ADD_RANDOM 10 /* Contributes to random pool */
584#define QUEUE_FLAG_SECERASE 11 /* supports secure erase */
585#define QUEUE_FLAG_SAME_FORCE 12 /* force complete on same CPU */
586#define QUEUE_FLAG_DEAD 13 /* queue tear-down finished */
587#define QUEUE_FLAG_INIT_DONE 14 /* queue is initialized */
588#define QUEUE_FLAG_POLL 16 /* IO polling enabled if set */
589#define QUEUE_FLAG_WC 17 /* Write back caching */
590#define QUEUE_FLAG_FUA 18 /* device supports FUA writes */
591#define QUEUE_FLAG_DAX 19 /* device supports DAX */
592#define QUEUE_FLAG_STATS 20 /* track IO start and completion times */
593#define QUEUE_FLAG_POLL_STATS 21 /* collecting stats for hybrid polling */
594#define QUEUE_FLAG_REGISTERED 22 /* queue has been registered to a disk */
595#define QUEUE_FLAG_SCSI_PASSTHROUGH 23 /* queue supports SCSI commands */
596#define QUEUE_FLAG_QUIESCED 24 /* queue has been quiesced */
597#define QUEUE_FLAG_PCI_P2PDMA 25 /* device supports PCI p2p requests */
598
599#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
600 (1 << QUEUE_FLAG_SAME_COMP))
601
602void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
603void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
604bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
605
606#define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
607#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
608#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
609#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
610#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
611#define blk_queue_noxmerges(q) \
612 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
613#define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
614#define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
615#define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
616#define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
617#define blk_queue_secure_erase(q) \
618 (test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
619#define blk_queue_dax(q) test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
620#define blk_queue_scsi_passthrough(q) \
621 test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
622#define blk_queue_pci_p2pdma(q) \
623 test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
624
625#define blk_noretry_request(rq) \
626 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
627 REQ_FAILFAST_DRIVER))
628#define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
629#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
630#define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
631
632extern void blk_set_pm_only(struct request_queue *q);
633extern void blk_clear_pm_only(struct request_queue *q);
634
635static inline bool blk_account_rq(struct request *rq)
636{
637 return (rq->rq_flags & RQF_STARTED) && !blk_rq_is_passthrough(rq);
638}
639
640#define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
641
642#define rq_data_dir(rq) (op_is_write(req_op(rq)) ? WRITE : READ)
643
644static inline bool queue_is_mq(struct request_queue *q)
645{
646 return q->mq_ops;
647}
648
649static inline enum blk_zoned_model
650blk_queue_zoned_model(struct request_queue *q)
651{
652 return q->limits.zoned;
653}
654
655static inline bool blk_queue_is_zoned(struct request_queue *q)
656{
657 switch (blk_queue_zoned_model(q)) {
658 case BLK_ZONED_HA:
659 case BLK_ZONED_HM:
660 return true;
661 default:
662 return false;
663 }
664}
665
666static inline unsigned int blk_queue_zone_sectors(struct request_queue *q)
667{
668 return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
669}
670
671#ifdef CONFIG_BLK_DEV_ZONED
672static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
673{
674 return blk_queue_is_zoned(q) ? q->nr_zones : 0;
675}
676
677static inline unsigned int blk_queue_zone_no(struct request_queue *q,
678 sector_t sector)
679{
680 if (!blk_queue_is_zoned(q))
681 return 0;
682 return sector >> ilog2(q->limits.chunk_sectors);
683}
684
685static inline bool blk_queue_zone_is_seq(struct request_queue *q,
686 sector_t sector)
687{
688 if (!blk_queue_is_zoned(q) || !q->seq_zones_bitmap)
689 return false;
690 return test_bit(blk_queue_zone_no(q, sector), q->seq_zones_bitmap);
691}
692#else /* CONFIG_BLK_DEV_ZONED */
693static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
694{
695 return 0;
696}
697#endif /* CONFIG_BLK_DEV_ZONED */
698
699static inline bool rq_is_sync(struct request *rq)
700{
701 return op_is_sync(rq->cmd_flags);
702}
703
704static inline bool rq_mergeable(struct request *rq)
705{
706 if (blk_rq_is_passthrough(rq))
707 return false;
708
709 if (req_op(rq) == REQ_OP_FLUSH)
710 return false;
711
712 if (req_op(rq) == REQ_OP_WRITE_ZEROES)
713 return false;
714
715 if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
716 return false;
717 if (rq->rq_flags & RQF_NOMERGE_FLAGS)
718 return false;
719
720 return true;
721}
722
723static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
724{
725 if (bio_page(a) == bio_page(b) &&
726 bio_offset(a) == bio_offset(b))
727 return true;
728
729 return false;
730}
731
732static inline unsigned int blk_queue_depth(struct request_queue *q)
733{
734 if (q->queue_depth)
735 return q->queue_depth;
736
737 return q->nr_requests;
738}
739
740extern unsigned long blk_max_low_pfn, blk_max_pfn;
741
742/*
743 * standard bounce addresses:
744 *
745 * BLK_BOUNCE_HIGH : bounce all highmem pages
746 * BLK_BOUNCE_ANY : don't bounce anything
747 * BLK_BOUNCE_ISA : bounce pages above ISA DMA boundary
748 */
749
750#if BITS_PER_LONG == 32
751#define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT)
752#else
753#define BLK_BOUNCE_HIGH -1ULL
754#endif
755#define BLK_BOUNCE_ANY (-1ULL)
756#define BLK_BOUNCE_ISA (DMA_BIT_MASK(24))
757
758/*
759 * default timeout for SG_IO if none specified
760 */
761#define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
762#define BLK_MIN_SG_TIMEOUT (7 * HZ)
763
764struct rq_map_data {
765 struct page **pages;
766 int page_order;
767 int nr_entries;
768 unsigned long offset;
769 int null_mapped;
770 int from_user;
771};
772
773struct req_iterator {
774 struct bvec_iter iter;
775 struct bio *bio;
776};
777
778/* This should not be used directly - use rq_for_each_segment */
779#define for_each_bio(_bio) \
780 for (; _bio; _bio = _bio->bi_next)
781#define __rq_for_each_bio(_bio, rq) \
782 if ((rq->bio)) \
783 for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
784
785#define rq_for_each_segment(bvl, _rq, _iter) \
786 __rq_for_each_bio(_iter.bio, _rq) \
787 bio_for_each_segment(bvl, _iter.bio, _iter.iter)
788
789#define rq_for_each_bvec(bvl, _rq, _iter) \
790 __rq_for_each_bio(_iter.bio, _rq) \
791 bio_for_each_bvec(bvl, _iter.bio, _iter.iter)
792
793#define rq_iter_last(bvec, _iter) \
794 (_iter.bio->bi_next == NULL && \
795 bio_iter_last(bvec, _iter.iter))
796
797#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
798# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
799#endif
800#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
801extern void rq_flush_dcache_pages(struct request *rq);
802#else
803static inline void rq_flush_dcache_pages(struct request *rq)
804{
805}
806#endif
807
808extern int blk_register_queue(struct gendisk *disk);
809extern void blk_unregister_queue(struct gendisk *disk);
810extern blk_qc_t generic_make_request(struct bio *bio);
811extern blk_qc_t direct_make_request(struct bio *bio);
812extern void blk_rq_init(struct request_queue *q, struct request *rq);
813extern void blk_init_request_from_bio(struct request *req, struct bio *bio);
814extern void blk_put_request(struct request *);
815extern struct request *blk_get_request(struct request_queue *, unsigned int op,
816 blk_mq_req_flags_t flags);
817extern int blk_lld_busy(struct request_queue *q);
818extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
819 struct bio_set *bs, gfp_t gfp_mask,
820 int (*bio_ctr)(struct bio *, struct bio *, void *),
821 void *data);
822extern void blk_rq_unprep_clone(struct request *rq);
823extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
824 struct request *rq);
825extern int blk_rq_append_bio(struct request *rq, struct bio **bio);
826extern void blk_queue_split(struct request_queue *, struct bio **);
827extern void blk_recount_segments(struct request_queue *, struct bio *);
828extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
829extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
830 unsigned int, void __user *);
831extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
832 unsigned int, void __user *);
833extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
834 struct scsi_ioctl_command __user *);
835
836extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
837extern void blk_queue_exit(struct request_queue *q);
838extern void blk_sync_queue(struct request_queue *q);
839extern int blk_rq_map_user(struct request_queue *, struct request *,
840 struct rq_map_data *, void __user *, unsigned long,
841 gfp_t);
842extern int blk_rq_unmap_user(struct bio *);
843extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
844extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
845 struct rq_map_data *, const struct iov_iter *,
846 gfp_t);
847extern void blk_execute_rq(struct request_queue *, struct gendisk *,
848 struct request *, int);
849extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
850 struct request *, int, rq_end_io_fn *);
851
852int blk_status_to_errno(blk_status_t status);
853blk_status_t errno_to_blk_status(int errno);
854
855int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin);
856
857static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
858{
859 return bdev->bd_disk->queue; /* this is never NULL */
860}
861
862/*
863 * The basic unit of block I/O is a sector. It is used in a number of contexts
864 * in Linux (blk, bio, genhd). The size of one sector is 512 = 2**9
865 * bytes. Variables of type sector_t represent an offset or size that is a
866 * multiple of 512 bytes. Hence these two constants.
867 */
868#ifndef SECTOR_SHIFT
869#define SECTOR_SHIFT 9
870#endif
871#ifndef SECTOR_SIZE
872#define SECTOR_SIZE (1 << SECTOR_SHIFT)
873#endif
874
875/*
876 * blk_rq_pos() : the current sector
877 * blk_rq_bytes() : bytes left in the entire request
878 * blk_rq_cur_bytes() : bytes left in the current segment
879 * blk_rq_err_bytes() : bytes left till the next error boundary
880 * blk_rq_sectors() : sectors left in the entire request
881 * blk_rq_cur_sectors() : sectors left in the current segment
882 */
883static inline sector_t blk_rq_pos(const struct request *rq)
884{
885 return rq->__sector;
886}
887
888static inline unsigned int blk_rq_bytes(const struct request *rq)
889{
890 return rq->__data_len;
891}
892
893static inline int blk_rq_cur_bytes(const struct request *rq)
894{
895 return rq->bio ? bio_cur_bytes(rq->bio) : 0;
896}
897
898extern unsigned int blk_rq_err_bytes(const struct request *rq);
899
900static inline unsigned int blk_rq_sectors(const struct request *rq)
901{
902 return blk_rq_bytes(rq) >> SECTOR_SHIFT;
903}
904
905static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
906{
907 return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
908}
909
910#ifdef CONFIG_BLK_DEV_ZONED
911static inline unsigned int blk_rq_zone_no(struct request *rq)
912{
913 return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
914}
915
916static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
917{
918 return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq));
919}
920#endif /* CONFIG_BLK_DEV_ZONED */
921
922/*
923 * Some commands like WRITE SAME have a payload or data transfer size which
924 * is different from the size of the request. Any driver that supports such
925 * commands using the RQF_SPECIAL_PAYLOAD flag needs to use this helper to
926 * calculate the data transfer size.
927 */
928static inline unsigned int blk_rq_payload_bytes(struct request *rq)
929{
930 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
931 return rq->special_vec.bv_len;
932 return blk_rq_bytes(rq);
933}
934
935static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
936 int op)
937{
938 if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
939 return min(q->limits.max_discard_sectors,
940 UINT_MAX >> SECTOR_SHIFT);
941
942 if (unlikely(op == REQ_OP_WRITE_SAME))
943 return q->limits.max_write_same_sectors;
944
945 if (unlikely(op == REQ_OP_WRITE_ZEROES))
946 return q->limits.max_write_zeroes_sectors;
947
948 return q->limits.max_sectors;
949}
950
951/*
952 * Return maximum size of a request at given offset. Only valid for
953 * file system requests.
954 */
955static inline unsigned int blk_max_size_offset(struct request_queue *q,
956 sector_t offset)
957{
958 if (!q->limits.chunk_sectors)
959 return q->limits.max_sectors;
960
961 return min(q->limits.max_sectors, (unsigned int)(q->limits.chunk_sectors -
962 (offset & (q->limits.chunk_sectors - 1))));
963}
964
965static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
966 sector_t offset)
967{
968 struct request_queue *q = rq->q;
969
970 if (blk_rq_is_passthrough(rq))
971 return q->limits.max_hw_sectors;
972
973 if (!q->limits.chunk_sectors ||
974 req_op(rq) == REQ_OP_DISCARD ||
975 req_op(rq) == REQ_OP_SECURE_ERASE)
976 return blk_queue_get_max_sectors(q, req_op(rq));
977
978 return min(blk_max_size_offset(q, offset),
979 blk_queue_get_max_sectors(q, req_op(rq)));
980}
981
982static inline unsigned int blk_rq_count_bios(struct request *rq)
983{
984 unsigned int nr_bios = 0;
985 struct bio *bio;
986
987 __rq_for_each_bio(bio, rq)
988 nr_bios++;
989
990 return nr_bios;
991}
992
993void blk_steal_bios(struct bio_list *list, struct request *rq);
994
995/*
996 * Request completion related functions.
997 *
998 * blk_update_request() completes given number of bytes and updates
999 * the request without completing it.
1000 *
1001 * blk_end_request() and friends. __blk_end_request() must be called
1002 * with the request queue spinlock acquired.
1003 *
1004 * Several drivers define their own end_request and call
1005 * blk_end_request() for parts of the original function.
1006 * This prevents code duplication in drivers.
1007 */
1008extern bool blk_update_request(struct request *rq, blk_status_t error,
1009 unsigned int nr_bytes);
1010extern void blk_end_request_all(struct request *rq, blk_status_t error);
1011extern bool __blk_end_request(struct request *rq, blk_status_t error,
1012 unsigned int nr_bytes);
1013extern void __blk_end_request_all(struct request *rq, blk_status_t error);
1014extern bool __blk_end_request_cur(struct request *rq, blk_status_t error);
1015
1016extern void __blk_complete_request(struct request *);
1017extern void blk_abort_request(struct request *);
1018
1019/*
1020 * Access functions for manipulating queue properties
1021 */
1022extern void blk_cleanup_queue(struct request_queue *);
1023extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
1024extern void blk_queue_bounce_limit(struct request_queue *, u64);
1025extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
1026extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
1027extern void blk_queue_max_segments(struct request_queue *, unsigned short);
1028extern void blk_queue_max_discard_segments(struct request_queue *,
1029 unsigned short);
1030extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
1031extern void blk_queue_max_discard_sectors(struct request_queue *q,
1032 unsigned int max_discard_sectors);
1033extern void blk_queue_max_write_same_sectors(struct request_queue *q,
1034 unsigned int max_write_same_sectors);
1035extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
1036 unsigned int max_write_same_sectors);
1037extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
1038extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1039extern void blk_queue_alignment_offset(struct request_queue *q,
1040 unsigned int alignment);
1041extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
1042extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
1043extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
1044extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
1045extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1046extern void blk_set_default_limits(struct queue_limits *lim);
1047extern void blk_set_stacking_limits(struct queue_limits *lim);
1048extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
1049 sector_t offset);
1050extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
1051 sector_t offset);
1052extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
1053 sector_t offset);
1054extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
1055extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
1056extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
1057extern int blk_queue_dma_drain(struct request_queue *q,
1058 dma_drain_needed_fn *dma_drain_needed,
1059 void *buf, unsigned int size);
1060extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1061extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1062extern void blk_queue_dma_alignment(struct request_queue *, int);
1063extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1064extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1065extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
1066
1067/*
1068 * Number of physical segments as sent to the device.
1069 *
1070 * Normally this is the number of discontiguous data segments sent by the
1071 * submitter. But for data-less command like discard we might have no
1072 * actual data segments submitted, but the driver might have to add it's
1073 * own special payload. In that case we still return 1 here so that this
1074 * special payload will be mapped.
1075 */
1076static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
1077{
1078 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1079 return 1;
1080 return rq->nr_phys_segments;
1081}
1082
1083/*
1084 * Number of discard segments (or ranges) the driver needs to fill in.
1085 * Each discard bio merged into a request is counted as one segment.
1086 */
1087static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
1088{
1089 return max_t(unsigned short, rq->nr_phys_segments, 1);
1090}
1091
1092extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
1093extern void blk_dump_rq_flags(struct request *, char *);
1094extern long nr_blockdev_pages(void);
1095
1096bool __must_check blk_get_queue(struct request_queue *);
1097struct request_queue *blk_alloc_queue(gfp_t);
1098struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id);
1099extern void blk_put_queue(struct request_queue *);
1100extern void blk_set_queue_dying(struct request_queue *);
1101
1102/*
1103 * blk_plug permits building a queue of related requests by holding the I/O
1104 * fragments for a short period. This allows merging of sequential requests
1105 * into single larger request. As the requests are moved from a per-task list to
1106 * the device's request_queue in a batch, this results in improved scalability
1107 * as the lock contention for request_queue lock is reduced.
1108 *
1109 * It is ok not to disable preemption when adding the request to the plug list
1110 * or when attempting a merge, because blk_schedule_flush_list() will only flush
1111 * the plug list when the task sleeps by itself. For details, please see
1112 * schedule() where blk_schedule_flush_plug() is called.
1113 */
1114struct blk_plug {
1115 struct list_head mq_list; /* blk-mq requests */
1116 struct list_head cb_list; /* md requires an unplug callback */
1117 unsigned short rq_count;
1118 bool multiple_queues;
1119};
1120#define BLK_MAX_REQUEST_COUNT 16
1121#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
1122
1123struct blk_plug_cb;
1124typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1125struct blk_plug_cb {
1126 struct list_head list;
1127 blk_plug_cb_fn callback;
1128 void *data;
1129};
1130extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
1131 void *data, int size);
1132extern void blk_start_plug(struct blk_plug *);
1133extern void blk_finish_plug(struct blk_plug *);
1134extern void blk_flush_plug_list(struct blk_plug *, bool);
1135
1136static inline void blk_flush_plug(struct task_struct *tsk)
1137{
1138 struct blk_plug *plug = tsk->plug;
1139
1140 if (plug)
1141 blk_flush_plug_list(plug, false);
1142}
1143
1144static inline void blk_schedule_flush_plug(struct task_struct *tsk)
1145{
1146 struct blk_plug *plug = tsk->plug;
1147
1148 if (plug)
1149 blk_flush_plug_list(plug, true);
1150}
1151
1152static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1153{
1154 struct blk_plug *plug = tsk->plug;
1155
1156 return plug &&
1157 (!list_empty(&plug->mq_list) ||
1158 !list_empty(&plug->cb_list));
1159}
1160
1161extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
1162extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
1163 sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1164
1165#define BLKDEV_DISCARD_SECURE (1 << 0) /* issue a secure erase */
1166
1167extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1168 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1169extern int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1170 sector_t nr_sects, gfp_t gfp_mask, int flags,
1171 struct bio **biop);
1172
1173#define BLKDEV_ZERO_NOUNMAP (1 << 0) /* do not free blocks */
1174#define BLKDEV_ZERO_NOFALLBACK (1 << 1) /* don't write explicit zeroes */
1175
1176extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1177 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1178 unsigned flags);
1179extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1180 sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1181
1182static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1183 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1184{
1185 return blkdev_issue_discard(sb->s_bdev,
1186 block << (sb->s_blocksize_bits -
1187 SECTOR_SHIFT),
1188 nr_blocks << (sb->s_blocksize_bits -
1189 SECTOR_SHIFT),
1190 gfp_mask, flags);
1191}
1192static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1193 sector_t nr_blocks, gfp_t gfp_mask)
1194{
1195 return blkdev_issue_zeroout(sb->s_bdev,
1196 block << (sb->s_blocksize_bits -
1197 SECTOR_SHIFT),
1198 nr_blocks << (sb->s_blocksize_bits -
1199 SECTOR_SHIFT),
1200 gfp_mask, 0);
1201}
1202
1203extern int blk_verify_command(unsigned char *cmd, fmode_t mode);
1204
1205enum blk_default_limits {
1206 BLK_MAX_SEGMENTS = 128,
1207 BLK_SAFE_MAX_SECTORS = 255,
1208 BLK_DEF_MAX_SECTORS = 2560,
1209 BLK_MAX_SEGMENT_SIZE = 65536,
1210 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1211};
1212
1213static inline unsigned long queue_segment_boundary(struct request_queue *q)
1214{
1215 return q->limits.seg_boundary_mask;
1216}
1217
1218static inline unsigned long queue_virt_boundary(struct request_queue *q)
1219{
1220 return q->limits.virt_boundary_mask;
1221}
1222
1223static inline unsigned int queue_max_sectors(struct request_queue *q)
1224{
1225 return q->limits.max_sectors;
1226}
1227
1228static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
1229{
1230 return q->limits.max_hw_sectors;
1231}
1232
1233static inline unsigned short queue_max_segments(struct request_queue *q)
1234{
1235 return q->limits.max_segments;
1236}
1237
1238static inline unsigned short queue_max_discard_segments(struct request_queue *q)
1239{
1240 return q->limits.max_discard_segments;
1241}
1242
1243static inline unsigned int queue_max_segment_size(struct request_queue *q)
1244{
1245 return q->limits.max_segment_size;
1246}
1247
1248static inline unsigned short queue_logical_block_size(struct request_queue *q)
1249{
1250 int retval = 512;
1251
1252 if (q && q->limits.logical_block_size)
1253 retval = q->limits.logical_block_size;
1254
1255 return retval;
1256}
1257
1258static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
1259{
1260 return queue_logical_block_size(bdev_get_queue(bdev));
1261}
1262
1263static inline unsigned int queue_physical_block_size(struct request_queue *q)
1264{
1265 return q->limits.physical_block_size;
1266}
1267
1268static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1269{
1270 return queue_physical_block_size(bdev_get_queue(bdev));
1271}
1272
1273static inline unsigned int queue_io_min(struct request_queue *q)
1274{
1275 return q->limits.io_min;
1276}
1277
1278static inline int bdev_io_min(struct block_device *bdev)
1279{
1280 return queue_io_min(bdev_get_queue(bdev));
1281}
1282
1283static inline unsigned int queue_io_opt(struct request_queue *q)
1284{
1285 return q->limits.io_opt;
1286}
1287
1288static inline int bdev_io_opt(struct block_device *bdev)
1289{
1290 return queue_io_opt(bdev_get_queue(bdev));
1291}
1292
1293static inline int queue_alignment_offset(struct request_queue *q)
1294{
1295 if (q->limits.misaligned)
1296 return -1;
1297
1298 return q->limits.alignment_offset;
1299}
1300
1301static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1302{
1303 unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1304 unsigned int alignment = sector_div(sector, granularity >> SECTOR_SHIFT)
1305 << SECTOR_SHIFT;
1306
1307 return (granularity + lim->alignment_offset - alignment) % granularity;
1308}
1309
1310static inline int bdev_alignment_offset(struct block_device *bdev)
1311{
1312 struct request_queue *q = bdev_get_queue(bdev);
1313
1314 if (q->limits.misaligned)
1315 return -1;
1316
1317 if (bdev != bdev->bd_contains)
1318 return bdev->bd_part->alignment_offset;
1319
1320 return q->limits.alignment_offset;
1321}
1322
1323static inline int queue_discard_alignment(struct request_queue *q)
1324{
1325 if (q->limits.discard_misaligned)
1326 return -1;
1327
1328 return q->limits.discard_alignment;
1329}
1330
1331static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1332{
1333 unsigned int alignment, granularity, offset;
1334
1335 if (!lim->max_discard_sectors)
1336 return 0;
1337
1338 /* Why are these in bytes, not sectors? */
1339 alignment = lim->discard_alignment >> SECTOR_SHIFT;
1340 granularity = lim->discard_granularity >> SECTOR_SHIFT;
1341 if (!granularity)
1342 return 0;
1343
1344 /* Offset of the partition start in 'granularity' sectors */
1345 offset = sector_div(sector, granularity);
1346
1347 /* And why do we do this modulus *again* in blkdev_issue_discard()? */
1348 offset = (granularity + alignment - offset) % granularity;
1349
1350 /* Turn it back into bytes, gaah */
1351 return offset << SECTOR_SHIFT;
1352}
1353
1354static inline int bdev_discard_alignment(struct block_device *bdev)
1355{
1356 struct request_queue *q = bdev_get_queue(bdev);
1357
1358 if (bdev != bdev->bd_contains)
1359 return bdev->bd_part->discard_alignment;
1360
1361 return q->limits.discard_alignment;
1362}
1363
1364static inline unsigned int bdev_write_same(struct block_device *bdev)
1365{
1366 struct request_queue *q = bdev_get_queue(bdev);
1367
1368 if (q)
1369 return q->limits.max_write_same_sectors;
1370
1371 return 0;
1372}
1373
1374static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1375{
1376 struct request_queue *q = bdev_get_queue(bdev);
1377
1378 if (q)
1379 return q->limits.max_write_zeroes_sectors;
1380
1381 return 0;
1382}
1383
1384static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
1385{
1386 struct request_queue *q = bdev_get_queue(bdev);
1387
1388 if (q)
1389 return blk_queue_zoned_model(q);
1390
1391 return BLK_ZONED_NONE;
1392}
1393
1394static inline bool bdev_is_zoned(struct block_device *bdev)
1395{
1396 struct request_queue *q = bdev_get_queue(bdev);
1397
1398 if (q)
1399 return blk_queue_is_zoned(q);
1400
1401 return false;
1402}
1403
1404static inline unsigned int bdev_zone_sectors(struct block_device *bdev)
1405{
1406 struct request_queue *q = bdev_get_queue(bdev);
1407
1408 if (q)
1409 return blk_queue_zone_sectors(q);
1410 return 0;
1411}
1412
1413static inline int queue_dma_alignment(struct request_queue *q)
1414{
1415 return q ? q->dma_alignment : 511;
1416}
1417
1418static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1419 unsigned int len)
1420{
1421 unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1422 return !(addr & alignment) && !(len & alignment);
1423}
1424
1425/* assumes size > 256 */
1426static inline unsigned int blksize_bits(unsigned int size)
1427{
1428 unsigned int bits = 8;
1429 do {
1430 bits++;
1431 size >>= 1;
1432 } while (size > 256);
1433 return bits;
1434}
1435
1436static inline unsigned int block_size(struct block_device *bdev)
1437{
1438 return bdev->bd_block_size;
1439}
1440
1441typedef struct {struct page *v;} Sector;
1442
1443unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);
1444
1445static inline void put_dev_sector(Sector p)
1446{
1447 put_page(p.v);
1448}
1449
1450int kblockd_schedule_work(struct work_struct *work);
1451int kblockd_schedule_work_on(int cpu, struct work_struct *work);
1452int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1453
1454#define MODULE_ALIAS_BLOCKDEV(major,minor) \
1455 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1456#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1457 MODULE_ALIAS("block-major-" __stringify(major) "-*")
1458
1459#if defined(CONFIG_BLK_DEV_INTEGRITY)
1460
1461enum blk_integrity_flags {
1462 BLK_INTEGRITY_VERIFY = 1 << 0,
1463 BLK_INTEGRITY_GENERATE = 1 << 1,
1464 BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2,
1465 BLK_INTEGRITY_IP_CHECKSUM = 1 << 3,
1466};
1467
1468struct blk_integrity_iter {
1469 void *prot_buf;
1470 void *data_buf;
1471 sector_t seed;
1472 unsigned int data_size;
1473 unsigned short interval;
1474 const char *disk_name;
1475};
1476
1477typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
1478
1479struct blk_integrity_profile {
1480 integrity_processing_fn *generate_fn;
1481 integrity_processing_fn *verify_fn;
1482 const char *name;
1483};
1484
1485extern void blk_integrity_register(struct gendisk *, struct blk_integrity *);
1486extern void blk_integrity_unregister(struct gendisk *);
1487extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1488extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
1489 struct scatterlist *);
1490extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
1491extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
1492 struct request *);
1493extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
1494 struct bio *);
1495
1496static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1497{
1498 struct blk_integrity *bi = &disk->queue->integrity;
1499
1500 if (!bi->profile)
1501 return NULL;
1502
1503 return bi;
1504}
1505
1506static inline
1507struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
1508{
1509 return blk_get_integrity(bdev->bd_disk);
1510}
1511
1512static inline bool blk_integrity_rq(struct request *rq)
1513{
1514 return rq->cmd_flags & REQ_INTEGRITY;
1515}
1516
1517static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1518 unsigned int segs)
1519{
1520 q->limits.max_integrity_segments = segs;
1521}
1522
1523static inline unsigned short
1524queue_max_integrity_segments(struct request_queue *q)
1525{
1526 return q->limits.max_integrity_segments;
1527}
1528
1529/**
1530 * bio_integrity_intervals - Return number of integrity intervals for a bio
1531 * @bi: blk_integrity profile for device
1532 * @sectors: Size of the bio in 512-byte sectors
1533 *
1534 * Description: The block layer calculates everything in 512 byte
1535 * sectors but integrity metadata is done in terms of the data integrity
1536 * interval size of the storage device. Convert the block layer sectors
1537 * to the appropriate number of integrity intervals.
1538 */
1539static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
1540 unsigned int sectors)
1541{
1542 return sectors >> (bi->interval_exp - 9);
1543}
1544
1545static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
1546 unsigned int sectors)
1547{
1548 return bio_integrity_intervals(bi, sectors) * bi->tuple_size;
1549}
1550
1551#else /* CONFIG_BLK_DEV_INTEGRITY */
1552
1553struct bio;
1554struct block_device;
1555struct gendisk;
1556struct blk_integrity;
1557
1558static inline int blk_integrity_rq(struct request *rq)
1559{
1560 return 0;
1561}
1562static inline int blk_rq_count_integrity_sg(struct request_queue *q,
1563 struct bio *b)
1564{
1565 return 0;
1566}
1567static inline int blk_rq_map_integrity_sg(struct request_queue *q,
1568 struct bio *b,
1569 struct scatterlist *s)
1570{
1571 return 0;
1572}
1573static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
1574{
1575 return NULL;
1576}
1577static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1578{
1579 return NULL;
1580}
1581static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
1582{
1583 return 0;
1584}
1585static inline void blk_integrity_register(struct gendisk *d,
1586 struct blk_integrity *b)
1587{
1588}
1589static inline void blk_integrity_unregister(struct gendisk *d)
1590{
1591}
1592static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1593 unsigned int segs)
1594{
1595}
1596static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
1597{
1598 return 0;
1599}
1600static inline bool blk_integrity_merge_rq(struct request_queue *rq,
1601 struct request *r1,
1602 struct request *r2)
1603{
1604 return true;
1605}
1606static inline bool blk_integrity_merge_bio(struct request_queue *rq,
1607 struct request *r,
1608 struct bio *b)
1609{
1610 return true;
1611}
1612
1613static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
1614 unsigned int sectors)
1615{
1616 return 0;
1617}
1618
1619static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
1620 unsigned int sectors)
1621{
1622 return 0;
1623}
1624
1625#endif /* CONFIG_BLK_DEV_INTEGRITY */
1626
1627struct block_device_operations {
1628 int (*open) (struct block_device *, fmode_t);
1629 void (*release) (struct gendisk *, fmode_t);
1630 int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int);
1631 int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1632 int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1633 unsigned int (*check_events) (struct gendisk *disk,
1634 unsigned int clearing);
1635 /* ->media_changed() is DEPRECATED, use ->check_events() instead */
1636 int (*media_changed) (struct gendisk *);
1637 void (*unlock_native_capacity) (struct gendisk *);
1638 int (*revalidate_disk) (struct gendisk *);
1639 int (*getgeo)(struct block_device *, struct hd_geometry *);
1640 /* this callback is with swap_lock and sometimes page table lock held */
1641 void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1642 int (*report_zones)(struct gendisk *, sector_t sector,
1643 struct blk_zone *zones, unsigned int *nr_zones,
1644 gfp_t gfp_mask);
1645 struct module *owner;
1646 const struct pr_ops *pr_ops;
1647};
1648
1649extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
1650 unsigned long);
1651extern int bdev_read_page(struct block_device *, sector_t, struct page *);
1652extern int bdev_write_page(struct block_device *, sector_t, struct page *,
1653 struct writeback_control *);
1654
1655#ifdef CONFIG_BLK_DEV_ZONED
1656bool blk_req_needs_zone_write_lock(struct request *rq);
1657void __blk_req_zone_write_lock(struct request *rq);
1658void __blk_req_zone_write_unlock(struct request *rq);
1659
1660static inline void blk_req_zone_write_lock(struct request *rq)
1661{
1662 if (blk_req_needs_zone_write_lock(rq))
1663 __blk_req_zone_write_lock(rq);
1664}
1665
1666static inline void blk_req_zone_write_unlock(struct request *rq)
1667{
1668 if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
1669 __blk_req_zone_write_unlock(rq);
1670}
1671
1672static inline bool blk_req_zone_is_write_locked(struct request *rq)
1673{
1674 return rq->q->seq_zones_wlock &&
1675 test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock);
1676}
1677
1678static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
1679{
1680 if (!blk_req_needs_zone_write_lock(rq))
1681 return true;
1682 return !blk_req_zone_is_write_locked(rq);
1683}
1684#else
1685static inline bool blk_req_needs_zone_write_lock(struct request *rq)
1686{
1687 return false;
1688}
1689
1690static inline void blk_req_zone_write_lock(struct request *rq)
1691{
1692}
1693
1694static inline void blk_req_zone_write_unlock(struct request *rq)
1695{
1696}
1697static inline bool blk_req_zone_is_write_locked(struct request *rq)
1698{
1699 return false;
1700}
1701
1702static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
1703{
1704 return true;
1705}
1706#endif /* CONFIG_BLK_DEV_ZONED */
1707
1708#else /* CONFIG_BLOCK */
1709
1710struct block_device;
1711
1712/*
1713 * stubs for when the block layer is configured out
1714 */
1715#define buffer_heads_over_limit 0
1716
1717static inline long nr_blockdev_pages(void)
1718{
1719 return 0;
1720}
1721
1722struct blk_plug {
1723};
1724
1725static inline void blk_start_plug(struct blk_plug *plug)
1726{
1727}
1728
1729static inline void blk_finish_plug(struct blk_plug *plug)
1730{
1731}
1732
1733static inline void blk_flush_plug(struct task_struct *task)
1734{
1735}
1736
1737static inline void blk_schedule_flush_plug(struct task_struct *task)
1738{
1739}
1740
1741
1742static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1743{
1744 return false;
1745}
1746
1747static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
1748 sector_t *error_sector)
1749{
1750 return 0;
1751}
1752
1753#endif /* CONFIG_BLOCK */
1754
1755static inline void blk_wake_io_task(struct task_struct *waiter)
1756{
1757 /*
1758 * If we're polling, the task itself is doing the completions. For
1759 * that case, we don't need to signal a wakeup, it's enough to just
1760 * mark us as RUNNING.
1761 */
1762 if (waiter == current)
1763 __set_current_state(TASK_RUNNING);
1764 else
1765 wake_up_process(waiter);
1766}
1767
1768#endif
1769