aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2018-10-22 12:46:08 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2018-10-22 12:46:08 -0400
commit6ab9e09238fdfd742fe23b81e2d385a1cab49d9b (patch)
tree38c27b6d70c68f1114df2f49f72f1f757b22cd07 /include/linux
parent528985117126f11beea339cf39120ee99da04cd2 (diff)
parent52990a5fb0c991ecafebdab43138b5ed41376852 (diff)
Merge tag 'for-4.20/block-20181021' of git://git.kernel.dk/linux-block
Pull block layer updates from Jens Axboe: "This is the main pull request for block changes for 4.20. This contains: - Series enabling runtime PM for blk-mq (Bart). - Two pull requests from Christoph for NVMe, with items such as; - Better AEN tracking - Multipath improvements - RDMA fixes - Rework of FC for target removal - Fixes for issues identified by static checkers - Fabric cleanups, as prep for TCP transport - Various cleanups and bug fixes - Block merging cleanups (Christoph) - Conversion of drivers to generic DMA mapping API (Christoph) - Series fixing ref count issues with blkcg (Dennis) - Series improving BFQ heuristics (Paolo, et al) - Series improving heuristics for the Kyber IO scheduler (Omar) - Removal of dangerous bio_rewind_iter() API (Ming) - Apply single queue IPI redirection logic to blk-mq (Ming) - Set of fixes and improvements for bcache (Coly et al) - Series closing a hotplug race with sysfs group attributes (Hannes) - Set of patches for lightnvm: - pblk trace support (Hans) - SPDX license header update (Javier) - Tons of refactoring patches to cleanly abstract the 1.2 and 2.0 specs behind a common core interface. (Javier, Matias) - Enable pblk to use a common interface to retrieve chunk metadata (Matias) - Bug fixes (Various) - Set of fixes and updates to the blk IO latency target (Josef) - blk-mq queue number updates fixes (Jianchao) - Convert a bunch of drivers from the old legacy IO interface to blk-mq. This will conclude with the removal of the legacy IO interface itself in 4.21, with the rest of the drivers (me, Omar) - Removal of the DAC960 driver. The SCSI tree will introduce two replacement drivers for this (Hannes)" * tag 'for-4.20/block-20181021' of git://git.kernel.dk/linux-block: (204 commits) block: setup bounce bio_sets properly blkcg: reassociate bios when make_request() is called recursively blkcg: fix edge case for blk_get_rl() under memory pressure nvme-fabrics: move controller options matching to fabrics nvme-rdma: always have a valid trsvcid mtip32xx: fully switch to the generic DMA API rsxx: switch to the generic DMA API umem: switch to the generic DMA API sx8: switch to the generic DMA API sx8: remove dead IF_64BIT_DMA_IS_POSSIBLE code skd: switch to the generic DMA API ubd: remove use of blk_rq_map_sg nvme-pci: remove duplicate check drivers/block: Remove DAC960 driver nvme-pci: fix hot removal during error handling nvmet-fcloop: suppress a compiler warning nvme-core: make implicit seed truncation explicit nvmet-fc: fix kernel-doc headers nvme-fc: rework the request initialization code nvme-fc: introduce struct nvme_fcp_op_w_sgl ...
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/amifd.h63
-rw-r--r--include/linux/amifdreg.h82
-rw-r--r--include/linux/bio.h78
-rw-r--r--include/linux/blk-cgroup.h145
-rw-r--r--include/linux/blk-mq.h4
-rw-r--r--include/linux/blk-pm.h24
-rw-r--r--include/linux/blk_types.h1
-rw-r--r--include/linux/blkdev.h164
-rw-r--r--include/linux/bvec.h3
-rw-r--r--include/linux/cgroup.h2
-rw-r--r--include/linux/elevator.h2
-rw-r--r--include/linux/genhd.h5
-rw-r--r--include/linux/lightnvm.h166
-rw-r--r--include/linux/mtd/blktrans.h5
-rw-r--r--include/linux/nvme.h1
-rw-r--r--include/linux/percpu-refcount.h1
-rw-r--r--include/linux/writeback.h5
17 files changed, 331 insertions, 420 deletions
diff --git a/include/linux/amifd.h b/include/linux/amifd.h
deleted file mode 100644
index 202a77dbe46d..000000000000
--- a/include/linux/amifd.h
+++ /dev/null
@@ -1,63 +0,0 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _AMIFD_H
3#define _AMIFD_H
4
5/* Definitions for the Amiga floppy driver */
6
7#include <linux/fd.h>
8
9#define FD_MAX_UNITS 4 /* Max. Number of drives */
10#define FLOPPY_MAX_SECTORS 22 /* Max. Number of sectors per track */
11
12#ifndef ASSEMBLER
13
14struct fd_data_type {
15 char *name; /* description of data type */
16 int sects; /* sectors per track */
17#ifdef __STDC__
18 int (*read_fkt)(int);
19 void (*write_fkt)(int);
20#else
21 int (*read_fkt)(); /* read whole track */
22 void (*write_fkt)(); /* write whole track */
23#endif
24};
25
26/*
27** Floppy type descriptions
28*/
29
30struct fd_drive_type {
31 unsigned long code; /* code returned from drive */
32 char *name; /* description of drive */
33 unsigned int tracks; /* number of tracks */
34 unsigned int heads; /* number of heads */
35 unsigned int read_size; /* raw read size for one track */
36 unsigned int write_size; /* raw write size for one track */
37 unsigned int sect_mult; /* sectors and gap multiplier (HD = 2) */
38 unsigned int precomp1; /* start track for precomp 1 */
39 unsigned int precomp2; /* start track for precomp 2 */
40 unsigned int step_delay; /* time (in ms) for delay after step */
41 unsigned int settle_time; /* time to settle after dir change */
42 unsigned int side_time; /* time needed to change sides */
43};
44
45struct amiga_floppy_struct {
46 struct fd_drive_type *type; /* type of floppy for this unit */
47 struct fd_data_type *dtype; /* type of floppy for this unit */
48 int track; /* current track (-1 == unknown) */
49 unsigned char *trackbuf; /* current track (kmaloc()'d */
50
51 int blocks; /* total # blocks on disk */
52
53 int changed; /* true when not known */
54 int disk; /* disk in drive (-1 == unknown) */
55 int motor; /* true when motor is at speed */
56 int busy; /* true when drive is active */
57 int dirty; /* true when trackbuf is not on disk */
58 int status; /* current error code for unit */
59 struct gendisk *gendisk;
60};
61#endif
62
63#endif
diff --git a/include/linux/amifdreg.h b/include/linux/amifdreg.h
deleted file mode 100644
index 9b514d05ec70..000000000000
--- a/include/linux/amifdreg.h
+++ /dev/null
@@ -1,82 +0,0 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_AMIFDREG_H
3#define _LINUX_AMIFDREG_H
4
5/*
6** CIAAPRA bits (read only)
7*/
8
9#define DSKRDY (0x1<<5) /* disk ready when low */
10#define DSKTRACK0 (0x1<<4) /* head at track zero when low */
11#define DSKPROT (0x1<<3) /* disk protected when low */
12#define DSKCHANGE (0x1<<2) /* low when disk removed */
13
14/*
15** CIAAPRB bits (read/write)
16*/
17
18#define DSKMOTOR (0x1<<7) /* motor on when low */
19#define DSKSEL3 (0x1<<6) /* select drive 3 when low */
20#define DSKSEL2 (0x1<<5) /* select drive 2 when low */
21#define DSKSEL1 (0x1<<4) /* select drive 1 when low */
22#define DSKSEL0 (0x1<<3) /* select drive 0 when low */
23#define DSKSIDE (0x1<<2) /* side selection: 0 = upper, 1 = lower */
24#define DSKDIREC (0x1<<1) /* step direction: 0=in, 1=out (to trk 0) */
25#define DSKSTEP (0x1) /* pulse low to step head 1 track */
26
27/*
28** DSKBYTR bits (read only)
29*/
30
31#define DSKBYT (1<<15) /* register contains valid byte when set */
32#define DMAON (1<<14) /* disk DMA enabled */
33#define DISKWRITE (1<<13) /* disk write bit in DSKLEN enabled */
34#define WORDEQUAL (1<<12) /* DSKSYNC register match when true */
35/* bits 7-0 are data */
36
37/*
38** ADKCON/ADKCONR bits
39*/
40
41#ifndef SETCLR
42#define ADK_SETCLR (1<<15) /* control bit */
43#endif
44#define ADK_PRECOMP1 (1<<14) /* precompensation selection */
45#define ADK_PRECOMP0 (1<<13) /* 00=none, 01=140ns, 10=280ns, 11=500ns */
46#define ADK_MFMPREC (1<<12) /* 0=GCR precomp., 1=MFM precomp. */
47#define ADK_WORDSYNC (1<<10) /* enable DSKSYNC auto DMA */
48#define ADK_MSBSYNC (1<<9) /* when 1, enable sync on MSbit (for GCR) */
49#define ADK_FAST (1<<8) /* bit cell: 0=2us (GCR), 1=1us (MFM) */
50
51/*
52** DSKLEN bits
53*/
54
55#define DSKLEN_DMAEN (1<<15)
56#define DSKLEN_WRITE (1<<14)
57
58/*
59** INTENA/INTREQ bits
60*/
61
62#define DSKINDEX (0x1<<4) /* DSKINDEX bit */
63
64/*
65** Misc
66*/
67
68#define MFM_SYNC 0x4489 /* standard MFM sync value */
69
70/* Values for FD_COMMAND */
71#define FD_RECALIBRATE 0x07 /* move to track 0 */
72#define FD_SEEK 0x0F /* seek track */
73#define FD_READ 0xE6 /* read with MT, MFM, SKip deleted */
74#define FD_WRITE 0xC5 /* write with MT, MFM */
75#define FD_SENSEI 0x08 /* Sense Interrupt Status */
76#define FD_SPECIFY 0x03 /* specify HUT etc */
77#define FD_FORMAT 0x4D /* format one track */
78#define FD_VERSION 0x10 /* get version code */
79#define FD_CONFIGURE 0x13 /* configure FIFO operation */
80#define FD_PERPENDICULAR 0x12 /* perpendicular r/w mode */
81
82#endif /* _LINUX_AMIFDREG_H */
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 51371740d2a8..b47c7f716731 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -21,12 +21,8 @@
21#include <linux/highmem.h> 21#include <linux/highmem.h>
22#include <linux/mempool.h> 22#include <linux/mempool.h>
23#include <linux/ioprio.h> 23#include <linux/ioprio.h>
24#include <linux/bug.h>
25 24
26#ifdef CONFIG_BLOCK 25#ifdef CONFIG_BLOCK
27
28#include <asm/io.h>
29
30/* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */ 26/* struct bio, bio_vec and BIO_* flags are defined in blk_types.h */
31#include <linux/blk_types.h> 27#include <linux/blk_types.h>
32 28
@@ -133,32 +129,6 @@ static inline bool bio_full(struct bio *bio)
133} 129}
134 130
135/* 131/*
136 * will die
137 */
138#define bvec_to_phys(bv) (page_to_phys((bv)->bv_page) + (unsigned long) (bv)->bv_offset)
139
140/*
141 * merge helpers etc
142 */
143
144/* Default implementation of BIOVEC_PHYS_MERGEABLE */
145#define __BIOVEC_PHYS_MERGEABLE(vec1, vec2) \
146 ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2)))
147
148/*
149 * allow arch override, for eg virtualized architectures (put in asm/io.h)
150 */
151#ifndef BIOVEC_PHYS_MERGEABLE
152#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \
153 __BIOVEC_PHYS_MERGEABLE(vec1, vec2)
154#endif
155
156#define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \
157 (((addr1) | (mask)) == (((addr2) - 1) | (mask)))
158#define BIOVEC_SEG_BOUNDARY(q, b1, b2) \
159 __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q)))
160
161/*
162 * drivers should _never_ use the all version - the bio may have been split 132 * drivers should _never_ use the all version - the bio may have been split
163 * before it got to the driver and the driver won't own all of it 133 * before it got to the driver and the driver won't own all of it
164 */ 134 */
@@ -170,27 +140,11 @@ static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
170{ 140{
171 iter->bi_sector += bytes >> 9; 141 iter->bi_sector += bytes >> 9;
172 142
173 if (bio_no_advance_iter(bio)) { 143 if (bio_no_advance_iter(bio))
174 iter->bi_size -= bytes; 144 iter->bi_size -= bytes;
175 iter->bi_done += bytes; 145 else
176 } else {
177 bvec_iter_advance(bio->bi_io_vec, iter, bytes); 146 bvec_iter_advance(bio->bi_io_vec, iter, bytes);
178 /* TODO: It is reasonable to complete bio with error here. */ 147 /* TODO: It is reasonable to complete bio with error here. */
179 }
180}
181
182static inline bool bio_rewind_iter(struct bio *bio, struct bvec_iter *iter,
183 unsigned int bytes)
184{
185 iter->bi_sector -= bytes >> 9;
186
187 if (bio_no_advance_iter(bio)) {
188 iter->bi_size += bytes;
189 iter->bi_done -= bytes;
190 return true;
191 }
192
193 return bvec_iter_rewind(bio->bi_io_vec, iter, bytes);
194} 148}
195 149
196#define __bio_for_each_segment(bvl, bio, iter, start) \ 150#define __bio_for_each_segment(bvl, bio, iter, start) \
@@ -353,6 +307,8 @@ struct bio_integrity_payload {
353 unsigned short bip_max_vcnt; /* integrity bio_vec slots */ 307 unsigned short bip_max_vcnt; /* integrity bio_vec slots */
354 unsigned short bip_flags; /* control flags */ 308 unsigned short bip_flags; /* control flags */
355 309
310 struct bvec_iter bio_iter; /* for rewinding parent bio */
311
356 struct work_struct bip_work; /* I/O completion */ 312 struct work_struct bip_work; /* I/O completion */
357 313
358 struct bio_vec *bip_vec; 314 struct bio_vec *bip_vec;
@@ -547,23 +503,31 @@ do { \
547 disk_devt((bio)->bi_disk) 503 disk_devt((bio)->bi_disk)
548 504
549#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) 505#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
550int bio_associate_blkcg_from_page(struct bio *bio, struct page *page); 506int bio_associate_blkg_from_page(struct bio *bio, struct page *page);
551#else 507#else
552static inline int bio_associate_blkcg_from_page(struct bio *bio, 508static inline int bio_associate_blkg_from_page(struct bio *bio,
553 struct page *page) { return 0; } 509 struct page *page) { return 0; }
554#endif 510#endif
555 511
556#ifdef CONFIG_BLK_CGROUP 512#ifdef CONFIG_BLK_CGROUP
557int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css);
558int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg); 513int bio_associate_blkg(struct bio *bio, struct blkcg_gq *blkg);
514int bio_associate_blkg_from_css(struct bio *bio,
515 struct cgroup_subsys_state *css);
516int bio_associate_create_blkg(struct request_queue *q, struct bio *bio);
517int bio_reassociate_blkg(struct request_queue *q, struct bio *bio);
559void bio_disassociate_task(struct bio *bio); 518void bio_disassociate_task(struct bio *bio);
560void bio_clone_blkcg_association(struct bio *dst, struct bio *src); 519void bio_clone_blkg_association(struct bio *dst, struct bio *src);
561#else /* CONFIG_BLK_CGROUP */ 520#else /* CONFIG_BLK_CGROUP */
562static inline int bio_associate_blkcg(struct bio *bio, 521static inline int bio_associate_blkg_from_css(struct bio *bio,
563 struct cgroup_subsys_state *blkcg_css) { return 0; } 522 struct cgroup_subsys_state *css)
523{ return 0; }
524static inline int bio_associate_create_blkg(struct request_queue *q,
525 struct bio *bio) { return 0; }
526static inline int bio_reassociate_blkg(struct request_queue *q, struct bio *bio)
527{ return 0; }
564static inline void bio_disassociate_task(struct bio *bio) { } 528static inline void bio_disassociate_task(struct bio *bio) { }
565static inline void bio_clone_blkcg_association(struct bio *dst, 529static inline void bio_clone_blkg_association(struct bio *dst,
566 struct bio *src) { } 530 struct bio *src) { }
567#endif /* CONFIG_BLK_CGROUP */ 531#endif /* CONFIG_BLK_CGROUP */
568 532
569#ifdef CONFIG_HIGHMEM 533#ifdef CONFIG_HIGHMEM
diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h
index 6d766a19f2bb..1e76ceebeb5d 100644
--- a/include/linux/blk-cgroup.h
+++ b/include/linux/blk-cgroup.h
@@ -126,7 +126,7 @@ struct blkcg_gq {
126 struct request_list rl; 126 struct request_list rl;
127 127
128 /* reference count */ 128 /* reference count */
129 atomic_t refcnt; 129 struct percpu_ref refcnt;
130 130
131 /* is this blkg online? protected by both blkcg and q locks */ 131 /* is this blkg online? protected by both blkcg and q locks */
132 bool online; 132 bool online;
@@ -184,6 +184,8 @@ extern struct cgroup_subsys_state * const blkcg_root_css;
184 184
185struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg, 185struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
186 struct request_queue *q, bool update_hint); 186 struct request_queue *q, bool update_hint);
187struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
188 struct request_queue *q);
187struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg, 189struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
188 struct request_queue *q); 190 struct request_queue *q);
189int blkcg_init_queue(struct request_queue *q); 191int blkcg_init_queue(struct request_queue *q);
@@ -230,22 +232,59 @@ int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
230 char *input, struct blkg_conf_ctx *ctx); 232 char *input, struct blkg_conf_ctx *ctx);
231void blkg_conf_finish(struct blkg_conf_ctx *ctx); 233void blkg_conf_finish(struct blkg_conf_ctx *ctx);
232 234
235/**
236 * blkcg_css - find the current css
237 *
238 * Find the css associated with either the kthread or the current task.
239 * This may return a dying css, so it is up to the caller to use tryget logic
240 * to confirm it is alive and well.
241 */
242static inline struct cgroup_subsys_state *blkcg_css(void)
243{
244 struct cgroup_subsys_state *css;
245
246 css = kthread_blkcg();
247 if (css)
248 return css;
249 return task_css(current, io_cgrp_id);
250}
233 251
234static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css) 252static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css)
235{ 253{
236 return css ? container_of(css, struct blkcg, css) : NULL; 254 return css ? container_of(css, struct blkcg, css) : NULL;
237} 255}
238 256
239static inline struct blkcg *bio_blkcg(struct bio *bio) 257/**
258 * __bio_blkcg - internal version of bio_blkcg for bfq and cfq
259 *
260 * DO NOT USE.
261 * There is a flaw using this version of the function. In particular, this was
262 * used in a broken paradigm where association was called on the given css. It
263 * is possible though that the returned css from task_css() is in the process
264 * of dying due to migration of the current task. So it is improper to assume
265 * *_get() is going to succeed. Both BFQ and CFQ rely on this logic and will
266 * take additional work to handle more gracefully.
267 */
268static inline struct blkcg *__bio_blkcg(struct bio *bio)
240{ 269{
241 struct cgroup_subsys_state *css; 270 if (bio && bio->bi_blkg)
271 return bio->bi_blkg->blkcg;
272 return css_to_blkcg(blkcg_css());
273}
242 274
243 if (bio && bio->bi_css) 275/**
244 return css_to_blkcg(bio->bi_css); 276 * bio_blkcg - grab the blkcg associated with a bio
245 css = kthread_blkcg(); 277 * @bio: target bio
246 if (css) 278 *
247 return css_to_blkcg(css); 279 * This returns the blkcg associated with a bio, NULL if not associated.
248 return css_to_blkcg(task_css(current, io_cgrp_id)); 280 * Callers are expected to either handle NULL or know association has been
281 * done prior to calling this.
282 */
283static inline struct blkcg *bio_blkcg(struct bio *bio)
284{
285 if (bio && bio->bi_blkg)
286 return bio->bi_blkg->blkcg;
287 return NULL;
249} 288}
250 289
251static inline bool blk_cgroup_congested(void) 290static inline bool blk_cgroup_congested(void)
@@ -451,26 +490,35 @@ static inline int blkg_path(struct blkcg_gq *blkg, char *buf, int buflen)
451 */ 490 */
452static inline void blkg_get(struct blkcg_gq *blkg) 491static inline void blkg_get(struct blkcg_gq *blkg)
453{ 492{
454 WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0); 493 percpu_ref_get(&blkg->refcnt);
455 atomic_inc(&blkg->refcnt);
456} 494}
457 495
458/** 496/**
459 * blkg_try_get - try and get a blkg reference 497 * blkg_tryget - try and get a blkg reference
460 * @blkg: blkg to get 498 * @blkg: blkg to get
461 * 499 *
462 * This is for use when doing an RCU lookup of the blkg. We may be in the midst 500 * This is for use when doing an RCU lookup of the blkg. We may be in the midst
463 * of freeing this blkg, so we can only use it if the refcnt is not zero. 501 * of freeing this blkg, so we can only use it if the refcnt is not zero.
464 */ 502 */
465static inline struct blkcg_gq *blkg_try_get(struct blkcg_gq *blkg) 503static inline bool blkg_tryget(struct blkcg_gq *blkg)
466{ 504{
467 if (atomic_inc_not_zero(&blkg->refcnt)) 505 return percpu_ref_tryget(&blkg->refcnt);
468 return blkg;
469 return NULL;
470} 506}
471 507
508/**
509 * blkg_tryget_closest - try and get a blkg ref on the closet blkg
510 * @blkg: blkg to get
511 *
512 * This walks up the blkg tree to find the closest non-dying blkg and returns
513 * the blkg that it did association with as it may not be the passed in blkg.
514 */
515static inline struct blkcg_gq *blkg_tryget_closest(struct blkcg_gq *blkg)
516{
517 while (!percpu_ref_tryget(&blkg->refcnt))
518 blkg = blkg->parent;
472 519
473void __blkg_release_rcu(struct rcu_head *rcu); 520 return blkg;
521}
474 522
475/** 523/**
476 * blkg_put - put a blkg reference 524 * blkg_put - put a blkg reference
@@ -478,9 +526,7 @@ void __blkg_release_rcu(struct rcu_head *rcu);
478 */ 526 */
479static inline void blkg_put(struct blkcg_gq *blkg) 527static inline void blkg_put(struct blkcg_gq *blkg)
480{ 528{
481 WARN_ON_ONCE(atomic_read(&blkg->refcnt) <= 0); 529 percpu_ref_put(&blkg->refcnt);
482 if (atomic_dec_and_test(&blkg->refcnt))
483 call_rcu(&blkg->rcu_head, __blkg_release_rcu);
484} 530}
485 531
486/** 532/**
@@ -533,25 +579,36 @@ static inline struct request_list *blk_get_rl(struct request_queue *q,
533 579
534 rcu_read_lock(); 580 rcu_read_lock();
535 581
536 blkcg = bio_blkcg(bio); 582 if (bio && bio->bi_blkg) {
583 blkcg = bio->bi_blkg->blkcg;
584 if (blkcg == &blkcg_root)
585 goto rl_use_root;
586
587 blkg_get(bio->bi_blkg);
588 rcu_read_unlock();
589 return &bio->bi_blkg->rl;
590 }
537 591
538 /* bypass blkg lookup and use @q->root_rl directly for root */ 592 blkcg = css_to_blkcg(blkcg_css());
539 if (blkcg == &blkcg_root) 593 if (blkcg == &blkcg_root)
540 goto root_rl; 594 goto rl_use_root;
541 595
542 /*
543 * Try to use blkg->rl. blkg lookup may fail under memory pressure
544 * or if either the blkcg or queue is going away. Fall back to
545 * root_rl in such cases.
546 */
547 blkg = blkg_lookup(blkcg, q); 596 blkg = blkg_lookup(blkcg, q);
548 if (unlikely(!blkg)) 597 if (unlikely(!blkg))
549 goto root_rl; 598 blkg = __blkg_lookup_create(blkcg, q);
599
600 if (blkg->blkcg == &blkcg_root || !blkg_tryget(blkg))
601 goto rl_use_root;
550 602
551 blkg_get(blkg);
552 rcu_read_unlock(); 603 rcu_read_unlock();
553 return &blkg->rl; 604 return &blkg->rl;
554root_rl: 605
606 /*
607 * Each blkg has its own request_list, however, the root blkcg
608 * uses the request_queue's root_rl. This is to avoid most
609 * overhead for the root blkcg.
610 */
611rl_use_root:
555 rcu_read_unlock(); 612 rcu_read_unlock();
556 return &q->root_rl; 613 return &q->root_rl;
557} 614}
@@ -797,32 +854,26 @@ static inline bool blk_throtl_bio(struct request_queue *q, struct blkcg_gq *blkg
797 struct bio *bio) { return false; } 854 struct bio *bio) { return false; }
798#endif 855#endif
799 856
857
858static inline void blkcg_bio_issue_init(struct bio *bio)
859{
860 bio_issue_init(&bio->bi_issue, bio_sectors(bio));
861}
862
800static inline bool blkcg_bio_issue_check(struct request_queue *q, 863static inline bool blkcg_bio_issue_check(struct request_queue *q,
801 struct bio *bio) 864 struct bio *bio)
802{ 865{
803 struct blkcg *blkcg;
804 struct blkcg_gq *blkg; 866 struct blkcg_gq *blkg;
805 bool throtl = false; 867 bool throtl = false;
806 868
807 rcu_read_lock(); 869 rcu_read_lock();
808 blkcg = bio_blkcg(bio);
809
810 /* associate blkcg if bio hasn't attached one */
811 bio_associate_blkcg(bio, &blkcg->css);
812 870
813 blkg = blkg_lookup(blkcg, q); 871 bio_associate_create_blkg(q, bio);
814 if (unlikely(!blkg)) { 872 blkg = bio->bi_blkg;
815 spin_lock_irq(q->queue_lock);
816 blkg = blkg_lookup_create(blkcg, q);
817 if (IS_ERR(blkg))
818 blkg = NULL;
819 spin_unlock_irq(q->queue_lock);
820 }
821 873
822 throtl = blk_throtl_bio(q, blkg, bio); 874 throtl = blk_throtl_bio(q, blkg, bio);
823 875
824 if (!throtl) { 876 if (!throtl) {
825 blkg = blkg ?: q->root_blkg;
826 /* 877 /*
827 * If the bio is flagged with BIO_QUEUE_ENTERED it means this 878 * If the bio is flagged with BIO_QUEUE_ENTERED it means this
828 * is a split bio and we would have already accounted for the 879 * is a split bio and we would have already accounted for the
@@ -834,6 +885,8 @@ static inline bool blkcg_bio_issue_check(struct request_queue *q,
834 blkg_rwstat_add(&blkg->stat_ios, bio->bi_opf, 1); 885 blkg_rwstat_add(&blkg->stat_ios, bio->bi_opf, 1);
835 } 886 }
836 887
888 blkcg_bio_issue_init(bio);
889
837 rcu_read_unlock(); 890 rcu_read_unlock();
838 return !throtl; 891 return !throtl;
839} 892}
@@ -930,6 +983,7 @@ static inline int blkcg_activate_policy(struct request_queue *q,
930static inline void blkcg_deactivate_policy(struct request_queue *q, 983static inline void blkcg_deactivate_policy(struct request_queue *q,
931 const struct blkcg_policy *pol) { } 984 const struct blkcg_policy *pol) { }
932 985
986static inline struct blkcg *__bio_blkcg(struct bio *bio) { return NULL; }
933static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; } 987static inline struct blkcg *bio_blkcg(struct bio *bio) { return NULL; }
934 988
935static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg, 989static inline struct blkg_policy_data *blkg_to_pd(struct blkcg_gq *blkg,
@@ -945,6 +999,7 @@ static inline void blk_put_rl(struct request_list *rl) { }
945static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl) { } 999static inline void blk_rq_set_rl(struct request *rq, struct request_list *rl) { }
946static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q->root_rl; } 1000static inline struct request_list *blk_rq_rl(struct request *rq) { return &rq->q->root_rl; }
947 1001
1002static inline void blkcg_bio_issue_init(struct bio *bio) { }
948static inline bool blkcg_bio_issue_check(struct request_queue *q, 1003static inline bool blkcg_bio_issue_check(struct request_queue *q,
949 struct bio *bio) { return true; } 1004 struct bio *bio) { return true; }
950 1005
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 1da59c16f637..2286dc12c6bc 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -203,6 +203,10 @@ enum {
203struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); 203struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *);
204struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, 204struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
205 struct request_queue *q); 205 struct request_queue *q);
206struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set,
207 const struct blk_mq_ops *ops,
208 unsigned int queue_depth,
209 unsigned int set_flags);
206int blk_mq_register_dev(struct device *, struct request_queue *); 210int blk_mq_register_dev(struct device *, struct request_queue *);
207void blk_mq_unregister_dev(struct device *, struct request_queue *); 211void blk_mq_unregister_dev(struct device *, struct request_queue *);
208 212
diff --git a/include/linux/blk-pm.h b/include/linux/blk-pm.h
new file mode 100644
index 000000000000..b80c65aba249
--- /dev/null
+++ b/include/linux/blk-pm.h
@@ -0,0 +1,24 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _BLK_PM_H_
4#define _BLK_PM_H_
5
6struct device;
7struct request_queue;
8
9/*
10 * block layer runtime pm functions
11 */
12#ifdef CONFIG_PM
13extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
14extern int blk_pre_runtime_suspend(struct request_queue *q);
15extern void blk_post_runtime_suspend(struct request_queue *q, int err);
16extern void blk_pre_runtime_resume(struct request_queue *q);
17extern void blk_post_runtime_resume(struct request_queue *q, int err);
18extern void blk_set_runtime_active(struct request_queue *q);
19#else
20static inline void blk_pm_runtime_init(struct request_queue *q,
21 struct device *dev) {}
22#endif
23
24#endif /* _BLK_PM_H_ */
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index f6dfb30737d8..9578c7ab1eb6 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -178,7 +178,6 @@ struct bio {
178 * release. Read comment on top of bio_associate_current(). 178 * release. Read comment on top of bio_associate_current().
179 */ 179 */
180 struct io_context *bi_ioc; 180 struct io_context *bi_ioc;
181 struct cgroup_subsys_state *bi_css;
182 struct blkcg_gq *bi_blkg; 181 struct blkcg_gq *bi_blkg;
183 struct bio_issue bi_issue; 182 struct bio_issue bi_issue;
184#endif 183#endif
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 6980014357d4..61207560e826 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -108,7 +108,7 @@ typedef __u32 __bitwise req_flags_t;
108#define RQF_QUIET ((__force req_flags_t)(1 << 11)) 108#define RQF_QUIET ((__force req_flags_t)(1 << 11))
109/* elevator private data attached */ 109/* elevator private data attached */
110#define RQF_ELVPRIV ((__force req_flags_t)(1 << 12)) 110#define RQF_ELVPRIV ((__force req_flags_t)(1 << 12))
111/* account I/O stat */ 111/* account into disk and partition IO statistics */
112#define RQF_IO_STAT ((__force req_flags_t)(1 << 13)) 112#define RQF_IO_STAT ((__force req_flags_t)(1 << 13))
113/* request came from our alloc pool */ 113/* request came from our alloc pool */
114#define RQF_ALLOCED ((__force req_flags_t)(1 << 14)) 114#define RQF_ALLOCED ((__force req_flags_t)(1 << 14))
@@ -116,7 +116,7 @@ typedef __u32 __bitwise req_flags_t;
116#define RQF_PM ((__force req_flags_t)(1 << 15)) 116#define RQF_PM ((__force req_flags_t)(1 << 15))
117/* on IO scheduler merge hash */ 117/* on IO scheduler merge hash */
118#define RQF_HASHED ((__force req_flags_t)(1 << 16)) 118#define RQF_HASHED ((__force req_flags_t)(1 << 16))
119/* IO stats tracking on */ 119/* track IO completion time */
120#define RQF_STATS ((__force req_flags_t)(1 << 17)) 120#define RQF_STATS ((__force req_flags_t)(1 << 17))
121/* Look at ->special_vec for the actual data payload instead of the 121/* Look at ->special_vec for the actual data payload instead of the
122 bio chain. */ 122 bio chain. */
@@ -504,6 +504,12 @@ struct request_queue {
504 * various queue flags, see QUEUE_* below 504 * various queue flags, see QUEUE_* below
505 */ 505 */
506 unsigned long queue_flags; 506 unsigned long queue_flags;
507 /*
508 * Number of contexts that have called blk_set_pm_only(). If this
509 * counter is above zero then only RQF_PM and RQF_PREEMPT requests are
510 * processed.
511 */
512 atomic_t pm_only;
507 513
508 /* 514 /*
509 * ida allocated id for this queue. Used to index queues from 515 * ida allocated id for this queue. Used to index queues from
@@ -679,7 +685,7 @@ struct request_queue {
679#define QUEUE_FLAG_FAIL_IO 7 /* fake timeout */ 685#define QUEUE_FLAG_FAIL_IO 7 /* fake timeout */
680#define QUEUE_FLAG_NONROT 9 /* non-rotational device (SSD) */ 686#define QUEUE_FLAG_NONROT 9 /* non-rotational device (SSD) */
681#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ 687#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */
682#define QUEUE_FLAG_IO_STAT 10 /* do IO stats */ 688#define QUEUE_FLAG_IO_STAT 10 /* do disk/partitions IO accounting */
683#define QUEUE_FLAG_DISCARD 11 /* supports DISCARD */ 689#define QUEUE_FLAG_DISCARD 11 /* supports DISCARD */
684#define QUEUE_FLAG_NOXMERGES 12 /* No extended merges */ 690#define QUEUE_FLAG_NOXMERGES 12 /* No extended merges */
685#define QUEUE_FLAG_ADD_RANDOM 13 /* Contributes to random pool */ 691#define QUEUE_FLAG_ADD_RANDOM 13 /* Contributes to random pool */
@@ -693,12 +699,11 @@ struct request_queue {
693#define QUEUE_FLAG_FUA 21 /* device supports FUA writes */ 699#define QUEUE_FLAG_FUA 21 /* device supports FUA writes */
694#define QUEUE_FLAG_FLUSH_NQ 22 /* flush not queueuable */ 700#define QUEUE_FLAG_FLUSH_NQ 22 /* flush not queueuable */
695#define QUEUE_FLAG_DAX 23 /* device supports DAX */ 701#define QUEUE_FLAG_DAX 23 /* device supports DAX */
696#define QUEUE_FLAG_STATS 24 /* track rq completion times */ 702#define QUEUE_FLAG_STATS 24 /* track IO start and completion times */
697#define QUEUE_FLAG_POLL_STATS 25 /* collecting stats for hybrid polling */ 703#define QUEUE_FLAG_POLL_STATS 25 /* collecting stats for hybrid polling */
698#define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */ 704#define QUEUE_FLAG_REGISTERED 26 /* queue has been registered to a disk */
699#define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */ 705#define QUEUE_FLAG_SCSI_PASSTHROUGH 27 /* queue supports SCSI commands */
700#define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */ 706#define QUEUE_FLAG_QUIESCED 28 /* queue has been quiesced */
701#define QUEUE_FLAG_PREEMPT_ONLY 29 /* only process REQ_PREEMPT requests */
702 707
703#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ 708#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
704 (1 << QUEUE_FLAG_SAME_COMP) | \ 709 (1 << QUEUE_FLAG_SAME_COMP) | \
@@ -736,12 +741,11 @@ bool blk_queue_flag_test_and_clear(unsigned int flag, struct request_queue *q);
736 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ 741 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
737 REQ_FAILFAST_DRIVER)) 742 REQ_FAILFAST_DRIVER))
738#define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags) 743#define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
739#define blk_queue_preempt_only(q) \ 744#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
740 test_bit(QUEUE_FLAG_PREEMPT_ONLY, &(q)->queue_flags)
741#define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags) 745#define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
742 746
743extern int blk_set_preempt_only(struct request_queue *q); 747extern void blk_set_pm_only(struct request_queue *q);
744extern void blk_clear_preempt_only(struct request_queue *q); 748extern void blk_clear_pm_only(struct request_queue *q);
745 749
746static inline int queue_in_flight(struct request_queue *q) 750static inline int queue_in_flight(struct request_queue *q)
747{ 751{
@@ -1281,29 +1285,6 @@ extern void blk_put_queue(struct request_queue *);
1281extern void blk_set_queue_dying(struct request_queue *); 1285extern void blk_set_queue_dying(struct request_queue *);
1282 1286
1283/* 1287/*
1284 * block layer runtime pm functions
1285 */
1286#ifdef CONFIG_PM
1287extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
1288extern int blk_pre_runtime_suspend(struct request_queue *q);
1289extern void blk_post_runtime_suspend(struct request_queue *q, int err);
1290extern void blk_pre_runtime_resume(struct request_queue *q);
1291extern void blk_post_runtime_resume(struct request_queue *q, int err);
1292extern void blk_set_runtime_active(struct request_queue *q);
1293#else
1294static inline void blk_pm_runtime_init(struct request_queue *q,
1295 struct device *dev) {}
1296static inline int blk_pre_runtime_suspend(struct request_queue *q)
1297{
1298 return -ENOSYS;
1299}
1300static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
1301static inline void blk_pre_runtime_resume(struct request_queue *q) {}
1302static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
1303static inline void blk_set_runtime_active(struct request_queue *q) {}
1304#endif
1305
1306/*
1307 * blk_plug permits building a queue of related requests by holding the I/O 1288 * blk_plug permits building a queue of related requests by holding the I/O
1308 * fragments for a short period. This allows merging of sequential requests 1289 * fragments for a short period. This allows merging of sequential requests
1309 * into single larger request. As the requests are moved from a per-task list to 1290 * into single larger request. As the requests are moved from a per-task list to
@@ -1676,94 +1657,6 @@ static inline void put_dev_sector(Sector p)
1676 put_page(p.v); 1657 put_page(p.v);
1677} 1658}
1678 1659
1679static inline bool __bvec_gap_to_prev(struct request_queue *q,
1680 struct bio_vec *bprv, unsigned int offset)
1681{
1682 return offset ||
1683 ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
1684}
1685
1686/*
1687 * Check if adding a bio_vec after bprv with offset would create a gap in
1688 * the SG list. Most drivers don't care about this, but some do.
1689 */
1690static inline bool bvec_gap_to_prev(struct request_queue *q,
1691 struct bio_vec *bprv, unsigned int offset)
1692{
1693 if (!queue_virt_boundary(q))
1694 return false;
1695 return __bvec_gap_to_prev(q, bprv, offset);
1696}
1697
1698/*
1699 * Check if the two bvecs from two bios can be merged to one segment.
1700 * If yes, no need to check gap between the two bios since the 1st bio
1701 * and the 1st bvec in the 2nd bio can be handled in one segment.
1702 */
1703static inline bool bios_segs_mergeable(struct request_queue *q,
1704 struct bio *prev, struct bio_vec *prev_last_bv,
1705 struct bio_vec *next_first_bv)
1706{
1707 if (!BIOVEC_PHYS_MERGEABLE(prev_last_bv, next_first_bv))
1708 return false;
1709 if (!BIOVEC_SEG_BOUNDARY(q, prev_last_bv, next_first_bv))
1710 return false;
1711 if (prev->bi_seg_back_size + next_first_bv->bv_len >
1712 queue_max_segment_size(q))
1713 return false;
1714 return true;
1715}
1716
1717static inline bool bio_will_gap(struct request_queue *q,
1718 struct request *prev_rq,
1719 struct bio *prev,
1720 struct bio *next)
1721{
1722 if (bio_has_data(prev) && queue_virt_boundary(q)) {
1723 struct bio_vec pb, nb;
1724
1725 /*
1726 * don't merge if the 1st bio starts with non-zero
1727 * offset, otherwise it is quite difficult to respect
1728 * sg gap limit. We work hard to merge a huge number of small
1729 * single bios in case of mkfs.
1730 */
1731 if (prev_rq)
1732 bio_get_first_bvec(prev_rq->bio, &pb);
1733 else
1734 bio_get_first_bvec(prev, &pb);
1735 if (pb.bv_offset)
1736 return true;
1737
1738 /*
1739 * We don't need to worry about the situation that the
1740 * merged segment ends in unaligned virt boundary:
1741 *
1742 * - if 'pb' ends aligned, the merged segment ends aligned
1743 * - if 'pb' ends unaligned, the next bio must include
1744 * one single bvec of 'nb', otherwise the 'nb' can't
1745 * merge with 'pb'
1746 */
1747 bio_get_last_bvec(prev, &pb);
1748 bio_get_first_bvec(next, &nb);
1749
1750 if (!bios_segs_mergeable(q, prev, &pb, &nb))
1751 return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
1752 }
1753
1754 return false;
1755}
1756
1757static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
1758{
1759 return bio_will_gap(req->q, req, req->biotail, bio);
1760}
1761
1762static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
1763{
1764 return bio_will_gap(req->q, NULL, bio, req->bio);
1765}
1766
1767int kblockd_schedule_work(struct work_struct *work); 1660int kblockd_schedule_work(struct work_struct *work);
1768int kblockd_schedule_work_on(int cpu, struct work_struct *work); 1661int kblockd_schedule_work_on(int cpu, struct work_struct *work);
1769int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay); 1662int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
@@ -1843,26 +1736,6 @@ queue_max_integrity_segments(struct request_queue *q)
1843 return q->limits.max_integrity_segments; 1736 return q->limits.max_integrity_segments;
1844} 1737}
1845 1738
1846static inline bool integrity_req_gap_back_merge(struct request *req,
1847 struct bio *next)
1848{
1849 struct bio_integrity_payload *bip = bio_integrity(req->bio);
1850 struct bio_integrity_payload *bip_next = bio_integrity(next);
1851
1852 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1853 bip_next->bip_vec[0].bv_offset);
1854}
1855
1856static inline bool integrity_req_gap_front_merge(struct request *req,
1857 struct bio *bio)
1858{
1859 struct bio_integrity_payload *bip = bio_integrity(bio);
1860 struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
1861
1862 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1863 bip_next->bip_vec[0].bv_offset);
1864}
1865
1866/** 1739/**
1867 * bio_integrity_intervals - Return number of integrity intervals for a bio 1740 * bio_integrity_intervals - Return number of integrity intervals for a bio
1868 * @bi: blk_integrity profile for device 1741 * @bi: blk_integrity profile for device
@@ -1947,17 +1820,6 @@ static inline bool blk_integrity_merge_bio(struct request_queue *rq,
1947 return true; 1820 return true;
1948} 1821}
1949 1822
1950static inline bool integrity_req_gap_back_merge(struct request *req,
1951 struct bio *next)
1952{
1953 return false;
1954}
1955static inline bool integrity_req_gap_front_merge(struct request *req,
1956 struct bio *bio)
1957{
1958 return false;
1959}
1960
1961static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, 1823static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
1962 unsigned int sectors) 1824 unsigned int sectors)
1963{ 1825{
diff --git a/include/linux/bvec.h b/include/linux/bvec.h
index fe7a22dd133b..02c73c6aa805 100644
--- a/include/linux/bvec.h
+++ b/include/linux/bvec.h
@@ -40,8 +40,6 @@ struct bvec_iter {
40 40
41 unsigned int bi_idx; /* current index into bvl_vec */ 41 unsigned int bi_idx; /* current index into bvl_vec */
42 42
43 unsigned int bi_done; /* number of bytes completed */
44
45 unsigned int bi_bvec_done; /* number of bytes completed in 43 unsigned int bi_bvec_done; /* number of bytes completed in
46 current bvec */ 44 current bvec */
47}; 45};
@@ -85,7 +83,6 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv,
85 bytes -= len; 83 bytes -= len;
86 iter->bi_size -= len; 84 iter->bi_size -= len;
87 iter->bi_bvec_done += len; 85 iter->bi_bvec_done += len;
88 iter->bi_done += len;
89 86
90 if (iter->bi_bvec_done == __bvec_iter_bvec(bv, *iter)->bv_len) { 87 if (iter->bi_bvec_done == __bvec_iter_bvec(bv, *iter)->bv_len) {
91 iter->bi_bvec_done = 0; 88 iter->bi_bvec_done = 0;
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index 32c553556bbd..b8bcbdeb2eac 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -93,6 +93,8 @@ extern struct css_set init_css_set;
93 93
94bool css_has_online_children(struct cgroup_subsys_state *css); 94bool css_has_online_children(struct cgroup_subsys_state *css);
95struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss); 95struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss);
96struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgroup,
97 struct cgroup_subsys *ss);
96struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgroup, 98struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgroup,
97 struct cgroup_subsys *ss); 99 struct cgroup_subsys *ss);
98struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry, 100struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry,
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index a02deea30185..015bb59c0331 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -111,7 +111,7 @@ struct elevator_mq_ops {
111 void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); 111 void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool);
112 struct request *(*dispatch_request)(struct blk_mq_hw_ctx *); 112 struct request *(*dispatch_request)(struct blk_mq_hw_ctx *);
113 bool (*has_work)(struct blk_mq_hw_ctx *); 113 bool (*has_work)(struct blk_mq_hw_ctx *);
114 void (*completed_request)(struct request *); 114 void (*completed_request)(struct request *, u64);
115 void (*started_request)(struct request *); 115 void (*started_request)(struct request *);
116 void (*requeue_request)(struct request *); 116 void (*requeue_request)(struct request *);
117 struct request *(*former_request)(struct request_queue *, struct request *); 117 struct request *(*former_request)(struct request_queue *, struct request *);
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 25c08c6c7f99..70fc838e6773 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -402,10 +402,11 @@ static inline void free_part_info(struct hd_struct *part)
402extern void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part); 402extern void part_round_stats(struct request_queue *q, int cpu, struct hd_struct *part);
403 403
404/* block/genhd.c */ 404/* block/genhd.c */
405extern void device_add_disk(struct device *parent, struct gendisk *disk); 405extern void device_add_disk(struct device *parent, struct gendisk *disk,
406 const struct attribute_group **groups);
406static inline void add_disk(struct gendisk *disk) 407static inline void add_disk(struct gendisk *disk)
407{ 408{
408 device_add_disk(NULL, disk); 409 device_add_disk(NULL, disk, NULL);
409} 410}
410extern void device_add_disk_no_queue_reg(struct device *parent, struct gendisk *disk); 411extern void device_add_disk_no_queue_reg(struct device *parent, struct gendisk *disk);
411static inline void add_disk_no_queue_reg(struct gendisk *disk) 412static inline void add_disk_no_queue_reg(struct gendisk *disk)
diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h
index e9e0d1c7eaf5..2fdeac1a420d 100644
--- a/include/linux/lightnvm.h
+++ b/include/linux/lightnvm.h
@@ -86,8 +86,8 @@ struct nvm_chk_meta;
86typedef int (nvm_id_fn)(struct nvm_dev *); 86typedef int (nvm_id_fn)(struct nvm_dev *);
87typedef int (nvm_op_bb_tbl_fn)(struct nvm_dev *, struct ppa_addr, u8 *); 87typedef int (nvm_op_bb_tbl_fn)(struct nvm_dev *, struct ppa_addr, u8 *);
88typedef int (nvm_op_set_bb_fn)(struct nvm_dev *, struct ppa_addr *, int, int); 88typedef int (nvm_op_set_bb_fn)(struct nvm_dev *, struct ppa_addr *, int, int);
89typedef int (nvm_get_chk_meta_fn)(struct nvm_dev *, struct nvm_chk_meta *, 89typedef int (nvm_get_chk_meta_fn)(struct nvm_dev *, sector_t, int,
90 sector_t, int); 90 struct nvm_chk_meta *);
91typedef int (nvm_submit_io_fn)(struct nvm_dev *, struct nvm_rq *); 91typedef int (nvm_submit_io_fn)(struct nvm_dev *, struct nvm_rq *);
92typedef int (nvm_submit_io_sync_fn)(struct nvm_dev *, struct nvm_rq *); 92typedef int (nvm_submit_io_sync_fn)(struct nvm_dev *, struct nvm_rq *);
93typedef void *(nvm_create_dma_pool_fn)(struct nvm_dev *, char *); 93typedef void *(nvm_create_dma_pool_fn)(struct nvm_dev *, char *);
@@ -305,6 +305,8 @@ struct nvm_rq {
305 u64 ppa_status; /* ppa media status */ 305 u64 ppa_status; /* ppa media status */
306 int error; 306 int error;
307 307
308 int is_seq; /* Sequential hint flag. 1.2 only */
309
308 void *private; 310 void *private;
309}; 311};
310 312
@@ -318,6 +320,11 @@ static inline void *nvm_rq_to_pdu(struct nvm_rq *rqdata)
318 return rqdata + 1; 320 return rqdata + 1;
319} 321}
320 322
323static inline struct ppa_addr *nvm_rq_to_ppa_list(struct nvm_rq *rqd)
324{
325 return (rqd->nr_ppas > 1) ? rqd->ppa_list : &rqd->ppa_addr;
326}
327
321enum { 328enum {
322 NVM_BLK_ST_FREE = 0x1, /* Free block */ 329 NVM_BLK_ST_FREE = 0x1, /* Free block */
323 NVM_BLK_ST_TGT = 0x2, /* Block in use by target */ 330 NVM_BLK_ST_TGT = 0x2, /* Block in use by target */
@@ -485,6 +492,144 @@ static inline struct ppa_addr dev_to_generic_addr(struct nvm_dev *dev,
485 return l; 492 return l;
486} 493}
487 494
495static inline u64 dev_to_chunk_addr(struct nvm_dev *dev, void *addrf,
496 struct ppa_addr p)
497{
498 struct nvm_geo *geo = &dev->geo;
499 u64 caddr;
500
501 if (geo->version == NVM_OCSSD_SPEC_12) {
502 struct nvm_addrf_12 *ppaf = (struct nvm_addrf_12 *)addrf;
503
504 caddr = (u64)p.g.pg << ppaf->pg_offset;
505 caddr |= (u64)p.g.pl << ppaf->pln_offset;
506 caddr |= (u64)p.g.sec << ppaf->sec_offset;
507 } else {
508 caddr = p.m.sec;
509 }
510
511 return caddr;
512}
513
514static inline struct ppa_addr nvm_ppa32_to_ppa64(struct nvm_dev *dev,
515 void *addrf, u32 ppa32)
516{
517 struct ppa_addr ppa64;
518
519 ppa64.ppa = 0;
520
521 if (ppa32 == -1) {
522 ppa64.ppa = ADDR_EMPTY;
523 } else if (ppa32 & (1U << 31)) {
524 ppa64.c.line = ppa32 & ((~0U) >> 1);
525 ppa64.c.is_cached = 1;
526 } else {
527 struct nvm_geo *geo = &dev->geo;
528
529 if (geo->version == NVM_OCSSD_SPEC_12) {
530 struct nvm_addrf_12 *ppaf = addrf;
531
532 ppa64.g.ch = (ppa32 & ppaf->ch_mask) >>
533 ppaf->ch_offset;
534 ppa64.g.lun = (ppa32 & ppaf->lun_mask) >>
535 ppaf->lun_offset;
536 ppa64.g.blk = (ppa32 & ppaf->blk_mask) >>
537 ppaf->blk_offset;
538 ppa64.g.pg = (ppa32 & ppaf->pg_mask) >>
539 ppaf->pg_offset;
540 ppa64.g.pl = (ppa32 & ppaf->pln_mask) >>
541 ppaf->pln_offset;
542 ppa64.g.sec = (ppa32 & ppaf->sec_mask) >>
543 ppaf->sec_offset;
544 } else {
545 struct nvm_addrf *lbaf = addrf;
546
547 ppa64.m.grp = (ppa32 & lbaf->ch_mask) >>
548 lbaf->ch_offset;
549 ppa64.m.pu = (ppa32 & lbaf->lun_mask) >>
550 lbaf->lun_offset;
551 ppa64.m.chk = (ppa32 & lbaf->chk_mask) >>
552 lbaf->chk_offset;
553 ppa64.m.sec = (ppa32 & lbaf->sec_mask) >>
554 lbaf->sec_offset;
555 }
556 }
557
558 return ppa64;
559}
560
561static inline u32 nvm_ppa64_to_ppa32(struct nvm_dev *dev,
562 void *addrf, struct ppa_addr ppa64)
563{
564 u32 ppa32 = 0;
565
566 if (ppa64.ppa == ADDR_EMPTY) {
567 ppa32 = ~0U;
568 } else if (ppa64.c.is_cached) {
569 ppa32 |= ppa64.c.line;
570 ppa32 |= 1U << 31;
571 } else {
572 struct nvm_geo *geo = &dev->geo;
573
574 if (geo->version == NVM_OCSSD_SPEC_12) {
575 struct nvm_addrf_12 *ppaf = addrf;
576
577 ppa32 |= ppa64.g.ch << ppaf->ch_offset;
578 ppa32 |= ppa64.g.lun << ppaf->lun_offset;
579 ppa32 |= ppa64.g.blk << ppaf->blk_offset;
580 ppa32 |= ppa64.g.pg << ppaf->pg_offset;
581 ppa32 |= ppa64.g.pl << ppaf->pln_offset;
582 ppa32 |= ppa64.g.sec << ppaf->sec_offset;
583 } else {
584 struct nvm_addrf *lbaf = addrf;
585
586 ppa32 |= ppa64.m.grp << lbaf->ch_offset;
587 ppa32 |= ppa64.m.pu << lbaf->lun_offset;
588 ppa32 |= ppa64.m.chk << lbaf->chk_offset;
589 ppa32 |= ppa64.m.sec << lbaf->sec_offset;
590 }
591 }
592
593 return ppa32;
594}
595
596static inline int nvm_next_ppa_in_chk(struct nvm_tgt_dev *dev,
597 struct ppa_addr *ppa)
598{
599 struct nvm_geo *geo = &dev->geo;
600 int last = 0;
601
602 if (geo->version == NVM_OCSSD_SPEC_12) {
603 int sec = ppa->g.sec;
604
605 sec++;
606 if (sec == geo->ws_min) {
607 int pg = ppa->g.pg;
608
609 sec = 0;
610 pg++;
611 if (pg == geo->num_pg) {
612 int pl = ppa->g.pl;
613
614 pg = 0;
615 pl++;
616 if (pl == geo->num_pln)
617 last = 1;
618
619 ppa->g.pl = pl;
620 }
621 ppa->g.pg = pg;
622 }
623 ppa->g.sec = sec;
624 } else {
625 ppa->m.sec++;
626 if (ppa->m.sec == geo->clba)
627 last = 1;
628 }
629
630 return last;
631}
632
488typedef blk_qc_t (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *); 633typedef blk_qc_t (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *);
489typedef sector_t (nvm_tgt_capacity_fn)(void *); 634typedef sector_t (nvm_tgt_capacity_fn)(void *);
490typedef void *(nvm_tgt_init_fn)(struct nvm_tgt_dev *, struct gendisk *, 635typedef void *(nvm_tgt_init_fn)(struct nvm_tgt_dev *, struct gendisk *,
@@ -493,9 +638,15 @@ typedef void (nvm_tgt_exit_fn)(void *, bool);
493typedef int (nvm_tgt_sysfs_init_fn)(struct gendisk *); 638typedef int (nvm_tgt_sysfs_init_fn)(struct gendisk *);
494typedef void (nvm_tgt_sysfs_exit_fn)(struct gendisk *); 639typedef void (nvm_tgt_sysfs_exit_fn)(struct gendisk *);
495 640
641enum {
642 NVM_TGT_F_DEV_L2P = 0,
643 NVM_TGT_F_HOST_L2P = 1 << 0,
644};
645
496struct nvm_tgt_type { 646struct nvm_tgt_type {
497 const char *name; 647 const char *name;
498 unsigned int version[3]; 648 unsigned int version[3];
649 int flags;
499 650
500 /* target entry points */ 651 /* target entry points */
501 nvm_tgt_make_rq_fn *make_rq; 652 nvm_tgt_make_rq_fn *make_rq;
@@ -524,18 +675,13 @@ extern struct nvm_dev *nvm_alloc_dev(int);
524extern int nvm_register(struct nvm_dev *); 675extern int nvm_register(struct nvm_dev *);
525extern void nvm_unregister(struct nvm_dev *); 676extern void nvm_unregister(struct nvm_dev *);
526 677
527 678extern int nvm_get_chunk_meta(struct nvm_tgt_dev *, struct ppa_addr,
528extern int nvm_get_chunk_meta(struct nvm_tgt_dev *tgt_dev, 679 int, struct nvm_chk_meta *);
529 struct nvm_chk_meta *meta, struct ppa_addr ppa, 680extern int nvm_set_chunk_meta(struct nvm_tgt_dev *, struct ppa_addr *,
530 int nchks);
531
532extern int nvm_set_tgt_bb_tbl(struct nvm_tgt_dev *, struct ppa_addr *,
533 int, int); 681 int, int);
534extern int nvm_submit_io(struct nvm_tgt_dev *, struct nvm_rq *); 682extern int nvm_submit_io(struct nvm_tgt_dev *, struct nvm_rq *);
535extern int nvm_submit_io_sync(struct nvm_tgt_dev *, struct nvm_rq *); 683extern int nvm_submit_io_sync(struct nvm_tgt_dev *, struct nvm_rq *);
536extern void nvm_end_io(struct nvm_rq *); 684extern void nvm_end_io(struct nvm_rq *);
537extern int nvm_bb_tbl_fold(struct nvm_dev *, u8 *, int);
538extern int nvm_get_tgt_bb_tbl(struct nvm_tgt_dev *, struct ppa_addr, u8 *);
539 685
540#else /* CONFIG_NVM */ 686#else /* CONFIG_NVM */
541struct nvm_dev_ops; 687struct nvm_dev_ops;
diff --git a/include/linux/mtd/blktrans.h b/include/linux/mtd/blktrans.h
index e93837f647de..1d3ade69d39a 100644
--- a/include/linux/mtd/blktrans.h
+++ b/include/linux/mtd/blktrans.h
@@ -23,7 +23,6 @@
23#include <linux/mutex.h> 23#include <linux/mutex.h>
24#include <linux/kref.h> 24#include <linux/kref.h>
25#include <linux/sysfs.h> 25#include <linux/sysfs.h>
26#include <linux/workqueue.h>
27 26
28struct hd_geometry; 27struct hd_geometry;
29struct mtd_info; 28struct mtd_info;
@@ -44,9 +43,9 @@ struct mtd_blktrans_dev {
44 struct kref ref; 43 struct kref ref;
45 struct gendisk *disk; 44 struct gendisk *disk;
46 struct attribute_group *disk_attributes; 45 struct attribute_group *disk_attributes;
47 struct workqueue_struct *wq;
48 struct work_struct work;
49 struct request_queue *rq; 46 struct request_queue *rq;
47 struct list_head rq_list;
48 struct blk_mq_tag_set *tag_set;
50 spinlock_t queue_lock; 49 spinlock_t queue_lock;
51 void *priv; 50 void *priv;
52 fmode_t file_mode; 51 fmode_t file_mode;
diff --git a/include/linux/nvme.h b/include/linux/nvme.h
index 68e91ef5494c..818dbe9331be 100644
--- a/include/linux/nvme.h
+++ b/include/linux/nvme.h
@@ -1241,6 +1241,7 @@ enum {
1241 NVME_SC_ANA_PERSISTENT_LOSS = 0x301, 1241 NVME_SC_ANA_PERSISTENT_LOSS = 0x301,
1242 NVME_SC_ANA_INACCESSIBLE = 0x302, 1242 NVME_SC_ANA_INACCESSIBLE = 0x302,
1243 NVME_SC_ANA_TRANSITION = 0x303, 1243 NVME_SC_ANA_TRANSITION = 0x303,
1244 NVME_SC_HOST_PATH_ERROR = 0x370,
1244 1245
1245 NVME_SC_DNR = 0x4000, 1246 NVME_SC_DNR = 0x4000,
1246}; 1247};
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index 009cdf3d65b6..b297cd1cd4f1 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -108,6 +108,7 @@ void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref);
108void percpu_ref_switch_to_percpu(struct percpu_ref *ref); 108void percpu_ref_switch_to_percpu(struct percpu_ref *ref);
109void percpu_ref_kill_and_confirm(struct percpu_ref *ref, 109void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
110 percpu_ref_func_t *confirm_kill); 110 percpu_ref_func_t *confirm_kill);
111void percpu_ref_resurrect(struct percpu_ref *ref);
111void percpu_ref_reinit(struct percpu_ref *ref); 112void percpu_ref_reinit(struct percpu_ref *ref);
112 113
113/** 114/**
diff --git a/include/linux/writeback.h b/include/linux/writeback.h
index fdfd04e348f6..738a0c24874f 100644
--- a/include/linux/writeback.h
+++ b/include/linux/writeback.h
@@ -246,7 +246,8 @@ static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc,
246 * 246 *
247 * @bio is a part of the writeback in progress controlled by @wbc. Perform 247 * @bio is a part of the writeback in progress controlled by @wbc. Perform
248 * writeback specific initialization. This is used to apply the cgroup 248 * writeback specific initialization. This is used to apply the cgroup
249 * writeback context. 249 * writeback context. Must be called after the bio has been associated with
250 * a device.
250 */ 251 */
251static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio) 252static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio)
252{ 253{
@@ -257,7 +258,7 @@ static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio)
257 * regular writeback instead of writing things out itself. 258 * regular writeback instead of writing things out itself.
258 */ 259 */
259 if (wbc->wb) 260 if (wbc->wb)
260 bio_associate_blkcg(bio, wbc->wb->blkcg_css); 261 bio_associate_blkg_from_css(bio, wbc->wb->blkcg_css);
261} 262}
262 263
263#else /* CONFIG_CGROUP_WRITEBACK */ 264#else /* CONFIG_CGROUP_WRITEBACK */
generated by cgit v1.2.2 (git 2.25.0) at 2026-05-30 19:03:55 -0400