diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2015-11-04 23:46:08 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2015-11-04 23:46:08 -0500 |
commit | effa04cc5a31b3f12cda6025ab93460f1f0e454e (patch) | |
tree | 4abea24fe619c2f7250d373af014c9693f350100 | |
parent | a9aa31cdc2a7be4a70b0ea24a451dfeb00ce0024 (diff) | |
parent | 5f436e5ef170e5d3301bf5777a3c7c048295db1c (diff) |
Merge branch 'for-4.4/lightnvm' of git://git.kernel.dk/linux-block
Pull lightnvm support from Jens Axboe:
"This adds support for lightnvm, and adds support to NVMe as well.
This is pretty exciting, in that it enables new and interesting use
cases for compatible flash devices. There's a LWN writeup about an
earlier posting here:
https://lwn.net/Articles/641247/
This has been underway for a while, and should be ready for merging at
this point"
* 'for-4.4/lightnvm' of git://git.kernel.dk/linux-block:
nvme: lightnvm: clean up a data type
lightnvm: refactor phys addrs type to u64
nvme: LightNVM support
rrpc: Round-robin sector target with cost-based gc
gennvm: Generic NVM manager
lightnvm: Support for Open-Channel SSDs
-rw-r--r-- | Documentation/ioctl/ioctl-number.txt | 1 | ||||
-rw-r--r-- | MAINTAINERS | 8 | ||||
-rw-r--r-- | drivers/Kconfig | 2 | ||||
-rw-r--r-- | drivers/Makefile | 1 | ||||
-rw-r--r-- | drivers/lightnvm/Kconfig | 42 | ||||
-rw-r--r-- | drivers/lightnvm/Makefile | 7 | ||||
-rw-r--r-- | drivers/lightnvm/core.c | 826 | ||||
-rw-r--r-- | drivers/lightnvm/gennvm.c | 485 | ||||
-rw-r--r-- | drivers/lightnvm/gennvm.h | 46 | ||||
-rw-r--r-- | drivers/lightnvm/rrpc.c | 1323 | ||||
-rw-r--r-- | drivers/lightnvm/rrpc.h | 239 | ||||
-rw-r--r-- | drivers/nvme/host/Makefile | 2 | ||||
-rw-r--r-- | drivers/nvme/host/lightnvm.c | 526 | ||||
-rw-r--r-- | drivers/nvme/host/nvme.h | 10 | ||||
-rw-r--r-- | drivers/nvme/host/pci.c | 39 | ||||
-rw-r--r-- | include/linux/lightnvm.h | 522 | ||||
-rw-r--r-- | include/uapi/linux/lightnvm.h | 130 |
17 files changed, 4197 insertions, 12 deletions
diff --git a/Documentation/ioctl/ioctl-number.txt b/Documentation/ioctl/ioctl-number.txt index df1b25eb8382..8a44d44cf901 100644 --- a/Documentation/ioctl/ioctl-number.txt +++ b/Documentation/ioctl/ioctl-number.txt | |||
@@ -149,6 +149,7 @@ Code Seq#(hex) Include File Comments | |||
149 | 'K' all linux/kd.h | 149 | 'K' all linux/kd.h |
150 | 'L' 00-1F linux/loop.h conflict! | 150 | 'L' 00-1F linux/loop.h conflict! |
151 | 'L' 10-1F drivers/scsi/mpt2sas/mpt2sas_ctl.h conflict! | 151 | 'L' 10-1F drivers/scsi/mpt2sas/mpt2sas_ctl.h conflict! |
152 | 'L' 20-2F linux/lightnvm.h | ||
152 | 'L' E0-FF linux/ppdd.h encrypted disk device driver | 153 | 'L' E0-FF linux/ppdd.h encrypted disk device driver |
153 | <http://linux01.gwdg.de/~alatham/ppdd.html> | 154 | <http://linux01.gwdg.de/~alatham/ppdd.html> |
154 | 'M' all linux/soundcard.h conflict! | 155 | 'M' all linux/soundcard.h conflict! |
diff --git a/MAINTAINERS b/MAINTAINERS index 249f96923642..ed6dccf2bfa0 100644 --- a/MAINTAINERS +++ b/MAINTAINERS | |||
@@ -6279,6 +6279,14 @@ F: drivers/nvdimm/pmem.c | |||
6279 | F: include/linux/pmem.h | 6279 | F: include/linux/pmem.h |
6280 | F: arch/*/include/asm/pmem.h | 6280 | F: arch/*/include/asm/pmem.h |
6281 | 6281 | ||
6282 | LIGHTNVM PLATFORM SUPPORT | ||
6283 | M: Matias Bjorling <mb@lightnvm.io> | ||
6284 | W: http://github/OpenChannelSSD | ||
6285 | S: Maintained | ||
6286 | F: drivers/lightnvm/ | ||
6287 | F: include/linux/lightnvm.h | ||
6288 | F: include/uapi/linux/lightnvm.h | ||
6289 | |||
6282 | LINUX FOR IBM pSERIES (RS/6000) | 6290 | LINUX FOR IBM pSERIES (RS/6000) |
6283 | M: Paul Mackerras <paulus@au.ibm.com> | 6291 | M: Paul Mackerras <paulus@au.ibm.com> |
6284 | W: http://www.ibm.com/linux/ltc/projects/ppc | 6292 | W: http://www.ibm.com/linux/ltc/projects/ppc |
diff --git a/drivers/Kconfig b/drivers/Kconfig index e69ec82ac80a..3a5ab4d5873d 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig | |||
@@ -44,6 +44,8 @@ source "drivers/net/Kconfig" | |||
44 | 44 | ||
45 | source "drivers/isdn/Kconfig" | 45 | source "drivers/isdn/Kconfig" |
46 | 46 | ||
47 | source "drivers/lightnvm/Kconfig" | ||
48 | |||
47 | # input before char - char/joystick depends on it. As does USB. | 49 | # input before char - char/joystick depends on it. As does USB. |
48 | 50 | ||
49 | source "drivers/input/Kconfig" | 51 | source "drivers/input/Kconfig" |
diff --git a/drivers/Makefile b/drivers/Makefile index 42f9dd5f07c8..7f1b7c5a1cfd 100644 --- a/drivers/Makefile +++ b/drivers/Makefile | |||
@@ -70,6 +70,7 @@ obj-$(CONFIG_NUBUS) += nubus/ | |||
70 | obj-y += macintosh/ | 70 | obj-y += macintosh/ |
71 | obj-$(CONFIG_IDE) += ide/ | 71 | obj-$(CONFIG_IDE) += ide/ |
72 | obj-$(CONFIG_SCSI) += scsi/ | 72 | obj-$(CONFIG_SCSI) += scsi/ |
73 | obj-$(CONFIG_NVM) += lightnvm/ | ||
73 | obj-y += nvme/ | 74 | obj-y += nvme/ |
74 | obj-$(CONFIG_ATA) += ata/ | 75 | obj-$(CONFIG_ATA) += ata/ |
75 | obj-$(CONFIG_TARGET_CORE) += target/ | 76 | obj-$(CONFIG_TARGET_CORE) += target/ |
diff --git a/drivers/lightnvm/Kconfig b/drivers/lightnvm/Kconfig new file mode 100644 index 000000000000..a16bf56d3f28 --- /dev/null +++ b/drivers/lightnvm/Kconfig | |||
@@ -0,0 +1,42 @@ | |||
1 | # | ||
2 | # Open-Channel SSD NVM configuration | ||
3 | # | ||
4 | |||
5 | menuconfig NVM | ||
6 | bool "Open-Channel SSD target support" | ||
7 | depends on BLOCK | ||
8 | help | ||
9 | Say Y here to get to enable Open-channel SSDs. | ||
10 | |||
11 | Open-Channel SSDs implement a set of extension to SSDs, that | ||
12 | exposes direct access to the underlying non-volatile memory. | ||
13 | |||
14 | If you say N, all options in this submenu will be skipped and disabled | ||
15 | only do this if you know what you are doing. | ||
16 | |||
17 | if NVM | ||
18 | |||
19 | config NVM_DEBUG | ||
20 | bool "Open-Channel SSD debugging support" | ||
21 | ---help--- | ||
22 | Exposes a debug management interface to create/remove targets at: | ||
23 | |||
24 | /sys/module/lnvm/parameters/configure_debug | ||
25 | |||
26 | It is required to create/remove targets without IOCTLs. | ||
27 | |||
28 | config NVM_GENNVM | ||
29 | tristate "Generic NVM manager for Open-Channel SSDs" | ||
30 | ---help--- | ||
31 | NVM media manager for Open-Channel SSDs that offload management | ||
32 | functionality to device, while keeping data placement and garbage | ||
33 | collection decisions on the host. | ||
34 | |||
35 | config NVM_RRPC | ||
36 | tristate "Round-robin Hybrid Open-Channel SSD target" | ||
37 | ---help--- | ||
38 | Allows an open-channel SSD to be exposed as a block device to the | ||
39 | host. The target is implemented using a linear mapping table and | ||
40 | cost-based garbage collection. It is optimized for 4K IO sizes. | ||
41 | |||
42 | endif # NVM | ||
diff --git a/drivers/lightnvm/Makefile b/drivers/lightnvm/Makefile new file mode 100644 index 000000000000..7e0f42acb737 --- /dev/null +++ b/drivers/lightnvm/Makefile | |||
@@ -0,0 +1,7 @@ | |||
1 | # | ||
2 | # Makefile for Open-Channel SSDs. | ||
3 | # | ||
4 | |||
5 | obj-$(CONFIG_NVM) := core.o | ||
6 | obj-$(CONFIG_NVM_GENNVM) += gennvm.o | ||
7 | obj-$(CONFIG_NVM_RRPC) += rrpc.o | ||
diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c new file mode 100644 index 000000000000..f659e605a406 --- /dev/null +++ b/drivers/lightnvm/core.c | |||
@@ -0,0 +1,826 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2015 IT University of Copenhagen. All rights reserved. | ||
3 | * Initial release: Matias Bjorling <m@bjorling.me> | ||
4 | * | ||
5 | * This program is free software; you can redistribute it and/or | ||
6 | * modify it under the terms of the GNU General Public License version | ||
7 | * 2 as published by the Free Software Foundation. | ||
8 | * | ||
9 | * This program is distributed in the hope that it will be useful, but | ||
10 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
12 | * General Public License for more details. | ||
13 | * | ||
14 | * You should have received a copy of the GNU General Public License | ||
15 | * along with this program; see the file COPYING. If not, write to | ||
16 | * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, | ||
17 | * USA. | ||
18 | * | ||
19 | */ | ||
20 | |||
21 | #include <linux/blkdev.h> | ||
22 | #include <linux/blk-mq.h> | ||
23 | #include <linux/list.h> | ||
24 | #include <linux/types.h> | ||
25 | #include <linux/sem.h> | ||
26 | #include <linux/bitmap.h> | ||
27 | #include <linux/module.h> | ||
28 | #include <linux/miscdevice.h> | ||
29 | #include <linux/lightnvm.h> | ||
30 | #include <uapi/linux/lightnvm.h> | ||
31 | |||
32 | static LIST_HEAD(nvm_targets); | ||
33 | static LIST_HEAD(nvm_mgrs); | ||
34 | static LIST_HEAD(nvm_devices); | ||
35 | static DECLARE_RWSEM(nvm_lock); | ||
36 | |||
37 | static struct nvm_tgt_type *nvm_find_target_type(const char *name) | ||
38 | { | ||
39 | struct nvm_tgt_type *tt; | ||
40 | |||
41 | list_for_each_entry(tt, &nvm_targets, list) | ||
42 | if (!strcmp(name, tt->name)) | ||
43 | return tt; | ||
44 | |||
45 | return NULL; | ||
46 | } | ||
47 | |||
48 | int nvm_register_target(struct nvm_tgt_type *tt) | ||
49 | { | ||
50 | int ret = 0; | ||
51 | |||
52 | down_write(&nvm_lock); | ||
53 | if (nvm_find_target_type(tt->name)) | ||
54 | ret = -EEXIST; | ||
55 | else | ||
56 | list_add(&tt->list, &nvm_targets); | ||
57 | up_write(&nvm_lock); | ||
58 | |||
59 | return ret; | ||
60 | } | ||
61 | EXPORT_SYMBOL(nvm_register_target); | ||
62 | |||
63 | void nvm_unregister_target(struct nvm_tgt_type *tt) | ||
64 | { | ||
65 | if (!tt) | ||
66 | return; | ||
67 | |||
68 | down_write(&nvm_lock); | ||
69 | list_del(&tt->list); | ||
70 | up_write(&nvm_lock); | ||
71 | } | ||
72 | EXPORT_SYMBOL(nvm_unregister_target); | ||
73 | |||
74 | void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags, | ||
75 | dma_addr_t *dma_handler) | ||
76 | { | ||
77 | return dev->ops->dev_dma_alloc(dev->q, dev->ppalist_pool, mem_flags, | ||
78 | dma_handler); | ||
79 | } | ||
80 | EXPORT_SYMBOL(nvm_dev_dma_alloc); | ||
81 | |||
82 | void nvm_dev_dma_free(struct nvm_dev *dev, void *ppa_list, | ||
83 | dma_addr_t dma_handler) | ||
84 | { | ||
85 | dev->ops->dev_dma_free(dev->ppalist_pool, ppa_list, dma_handler); | ||
86 | } | ||
87 | EXPORT_SYMBOL(nvm_dev_dma_free); | ||
88 | |||
89 | static struct nvmm_type *nvm_find_mgr_type(const char *name) | ||
90 | { | ||
91 | struct nvmm_type *mt; | ||
92 | |||
93 | list_for_each_entry(mt, &nvm_mgrs, list) | ||
94 | if (!strcmp(name, mt->name)) | ||
95 | return mt; | ||
96 | |||
97 | return NULL; | ||
98 | } | ||
99 | |||
100 | int nvm_register_mgr(struct nvmm_type *mt) | ||
101 | { | ||
102 | int ret = 0; | ||
103 | |||
104 | down_write(&nvm_lock); | ||
105 | if (nvm_find_mgr_type(mt->name)) | ||
106 | ret = -EEXIST; | ||
107 | else | ||
108 | list_add(&mt->list, &nvm_mgrs); | ||
109 | up_write(&nvm_lock); | ||
110 | |||
111 | return ret; | ||
112 | } | ||
113 | EXPORT_SYMBOL(nvm_register_mgr); | ||
114 | |||
115 | void nvm_unregister_mgr(struct nvmm_type *mt) | ||
116 | { | ||
117 | if (!mt) | ||
118 | return; | ||
119 | |||
120 | down_write(&nvm_lock); | ||
121 | list_del(&mt->list); | ||
122 | up_write(&nvm_lock); | ||
123 | } | ||
124 | EXPORT_SYMBOL(nvm_unregister_mgr); | ||
125 | |||
126 | static struct nvm_dev *nvm_find_nvm_dev(const char *name) | ||
127 | { | ||
128 | struct nvm_dev *dev; | ||
129 | |||
130 | list_for_each_entry(dev, &nvm_devices, devices) | ||
131 | if (!strcmp(name, dev->name)) | ||
132 | return dev; | ||
133 | |||
134 | return NULL; | ||
135 | } | ||
136 | |||
137 | struct nvm_block *nvm_get_blk(struct nvm_dev *dev, struct nvm_lun *lun, | ||
138 | unsigned long flags) | ||
139 | { | ||
140 | return dev->mt->get_blk(dev, lun, flags); | ||
141 | } | ||
142 | EXPORT_SYMBOL(nvm_get_blk); | ||
143 | |||
144 | /* Assumes that all valid pages have already been moved on release to bm */ | ||
145 | void nvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk) | ||
146 | { | ||
147 | return dev->mt->put_blk(dev, blk); | ||
148 | } | ||
149 | EXPORT_SYMBOL(nvm_put_blk); | ||
150 | |||
151 | int nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) | ||
152 | { | ||
153 | return dev->mt->submit_io(dev, rqd); | ||
154 | } | ||
155 | EXPORT_SYMBOL(nvm_submit_io); | ||
156 | |||
157 | int nvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk) | ||
158 | { | ||
159 | return dev->mt->erase_blk(dev, blk, 0); | ||
160 | } | ||
161 | EXPORT_SYMBOL(nvm_erase_blk); | ||
162 | |||
163 | static void nvm_core_free(struct nvm_dev *dev) | ||
164 | { | ||
165 | kfree(dev); | ||
166 | } | ||
167 | |||
168 | static int nvm_core_init(struct nvm_dev *dev) | ||
169 | { | ||
170 | struct nvm_id *id = &dev->identity; | ||
171 | struct nvm_id_group *grp = &id->groups[0]; | ||
172 | |||
173 | /* device values */ | ||
174 | dev->nr_chnls = grp->num_ch; | ||
175 | dev->luns_per_chnl = grp->num_lun; | ||
176 | dev->pgs_per_blk = grp->num_pg; | ||
177 | dev->blks_per_lun = grp->num_blk; | ||
178 | dev->nr_planes = grp->num_pln; | ||
179 | dev->sec_size = grp->csecs; | ||
180 | dev->oob_size = grp->sos; | ||
181 | dev->sec_per_pg = grp->fpg_sz / grp->csecs; | ||
182 | dev->addr_mode = id->ppat; | ||
183 | dev->addr_format = id->ppaf; | ||
184 | |||
185 | dev->plane_mode = NVM_PLANE_SINGLE; | ||
186 | dev->max_rq_size = dev->ops->max_phys_sect * dev->sec_size; | ||
187 | |||
188 | if (grp->mpos & 0x020202) | ||
189 | dev->plane_mode = NVM_PLANE_DOUBLE; | ||
190 | if (grp->mpos & 0x040404) | ||
191 | dev->plane_mode = NVM_PLANE_QUAD; | ||
192 | |||
193 | /* calculated values */ | ||
194 | dev->sec_per_pl = dev->sec_per_pg * dev->nr_planes; | ||
195 | dev->sec_per_blk = dev->sec_per_pl * dev->pgs_per_blk; | ||
196 | dev->sec_per_lun = dev->sec_per_blk * dev->blks_per_lun; | ||
197 | dev->nr_luns = dev->luns_per_chnl * dev->nr_chnls; | ||
198 | |||
199 | dev->total_blocks = dev->nr_planes * | ||
200 | dev->blks_per_lun * | ||
201 | dev->luns_per_chnl * | ||
202 | dev->nr_chnls; | ||
203 | dev->total_pages = dev->total_blocks * dev->pgs_per_blk; | ||
204 | INIT_LIST_HEAD(&dev->online_targets); | ||
205 | |||
206 | return 0; | ||
207 | } | ||
208 | |||
209 | static void nvm_free(struct nvm_dev *dev) | ||
210 | { | ||
211 | if (!dev) | ||
212 | return; | ||
213 | |||
214 | if (dev->mt) | ||
215 | dev->mt->unregister_mgr(dev); | ||
216 | |||
217 | nvm_core_free(dev); | ||
218 | } | ||
219 | |||
220 | static int nvm_init(struct nvm_dev *dev) | ||
221 | { | ||
222 | struct nvmm_type *mt; | ||
223 | int ret = 0; | ||
224 | |||
225 | if (!dev->q || !dev->ops) | ||
226 | return -EINVAL; | ||
227 | |||
228 | if (dev->ops->identity(dev->q, &dev->identity)) { | ||
229 | pr_err("nvm: device could not be identified\n"); | ||
230 | ret = -EINVAL; | ||
231 | goto err; | ||
232 | } | ||
233 | |||
234 | pr_debug("nvm: ver:%x nvm_vendor:%x groups:%u\n", | ||
235 | dev->identity.ver_id, dev->identity.vmnt, | ||
236 | dev->identity.cgrps); | ||
237 | |||
238 | if (dev->identity.ver_id != 1) { | ||
239 | pr_err("nvm: device not supported by kernel."); | ||
240 | goto err; | ||
241 | } | ||
242 | |||
243 | if (dev->identity.cgrps != 1) { | ||
244 | pr_err("nvm: only one group configuration supported."); | ||
245 | goto err; | ||
246 | } | ||
247 | |||
248 | ret = nvm_core_init(dev); | ||
249 | if (ret) { | ||
250 | pr_err("nvm: could not initialize core structures.\n"); | ||
251 | goto err; | ||
252 | } | ||
253 | |||
254 | /* register with device with a supported manager */ | ||
255 | list_for_each_entry(mt, &nvm_mgrs, list) { | ||
256 | ret = mt->register_mgr(dev); | ||
257 | if (ret < 0) | ||
258 | goto err; /* initialization failed */ | ||
259 | if (ret > 0) { | ||
260 | dev->mt = mt; | ||
261 | break; /* successfully initialized */ | ||
262 | } | ||
263 | } | ||
264 | |||
265 | if (!ret) { | ||
266 | pr_info("nvm: no compatible manager found.\n"); | ||
267 | return 0; | ||
268 | } | ||
269 | |||
270 | pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n", | ||
271 | dev->name, dev->sec_per_pg, dev->nr_planes, | ||
272 | dev->pgs_per_blk, dev->blks_per_lun, dev->nr_luns, | ||
273 | dev->nr_chnls); | ||
274 | return 0; | ||
275 | err: | ||
276 | nvm_free(dev); | ||
277 | pr_err("nvm: failed to initialize nvm\n"); | ||
278 | return ret; | ||
279 | } | ||
280 | |||
281 | static void nvm_exit(struct nvm_dev *dev) | ||
282 | { | ||
283 | if (dev->ppalist_pool) | ||
284 | dev->ops->destroy_dma_pool(dev->ppalist_pool); | ||
285 | nvm_free(dev); | ||
286 | |||
287 | pr_info("nvm: successfully unloaded\n"); | ||
288 | } | ||
289 | |||
290 | int nvm_register(struct request_queue *q, char *disk_name, | ||
291 | struct nvm_dev_ops *ops) | ||
292 | { | ||
293 | struct nvm_dev *dev; | ||
294 | int ret; | ||
295 | |||
296 | if (!ops->identity) | ||
297 | return -EINVAL; | ||
298 | |||
299 | dev = kzalloc(sizeof(struct nvm_dev), GFP_KERNEL); | ||
300 | if (!dev) | ||
301 | return -ENOMEM; | ||
302 | |||
303 | dev->q = q; | ||
304 | dev->ops = ops; | ||
305 | strncpy(dev->name, disk_name, DISK_NAME_LEN); | ||
306 | |||
307 | ret = nvm_init(dev); | ||
308 | if (ret) | ||
309 | goto err_init; | ||
310 | |||
311 | down_write(&nvm_lock); | ||
312 | list_add(&dev->devices, &nvm_devices); | ||
313 | up_write(&nvm_lock); | ||
314 | |||
315 | if (dev->ops->max_phys_sect > 1) { | ||
316 | dev->ppalist_pool = dev->ops->create_dma_pool(dev->q, | ||
317 | "ppalist"); | ||
318 | if (!dev->ppalist_pool) { | ||
319 | pr_err("nvm: could not create ppa pool\n"); | ||
320 | return -ENOMEM; | ||
321 | } | ||
322 | } else if (dev->ops->max_phys_sect > 256) { | ||
323 | pr_info("nvm: max sectors supported is 256.\n"); | ||
324 | return -EINVAL; | ||
325 | } | ||
326 | |||
327 | return 0; | ||
328 | err_init: | ||
329 | kfree(dev); | ||
330 | return ret; | ||
331 | } | ||
332 | EXPORT_SYMBOL(nvm_register); | ||
333 | |||
334 | void nvm_unregister(char *disk_name) | ||
335 | { | ||
336 | struct nvm_dev *dev = nvm_find_nvm_dev(disk_name); | ||
337 | |||
338 | if (!dev) { | ||
339 | pr_err("nvm: could not find device %s to unregister\n", | ||
340 | disk_name); | ||
341 | return; | ||
342 | } | ||
343 | |||
344 | nvm_exit(dev); | ||
345 | |||
346 | down_write(&nvm_lock); | ||
347 | list_del(&dev->devices); | ||
348 | up_write(&nvm_lock); | ||
349 | } | ||
350 | EXPORT_SYMBOL(nvm_unregister); | ||
351 | |||
352 | static const struct block_device_operations nvm_fops = { | ||
353 | .owner = THIS_MODULE, | ||
354 | }; | ||
355 | |||
356 | static int nvm_create_target(struct nvm_dev *dev, | ||
357 | struct nvm_ioctl_create *create) | ||
358 | { | ||
359 | struct nvm_ioctl_create_simple *s = &create->conf.s; | ||
360 | struct request_queue *tqueue; | ||
361 | struct nvmm_type *mt; | ||
362 | struct gendisk *tdisk; | ||
363 | struct nvm_tgt_type *tt; | ||
364 | struct nvm_target *t; | ||
365 | void *targetdata; | ||
366 | int ret = 0; | ||
367 | |||
368 | if (!dev->mt) { | ||
369 | /* register with device with a supported NVM manager */ | ||
370 | list_for_each_entry(mt, &nvm_mgrs, list) { | ||
371 | ret = mt->register_mgr(dev); | ||
372 | if (ret < 0) | ||
373 | return ret; /* initialization failed */ | ||
374 | if (ret > 0) { | ||
375 | dev->mt = mt; | ||
376 | break; /* successfully initialized */ | ||
377 | } | ||
378 | } | ||
379 | |||
380 | if (!ret) { | ||
381 | pr_info("nvm: no compatible nvm manager found.\n"); | ||
382 | return -ENODEV; | ||
383 | } | ||
384 | } | ||
385 | |||
386 | tt = nvm_find_target_type(create->tgttype); | ||
387 | if (!tt) { | ||
388 | pr_err("nvm: target type %s not found\n", create->tgttype); | ||
389 | return -EINVAL; | ||
390 | } | ||
391 | |||
392 | down_write(&nvm_lock); | ||
393 | list_for_each_entry(t, &dev->online_targets, list) { | ||
394 | if (!strcmp(create->tgtname, t->disk->disk_name)) { | ||
395 | pr_err("nvm: target name already exists.\n"); | ||
396 | up_write(&nvm_lock); | ||
397 | return -EINVAL; | ||
398 | } | ||
399 | } | ||
400 | up_write(&nvm_lock); | ||
401 | |||
402 | t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL); | ||
403 | if (!t) | ||
404 | return -ENOMEM; | ||
405 | |||
406 | tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node); | ||
407 | if (!tqueue) | ||
408 | goto err_t; | ||
409 | blk_queue_make_request(tqueue, tt->make_rq); | ||
410 | |||
411 | tdisk = alloc_disk(0); | ||
412 | if (!tdisk) | ||
413 | goto err_queue; | ||
414 | |||
415 | sprintf(tdisk->disk_name, "%s", create->tgtname); | ||
416 | tdisk->flags = GENHD_FL_EXT_DEVT; | ||
417 | tdisk->major = 0; | ||
418 | tdisk->first_minor = 0; | ||
419 | tdisk->fops = &nvm_fops; | ||
420 | tdisk->queue = tqueue; | ||
421 | |||
422 | targetdata = tt->init(dev, tdisk, s->lun_begin, s->lun_end); | ||
423 | if (IS_ERR(targetdata)) | ||
424 | goto err_init; | ||
425 | |||
426 | tdisk->private_data = targetdata; | ||
427 | tqueue->queuedata = targetdata; | ||
428 | |||
429 | blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect); | ||
430 | |||
431 | set_capacity(tdisk, tt->capacity(targetdata)); | ||
432 | add_disk(tdisk); | ||
433 | |||
434 | t->type = tt; | ||
435 | t->disk = tdisk; | ||
436 | |||
437 | down_write(&nvm_lock); | ||
438 | list_add_tail(&t->list, &dev->online_targets); | ||
439 | up_write(&nvm_lock); | ||
440 | |||
441 | return 0; | ||
442 | err_init: | ||
443 | put_disk(tdisk); | ||
444 | err_queue: | ||
445 | blk_cleanup_queue(tqueue); | ||
446 | err_t: | ||
447 | kfree(t); | ||
448 | return -ENOMEM; | ||
449 | } | ||
450 | |||
451 | static void nvm_remove_target(struct nvm_target *t) | ||
452 | { | ||
453 | struct nvm_tgt_type *tt = t->type; | ||
454 | struct gendisk *tdisk = t->disk; | ||
455 | struct request_queue *q = tdisk->queue; | ||
456 | |||
457 | lockdep_assert_held(&nvm_lock); | ||
458 | |||
459 | del_gendisk(tdisk); | ||
460 | if (tt->exit) | ||
461 | tt->exit(tdisk->private_data); | ||
462 | |||
463 | blk_cleanup_queue(q); | ||
464 | |||
465 | put_disk(tdisk); | ||
466 | |||
467 | list_del(&t->list); | ||
468 | kfree(t); | ||
469 | } | ||
470 | |||
471 | static int __nvm_configure_create(struct nvm_ioctl_create *create) | ||
472 | { | ||
473 | struct nvm_dev *dev; | ||
474 | struct nvm_ioctl_create_simple *s; | ||
475 | |||
476 | dev = nvm_find_nvm_dev(create->dev); | ||
477 | if (!dev) { | ||
478 | pr_err("nvm: device not found\n"); | ||
479 | return -EINVAL; | ||
480 | } | ||
481 | |||
482 | if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) { | ||
483 | pr_err("nvm: config type not valid\n"); | ||
484 | return -EINVAL; | ||
485 | } | ||
486 | s = &create->conf.s; | ||
487 | |||
488 | if (s->lun_begin > s->lun_end || s->lun_end > dev->nr_luns) { | ||
489 | pr_err("nvm: lun out of bound (%u:%u > %u)\n", | ||
490 | s->lun_begin, s->lun_end, dev->nr_luns); | ||
491 | return -EINVAL; | ||
492 | } | ||
493 | |||
494 | return nvm_create_target(dev, create); | ||
495 | } | ||
496 | |||
497 | static int __nvm_configure_remove(struct nvm_ioctl_remove *remove) | ||
498 | { | ||
499 | struct nvm_target *t = NULL; | ||
500 | struct nvm_dev *dev; | ||
501 | int ret = -1; | ||
502 | |||
503 | down_write(&nvm_lock); | ||
504 | list_for_each_entry(dev, &nvm_devices, devices) | ||
505 | list_for_each_entry(t, &dev->online_targets, list) { | ||
506 | if (!strcmp(remove->tgtname, t->disk->disk_name)) { | ||
507 | nvm_remove_target(t); | ||
508 | ret = 0; | ||
509 | break; | ||
510 | } | ||
511 | } | ||
512 | up_write(&nvm_lock); | ||
513 | |||
514 | if (ret) { | ||
515 | pr_err("nvm: target \"%s\" doesn't exist.\n", remove->tgtname); | ||
516 | return -EINVAL; | ||
517 | } | ||
518 | |||
519 | return 0; | ||
520 | } | ||
521 | |||
522 | #ifdef CONFIG_NVM_DEBUG | ||
523 | static int nvm_configure_show(const char *val) | ||
524 | { | ||
525 | struct nvm_dev *dev; | ||
526 | char opcode, devname[DISK_NAME_LEN]; | ||
527 | int ret; | ||
528 | |||
529 | ret = sscanf(val, "%c %32s", &opcode, devname); | ||
530 | if (ret != 2) { | ||
531 | pr_err("nvm: invalid command. Use \"opcode devicename\".\n"); | ||
532 | return -EINVAL; | ||
533 | } | ||
534 | |||
535 | dev = nvm_find_nvm_dev(devname); | ||
536 | if (!dev) { | ||
537 | pr_err("nvm: device not found\n"); | ||
538 | return -EINVAL; | ||
539 | } | ||
540 | |||
541 | if (!dev->mt) | ||
542 | return 0; | ||
543 | |||
544 | dev->mt->free_blocks_print(dev); | ||
545 | |||
546 | return 0; | ||
547 | } | ||
548 | |||
549 | static int nvm_configure_remove(const char *val) | ||
550 | { | ||
551 | struct nvm_ioctl_remove remove; | ||
552 | char opcode; | ||
553 | int ret; | ||
554 | |||
555 | ret = sscanf(val, "%c %256s", &opcode, remove.tgtname); | ||
556 | if (ret != 2) { | ||
557 | pr_err("nvm: invalid command. Use \"d targetname\".\n"); | ||
558 | return -EINVAL; | ||
559 | } | ||
560 | |||
561 | remove.flags = 0; | ||
562 | |||
563 | return __nvm_configure_remove(&remove); | ||
564 | } | ||
565 | |||
566 | static int nvm_configure_create(const char *val) | ||
567 | { | ||
568 | struct nvm_ioctl_create create; | ||
569 | char opcode; | ||
570 | int lun_begin, lun_end, ret; | ||
571 | |||
572 | ret = sscanf(val, "%c %256s %256s %48s %u:%u", &opcode, create.dev, | ||
573 | create.tgtname, create.tgttype, | ||
574 | &lun_begin, &lun_end); | ||
575 | if (ret != 6) { | ||
576 | pr_err("nvm: invalid command. Use \"opcode device name tgttype lun_begin:lun_end\".\n"); | ||
577 | return -EINVAL; | ||
578 | } | ||
579 | |||
580 | create.flags = 0; | ||
581 | create.conf.type = NVM_CONFIG_TYPE_SIMPLE; | ||
582 | create.conf.s.lun_begin = lun_begin; | ||
583 | create.conf.s.lun_end = lun_end; | ||
584 | |||
585 | return __nvm_configure_create(&create); | ||
586 | } | ||
587 | |||
588 | |||
589 | /* Exposes administrative interface through /sys/module/lnvm/configure_by_str */ | ||
590 | static int nvm_configure_by_str_event(const char *val, | ||
591 | const struct kernel_param *kp) | ||
592 | { | ||
593 | char opcode; | ||
594 | int ret; | ||
595 | |||
596 | ret = sscanf(val, "%c", &opcode); | ||
597 | if (ret != 1) { | ||
598 | pr_err("nvm: string must have the format of \"cmd ...\"\n"); | ||
599 | return -EINVAL; | ||
600 | } | ||
601 | |||
602 | switch (opcode) { | ||
603 | case 'a': | ||
604 | return nvm_configure_create(val); | ||
605 | case 'd': | ||
606 | return nvm_configure_remove(val); | ||
607 | case 's': | ||
608 | return nvm_configure_show(val); | ||
609 | default: | ||
610 | pr_err("nvm: invalid command\n"); | ||
611 | return -EINVAL; | ||
612 | } | ||
613 | |||
614 | return 0; | ||
615 | } | ||
616 | |||
617 | static int nvm_configure_get(char *buf, const struct kernel_param *kp) | ||
618 | { | ||
619 | int sz = 0; | ||
620 | char *buf_start = buf; | ||
621 | struct nvm_dev *dev; | ||
622 | |||
623 | buf += sprintf(buf, "available devices:\n"); | ||
624 | down_write(&nvm_lock); | ||
625 | list_for_each_entry(dev, &nvm_devices, devices) { | ||
626 | if (sz > 4095 - DISK_NAME_LEN) | ||
627 | break; | ||
628 | buf += sprintf(buf, " %32s\n", dev->name); | ||
629 | } | ||
630 | up_write(&nvm_lock); | ||
631 | |||
632 | return buf - buf_start - 1; | ||
633 | } | ||
634 | |||
635 | static const struct kernel_param_ops nvm_configure_by_str_event_param_ops = { | ||
636 | .set = nvm_configure_by_str_event, | ||
637 | .get = nvm_configure_get, | ||
638 | }; | ||
639 | |||
640 | #undef MODULE_PARAM_PREFIX | ||
641 | #define MODULE_PARAM_PREFIX "lnvm." | ||
642 | |||
643 | module_param_cb(configure_debug, &nvm_configure_by_str_event_param_ops, NULL, | ||
644 | 0644); | ||
645 | |||
646 | #endif /* CONFIG_NVM_DEBUG */ | ||
647 | |||
648 | static long nvm_ioctl_info(struct file *file, void __user *arg) | ||
649 | { | ||
650 | struct nvm_ioctl_info *info; | ||
651 | struct nvm_tgt_type *tt; | ||
652 | int tgt_iter = 0; | ||
653 | |||
654 | if (!capable(CAP_SYS_ADMIN)) | ||
655 | return -EPERM; | ||
656 | |||
657 | info = memdup_user(arg, sizeof(struct nvm_ioctl_info)); | ||
658 | if (IS_ERR(info)) | ||
659 | return -EFAULT; | ||
660 | |||
661 | info->version[0] = NVM_VERSION_MAJOR; | ||
662 | info->version[1] = NVM_VERSION_MINOR; | ||
663 | info->version[2] = NVM_VERSION_PATCH; | ||
664 | |||
665 | down_write(&nvm_lock); | ||
666 | list_for_each_entry(tt, &nvm_targets, list) { | ||
667 | struct nvm_ioctl_info_tgt *tgt = &info->tgts[tgt_iter]; | ||
668 | |||
669 | tgt->version[0] = tt->version[0]; | ||
670 | tgt->version[1] = tt->version[1]; | ||
671 | tgt->version[2] = tt->version[2]; | ||
672 | strncpy(tgt->tgtname, tt->name, NVM_TTYPE_NAME_MAX); | ||
673 | |||
674 | tgt_iter++; | ||
675 | } | ||
676 | |||
677 | info->tgtsize = tgt_iter; | ||
678 | up_write(&nvm_lock); | ||
679 | |||
680 | if (copy_to_user(arg, info, sizeof(struct nvm_ioctl_info))) | ||
681 | return -EFAULT; | ||
682 | |||
683 | kfree(info); | ||
684 | return 0; | ||
685 | } | ||
686 | |||
687 | static long nvm_ioctl_get_devices(struct file *file, void __user *arg) | ||
688 | { | ||
689 | struct nvm_ioctl_get_devices *devices; | ||
690 | struct nvm_dev *dev; | ||
691 | int i = 0; | ||
692 | |||
693 | if (!capable(CAP_SYS_ADMIN)) | ||
694 | return -EPERM; | ||
695 | |||
696 | devices = kzalloc(sizeof(struct nvm_ioctl_get_devices), GFP_KERNEL); | ||
697 | if (!devices) | ||
698 | return -ENOMEM; | ||
699 | |||
700 | down_write(&nvm_lock); | ||
701 | list_for_each_entry(dev, &nvm_devices, devices) { | ||
702 | struct nvm_ioctl_device_info *info = &devices->info[i]; | ||
703 | |||
704 | sprintf(info->devname, "%s", dev->name); | ||
705 | if (dev->mt) { | ||
706 | info->bmversion[0] = dev->mt->version[0]; | ||
707 | info->bmversion[1] = dev->mt->version[1]; | ||
708 | info->bmversion[2] = dev->mt->version[2]; | ||
709 | sprintf(info->bmname, "%s", dev->mt->name); | ||
710 | } else { | ||
711 | sprintf(info->bmname, "none"); | ||
712 | } | ||
713 | |||
714 | i++; | ||
715 | if (i > 31) { | ||
716 | pr_err("nvm: max 31 devices can be reported.\n"); | ||
717 | break; | ||
718 | } | ||
719 | } | ||
720 | up_write(&nvm_lock); | ||
721 | |||
722 | devices->nr_devices = i; | ||
723 | |||
724 | if (copy_to_user(arg, devices, sizeof(struct nvm_ioctl_get_devices))) | ||
725 | return -EFAULT; | ||
726 | |||
727 | kfree(devices); | ||
728 | return 0; | ||
729 | } | ||
730 | |||
731 | static long nvm_ioctl_dev_create(struct file *file, void __user *arg) | ||
732 | { | ||
733 | struct nvm_ioctl_create create; | ||
734 | |||
735 | if (!capable(CAP_SYS_ADMIN)) | ||
736 | return -EPERM; | ||
737 | |||
738 | if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create))) | ||
739 | return -EFAULT; | ||
740 | |||
741 | create.dev[DISK_NAME_LEN - 1] = '\0'; | ||
742 | create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0'; | ||
743 | create.tgtname[DISK_NAME_LEN - 1] = '\0'; | ||
744 | |||
745 | if (create.flags != 0) { | ||
746 | pr_err("nvm: no flags supported\n"); | ||
747 | return -EINVAL; | ||
748 | } | ||
749 | |||
750 | return __nvm_configure_create(&create); | ||
751 | } | ||
752 | |||
753 | static long nvm_ioctl_dev_remove(struct file *file, void __user *arg) | ||
754 | { | ||
755 | struct nvm_ioctl_remove remove; | ||
756 | |||
757 | if (!capable(CAP_SYS_ADMIN)) | ||
758 | return -EPERM; | ||
759 | |||
760 | if (copy_from_user(&remove, arg, sizeof(struct nvm_ioctl_remove))) | ||
761 | return -EFAULT; | ||
762 | |||
763 | remove.tgtname[DISK_NAME_LEN - 1] = '\0'; | ||
764 | |||
765 | if (remove.flags != 0) { | ||
766 | pr_err("nvm: no flags supported\n"); | ||
767 | return -EINVAL; | ||
768 | } | ||
769 | |||
770 | return __nvm_configure_remove(&remove); | ||
771 | } | ||
772 | |||
773 | static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg) | ||
774 | { | ||
775 | void __user *argp = (void __user *)arg; | ||
776 | |||
777 | switch (cmd) { | ||
778 | case NVM_INFO: | ||
779 | return nvm_ioctl_info(file, argp); | ||
780 | case NVM_GET_DEVICES: | ||
781 | return nvm_ioctl_get_devices(file, argp); | ||
782 | case NVM_DEV_CREATE: | ||
783 | return nvm_ioctl_dev_create(file, argp); | ||
784 | case NVM_DEV_REMOVE: | ||
785 | return nvm_ioctl_dev_remove(file, argp); | ||
786 | } | ||
787 | return 0; | ||
788 | } | ||
789 | |||
790 | static const struct file_operations _ctl_fops = { | ||
791 | .open = nonseekable_open, | ||
792 | .unlocked_ioctl = nvm_ctl_ioctl, | ||
793 | .owner = THIS_MODULE, | ||
794 | .llseek = noop_llseek, | ||
795 | }; | ||
796 | |||
797 | static struct miscdevice _nvm_misc = { | ||
798 | .minor = MISC_DYNAMIC_MINOR, | ||
799 | .name = "lightnvm", | ||
800 | .nodename = "lightnvm/control", | ||
801 | .fops = &_ctl_fops, | ||
802 | }; | ||
803 | |||
804 | MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR); | ||
805 | |||
806 | static int __init nvm_mod_init(void) | ||
807 | { | ||
808 | int ret; | ||
809 | |||
810 | ret = misc_register(&_nvm_misc); | ||
811 | if (ret) | ||
812 | pr_err("nvm: misc_register failed for control device"); | ||
813 | |||
814 | return ret; | ||
815 | } | ||
816 | |||
817 | static void __exit nvm_mod_exit(void) | ||
818 | { | ||
819 | misc_deregister(&_nvm_misc); | ||
820 | } | ||
821 | |||
822 | MODULE_AUTHOR("Matias Bjorling <m@bjorling.me>"); | ||
823 | MODULE_LICENSE("GPL v2"); | ||
824 | MODULE_VERSION("0.1"); | ||
825 | module_init(nvm_mod_init); | ||
826 | module_exit(nvm_mod_exit); | ||
diff --git a/drivers/lightnvm/gennvm.c b/drivers/lightnvm/gennvm.c new file mode 100644 index 000000000000..ae1fb2bdc5f4 --- /dev/null +++ b/drivers/lightnvm/gennvm.c | |||
@@ -0,0 +1,485 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2015 Matias Bjorling <m@bjorling.me> | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or | ||
5 | * modify it under the terms of the GNU General Public License version | ||
6 | * 2 as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope that it will be useful, but | ||
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
11 | * General Public License for more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program; see the file COPYING. If not, write to | ||
15 | * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, | ||
16 | * USA. | ||
17 | * | ||
18 | * Implementation of a generic nvm manager for Open-Channel SSDs. | ||
19 | */ | ||
20 | |||
21 | #include "gennvm.h" | ||
22 | |||
23 | static void gennvm_blocks_free(struct nvm_dev *dev) | ||
24 | { | ||
25 | struct gen_nvm *gn = dev->mp; | ||
26 | struct gen_lun *lun; | ||
27 | int i; | ||
28 | |||
29 | gennvm_for_each_lun(gn, lun, i) { | ||
30 | if (!lun->vlun.blocks) | ||
31 | break; | ||
32 | vfree(lun->vlun.blocks); | ||
33 | } | ||
34 | } | ||
35 | |||
36 | static void gennvm_luns_free(struct nvm_dev *dev) | ||
37 | { | ||
38 | struct gen_nvm *gn = dev->mp; | ||
39 | |||
40 | kfree(gn->luns); | ||
41 | } | ||
42 | |||
43 | static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn) | ||
44 | { | ||
45 | struct gen_lun *lun; | ||
46 | int i; | ||
47 | |||
48 | gn->luns = kcalloc(dev->nr_luns, sizeof(struct gen_lun), GFP_KERNEL); | ||
49 | if (!gn->luns) | ||
50 | return -ENOMEM; | ||
51 | |||
52 | gennvm_for_each_lun(gn, lun, i) { | ||
53 | spin_lock_init(&lun->vlun.lock); | ||
54 | INIT_LIST_HEAD(&lun->free_list); | ||
55 | INIT_LIST_HEAD(&lun->used_list); | ||
56 | INIT_LIST_HEAD(&lun->bb_list); | ||
57 | |||
58 | lun->reserved_blocks = 2; /* for GC only */ | ||
59 | lun->vlun.id = i; | ||
60 | lun->vlun.lun_id = i % dev->luns_per_chnl; | ||
61 | lun->vlun.chnl_id = i / dev->luns_per_chnl; | ||
62 | lun->vlun.nr_free_blocks = dev->blks_per_lun; | ||
63 | } | ||
64 | return 0; | ||
65 | } | ||
66 | |||
67 | static int gennvm_block_bb(u32 lun_id, void *bb_bitmap, unsigned int nr_blocks, | ||
68 | void *private) | ||
69 | { | ||
70 | struct gen_nvm *gn = private; | ||
71 | struct gen_lun *lun = &gn->luns[lun_id]; | ||
72 | struct nvm_block *blk; | ||
73 | int i; | ||
74 | |||
75 | if (unlikely(bitmap_empty(bb_bitmap, nr_blocks))) | ||
76 | return 0; | ||
77 | |||
78 | i = -1; | ||
79 | while ((i = find_next_bit(bb_bitmap, nr_blocks, i + 1)) < nr_blocks) { | ||
80 | blk = &lun->vlun.blocks[i]; | ||
81 | if (!blk) { | ||
82 | pr_err("gennvm: BB data is out of bounds.\n"); | ||
83 | return -EINVAL; | ||
84 | } | ||
85 | |||
86 | list_move_tail(&blk->list, &lun->bb_list); | ||
87 | } | ||
88 | |||
89 | return 0; | ||
90 | } | ||
91 | |||
92 | static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private) | ||
93 | { | ||
94 | struct nvm_dev *dev = private; | ||
95 | struct gen_nvm *gn = dev->mp; | ||
96 | sector_t max_pages = dev->total_pages * (dev->sec_size >> 9); | ||
97 | u64 elba = slba + nlb; | ||
98 | struct gen_lun *lun; | ||
99 | struct nvm_block *blk; | ||
100 | u64 i; | ||
101 | int lun_id; | ||
102 | |||
103 | if (unlikely(elba > dev->total_pages)) { | ||
104 | pr_err("gennvm: L2P data from device is out of bounds!\n"); | ||
105 | return -EINVAL; | ||
106 | } | ||
107 | |||
108 | for (i = 0; i < nlb; i++) { | ||
109 | u64 pba = le64_to_cpu(entries[i]); | ||
110 | |||
111 | if (unlikely(pba >= max_pages && pba != U64_MAX)) { | ||
112 | pr_err("gennvm: L2P data entry is out of bounds!\n"); | ||
113 | return -EINVAL; | ||
114 | } | ||
115 | |||
116 | /* Address zero is a special one. The first page on a disk is | ||
117 | * protected. It often holds internal device boot | ||
118 | * information. | ||
119 | */ | ||
120 | if (!pba) | ||
121 | continue; | ||
122 | |||
123 | /* resolve block from physical address */ | ||
124 | lun_id = div_u64(pba, dev->sec_per_lun); | ||
125 | lun = &gn->luns[lun_id]; | ||
126 | |||
127 | /* Calculate block offset into lun */ | ||
128 | pba = pba - (dev->sec_per_lun * lun_id); | ||
129 | blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)]; | ||
130 | |||
131 | if (!blk->type) { | ||
132 | /* at this point, we don't know anything about the | ||
133 | * block. It's up to the FTL on top to re-etablish the | ||
134 | * block state | ||
135 | */ | ||
136 | list_move_tail(&blk->list, &lun->used_list); | ||
137 | blk->type = 1; | ||
138 | lun->vlun.nr_free_blocks--; | ||
139 | } | ||
140 | } | ||
141 | |||
142 | return 0; | ||
143 | } | ||
144 | |||
145 | static int gennvm_blocks_init(struct nvm_dev *dev, struct gen_nvm *gn) | ||
146 | { | ||
147 | struct gen_lun *lun; | ||
148 | struct nvm_block *block; | ||
149 | sector_t lun_iter, blk_iter, cur_block_id = 0; | ||
150 | int ret; | ||
151 | |||
152 | gennvm_for_each_lun(gn, lun, lun_iter) { | ||
153 | lun->vlun.blocks = vzalloc(sizeof(struct nvm_block) * | ||
154 | dev->blks_per_lun); | ||
155 | if (!lun->vlun.blocks) | ||
156 | return -ENOMEM; | ||
157 | |||
158 | for (blk_iter = 0; blk_iter < dev->blks_per_lun; blk_iter++) { | ||
159 | block = &lun->vlun.blocks[blk_iter]; | ||
160 | |||
161 | INIT_LIST_HEAD(&block->list); | ||
162 | |||
163 | block->lun = &lun->vlun; | ||
164 | block->id = cur_block_id++; | ||
165 | |||
166 | /* First block is reserved for device */ | ||
167 | if (unlikely(lun_iter == 0 && blk_iter == 0)) | ||
168 | continue; | ||
169 | |||
170 | list_add_tail(&block->list, &lun->free_list); | ||
171 | } | ||
172 | |||
173 | if (dev->ops->get_bb_tbl) { | ||
174 | ret = dev->ops->get_bb_tbl(dev->q, lun->vlun.id, | ||
175 | dev->blks_per_lun, gennvm_block_bb, gn); | ||
176 | if (ret) | ||
177 | pr_err("gennvm: could not read BB table\n"); | ||
178 | } | ||
179 | } | ||
180 | |||
181 | if (dev->ops->get_l2p_tbl) { | ||
182 | ret = dev->ops->get_l2p_tbl(dev->q, 0, dev->total_pages, | ||
183 | gennvm_block_map, dev); | ||
184 | if (ret) { | ||
185 | pr_err("gennvm: could not read L2P table.\n"); | ||
186 | pr_warn("gennvm: default block initialization"); | ||
187 | } | ||
188 | } | ||
189 | |||
190 | return 0; | ||
191 | } | ||
192 | |||
193 | static int gennvm_register(struct nvm_dev *dev) | ||
194 | { | ||
195 | struct gen_nvm *gn; | ||
196 | int ret; | ||
197 | |||
198 | gn = kzalloc(sizeof(struct gen_nvm), GFP_KERNEL); | ||
199 | if (!gn) | ||
200 | return -ENOMEM; | ||
201 | |||
202 | gn->nr_luns = dev->nr_luns; | ||
203 | dev->mp = gn; | ||
204 | |||
205 | ret = gennvm_luns_init(dev, gn); | ||
206 | if (ret) { | ||
207 | pr_err("gennvm: could not initialize luns\n"); | ||
208 | goto err; | ||
209 | } | ||
210 | |||
211 | ret = gennvm_blocks_init(dev, gn); | ||
212 | if (ret) { | ||
213 | pr_err("gennvm: could not initialize blocks\n"); | ||
214 | goto err; | ||
215 | } | ||
216 | |||
217 | return 1; | ||
218 | err: | ||
219 | kfree(gn); | ||
220 | return ret; | ||
221 | } | ||
222 | |||
223 | static void gennvm_unregister(struct nvm_dev *dev) | ||
224 | { | ||
225 | gennvm_blocks_free(dev); | ||
226 | gennvm_luns_free(dev); | ||
227 | kfree(dev->mp); | ||
228 | dev->mp = NULL; | ||
229 | } | ||
230 | |||
231 | static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev, | ||
232 | struct nvm_lun *vlun, unsigned long flags) | ||
233 | { | ||
234 | struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); | ||
235 | struct nvm_block *blk = NULL; | ||
236 | int is_gc = flags & NVM_IOTYPE_GC; | ||
237 | |||
238 | spin_lock(&vlun->lock); | ||
239 | |||
240 | if (list_empty(&lun->free_list)) { | ||
241 | pr_err_ratelimited("gennvm: lun %u have no free pages available", | ||
242 | lun->vlun.id); | ||
243 | spin_unlock(&vlun->lock); | ||
244 | goto out; | ||
245 | } | ||
246 | |||
247 | while (!is_gc && lun->vlun.nr_free_blocks < lun->reserved_blocks) { | ||
248 | spin_unlock(&vlun->lock); | ||
249 | goto out; | ||
250 | } | ||
251 | |||
252 | blk = list_first_entry(&lun->free_list, struct nvm_block, list); | ||
253 | list_move_tail(&blk->list, &lun->used_list); | ||
254 | blk->type = 1; | ||
255 | |||
256 | lun->vlun.nr_free_blocks--; | ||
257 | |||
258 | spin_unlock(&vlun->lock); | ||
259 | out: | ||
260 | return blk; | ||
261 | } | ||
262 | |||
263 | static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk) | ||
264 | { | ||
265 | struct nvm_lun *vlun = blk->lun; | ||
266 | struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun); | ||
267 | |||
268 | spin_lock(&vlun->lock); | ||
269 | |||
270 | switch (blk->type) { | ||
271 | case 1: | ||
272 | list_move_tail(&blk->list, &lun->free_list); | ||
273 | lun->vlun.nr_free_blocks++; | ||
274 | blk->type = 0; | ||
275 | break; | ||
276 | case 2: | ||
277 | list_move_tail(&blk->list, &lun->bb_list); | ||
278 | break; | ||
279 | default: | ||
280 | WARN_ON_ONCE(1); | ||
281 | pr_err("gennvm: erroneous block type (%lu -> %u)\n", | ||
282 | blk->id, blk->type); | ||
283 | list_move_tail(&blk->list, &lun->bb_list); | ||
284 | } | ||
285 | |||
286 | spin_unlock(&vlun->lock); | ||
287 | } | ||
288 | |||
289 | static void gennvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd) | ||
290 | { | ||
291 | int i; | ||
292 | |||
293 | if (rqd->nr_pages > 1) { | ||
294 | for (i = 0; i < rqd->nr_pages; i++) | ||
295 | rqd->ppa_list[i] = addr_to_generic_mode(dev, | ||
296 | rqd->ppa_list[i]); | ||
297 | } else { | ||
298 | rqd->ppa_addr = addr_to_generic_mode(dev, rqd->ppa_addr); | ||
299 | } | ||
300 | } | ||
301 | |||
302 | static void gennvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd) | ||
303 | { | ||
304 | int i; | ||
305 | |||
306 | if (rqd->nr_pages > 1) { | ||
307 | for (i = 0; i < rqd->nr_pages; i++) | ||
308 | rqd->ppa_list[i] = generic_to_addr_mode(dev, | ||
309 | rqd->ppa_list[i]); | ||
310 | } else { | ||
311 | rqd->ppa_addr = generic_to_addr_mode(dev, rqd->ppa_addr); | ||
312 | } | ||
313 | } | ||
314 | |||
315 | static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) | ||
316 | { | ||
317 | if (!dev->ops->submit_io) | ||
318 | return 0; | ||
319 | |||
320 | /* Convert address space */ | ||
321 | gennvm_generic_to_addr_mode(dev, rqd); | ||
322 | |||
323 | rqd->dev = dev; | ||
324 | return dev->ops->submit_io(dev->q, rqd); | ||
325 | } | ||
326 | |||
327 | static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa, | ||
328 | int type) | ||
329 | { | ||
330 | struct gen_nvm *gn = dev->mp; | ||
331 | struct gen_lun *lun; | ||
332 | struct nvm_block *blk; | ||
333 | |||
334 | if (unlikely(ppa->g.ch > dev->nr_chnls || | ||
335 | ppa->g.lun > dev->luns_per_chnl || | ||
336 | ppa->g.blk > dev->blks_per_lun)) { | ||
337 | WARN_ON_ONCE(1); | ||
338 | pr_err("gennvm: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u", | ||
339 | ppa->g.ch, dev->nr_chnls, | ||
340 | ppa->g.lun, dev->luns_per_chnl, | ||
341 | ppa->g.blk, dev->blks_per_lun); | ||
342 | return; | ||
343 | } | ||
344 | |||
345 | lun = &gn->luns[ppa->g.lun * ppa->g.ch]; | ||
346 | blk = &lun->vlun.blocks[ppa->g.blk]; | ||
347 | |||
348 | /* will be moved to bb list on put_blk from target */ | ||
349 | blk->type = type; | ||
350 | } | ||
351 | |||
352 | /* mark block bad. It is expected the target recover from the error. */ | ||
353 | static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd) | ||
354 | { | ||
355 | int i; | ||
356 | |||
357 | if (!dev->ops->set_bb) | ||
358 | return; | ||
359 | |||
360 | if (dev->ops->set_bb(dev->q, rqd, 1)) | ||
361 | return; | ||
362 | |||
363 | gennvm_addr_to_generic_mode(dev, rqd); | ||
364 | |||
365 | /* look up blocks and mark them as bad */ | ||
366 | if (rqd->nr_pages > 1) | ||
367 | for (i = 0; i < rqd->nr_pages; i++) | ||
368 | gennvm_blk_set_type(dev, &rqd->ppa_list[i], 2); | ||
369 | else | ||
370 | gennvm_blk_set_type(dev, &rqd->ppa_addr, 2); | ||
371 | } | ||
372 | |||
373 | static int gennvm_end_io(struct nvm_rq *rqd, int error) | ||
374 | { | ||
375 | struct nvm_tgt_instance *ins = rqd->ins; | ||
376 | int ret = 0; | ||
377 | |||
378 | switch (error) { | ||
379 | case NVM_RSP_SUCCESS: | ||
380 | break; | ||
381 | case NVM_RSP_ERR_EMPTYPAGE: | ||
382 | break; | ||
383 | case NVM_RSP_ERR_FAILWRITE: | ||
384 | gennvm_mark_blk_bad(rqd->dev, rqd); | ||
385 | default: | ||
386 | ret++; | ||
387 | } | ||
388 | |||
389 | ret += ins->tt->end_io(rqd, error); | ||
390 | |||
391 | return ret; | ||
392 | } | ||
393 | |||
394 | static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk, | ||
395 | unsigned long flags) | ||
396 | { | ||
397 | int plane_cnt = 0, pl_idx, ret; | ||
398 | struct ppa_addr addr; | ||
399 | struct nvm_rq rqd; | ||
400 | |||
401 | if (!dev->ops->erase_block) | ||
402 | return 0; | ||
403 | |||
404 | addr = block_to_ppa(dev, blk); | ||
405 | |||
406 | if (dev->plane_mode == NVM_PLANE_SINGLE) { | ||
407 | rqd.nr_pages = 1; | ||
408 | rqd.ppa_addr = addr; | ||
409 | } else { | ||
410 | plane_cnt = (1 << dev->plane_mode); | ||
411 | rqd.nr_pages = plane_cnt; | ||
412 | |||
413 | rqd.ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL, | ||
414 | &rqd.dma_ppa_list); | ||
415 | if (!rqd.ppa_list) { | ||
416 | pr_err("gennvm: failed to allocate dma memory\n"); | ||
417 | return -ENOMEM; | ||
418 | } | ||
419 | |||
420 | for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) { | ||
421 | addr.g.pl = pl_idx; | ||
422 | rqd.ppa_list[pl_idx] = addr; | ||
423 | } | ||
424 | } | ||
425 | |||
426 | gennvm_generic_to_addr_mode(dev, &rqd); | ||
427 | |||
428 | ret = dev->ops->erase_block(dev->q, &rqd); | ||
429 | |||
430 | if (plane_cnt) | ||
431 | nvm_dev_dma_free(dev, rqd.ppa_list, rqd.dma_ppa_list); | ||
432 | |||
433 | return ret; | ||
434 | } | ||
435 | |||
436 | static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid) | ||
437 | { | ||
438 | struct gen_nvm *gn = dev->mp; | ||
439 | |||
440 | return &gn->luns[lunid].vlun; | ||
441 | } | ||
442 | |||
443 | static void gennvm_free_blocks_print(struct nvm_dev *dev) | ||
444 | { | ||
445 | struct gen_nvm *gn = dev->mp; | ||
446 | struct gen_lun *lun; | ||
447 | unsigned int i; | ||
448 | |||
449 | gennvm_for_each_lun(gn, lun, i) | ||
450 | pr_info("%s: lun%8u\t%u\n", | ||
451 | dev->name, i, lun->vlun.nr_free_blocks); | ||
452 | } | ||
453 | |||
454 | static struct nvmm_type gennvm = { | ||
455 | .name = "gennvm", | ||
456 | .version = {0, 1, 0}, | ||
457 | |||
458 | .register_mgr = gennvm_register, | ||
459 | .unregister_mgr = gennvm_unregister, | ||
460 | |||
461 | .get_blk = gennvm_get_blk, | ||
462 | .put_blk = gennvm_put_blk, | ||
463 | |||
464 | .submit_io = gennvm_submit_io, | ||
465 | .end_io = gennvm_end_io, | ||
466 | .erase_blk = gennvm_erase_blk, | ||
467 | |||
468 | .get_lun = gennvm_get_lun, | ||
469 | .free_blocks_print = gennvm_free_blocks_print, | ||
470 | }; | ||
471 | |||
472 | static int __init gennvm_module_init(void) | ||
473 | { | ||
474 | return nvm_register_mgr(&gennvm); | ||
475 | } | ||
476 | |||
477 | static void gennvm_module_exit(void) | ||
478 | { | ||
479 | nvm_unregister_mgr(&gennvm); | ||
480 | } | ||
481 | |||
482 | module_init(gennvm_module_init); | ||
483 | module_exit(gennvm_module_exit); | ||
484 | MODULE_LICENSE("GPL v2"); | ||
485 | MODULE_DESCRIPTION("Generic media manager for Open-Channel SSDs"); | ||
diff --git a/drivers/lightnvm/gennvm.h b/drivers/lightnvm/gennvm.h new file mode 100644 index 000000000000..d23bd3501ddc --- /dev/null +++ b/drivers/lightnvm/gennvm.h | |||
@@ -0,0 +1,46 @@ | |||
1 | /* | ||
2 | * Copyright: Matias Bjorling <mb@bjorling.me> | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or | ||
5 | * modify it under the terms of the GNU General Public License version | ||
6 | * 2 as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope that it will be useful, but | ||
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
11 | * General Public License for more details. | ||
12 | * | ||
13 | */ | ||
14 | |||
15 | #ifndef GENNVM_H_ | ||
16 | #define GENNVM_H_ | ||
17 | |||
18 | #include <linux/module.h> | ||
19 | #include <linux/vmalloc.h> | ||
20 | |||
21 | #include <linux/lightnvm.h> | ||
22 | |||
23 | struct gen_lun { | ||
24 | struct nvm_lun vlun; | ||
25 | |||
26 | int reserved_blocks; | ||
27 | /* lun block lists */ | ||
28 | struct list_head used_list; /* In-use blocks */ | ||
29 | struct list_head free_list; /* Not used blocks i.e. released | ||
30 | * and ready for use | ||
31 | */ | ||
32 | struct list_head bb_list; /* Bad blocks. Mutually exclusive with | ||
33 | * free_list and used_list | ||
34 | */ | ||
35 | }; | ||
36 | |||
37 | struct gen_nvm { | ||
38 | int nr_luns; | ||
39 | struct gen_lun *luns; | ||
40 | }; | ||
41 | |||
42 | #define gennvm_for_each_lun(bm, lun, i) \ | ||
43 | for ((i) = 0, lun = &(bm)->luns[0]; \ | ||
44 | (i) < (bm)->nr_luns; (i)++, lun = &(bm)->luns[(i)]) | ||
45 | |||
46 | #endif /* GENNVM_H_ */ | ||
diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c new file mode 100644 index 000000000000..64a888a5e9b3 --- /dev/null +++ b/drivers/lightnvm/rrpc.c | |||
@@ -0,0 +1,1323 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2015 IT University of Copenhagen | ||
3 | * Initial release: Matias Bjorling <m@bjorling.me> | ||
4 | * | ||
5 | * This program is free software; you can redistribute it and/or | ||
6 | * modify it under the terms of the GNU General Public License version | ||
7 | * 2 as published by the Free Software Foundation. | ||
8 | * | ||
9 | * This program is distributed in the hope that it will be useful, but | ||
10 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
12 | * General Public License for more details. | ||
13 | * | ||
14 | * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs. | ||
15 | */ | ||
16 | |||
17 | #include "rrpc.h" | ||
18 | |||
19 | static struct kmem_cache *rrpc_gcb_cache, *rrpc_rq_cache; | ||
20 | static DECLARE_RWSEM(rrpc_lock); | ||
21 | |||
22 | static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio, | ||
23 | struct nvm_rq *rqd, unsigned long flags); | ||
24 | |||
25 | #define rrpc_for_each_lun(rrpc, rlun, i) \ | ||
26 | for ((i) = 0, rlun = &(rrpc)->luns[0]; \ | ||
27 | (i) < (rrpc)->nr_luns; (i)++, rlun = &(rrpc)->luns[(i)]) | ||
28 | |||
29 | static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a) | ||
30 | { | ||
31 | struct rrpc_block *rblk = a->rblk; | ||
32 | unsigned int pg_offset; | ||
33 | |||
34 | lockdep_assert_held(&rrpc->rev_lock); | ||
35 | |||
36 | if (a->addr == ADDR_EMPTY || !rblk) | ||
37 | return; | ||
38 | |||
39 | spin_lock(&rblk->lock); | ||
40 | |||
41 | div_u64_rem(a->addr, rrpc->dev->pgs_per_blk, &pg_offset); | ||
42 | WARN_ON(test_and_set_bit(pg_offset, rblk->invalid_pages)); | ||
43 | rblk->nr_invalid_pages++; | ||
44 | |||
45 | spin_unlock(&rblk->lock); | ||
46 | |||
47 | rrpc->rev_trans_map[a->addr - rrpc->poffset].addr = ADDR_EMPTY; | ||
48 | } | ||
49 | |||
50 | static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba, | ||
51 | unsigned len) | ||
52 | { | ||
53 | sector_t i; | ||
54 | |||
55 | spin_lock(&rrpc->rev_lock); | ||
56 | for (i = slba; i < slba + len; i++) { | ||
57 | struct rrpc_addr *gp = &rrpc->trans_map[i]; | ||
58 | |||
59 | rrpc_page_invalidate(rrpc, gp); | ||
60 | gp->rblk = NULL; | ||
61 | } | ||
62 | spin_unlock(&rrpc->rev_lock); | ||
63 | } | ||
64 | |||
65 | static struct nvm_rq *rrpc_inflight_laddr_acquire(struct rrpc *rrpc, | ||
66 | sector_t laddr, unsigned int pages) | ||
67 | { | ||
68 | struct nvm_rq *rqd; | ||
69 | struct rrpc_inflight_rq *inf; | ||
70 | |||
71 | rqd = mempool_alloc(rrpc->rq_pool, GFP_ATOMIC); | ||
72 | if (!rqd) | ||
73 | return ERR_PTR(-ENOMEM); | ||
74 | |||
75 | inf = rrpc_get_inflight_rq(rqd); | ||
76 | if (rrpc_lock_laddr(rrpc, laddr, pages, inf)) { | ||
77 | mempool_free(rqd, rrpc->rq_pool); | ||
78 | return NULL; | ||
79 | } | ||
80 | |||
81 | return rqd; | ||
82 | } | ||
83 | |||
84 | static void rrpc_inflight_laddr_release(struct rrpc *rrpc, struct nvm_rq *rqd) | ||
85 | { | ||
86 | struct rrpc_inflight_rq *inf = rrpc_get_inflight_rq(rqd); | ||
87 | |||
88 | rrpc_unlock_laddr(rrpc, inf); | ||
89 | |||
90 | mempool_free(rqd, rrpc->rq_pool); | ||
91 | } | ||
92 | |||
93 | static void rrpc_discard(struct rrpc *rrpc, struct bio *bio) | ||
94 | { | ||
95 | sector_t slba = bio->bi_iter.bi_sector / NR_PHY_IN_LOG; | ||
96 | sector_t len = bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE; | ||
97 | struct nvm_rq *rqd; | ||
98 | |||
99 | do { | ||
100 | rqd = rrpc_inflight_laddr_acquire(rrpc, slba, len); | ||
101 | schedule(); | ||
102 | } while (!rqd); | ||
103 | |||
104 | if (IS_ERR(rqd)) { | ||
105 | pr_err("rrpc: unable to acquire inflight IO\n"); | ||
106 | bio_io_error(bio); | ||
107 | return; | ||
108 | } | ||
109 | |||
110 | rrpc_invalidate_range(rrpc, slba, len); | ||
111 | rrpc_inflight_laddr_release(rrpc, rqd); | ||
112 | } | ||
113 | |||
114 | static int block_is_full(struct rrpc *rrpc, struct rrpc_block *rblk) | ||
115 | { | ||
116 | return (rblk->next_page == rrpc->dev->pgs_per_blk); | ||
117 | } | ||
118 | |||
119 | static u64 block_to_addr(struct rrpc *rrpc, struct rrpc_block *rblk) | ||
120 | { | ||
121 | struct nvm_block *blk = rblk->parent; | ||
122 | |||
123 | return blk->id * rrpc->dev->pgs_per_blk; | ||
124 | } | ||
125 | |||
126 | static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_dev *dev, u64 addr) | ||
127 | { | ||
128 | struct ppa_addr paddr; | ||
129 | |||
130 | paddr.ppa = addr; | ||
131 | return __linear_to_generic_addr(dev, paddr); | ||
132 | } | ||
133 | |||
134 | /* requires lun->lock taken */ | ||
135 | static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *rblk) | ||
136 | { | ||
137 | struct rrpc *rrpc = rlun->rrpc; | ||
138 | |||
139 | BUG_ON(!rblk); | ||
140 | |||
141 | if (rlun->cur) { | ||
142 | spin_lock(&rlun->cur->lock); | ||
143 | WARN_ON(!block_is_full(rrpc, rlun->cur)); | ||
144 | spin_unlock(&rlun->cur->lock); | ||
145 | } | ||
146 | rlun->cur = rblk; | ||
147 | } | ||
148 | |||
149 | static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun, | ||
150 | unsigned long flags) | ||
151 | { | ||
152 | struct nvm_block *blk; | ||
153 | struct rrpc_block *rblk; | ||
154 | |||
155 | blk = nvm_get_blk(rrpc->dev, rlun->parent, 0); | ||
156 | if (!blk) | ||
157 | return NULL; | ||
158 | |||
159 | rblk = &rlun->blocks[blk->id]; | ||
160 | blk->priv = rblk; | ||
161 | |||
162 | bitmap_zero(rblk->invalid_pages, rrpc->dev->pgs_per_blk); | ||
163 | rblk->next_page = 0; | ||
164 | rblk->nr_invalid_pages = 0; | ||
165 | atomic_set(&rblk->data_cmnt_size, 0); | ||
166 | |||
167 | return rblk; | ||
168 | } | ||
169 | |||
170 | static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk) | ||
171 | { | ||
172 | nvm_put_blk(rrpc->dev, rblk->parent); | ||
173 | } | ||
174 | |||
175 | static struct rrpc_lun *get_next_lun(struct rrpc *rrpc) | ||
176 | { | ||
177 | int next = atomic_inc_return(&rrpc->next_lun); | ||
178 | |||
179 | return &rrpc->luns[next % rrpc->nr_luns]; | ||
180 | } | ||
181 | |||
182 | static void rrpc_gc_kick(struct rrpc *rrpc) | ||
183 | { | ||
184 | struct rrpc_lun *rlun; | ||
185 | unsigned int i; | ||
186 | |||
187 | for (i = 0; i < rrpc->nr_luns; i++) { | ||
188 | rlun = &rrpc->luns[i]; | ||
189 | queue_work(rrpc->krqd_wq, &rlun->ws_gc); | ||
190 | } | ||
191 | } | ||
192 | |||
193 | /* | ||
194 | * timed GC every interval. | ||
195 | */ | ||
196 | static void rrpc_gc_timer(unsigned long data) | ||
197 | { | ||
198 | struct rrpc *rrpc = (struct rrpc *)data; | ||
199 | |||
200 | rrpc_gc_kick(rrpc); | ||
201 | mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10)); | ||
202 | } | ||
203 | |||
204 | static void rrpc_end_sync_bio(struct bio *bio) | ||
205 | { | ||
206 | struct completion *waiting = bio->bi_private; | ||
207 | |||
208 | if (bio->bi_error) | ||
209 | pr_err("nvm: gc request failed (%u).\n", bio->bi_error); | ||
210 | |||
211 | complete(waiting); | ||
212 | } | ||
213 | |||
214 | /* | ||
215 | * rrpc_move_valid_pages -- migrate live data off the block | ||
216 | * @rrpc: the 'rrpc' structure | ||
217 | * @block: the block from which to migrate live pages | ||
218 | * | ||
219 | * Description: | ||
220 | * GC algorithms may call this function to migrate remaining live | ||
221 | * pages off the block prior to erasing it. This function blocks | ||
222 | * further execution until the operation is complete. | ||
223 | */ | ||
224 | static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk) | ||
225 | { | ||
226 | struct request_queue *q = rrpc->dev->q; | ||
227 | struct rrpc_rev_addr *rev; | ||
228 | struct nvm_rq *rqd; | ||
229 | struct bio *bio; | ||
230 | struct page *page; | ||
231 | int slot; | ||
232 | int nr_pgs_per_blk = rrpc->dev->pgs_per_blk; | ||
233 | u64 phys_addr; | ||
234 | DECLARE_COMPLETION_ONSTACK(wait); | ||
235 | |||
236 | if (bitmap_full(rblk->invalid_pages, nr_pgs_per_blk)) | ||
237 | return 0; | ||
238 | |||
239 | bio = bio_alloc(GFP_NOIO, 1); | ||
240 | if (!bio) { | ||
241 | pr_err("nvm: could not alloc bio to gc\n"); | ||
242 | return -ENOMEM; | ||
243 | } | ||
244 | |||
245 | page = mempool_alloc(rrpc->page_pool, GFP_NOIO); | ||
246 | |||
247 | while ((slot = find_first_zero_bit(rblk->invalid_pages, | ||
248 | nr_pgs_per_blk)) < nr_pgs_per_blk) { | ||
249 | |||
250 | /* Lock laddr */ | ||
251 | phys_addr = (rblk->parent->id * nr_pgs_per_blk) + slot; | ||
252 | |||
253 | try: | ||
254 | spin_lock(&rrpc->rev_lock); | ||
255 | /* Get logical address from physical to logical table */ | ||
256 | rev = &rrpc->rev_trans_map[phys_addr - rrpc->poffset]; | ||
257 | /* already updated by previous regular write */ | ||
258 | if (rev->addr == ADDR_EMPTY) { | ||
259 | spin_unlock(&rrpc->rev_lock); | ||
260 | continue; | ||
261 | } | ||
262 | |||
263 | rqd = rrpc_inflight_laddr_acquire(rrpc, rev->addr, 1); | ||
264 | if (IS_ERR_OR_NULL(rqd)) { | ||
265 | spin_unlock(&rrpc->rev_lock); | ||
266 | schedule(); | ||
267 | goto try; | ||
268 | } | ||
269 | |||
270 | spin_unlock(&rrpc->rev_lock); | ||
271 | |||
272 | /* Perform read to do GC */ | ||
273 | bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr); | ||
274 | bio->bi_rw = READ; | ||
275 | bio->bi_private = &wait; | ||
276 | bio->bi_end_io = rrpc_end_sync_bio; | ||
277 | |||
278 | /* TODO: may fail when EXP_PG_SIZE > PAGE_SIZE */ | ||
279 | bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0); | ||
280 | |||
281 | if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) { | ||
282 | pr_err("rrpc: gc read failed.\n"); | ||
283 | rrpc_inflight_laddr_release(rrpc, rqd); | ||
284 | goto finished; | ||
285 | } | ||
286 | wait_for_completion_io(&wait); | ||
287 | |||
288 | bio_reset(bio); | ||
289 | reinit_completion(&wait); | ||
290 | |||
291 | bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr); | ||
292 | bio->bi_rw = WRITE; | ||
293 | bio->bi_private = &wait; | ||
294 | bio->bi_end_io = rrpc_end_sync_bio; | ||
295 | |||
296 | bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0); | ||
297 | |||
298 | /* turn the command around and write the data back to a new | ||
299 | * address | ||
300 | */ | ||
301 | if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) { | ||
302 | pr_err("rrpc: gc write failed.\n"); | ||
303 | rrpc_inflight_laddr_release(rrpc, rqd); | ||
304 | goto finished; | ||
305 | } | ||
306 | wait_for_completion_io(&wait); | ||
307 | |||
308 | rrpc_inflight_laddr_release(rrpc, rqd); | ||
309 | |||
310 | bio_reset(bio); | ||
311 | } | ||
312 | |||
313 | finished: | ||
314 | mempool_free(page, rrpc->page_pool); | ||
315 | bio_put(bio); | ||
316 | |||
317 | if (!bitmap_full(rblk->invalid_pages, nr_pgs_per_blk)) { | ||
318 | pr_err("nvm: failed to garbage collect block\n"); | ||
319 | return -EIO; | ||
320 | } | ||
321 | |||
322 | return 0; | ||
323 | } | ||
324 | |||
325 | static void rrpc_block_gc(struct work_struct *work) | ||
326 | { | ||
327 | struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc, | ||
328 | ws_gc); | ||
329 | struct rrpc *rrpc = gcb->rrpc; | ||
330 | struct rrpc_block *rblk = gcb->rblk; | ||
331 | struct nvm_dev *dev = rrpc->dev; | ||
332 | |||
333 | pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id); | ||
334 | |||
335 | if (rrpc_move_valid_pages(rrpc, rblk)) | ||
336 | goto done; | ||
337 | |||
338 | nvm_erase_blk(dev, rblk->parent); | ||
339 | rrpc_put_blk(rrpc, rblk); | ||
340 | done: | ||
341 | mempool_free(gcb, rrpc->gcb_pool); | ||
342 | } | ||
343 | |||
344 | /* the block with highest number of invalid pages, will be in the beginning | ||
345 | * of the list | ||
346 | */ | ||
347 | static struct rrpc_block *rblock_max_invalid(struct rrpc_block *ra, | ||
348 | struct rrpc_block *rb) | ||
349 | { | ||
350 | if (ra->nr_invalid_pages == rb->nr_invalid_pages) | ||
351 | return ra; | ||
352 | |||
353 | return (ra->nr_invalid_pages < rb->nr_invalid_pages) ? rb : ra; | ||
354 | } | ||
355 | |||
356 | /* linearly find the block with highest number of invalid pages | ||
357 | * requires lun->lock | ||
358 | */ | ||
359 | static struct rrpc_block *block_prio_find_max(struct rrpc_lun *rlun) | ||
360 | { | ||
361 | struct list_head *prio_list = &rlun->prio_list; | ||
362 | struct rrpc_block *rblock, *max; | ||
363 | |||
364 | BUG_ON(list_empty(prio_list)); | ||
365 | |||
366 | max = list_first_entry(prio_list, struct rrpc_block, prio); | ||
367 | list_for_each_entry(rblock, prio_list, prio) | ||
368 | max = rblock_max_invalid(max, rblock); | ||
369 | |||
370 | return max; | ||
371 | } | ||
372 | |||
373 | static void rrpc_lun_gc(struct work_struct *work) | ||
374 | { | ||
375 | struct rrpc_lun *rlun = container_of(work, struct rrpc_lun, ws_gc); | ||
376 | struct rrpc *rrpc = rlun->rrpc; | ||
377 | struct nvm_lun *lun = rlun->parent; | ||
378 | struct rrpc_block_gc *gcb; | ||
379 | unsigned int nr_blocks_need; | ||
380 | |||
381 | nr_blocks_need = rrpc->dev->blks_per_lun / GC_LIMIT_INVERSE; | ||
382 | |||
383 | if (nr_blocks_need < rrpc->nr_luns) | ||
384 | nr_blocks_need = rrpc->nr_luns; | ||
385 | |||
386 | spin_lock(&lun->lock); | ||
387 | while (nr_blocks_need > lun->nr_free_blocks && | ||
388 | !list_empty(&rlun->prio_list)) { | ||
389 | struct rrpc_block *rblock = block_prio_find_max(rlun); | ||
390 | struct nvm_block *block = rblock->parent; | ||
391 | |||
392 | if (!rblock->nr_invalid_pages) | ||
393 | break; | ||
394 | |||
395 | list_del_init(&rblock->prio); | ||
396 | |||
397 | BUG_ON(!block_is_full(rrpc, rblock)); | ||
398 | |||
399 | pr_debug("rrpc: selected block '%lu' for GC\n", block->id); | ||
400 | |||
401 | gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC); | ||
402 | if (!gcb) | ||
403 | break; | ||
404 | |||
405 | gcb->rrpc = rrpc; | ||
406 | gcb->rblk = rblock; | ||
407 | INIT_WORK(&gcb->ws_gc, rrpc_block_gc); | ||
408 | |||
409 | queue_work(rrpc->kgc_wq, &gcb->ws_gc); | ||
410 | |||
411 | nr_blocks_need--; | ||
412 | } | ||
413 | spin_unlock(&lun->lock); | ||
414 | |||
415 | /* TODO: Hint that request queue can be started again */ | ||
416 | } | ||
417 | |||
418 | static void rrpc_gc_queue(struct work_struct *work) | ||
419 | { | ||
420 | struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc, | ||
421 | ws_gc); | ||
422 | struct rrpc *rrpc = gcb->rrpc; | ||
423 | struct rrpc_block *rblk = gcb->rblk; | ||
424 | struct nvm_lun *lun = rblk->parent->lun; | ||
425 | struct rrpc_lun *rlun = &rrpc->luns[lun->id - rrpc->lun_offset]; | ||
426 | |||
427 | spin_lock(&rlun->lock); | ||
428 | list_add_tail(&rblk->prio, &rlun->prio_list); | ||
429 | spin_unlock(&rlun->lock); | ||
430 | |||
431 | mempool_free(gcb, rrpc->gcb_pool); | ||
432 | pr_debug("nvm: block '%lu' is full, allow GC (sched)\n", | ||
433 | rblk->parent->id); | ||
434 | } | ||
435 | |||
436 | static const struct block_device_operations rrpc_fops = { | ||
437 | .owner = THIS_MODULE, | ||
438 | }; | ||
439 | |||
440 | static struct rrpc_lun *rrpc_get_lun_rr(struct rrpc *rrpc, int is_gc) | ||
441 | { | ||
442 | unsigned int i; | ||
443 | struct rrpc_lun *rlun, *max_free; | ||
444 | |||
445 | if (!is_gc) | ||
446 | return get_next_lun(rrpc); | ||
447 | |||
448 | /* during GC, we don't care about RR, instead we want to make | ||
449 | * sure that we maintain evenness between the block luns. | ||
450 | */ | ||
451 | max_free = &rrpc->luns[0]; | ||
452 | /* prevent GC-ing lun from devouring pages of a lun with | ||
453 | * little free blocks. We don't take the lock as we only need an | ||
454 | * estimate. | ||
455 | */ | ||
456 | rrpc_for_each_lun(rrpc, rlun, i) { | ||
457 | if (rlun->parent->nr_free_blocks > | ||
458 | max_free->parent->nr_free_blocks) | ||
459 | max_free = rlun; | ||
460 | } | ||
461 | |||
462 | return max_free; | ||
463 | } | ||
464 | |||
465 | static struct rrpc_addr *rrpc_update_map(struct rrpc *rrpc, sector_t laddr, | ||
466 | struct rrpc_block *rblk, u64 paddr) | ||
467 | { | ||
468 | struct rrpc_addr *gp; | ||
469 | struct rrpc_rev_addr *rev; | ||
470 | |||
471 | BUG_ON(laddr >= rrpc->nr_pages); | ||
472 | |||
473 | gp = &rrpc->trans_map[laddr]; | ||
474 | spin_lock(&rrpc->rev_lock); | ||
475 | if (gp->rblk) | ||
476 | rrpc_page_invalidate(rrpc, gp); | ||
477 | |||
478 | gp->addr = paddr; | ||
479 | gp->rblk = rblk; | ||
480 | |||
481 | rev = &rrpc->rev_trans_map[gp->addr - rrpc->poffset]; | ||
482 | rev->addr = laddr; | ||
483 | spin_unlock(&rrpc->rev_lock); | ||
484 | |||
485 | return gp; | ||
486 | } | ||
487 | |||
488 | static u64 rrpc_alloc_addr(struct rrpc *rrpc, struct rrpc_block *rblk) | ||
489 | { | ||
490 | u64 addr = ADDR_EMPTY; | ||
491 | |||
492 | spin_lock(&rblk->lock); | ||
493 | if (block_is_full(rrpc, rblk)) | ||
494 | goto out; | ||
495 | |||
496 | addr = block_to_addr(rrpc, rblk) + rblk->next_page; | ||
497 | |||
498 | rblk->next_page++; | ||
499 | out: | ||
500 | spin_unlock(&rblk->lock); | ||
501 | return addr; | ||
502 | } | ||
503 | |||
504 | /* Simple round-robin Logical to physical address translation. | ||
505 | * | ||
506 | * Retrieve the mapping using the active append point. Then update the ap for | ||
507 | * the next write to the disk. | ||
508 | * | ||
509 | * Returns rrpc_addr with the physical address and block. Remember to return to | ||
510 | * rrpc->addr_cache when request is finished. | ||
511 | */ | ||
512 | static struct rrpc_addr *rrpc_map_page(struct rrpc *rrpc, sector_t laddr, | ||
513 | int is_gc) | ||
514 | { | ||
515 | struct rrpc_lun *rlun; | ||
516 | struct rrpc_block *rblk; | ||
517 | struct nvm_lun *lun; | ||
518 | u64 paddr; | ||
519 | |||
520 | rlun = rrpc_get_lun_rr(rrpc, is_gc); | ||
521 | lun = rlun->parent; | ||
522 | |||
523 | if (!is_gc && lun->nr_free_blocks < rrpc->nr_luns * 4) | ||
524 | return NULL; | ||
525 | |||
526 | spin_lock(&rlun->lock); | ||
527 | |||
528 | rblk = rlun->cur; | ||
529 | retry: | ||
530 | paddr = rrpc_alloc_addr(rrpc, rblk); | ||
531 | |||
532 | if (paddr == ADDR_EMPTY) { | ||
533 | rblk = rrpc_get_blk(rrpc, rlun, 0); | ||
534 | if (rblk) { | ||
535 | rrpc_set_lun_cur(rlun, rblk); | ||
536 | goto retry; | ||
537 | } | ||
538 | |||
539 | if (is_gc) { | ||
540 | /* retry from emergency gc block */ | ||
541 | paddr = rrpc_alloc_addr(rrpc, rlun->gc_cur); | ||
542 | if (paddr == ADDR_EMPTY) { | ||
543 | rblk = rrpc_get_blk(rrpc, rlun, 1); | ||
544 | if (!rblk) { | ||
545 | pr_err("rrpc: no more blocks"); | ||
546 | goto err; | ||
547 | } | ||
548 | |||
549 | rlun->gc_cur = rblk; | ||
550 | paddr = rrpc_alloc_addr(rrpc, rlun->gc_cur); | ||
551 | } | ||
552 | rblk = rlun->gc_cur; | ||
553 | } | ||
554 | } | ||
555 | |||
556 | spin_unlock(&rlun->lock); | ||
557 | return rrpc_update_map(rrpc, laddr, rblk, paddr); | ||
558 | err: | ||
559 | spin_unlock(&rlun->lock); | ||
560 | return NULL; | ||
561 | } | ||
562 | |||
563 | static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk) | ||
564 | { | ||
565 | struct rrpc_block_gc *gcb; | ||
566 | |||
567 | gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC); | ||
568 | if (!gcb) { | ||
569 | pr_err("rrpc: unable to queue block for gc."); | ||
570 | return; | ||
571 | } | ||
572 | |||
573 | gcb->rrpc = rrpc; | ||
574 | gcb->rblk = rblk; | ||
575 | |||
576 | INIT_WORK(&gcb->ws_gc, rrpc_gc_queue); | ||
577 | queue_work(rrpc->kgc_wq, &gcb->ws_gc); | ||
578 | } | ||
579 | |||
580 | static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd, | ||
581 | sector_t laddr, uint8_t npages) | ||
582 | { | ||
583 | struct rrpc_addr *p; | ||
584 | struct rrpc_block *rblk; | ||
585 | struct nvm_lun *lun; | ||
586 | int cmnt_size, i; | ||
587 | |||
588 | for (i = 0; i < npages; i++) { | ||
589 | p = &rrpc->trans_map[laddr + i]; | ||
590 | rblk = p->rblk; | ||
591 | lun = rblk->parent->lun; | ||
592 | |||
593 | cmnt_size = atomic_inc_return(&rblk->data_cmnt_size); | ||
594 | if (unlikely(cmnt_size == rrpc->dev->pgs_per_blk)) | ||
595 | rrpc_run_gc(rrpc, rblk); | ||
596 | } | ||
597 | } | ||
598 | |||
599 | static int rrpc_end_io(struct nvm_rq *rqd, int error) | ||
600 | { | ||
601 | struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance); | ||
602 | struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); | ||
603 | uint8_t npages = rqd->nr_pages; | ||
604 | sector_t laddr = rrpc_get_laddr(rqd->bio) - npages; | ||
605 | |||
606 | if (bio_data_dir(rqd->bio) == WRITE) | ||
607 | rrpc_end_io_write(rrpc, rrqd, laddr, npages); | ||
608 | |||
609 | if (rrqd->flags & NVM_IOTYPE_GC) | ||
610 | return 0; | ||
611 | |||
612 | rrpc_unlock_rq(rrpc, rqd); | ||
613 | bio_put(rqd->bio); | ||
614 | |||
615 | if (npages > 1) | ||
616 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); | ||
617 | if (rqd->metadata) | ||
618 | nvm_dev_dma_free(rrpc->dev, rqd->metadata, rqd->dma_metadata); | ||
619 | |||
620 | mempool_free(rqd, rrpc->rq_pool); | ||
621 | |||
622 | return 0; | ||
623 | } | ||
624 | |||
625 | static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio, | ||
626 | struct nvm_rq *rqd, unsigned long flags, int npages) | ||
627 | { | ||
628 | struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); | ||
629 | struct rrpc_addr *gp; | ||
630 | sector_t laddr = rrpc_get_laddr(bio); | ||
631 | int is_gc = flags & NVM_IOTYPE_GC; | ||
632 | int i; | ||
633 | |||
634 | if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) { | ||
635 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); | ||
636 | return NVM_IO_REQUEUE; | ||
637 | } | ||
638 | |||
639 | for (i = 0; i < npages; i++) { | ||
640 | /* We assume that mapping occurs at 4KB granularity */ | ||
641 | BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_pages)); | ||
642 | gp = &rrpc->trans_map[laddr + i]; | ||
643 | |||
644 | if (gp->rblk) { | ||
645 | rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev, | ||
646 | gp->addr); | ||
647 | } else { | ||
648 | BUG_ON(is_gc); | ||
649 | rrpc_unlock_laddr(rrpc, r); | ||
650 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, | ||
651 | rqd->dma_ppa_list); | ||
652 | return NVM_IO_DONE; | ||
653 | } | ||
654 | } | ||
655 | |||
656 | rqd->opcode = NVM_OP_HBREAD; | ||
657 | |||
658 | return NVM_IO_OK; | ||
659 | } | ||
660 | |||
661 | static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd, | ||
662 | unsigned long flags) | ||
663 | { | ||
664 | struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); | ||
665 | int is_gc = flags & NVM_IOTYPE_GC; | ||
666 | sector_t laddr = rrpc_get_laddr(bio); | ||
667 | struct rrpc_addr *gp; | ||
668 | |||
669 | if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) | ||
670 | return NVM_IO_REQUEUE; | ||
671 | |||
672 | BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_pages)); | ||
673 | gp = &rrpc->trans_map[laddr]; | ||
674 | |||
675 | if (gp->rblk) { | ||
676 | rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr); | ||
677 | } else { | ||
678 | BUG_ON(is_gc); | ||
679 | rrpc_unlock_rq(rrpc, rqd); | ||
680 | return NVM_IO_DONE; | ||
681 | } | ||
682 | |||
683 | rqd->opcode = NVM_OP_HBREAD; | ||
684 | rrqd->addr = gp; | ||
685 | |||
686 | return NVM_IO_OK; | ||
687 | } | ||
688 | |||
689 | static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio, | ||
690 | struct nvm_rq *rqd, unsigned long flags, int npages) | ||
691 | { | ||
692 | struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); | ||
693 | struct rrpc_addr *p; | ||
694 | sector_t laddr = rrpc_get_laddr(bio); | ||
695 | int is_gc = flags & NVM_IOTYPE_GC; | ||
696 | int i; | ||
697 | |||
698 | if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) { | ||
699 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list); | ||
700 | return NVM_IO_REQUEUE; | ||
701 | } | ||
702 | |||
703 | for (i = 0; i < npages; i++) { | ||
704 | /* We assume that mapping occurs at 4KB granularity */ | ||
705 | p = rrpc_map_page(rrpc, laddr + i, is_gc); | ||
706 | if (!p) { | ||
707 | BUG_ON(is_gc); | ||
708 | rrpc_unlock_laddr(rrpc, r); | ||
709 | nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, | ||
710 | rqd->dma_ppa_list); | ||
711 | rrpc_gc_kick(rrpc); | ||
712 | return NVM_IO_REQUEUE; | ||
713 | } | ||
714 | |||
715 | rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev, | ||
716 | p->addr); | ||
717 | } | ||
718 | |||
719 | rqd->opcode = NVM_OP_HBWRITE; | ||
720 | |||
721 | return NVM_IO_OK; | ||
722 | } | ||
723 | |||
724 | static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio, | ||
725 | struct nvm_rq *rqd, unsigned long flags) | ||
726 | { | ||
727 | struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); | ||
728 | struct rrpc_addr *p; | ||
729 | int is_gc = flags & NVM_IOTYPE_GC; | ||
730 | sector_t laddr = rrpc_get_laddr(bio); | ||
731 | |||
732 | if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) | ||
733 | return NVM_IO_REQUEUE; | ||
734 | |||
735 | p = rrpc_map_page(rrpc, laddr, is_gc); | ||
736 | if (!p) { | ||
737 | BUG_ON(is_gc); | ||
738 | rrpc_unlock_rq(rrpc, rqd); | ||
739 | rrpc_gc_kick(rrpc); | ||
740 | return NVM_IO_REQUEUE; | ||
741 | } | ||
742 | |||
743 | rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, p->addr); | ||
744 | rqd->opcode = NVM_OP_HBWRITE; | ||
745 | rrqd->addr = p; | ||
746 | |||
747 | return NVM_IO_OK; | ||
748 | } | ||
749 | |||
750 | static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio, | ||
751 | struct nvm_rq *rqd, unsigned long flags, uint8_t npages) | ||
752 | { | ||
753 | if (npages > 1) { | ||
754 | rqd->ppa_list = nvm_dev_dma_alloc(rrpc->dev, GFP_KERNEL, | ||
755 | &rqd->dma_ppa_list); | ||
756 | if (!rqd->ppa_list) { | ||
757 | pr_err("rrpc: not able to allocate ppa list\n"); | ||
758 | return NVM_IO_ERR; | ||
759 | } | ||
760 | |||
761 | if (bio_rw(bio) == WRITE) | ||
762 | return rrpc_write_ppalist_rq(rrpc, bio, rqd, flags, | ||
763 | npages); | ||
764 | |||
765 | return rrpc_read_ppalist_rq(rrpc, bio, rqd, flags, npages); | ||
766 | } | ||
767 | |||
768 | if (bio_rw(bio) == WRITE) | ||
769 | return rrpc_write_rq(rrpc, bio, rqd, flags); | ||
770 | |||
771 | return rrpc_read_rq(rrpc, bio, rqd, flags); | ||
772 | } | ||
773 | |||
774 | static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio, | ||
775 | struct nvm_rq *rqd, unsigned long flags) | ||
776 | { | ||
777 | int err; | ||
778 | struct rrpc_rq *rrq = nvm_rq_to_pdu(rqd); | ||
779 | uint8_t nr_pages = rrpc_get_pages(bio); | ||
780 | int bio_size = bio_sectors(bio) << 9; | ||
781 | |||
782 | if (bio_size < rrpc->dev->sec_size) | ||
783 | return NVM_IO_ERR; | ||
784 | else if (bio_size > rrpc->dev->max_rq_size) | ||
785 | return NVM_IO_ERR; | ||
786 | |||
787 | err = rrpc_setup_rq(rrpc, bio, rqd, flags, nr_pages); | ||
788 | if (err) | ||
789 | return err; | ||
790 | |||
791 | bio_get(bio); | ||
792 | rqd->bio = bio; | ||
793 | rqd->ins = &rrpc->instance; | ||
794 | rqd->nr_pages = nr_pages; | ||
795 | rrq->flags = flags; | ||
796 | |||
797 | err = nvm_submit_io(rrpc->dev, rqd); | ||
798 | if (err) { | ||
799 | pr_err("rrpc: I/O submission failed: %d\n", err); | ||
800 | return NVM_IO_ERR; | ||
801 | } | ||
802 | |||
803 | return NVM_IO_OK; | ||
804 | } | ||
805 | |||
806 | static void rrpc_make_rq(struct request_queue *q, struct bio *bio) | ||
807 | { | ||
808 | struct rrpc *rrpc = q->queuedata; | ||
809 | struct nvm_rq *rqd; | ||
810 | int err; | ||
811 | |||
812 | if (bio->bi_rw & REQ_DISCARD) { | ||
813 | rrpc_discard(rrpc, bio); | ||
814 | return; | ||
815 | } | ||
816 | |||
817 | rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL); | ||
818 | if (!rqd) { | ||
819 | pr_err_ratelimited("rrpc: not able to queue bio."); | ||
820 | bio_io_error(bio); | ||
821 | return; | ||
822 | } | ||
823 | memset(rqd, 0, sizeof(struct nvm_rq)); | ||
824 | |||
825 | err = rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_NONE); | ||
826 | switch (err) { | ||
827 | case NVM_IO_OK: | ||
828 | return; | ||
829 | case NVM_IO_ERR: | ||
830 | bio_io_error(bio); | ||
831 | break; | ||
832 | case NVM_IO_DONE: | ||
833 | bio_endio(bio); | ||
834 | break; | ||
835 | case NVM_IO_REQUEUE: | ||
836 | spin_lock(&rrpc->bio_lock); | ||
837 | bio_list_add(&rrpc->requeue_bios, bio); | ||
838 | spin_unlock(&rrpc->bio_lock); | ||
839 | queue_work(rrpc->kgc_wq, &rrpc->ws_requeue); | ||
840 | break; | ||
841 | } | ||
842 | |||
843 | mempool_free(rqd, rrpc->rq_pool); | ||
844 | } | ||
845 | |||
846 | static void rrpc_requeue(struct work_struct *work) | ||
847 | { | ||
848 | struct rrpc *rrpc = container_of(work, struct rrpc, ws_requeue); | ||
849 | struct bio_list bios; | ||
850 | struct bio *bio; | ||
851 | |||
852 | bio_list_init(&bios); | ||
853 | |||
854 | spin_lock(&rrpc->bio_lock); | ||
855 | bio_list_merge(&bios, &rrpc->requeue_bios); | ||
856 | bio_list_init(&rrpc->requeue_bios); | ||
857 | spin_unlock(&rrpc->bio_lock); | ||
858 | |||
859 | while ((bio = bio_list_pop(&bios))) | ||
860 | rrpc_make_rq(rrpc->disk->queue, bio); | ||
861 | } | ||
862 | |||
863 | static void rrpc_gc_free(struct rrpc *rrpc) | ||
864 | { | ||
865 | struct rrpc_lun *rlun; | ||
866 | int i; | ||
867 | |||
868 | if (rrpc->krqd_wq) | ||
869 | destroy_workqueue(rrpc->krqd_wq); | ||
870 | |||
871 | if (rrpc->kgc_wq) | ||
872 | destroy_workqueue(rrpc->kgc_wq); | ||
873 | |||
874 | if (!rrpc->luns) | ||
875 | return; | ||
876 | |||
877 | for (i = 0; i < rrpc->nr_luns; i++) { | ||
878 | rlun = &rrpc->luns[i]; | ||
879 | |||
880 | if (!rlun->blocks) | ||
881 | break; | ||
882 | vfree(rlun->blocks); | ||
883 | } | ||
884 | } | ||
885 | |||
886 | static int rrpc_gc_init(struct rrpc *rrpc) | ||
887 | { | ||
888 | rrpc->krqd_wq = alloc_workqueue("rrpc-lun", WQ_MEM_RECLAIM|WQ_UNBOUND, | ||
889 | rrpc->nr_luns); | ||
890 | if (!rrpc->krqd_wq) | ||
891 | return -ENOMEM; | ||
892 | |||
893 | rrpc->kgc_wq = alloc_workqueue("rrpc-bg", WQ_MEM_RECLAIM, 1); | ||
894 | if (!rrpc->kgc_wq) | ||
895 | return -ENOMEM; | ||
896 | |||
897 | setup_timer(&rrpc->gc_timer, rrpc_gc_timer, (unsigned long)rrpc); | ||
898 | |||
899 | return 0; | ||
900 | } | ||
901 | |||
902 | static void rrpc_map_free(struct rrpc *rrpc) | ||
903 | { | ||
904 | vfree(rrpc->rev_trans_map); | ||
905 | vfree(rrpc->trans_map); | ||
906 | } | ||
907 | |||
908 | static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private) | ||
909 | { | ||
910 | struct rrpc *rrpc = (struct rrpc *)private; | ||
911 | struct nvm_dev *dev = rrpc->dev; | ||
912 | struct rrpc_addr *addr = rrpc->trans_map + slba; | ||
913 | struct rrpc_rev_addr *raddr = rrpc->rev_trans_map; | ||
914 | sector_t max_pages = dev->total_pages * (dev->sec_size >> 9); | ||
915 | u64 elba = slba + nlb; | ||
916 | u64 i; | ||
917 | |||
918 | if (unlikely(elba > dev->total_pages)) { | ||
919 | pr_err("nvm: L2P data from device is out of bounds!\n"); | ||
920 | return -EINVAL; | ||
921 | } | ||
922 | |||
923 | for (i = 0; i < nlb; i++) { | ||
924 | u64 pba = le64_to_cpu(entries[i]); | ||
925 | /* LNVM treats address-spaces as silos, LBA and PBA are | ||
926 | * equally large and zero-indexed. | ||
927 | */ | ||
928 | if (unlikely(pba >= max_pages && pba != U64_MAX)) { | ||
929 | pr_err("nvm: L2P data entry is out of bounds!\n"); | ||
930 | return -EINVAL; | ||
931 | } | ||
932 | |||
933 | /* Address zero is a special one. The first page on a disk is | ||
934 | * protected. As it often holds internal device boot | ||
935 | * information. | ||
936 | */ | ||
937 | if (!pba) | ||
938 | continue; | ||
939 | |||
940 | addr[i].addr = pba; | ||
941 | raddr[pba].addr = slba + i; | ||
942 | } | ||
943 | |||
944 | return 0; | ||
945 | } | ||
946 | |||
947 | static int rrpc_map_init(struct rrpc *rrpc) | ||
948 | { | ||
949 | struct nvm_dev *dev = rrpc->dev; | ||
950 | sector_t i; | ||
951 | int ret; | ||
952 | |||
953 | rrpc->trans_map = vzalloc(sizeof(struct rrpc_addr) * rrpc->nr_pages); | ||
954 | if (!rrpc->trans_map) | ||
955 | return -ENOMEM; | ||
956 | |||
957 | rrpc->rev_trans_map = vmalloc(sizeof(struct rrpc_rev_addr) | ||
958 | * rrpc->nr_pages); | ||
959 | if (!rrpc->rev_trans_map) | ||
960 | return -ENOMEM; | ||
961 | |||
962 | for (i = 0; i < rrpc->nr_pages; i++) { | ||
963 | struct rrpc_addr *p = &rrpc->trans_map[i]; | ||
964 | struct rrpc_rev_addr *r = &rrpc->rev_trans_map[i]; | ||
965 | |||
966 | p->addr = ADDR_EMPTY; | ||
967 | r->addr = ADDR_EMPTY; | ||
968 | } | ||
969 | |||
970 | if (!dev->ops->get_l2p_tbl) | ||
971 | return 0; | ||
972 | |||
973 | /* Bring up the mapping table from device */ | ||
974 | ret = dev->ops->get_l2p_tbl(dev->q, 0, dev->total_pages, | ||
975 | rrpc_l2p_update, rrpc); | ||
976 | if (ret) { | ||
977 | pr_err("nvm: rrpc: could not read L2P table.\n"); | ||
978 | return -EINVAL; | ||
979 | } | ||
980 | |||
981 | return 0; | ||
982 | } | ||
983 | |||
984 | |||
985 | /* Minimum pages needed within a lun */ | ||
986 | #define PAGE_POOL_SIZE 16 | ||
987 | #define ADDR_POOL_SIZE 64 | ||
988 | |||
989 | static int rrpc_core_init(struct rrpc *rrpc) | ||
990 | { | ||
991 | down_write(&rrpc_lock); | ||
992 | if (!rrpc_gcb_cache) { | ||
993 | rrpc_gcb_cache = kmem_cache_create("rrpc_gcb", | ||
994 | sizeof(struct rrpc_block_gc), 0, 0, NULL); | ||
995 | if (!rrpc_gcb_cache) { | ||
996 | up_write(&rrpc_lock); | ||
997 | return -ENOMEM; | ||
998 | } | ||
999 | |||
1000 | rrpc_rq_cache = kmem_cache_create("rrpc_rq", | ||
1001 | sizeof(struct nvm_rq) + sizeof(struct rrpc_rq), | ||
1002 | 0, 0, NULL); | ||
1003 | if (!rrpc_rq_cache) { | ||
1004 | kmem_cache_destroy(rrpc_gcb_cache); | ||
1005 | up_write(&rrpc_lock); | ||
1006 | return -ENOMEM; | ||
1007 | } | ||
1008 | } | ||
1009 | up_write(&rrpc_lock); | ||
1010 | |||
1011 | rrpc->page_pool = mempool_create_page_pool(PAGE_POOL_SIZE, 0); | ||
1012 | if (!rrpc->page_pool) | ||
1013 | return -ENOMEM; | ||
1014 | |||
1015 | rrpc->gcb_pool = mempool_create_slab_pool(rrpc->dev->nr_luns, | ||
1016 | rrpc_gcb_cache); | ||
1017 | if (!rrpc->gcb_pool) | ||
1018 | return -ENOMEM; | ||
1019 | |||
1020 | rrpc->rq_pool = mempool_create_slab_pool(64, rrpc_rq_cache); | ||
1021 | if (!rrpc->rq_pool) | ||
1022 | return -ENOMEM; | ||
1023 | |||
1024 | spin_lock_init(&rrpc->inflights.lock); | ||
1025 | INIT_LIST_HEAD(&rrpc->inflights.reqs); | ||
1026 | |||
1027 | return 0; | ||
1028 | } | ||
1029 | |||
1030 | static void rrpc_core_free(struct rrpc *rrpc) | ||
1031 | { | ||
1032 | mempool_destroy(rrpc->page_pool); | ||
1033 | mempool_destroy(rrpc->gcb_pool); | ||
1034 | mempool_destroy(rrpc->rq_pool); | ||
1035 | } | ||
1036 | |||
1037 | static void rrpc_luns_free(struct rrpc *rrpc) | ||
1038 | { | ||
1039 | kfree(rrpc->luns); | ||
1040 | } | ||
1041 | |||
1042 | static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end) | ||
1043 | { | ||
1044 | struct nvm_dev *dev = rrpc->dev; | ||
1045 | struct rrpc_lun *rlun; | ||
1046 | int i, j; | ||
1047 | |||
1048 | spin_lock_init(&rrpc->rev_lock); | ||
1049 | |||
1050 | rrpc->luns = kcalloc(rrpc->nr_luns, sizeof(struct rrpc_lun), | ||
1051 | GFP_KERNEL); | ||
1052 | if (!rrpc->luns) | ||
1053 | return -ENOMEM; | ||
1054 | |||
1055 | /* 1:1 mapping */ | ||
1056 | for (i = 0; i < rrpc->nr_luns; i++) { | ||
1057 | struct nvm_lun *lun = dev->mt->get_lun(dev, lun_begin + i); | ||
1058 | |||
1059 | if (dev->pgs_per_blk > | ||
1060 | MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) { | ||
1061 | pr_err("rrpc: number of pages per block too high."); | ||
1062 | goto err; | ||
1063 | } | ||
1064 | |||
1065 | rlun = &rrpc->luns[i]; | ||
1066 | rlun->rrpc = rrpc; | ||
1067 | rlun->parent = lun; | ||
1068 | INIT_LIST_HEAD(&rlun->prio_list); | ||
1069 | INIT_WORK(&rlun->ws_gc, rrpc_lun_gc); | ||
1070 | spin_lock_init(&rlun->lock); | ||
1071 | |||
1072 | rrpc->total_blocks += dev->blks_per_lun; | ||
1073 | rrpc->nr_pages += dev->sec_per_lun; | ||
1074 | |||
1075 | rlun->blocks = vzalloc(sizeof(struct rrpc_block) * | ||
1076 | rrpc->dev->blks_per_lun); | ||
1077 | if (!rlun->blocks) | ||
1078 | goto err; | ||
1079 | |||
1080 | for (j = 0; j < rrpc->dev->blks_per_lun; j++) { | ||
1081 | struct rrpc_block *rblk = &rlun->blocks[j]; | ||
1082 | struct nvm_block *blk = &lun->blocks[j]; | ||
1083 | |||
1084 | rblk->parent = blk; | ||
1085 | INIT_LIST_HEAD(&rblk->prio); | ||
1086 | spin_lock_init(&rblk->lock); | ||
1087 | } | ||
1088 | } | ||
1089 | |||
1090 | return 0; | ||
1091 | err: | ||
1092 | return -ENOMEM; | ||
1093 | } | ||
1094 | |||
1095 | static void rrpc_free(struct rrpc *rrpc) | ||
1096 | { | ||
1097 | rrpc_gc_free(rrpc); | ||
1098 | rrpc_map_free(rrpc); | ||
1099 | rrpc_core_free(rrpc); | ||
1100 | rrpc_luns_free(rrpc); | ||
1101 | |||
1102 | kfree(rrpc); | ||
1103 | } | ||
1104 | |||
1105 | static void rrpc_exit(void *private) | ||
1106 | { | ||
1107 | struct rrpc *rrpc = private; | ||
1108 | |||
1109 | del_timer(&rrpc->gc_timer); | ||
1110 | |||
1111 | flush_workqueue(rrpc->krqd_wq); | ||
1112 | flush_workqueue(rrpc->kgc_wq); | ||
1113 | |||
1114 | rrpc_free(rrpc); | ||
1115 | } | ||
1116 | |||
1117 | static sector_t rrpc_capacity(void *private) | ||
1118 | { | ||
1119 | struct rrpc *rrpc = private; | ||
1120 | struct nvm_dev *dev = rrpc->dev; | ||
1121 | sector_t reserved, provisioned; | ||
1122 | |||
1123 | /* cur, gc, and two emergency blocks for each lun */ | ||
1124 | reserved = rrpc->nr_luns * dev->max_pages_per_blk * 4; | ||
1125 | provisioned = rrpc->nr_pages - reserved; | ||
1126 | |||
1127 | if (reserved > rrpc->nr_pages) { | ||
1128 | pr_err("rrpc: not enough space available to expose storage.\n"); | ||
1129 | return 0; | ||
1130 | } | ||
1131 | |||
1132 | sector_div(provisioned, 10); | ||
1133 | return provisioned * 9 * NR_PHY_IN_LOG; | ||
1134 | } | ||
1135 | |||
1136 | /* | ||
1137 | * Looks up the logical address from reverse trans map and check if its valid by | ||
1138 | * comparing the logical to physical address with the physical address. | ||
1139 | * Returns 0 on free, otherwise 1 if in use | ||
1140 | */ | ||
1141 | static void rrpc_block_map_update(struct rrpc *rrpc, struct rrpc_block *rblk) | ||
1142 | { | ||
1143 | struct nvm_dev *dev = rrpc->dev; | ||
1144 | int offset; | ||
1145 | struct rrpc_addr *laddr; | ||
1146 | u64 paddr, pladdr; | ||
1147 | |||
1148 | for (offset = 0; offset < dev->pgs_per_blk; offset++) { | ||
1149 | paddr = block_to_addr(rrpc, rblk) + offset; | ||
1150 | |||
1151 | pladdr = rrpc->rev_trans_map[paddr].addr; | ||
1152 | if (pladdr == ADDR_EMPTY) | ||
1153 | continue; | ||
1154 | |||
1155 | laddr = &rrpc->trans_map[pladdr]; | ||
1156 | |||
1157 | if (paddr == laddr->addr) { | ||
1158 | laddr->rblk = rblk; | ||
1159 | } else { | ||
1160 | set_bit(offset, rblk->invalid_pages); | ||
1161 | rblk->nr_invalid_pages++; | ||
1162 | } | ||
1163 | } | ||
1164 | } | ||
1165 | |||
1166 | static int rrpc_blocks_init(struct rrpc *rrpc) | ||
1167 | { | ||
1168 | struct rrpc_lun *rlun; | ||
1169 | struct rrpc_block *rblk; | ||
1170 | int lun_iter, blk_iter; | ||
1171 | |||
1172 | for (lun_iter = 0; lun_iter < rrpc->nr_luns; lun_iter++) { | ||
1173 | rlun = &rrpc->luns[lun_iter]; | ||
1174 | |||
1175 | for (blk_iter = 0; blk_iter < rrpc->dev->blks_per_lun; | ||
1176 | blk_iter++) { | ||
1177 | rblk = &rlun->blocks[blk_iter]; | ||
1178 | rrpc_block_map_update(rrpc, rblk); | ||
1179 | } | ||
1180 | } | ||
1181 | |||
1182 | return 0; | ||
1183 | } | ||
1184 | |||
1185 | static int rrpc_luns_configure(struct rrpc *rrpc) | ||
1186 | { | ||
1187 | struct rrpc_lun *rlun; | ||
1188 | struct rrpc_block *rblk; | ||
1189 | int i; | ||
1190 | |||
1191 | for (i = 0; i < rrpc->nr_luns; i++) { | ||
1192 | rlun = &rrpc->luns[i]; | ||
1193 | |||
1194 | rblk = rrpc_get_blk(rrpc, rlun, 0); | ||
1195 | if (!rblk) | ||
1196 | return -EINVAL; | ||
1197 | |||
1198 | rrpc_set_lun_cur(rlun, rblk); | ||
1199 | |||
1200 | /* Emergency gc block */ | ||
1201 | rblk = rrpc_get_blk(rrpc, rlun, 1); | ||
1202 | if (!rblk) | ||
1203 | return -EINVAL; | ||
1204 | rlun->gc_cur = rblk; | ||
1205 | } | ||
1206 | |||
1207 | return 0; | ||
1208 | } | ||
1209 | |||
1210 | static struct nvm_tgt_type tt_rrpc; | ||
1211 | |||
1212 | static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk, | ||
1213 | int lun_begin, int lun_end) | ||
1214 | { | ||
1215 | struct request_queue *bqueue = dev->q; | ||
1216 | struct request_queue *tqueue = tdisk->queue; | ||
1217 | struct rrpc *rrpc; | ||
1218 | int ret; | ||
1219 | |||
1220 | if (!(dev->identity.dom & NVM_RSP_L2P)) { | ||
1221 | pr_err("nvm: rrpc: device does not support l2p (%x)\n", | ||
1222 | dev->identity.dom); | ||
1223 | return ERR_PTR(-EINVAL); | ||
1224 | } | ||
1225 | |||
1226 | rrpc = kzalloc(sizeof(struct rrpc), GFP_KERNEL); | ||
1227 | if (!rrpc) | ||
1228 | return ERR_PTR(-ENOMEM); | ||
1229 | |||
1230 | rrpc->instance.tt = &tt_rrpc; | ||
1231 | rrpc->dev = dev; | ||
1232 | rrpc->disk = tdisk; | ||
1233 | |||
1234 | bio_list_init(&rrpc->requeue_bios); | ||
1235 | spin_lock_init(&rrpc->bio_lock); | ||
1236 | INIT_WORK(&rrpc->ws_requeue, rrpc_requeue); | ||
1237 | |||
1238 | rrpc->nr_luns = lun_end - lun_begin + 1; | ||
1239 | |||
1240 | /* simple round-robin strategy */ | ||
1241 | atomic_set(&rrpc->next_lun, -1); | ||
1242 | |||
1243 | ret = rrpc_luns_init(rrpc, lun_begin, lun_end); | ||
1244 | if (ret) { | ||
1245 | pr_err("nvm: rrpc: could not initialize luns\n"); | ||
1246 | goto err; | ||
1247 | } | ||
1248 | |||
1249 | rrpc->poffset = dev->sec_per_lun * lun_begin; | ||
1250 | rrpc->lun_offset = lun_begin; | ||
1251 | |||
1252 | ret = rrpc_core_init(rrpc); | ||
1253 | if (ret) { | ||
1254 | pr_err("nvm: rrpc: could not initialize core\n"); | ||
1255 | goto err; | ||
1256 | } | ||
1257 | |||
1258 | ret = rrpc_map_init(rrpc); | ||
1259 | if (ret) { | ||
1260 | pr_err("nvm: rrpc: could not initialize maps\n"); | ||
1261 | goto err; | ||
1262 | } | ||
1263 | |||
1264 | ret = rrpc_blocks_init(rrpc); | ||
1265 | if (ret) { | ||
1266 | pr_err("nvm: rrpc: could not initialize state for blocks\n"); | ||
1267 | goto err; | ||
1268 | } | ||
1269 | |||
1270 | ret = rrpc_luns_configure(rrpc); | ||
1271 | if (ret) { | ||
1272 | pr_err("nvm: rrpc: not enough blocks available in LUNs.\n"); | ||
1273 | goto err; | ||
1274 | } | ||
1275 | |||
1276 | ret = rrpc_gc_init(rrpc); | ||
1277 | if (ret) { | ||
1278 | pr_err("nvm: rrpc: could not initialize gc\n"); | ||
1279 | goto err; | ||
1280 | } | ||
1281 | |||
1282 | /* inherit the size from the underlying device */ | ||
1283 | blk_queue_logical_block_size(tqueue, queue_physical_block_size(bqueue)); | ||
1284 | blk_queue_max_hw_sectors(tqueue, queue_max_hw_sectors(bqueue)); | ||
1285 | |||
1286 | pr_info("nvm: rrpc initialized with %u luns and %llu pages.\n", | ||
1287 | rrpc->nr_luns, (unsigned long long)rrpc->nr_pages); | ||
1288 | |||
1289 | mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10)); | ||
1290 | |||
1291 | return rrpc; | ||
1292 | err: | ||
1293 | rrpc_free(rrpc); | ||
1294 | return ERR_PTR(ret); | ||
1295 | } | ||
1296 | |||
1297 | /* round robin, page-based FTL, and cost-based GC */ | ||
1298 | static struct nvm_tgt_type tt_rrpc = { | ||
1299 | .name = "rrpc", | ||
1300 | .version = {1, 0, 0}, | ||
1301 | |||
1302 | .make_rq = rrpc_make_rq, | ||
1303 | .capacity = rrpc_capacity, | ||
1304 | .end_io = rrpc_end_io, | ||
1305 | |||
1306 | .init = rrpc_init, | ||
1307 | .exit = rrpc_exit, | ||
1308 | }; | ||
1309 | |||
1310 | static int __init rrpc_module_init(void) | ||
1311 | { | ||
1312 | return nvm_register_target(&tt_rrpc); | ||
1313 | } | ||
1314 | |||
1315 | static void rrpc_module_exit(void) | ||
1316 | { | ||
1317 | nvm_unregister_target(&tt_rrpc); | ||
1318 | } | ||
1319 | |||
1320 | module_init(rrpc_module_init); | ||
1321 | module_exit(rrpc_module_exit); | ||
1322 | MODULE_LICENSE("GPL v2"); | ||
1323 | MODULE_DESCRIPTION("Block-Device Target for Open-Channel SSDs"); | ||
diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h new file mode 100644 index 000000000000..a9696a06c38c --- /dev/null +++ b/drivers/lightnvm/rrpc.h | |||
@@ -0,0 +1,239 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2015 IT University of Copenhagen | ||
3 | * Initial release: Matias Bjorling <m@bjorling.me> | ||
4 | * | ||
5 | * This program is free software; you can redistribute it and/or | ||
6 | * modify it under the terms of the GNU General Public License version | ||
7 | * 2 as published by the Free Software Foundation. | ||
8 | * | ||
9 | * This program is distributed in the hope that it will be useful, but | ||
10 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
12 | * General Public License for more details. | ||
13 | * | ||
14 | * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs. | ||
15 | */ | ||
16 | |||
17 | #ifndef RRPC_H_ | ||
18 | #define RRPC_H_ | ||
19 | |||
20 | #include <linux/blkdev.h> | ||
21 | #include <linux/blk-mq.h> | ||
22 | #include <linux/bio.h> | ||
23 | #include <linux/module.h> | ||
24 | #include <linux/kthread.h> | ||
25 | #include <linux/vmalloc.h> | ||
26 | |||
27 | #include <linux/lightnvm.h> | ||
28 | |||
29 | /* Run only GC if less than 1/X blocks are free */ | ||
30 | #define GC_LIMIT_INVERSE 10 | ||
31 | #define GC_TIME_SECS 100 | ||
32 | |||
33 | #define RRPC_SECTOR (512) | ||
34 | #define RRPC_EXPOSED_PAGE_SIZE (4096) | ||
35 | |||
36 | #define NR_PHY_IN_LOG (RRPC_EXPOSED_PAGE_SIZE / RRPC_SECTOR) | ||
37 | |||
38 | struct rrpc_inflight { | ||
39 | struct list_head reqs; | ||
40 | spinlock_t lock; | ||
41 | }; | ||
42 | |||
43 | struct rrpc_inflight_rq { | ||
44 | struct list_head list; | ||
45 | sector_t l_start; | ||
46 | sector_t l_end; | ||
47 | }; | ||
48 | |||
49 | struct rrpc_rq { | ||
50 | struct rrpc_inflight_rq inflight_rq; | ||
51 | struct rrpc_addr *addr; | ||
52 | unsigned long flags; | ||
53 | }; | ||
54 | |||
55 | struct rrpc_block { | ||
56 | struct nvm_block *parent; | ||
57 | struct list_head prio; | ||
58 | |||
59 | #define MAX_INVALID_PAGES_STORAGE 8 | ||
60 | /* Bitmap for invalid page intries */ | ||
61 | unsigned long invalid_pages[MAX_INVALID_PAGES_STORAGE]; | ||
62 | /* points to the next writable page within a block */ | ||
63 | unsigned int next_page; | ||
64 | /* number of pages that are invalid, wrt host page size */ | ||
65 | unsigned int nr_invalid_pages; | ||
66 | |||
67 | spinlock_t lock; | ||
68 | atomic_t data_cmnt_size; /* data pages committed to stable storage */ | ||
69 | }; | ||
70 | |||
71 | struct rrpc_lun { | ||
72 | struct rrpc *rrpc; | ||
73 | struct nvm_lun *parent; | ||
74 | struct rrpc_block *cur, *gc_cur; | ||
75 | struct rrpc_block *blocks; /* Reference to block allocation */ | ||
76 | struct list_head prio_list; /* Blocks that may be GC'ed */ | ||
77 | struct work_struct ws_gc; | ||
78 | |||
79 | spinlock_t lock; | ||
80 | }; | ||
81 | |||
82 | struct rrpc { | ||
83 | /* instance must be kept in top to resolve rrpc in unprep */ | ||
84 | struct nvm_tgt_instance instance; | ||
85 | |||
86 | struct nvm_dev *dev; | ||
87 | struct gendisk *disk; | ||
88 | |||
89 | u64 poffset; /* physical page offset */ | ||
90 | int lun_offset; | ||
91 | |||
92 | int nr_luns; | ||
93 | struct rrpc_lun *luns; | ||
94 | |||
95 | /* calculated values */ | ||
96 | unsigned long long nr_pages; | ||
97 | unsigned long total_blocks; | ||
98 | |||
99 | /* Write strategy variables. Move these into each for structure for each | ||
100 | * strategy | ||
101 | */ | ||
102 | atomic_t next_lun; /* Whenever a page is written, this is updated | ||
103 | * to point to the next write lun | ||
104 | */ | ||
105 | |||
106 | spinlock_t bio_lock; | ||
107 | struct bio_list requeue_bios; | ||
108 | struct work_struct ws_requeue; | ||
109 | |||
110 | /* Simple translation map of logical addresses to physical addresses. | ||
111 | * The logical addresses is known by the host system, while the physical | ||
112 | * addresses are used when writing to the disk block device. | ||
113 | */ | ||
114 | struct rrpc_addr *trans_map; | ||
115 | /* also store a reverse map for garbage collection */ | ||
116 | struct rrpc_rev_addr *rev_trans_map; | ||
117 | spinlock_t rev_lock; | ||
118 | |||
119 | struct rrpc_inflight inflights; | ||
120 | |||
121 | mempool_t *addr_pool; | ||
122 | mempool_t *page_pool; | ||
123 | mempool_t *gcb_pool; | ||
124 | mempool_t *rq_pool; | ||
125 | |||
126 | struct timer_list gc_timer; | ||
127 | struct workqueue_struct *krqd_wq; | ||
128 | struct workqueue_struct *kgc_wq; | ||
129 | }; | ||
130 | |||
131 | struct rrpc_block_gc { | ||
132 | struct rrpc *rrpc; | ||
133 | struct rrpc_block *rblk; | ||
134 | struct work_struct ws_gc; | ||
135 | }; | ||
136 | |||
137 | /* Logical to physical mapping */ | ||
138 | struct rrpc_addr { | ||
139 | u64 addr; | ||
140 | struct rrpc_block *rblk; | ||
141 | }; | ||
142 | |||
143 | /* Physical to logical mapping */ | ||
144 | struct rrpc_rev_addr { | ||
145 | u64 addr; | ||
146 | }; | ||
147 | |||
148 | static inline sector_t rrpc_get_laddr(struct bio *bio) | ||
149 | { | ||
150 | return bio->bi_iter.bi_sector / NR_PHY_IN_LOG; | ||
151 | } | ||
152 | |||
153 | static inline unsigned int rrpc_get_pages(struct bio *bio) | ||
154 | { | ||
155 | return bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE; | ||
156 | } | ||
157 | |||
158 | static inline sector_t rrpc_get_sector(sector_t laddr) | ||
159 | { | ||
160 | return laddr * NR_PHY_IN_LOG; | ||
161 | } | ||
162 | |||
163 | static inline int request_intersects(struct rrpc_inflight_rq *r, | ||
164 | sector_t laddr_start, sector_t laddr_end) | ||
165 | { | ||
166 | return (laddr_end >= r->l_start && laddr_end <= r->l_end) && | ||
167 | (laddr_start >= r->l_start && laddr_start <= r->l_end); | ||
168 | } | ||
169 | |||
170 | static int __rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr, | ||
171 | unsigned pages, struct rrpc_inflight_rq *r) | ||
172 | { | ||
173 | sector_t laddr_end = laddr + pages - 1; | ||
174 | struct rrpc_inflight_rq *rtmp; | ||
175 | |||
176 | spin_lock_irq(&rrpc->inflights.lock); | ||
177 | list_for_each_entry(rtmp, &rrpc->inflights.reqs, list) { | ||
178 | if (unlikely(request_intersects(rtmp, laddr, laddr_end))) { | ||
179 | /* existing, overlapping request, come back later */ | ||
180 | spin_unlock_irq(&rrpc->inflights.lock); | ||
181 | return 1; | ||
182 | } | ||
183 | } | ||
184 | |||
185 | r->l_start = laddr; | ||
186 | r->l_end = laddr_end; | ||
187 | |||
188 | list_add_tail(&r->list, &rrpc->inflights.reqs); | ||
189 | spin_unlock_irq(&rrpc->inflights.lock); | ||
190 | return 0; | ||
191 | } | ||
192 | |||
193 | static inline int rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr, | ||
194 | unsigned pages, | ||
195 | struct rrpc_inflight_rq *r) | ||
196 | { | ||
197 | BUG_ON((laddr + pages) > rrpc->nr_pages); | ||
198 | |||
199 | return __rrpc_lock_laddr(rrpc, laddr, pages, r); | ||
200 | } | ||
201 | |||
202 | static inline struct rrpc_inflight_rq *rrpc_get_inflight_rq(struct nvm_rq *rqd) | ||
203 | { | ||
204 | struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd); | ||
205 | |||
206 | return &rrqd->inflight_rq; | ||
207 | } | ||
208 | |||
209 | static inline int rrpc_lock_rq(struct rrpc *rrpc, struct bio *bio, | ||
210 | struct nvm_rq *rqd) | ||
211 | { | ||
212 | sector_t laddr = rrpc_get_laddr(bio); | ||
213 | unsigned int pages = rrpc_get_pages(bio); | ||
214 | struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); | ||
215 | |||
216 | return rrpc_lock_laddr(rrpc, laddr, pages, r); | ||
217 | } | ||
218 | |||
219 | static inline void rrpc_unlock_laddr(struct rrpc *rrpc, | ||
220 | struct rrpc_inflight_rq *r) | ||
221 | { | ||
222 | unsigned long flags; | ||
223 | |||
224 | spin_lock_irqsave(&rrpc->inflights.lock, flags); | ||
225 | list_del_init(&r->list); | ||
226 | spin_unlock_irqrestore(&rrpc->inflights.lock, flags); | ||
227 | } | ||
228 | |||
229 | static inline void rrpc_unlock_rq(struct rrpc *rrpc, struct nvm_rq *rqd) | ||
230 | { | ||
231 | struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd); | ||
232 | uint8_t pages = rqd->nr_pages; | ||
233 | |||
234 | BUG_ON((r->l_start + pages) > rrpc->nr_pages); | ||
235 | |||
236 | rrpc_unlock_laddr(rrpc, r); | ||
237 | } | ||
238 | |||
239 | #endif /* RRPC_H_ */ | ||
diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile index cfb6679ec245..219dc206fa5f 100644 --- a/drivers/nvme/host/Makefile +++ b/drivers/nvme/host/Makefile | |||
@@ -1,4 +1,4 @@ | |||
1 | 1 | ||
2 | obj-$(CONFIG_BLK_DEV_NVME) += nvme.o | 2 | obj-$(CONFIG_BLK_DEV_NVME) += nvme.o |
3 | 3 | ||
4 | nvme-y += pci.o scsi.o | 4 | nvme-y += pci.o scsi.o lightnvm.o |
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c new file mode 100644 index 000000000000..e0b7b95813bc --- /dev/null +++ b/drivers/nvme/host/lightnvm.c | |||
@@ -0,0 +1,526 @@ | |||
1 | /* | ||
2 | * nvme-lightnvm.c - LightNVM NVMe device | ||
3 | * | ||
4 | * Copyright (C) 2014-2015 IT University of Copenhagen | ||
5 | * Initial release: Matias Bjorling <mb@lightnvm.io> | ||
6 | * | ||
7 | * This program is free software; you can redistribute it and/or | ||
8 | * modify it under the terms of the GNU General Public License version | ||
9 | * 2 as published by the Free Software Foundation. | ||
10 | * | ||
11 | * This program is distributed in the hope that it will be useful, but | ||
12 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
14 | * General Public License for more details. | ||
15 | * | ||
16 | * You should have received a copy of the GNU General Public License | ||
17 | * along with this program; see the file COPYING. If not, write to | ||
18 | * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, | ||
19 | * USA. | ||
20 | * | ||
21 | */ | ||
22 | |||
23 | #include "nvme.h" | ||
24 | |||
25 | #ifdef CONFIG_NVM | ||
26 | |||
27 | #include <linux/nvme.h> | ||
28 | #include <linux/bitops.h> | ||
29 | #include <linux/lightnvm.h> | ||
30 | #include <linux/vmalloc.h> | ||
31 | |||
32 | enum nvme_nvm_admin_opcode { | ||
33 | nvme_nvm_admin_identity = 0xe2, | ||
34 | nvme_nvm_admin_get_l2p_tbl = 0xea, | ||
35 | nvme_nvm_admin_get_bb_tbl = 0xf2, | ||
36 | nvme_nvm_admin_set_bb_tbl = 0xf1, | ||
37 | }; | ||
38 | |||
39 | struct nvme_nvm_hb_rw { | ||
40 | __u8 opcode; | ||
41 | __u8 flags; | ||
42 | __u16 command_id; | ||
43 | __le32 nsid; | ||
44 | __u64 rsvd2; | ||
45 | __le64 metadata; | ||
46 | __le64 prp1; | ||
47 | __le64 prp2; | ||
48 | __le64 spba; | ||
49 | __le16 length; | ||
50 | __le16 control; | ||
51 | __le32 dsmgmt; | ||
52 | __le64 slba; | ||
53 | }; | ||
54 | |||
55 | struct nvme_nvm_ph_rw { | ||
56 | __u8 opcode; | ||
57 | __u8 flags; | ||
58 | __u16 command_id; | ||
59 | __le32 nsid; | ||
60 | __u64 rsvd2; | ||
61 | __le64 metadata; | ||
62 | __le64 prp1; | ||
63 | __le64 prp2; | ||
64 | __le64 spba; | ||
65 | __le16 length; | ||
66 | __le16 control; | ||
67 | __le32 dsmgmt; | ||
68 | __le64 resv; | ||
69 | }; | ||
70 | |||
71 | struct nvme_nvm_identity { | ||
72 | __u8 opcode; | ||
73 | __u8 flags; | ||
74 | __u16 command_id; | ||
75 | __le32 nsid; | ||
76 | __u64 rsvd[2]; | ||
77 | __le64 prp1; | ||
78 | __le64 prp2; | ||
79 | __le32 chnl_off; | ||
80 | __u32 rsvd11[5]; | ||
81 | }; | ||
82 | |||
83 | struct nvme_nvm_l2ptbl { | ||
84 | __u8 opcode; | ||
85 | __u8 flags; | ||
86 | __u16 command_id; | ||
87 | __le32 nsid; | ||
88 | __le32 cdw2[4]; | ||
89 | __le64 prp1; | ||
90 | __le64 prp2; | ||
91 | __le64 slba; | ||
92 | __le32 nlb; | ||
93 | __le16 cdw14[6]; | ||
94 | }; | ||
95 | |||
96 | struct nvme_nvm_bbtbl { | ||
97 | __u8 opcode; | ||
98 | __u8 flags; | ||
99 | __u16 command_id; | ||
100 | __le32 nsid; | ||
101 | __u64 rsvd[2]; | ||
102 | __le64 prp1; | ||
103 | __le64 prp2; | ||
104 | __le32 prp1_len; | ||
105 | __le32 prp2_len; | ||
106 | __le32 lbb; | ||
107 | __u32 rsvd11[3]; | ||
108 | }; | ||
109 | |||
110 | struct nvme_nvm_erase_blk { | ||
111 | __u8 opcode; | ||
112 | __u8 flags; | ||
113 | __u16 command_id; | ||
114 | __le32 nsid; | ||
115 | __u64 rsvd[2]; | ||
116 | __le64 prp1; | ||
117 | __le64 prp2; | ||
118 | __le64 spba; | ||
119 | __le16 length; | ||
120 | __le16 control; | ||
121 | __le32 dsmgmt; | ||
122 | __le64 resv; | ||
123 | }; | ||
124 | |||
125 | struct nvme_nvm_command { | ||
126 | union { | ||
127 | struct nvme_common_command common; | ||
128 | struct nvme_nvm_identity identity; | ||
129 | struct nvme_nvm_hb_rw hb_rw; | ||
130 | struct nvme_nvm_ph_rw ph_rw; | ||
131 | struct nvme_nvm_l2ptbl l2p; | ||
132 | struct nvme_nvm_bbtbl get_bb; | ||
133 | struct nvme_nvm_bbtbl set_bb; | ||
134 | struct nvme_nvm_erase_blk erase; | ||
135 | }; | ||
136 | }; | ||
137 | |||
138 | struct nvme_nvm_id_group { | ||
139 | __u8 mtype; | ||
140 | __u8 fmtype; | ||
141 | __le16 res16; | ||
142 | __u8 num_ch; | ||
143 | __u8 num_lun; | ||
144 | __u8 num_pln; | ||
145 | __le16 num_blk; | ||
146 | __le16 num_pg; | ||
147 | __le16 fpg_sz; | ||
148 | __le16 csecs; | ||
149 | __le16 sos; | ||
150 | __le32 trdt; | ||
151 | __le32 trdm; | ||
152 | __le32 tprt; | ||
153 | __le32 tprm; | ||
154 | __le32 tbet; | ||
155 | __le32 tbem; | ||
156 | __le32 mpos; | ||
157 | __le16 cpar; | ||
158 | __u8 reserved[913]; | ||
159 | } __packed; | ||
160 | |||
161 | struct nvme_nvm_addr_format { | ||
162 | __u8 ch_offset; | ||
163 | __u8 ch_len; | ||
164 | __u8 lun_offset; | ||
165 | __u8 lun_len; | ||
166 | __u8 pln_offset; | ||
167 | __u8 pln_len; | ||
168 | __u8 blk_offset; | ||
169 | __u8 blk_len; | ||
170 | __u8 pg_offset; | ||
171 | __u8 pg_len; | ||
172 | __u8 sect_offset; | ||
173 | __u8 sect_len; | ||
174 | __u8 res[4]; | ||
175 | } __packed; | ||
176 | |||
177 | struct nvme_nvm_id { | ||
178 | __u8 ver_id; | ||
179 | __u8 vmnt; | ||
180 | __u8 cgrps; | ||
181 | __u8 res[5]; | ||
182 | __le32 cap; | ||
183 | __le32 dom; | ||
184 | struct nvme_nvm_addr_format ppaf; | ||
185 | __u8 ppat; | ||
186 | __u8 resv[223]; | ||
187 | struct nvme_nvm_id_group groups[4]; | ||
188 | } __packed; | ||
189 | |||
190 | /* | ||
191 | * Check we didn't inadvertently grow the command struct | ||
192 | */ | ||
193 | static inline void _nvme_nvm_check_size(void) | ||
194 | { | ||
195 | BUILD_BUG_ON(sizeof(struct nvme_nvm_identity) != 64); | ||
196 | BUILD_BUG_ON(sizeof(struct nvme_nvm_hb_rw) != 64); | ||
197 | BUILD_BUG_ON(sizeof(struct nvme_nvm_ph_rw) != 64); | ||
198 | BUILD_BUG_ON(sizeof(struct nvme_nvm_bbtbl) != 64); | ||
199 | BUILD_BUG_ON(sizeof(struct nvme_nvm_l2ptbl) != 64); | ||
200 | BUILD_BUG_ON(sizeof(struct nvme_nvm_erase_blk) != 64); | ||
201 | BUILD_BUG_ON(sizeof(struct nvme_nvm_id_group) != 960); | ||
202 | BUILD_BUG_ON(sizeof(struct nvme_nvm_addr_format) != 128); | ||
203 | BUILD_BUG_ON(sizeof(struct nvme_nvm_id) != 4096); | ||
204 | } | ||
205 | |||
206 | static int init_grps(struct nvm_id *nvm_id, struct nvme_nvm_id *nvme_nvm_id) | ||
207 | { | ||
208 | struct nvme_nvm_id_group *src; | ||
209 | struct nvm_id_group *dst; | ||
210 | int i, end; | ||
211 | |||
212 | end = min_t(u32, 4, nvm_id->cgrps); | ||
213 | |||
214 | for (i = 0; i < end; i++) { | ||
215 | src = &nvme_nvm_id->groups[i]; | ||
216 | dst = &nvm_id->groups[i]; | ||
217 | |||
218 | dst->mtype = src->mtype; | ||
219 | dst->fmtype = src->fmtype; | ||
220 | dst->num_ch = src->num_ch; | ||
221 | dst->num_lun = src->num_lun; | ||
222 | dst->num_pln = src->num_pln; | ||
223 | |||
224 | dst->num_pg = le16_to_cpu(src->num_pg); | ||
225 | dst->num_blk = le16_to_cpu(src->num_blk); | ||
226 | dst->fpg_sz = le16_to_cpu(src->fpg_sz); | ||
227 | dst->csecs = le16_to_cpu(src->csecs); | ||
228 | dst->sos = le16_to_cpu(src->sos); | ||
229 | |||
230 | dst->trdt = le32_to_cpu(src->trdt); | ||
231 | dst->trdm = le32_to_cpu(src->trdm); | ||
232 | dst->tprt = le32_to_cpu(src->tprt); | ||
233 | dst->tprm = le32_to_cpu(src->tprm); | ||
234 | dst->tbet = le32_to_cpu(src->tbet); | ||
235 | dst->tbem = le32_to_cpu(src->tbem); | ||
236 | dst->mpos = le32_to_cpu(src->mpos); | ||
237 | |||
238 | dst->cpar = le16_to_cpu(src->cpar); | ||
239 | } | ||
240 | |||
241 | return 0; | ||
242 | } | ||
243 | |||
244 | static int nvme_nvm_identity(struct request_queue *q, struct nvm_id *nvm_id) | ||
245 | { | ||
246 | struct nvme_ns *ns = q->queuedata; | ||
247 | struct nvme_nvm_id *nvme_nvm_id; | ||
248 | struct nvme_nvm_command c = {}; | ||
249 | int ret; | ||
250 | |||
251 | c.identity.opcode = nvme_nvm_admin_identity; | ||
252 | c.identity.nsid = cpu_to_le32(ns->ns_id); | ||
253 | c.identity.chnl_off = 0; | ||
254 | |||
255 | nvme_nvm_id = kmalloc(sizeof(struct nvme_nvm_id), GFP_KERNEL); | ||
256 | if (!nvme_nvm_id) | ||
257 | return -ENOMEM; | ||
258 | |||
259 | ret = nvme_submit_sync_cmd(q, (struct nvme_command *)&c, nvme_nvm_id, | ||
260 | sizeof(struct nvme_nvm_id)); | ||
261 | if (ret) { | ||
262 | ret = -EIO; | ||
263 | goto out; | ||
264 | } | ||
265 | |||
266 | nvm_id->ver_id = nvme_nvm_id->ver_id; | ||
267 | nvm_id->vmnt = nvme_nvm_id->vmnt; | ||
268 | nvm_id->cgrps = nvme_nvm_id->cgrps; | ||
269 | nvm_id->cap = le32_to_cpu(nvme_nvm_id->cap); | ||
270 | nvm_id->dom = le32_to_cpu(nvme_nvm_id->dom); | ||
271 | |||
272 | ret = init_grps(nvm_id, nvme_nvm_id); | ||
273 | out: | ||
274 | kfree(nvme_nvm_id); | ||
275 | return ret; | ||
276 | } | ||
277 | |||
278 | static int nvme_nvm_get_l2p_tbl(struct request_queue *q, u64 slba, u32 nlb, | ||
279 | nvm_l2p_update_fn *update_l2p, void *priv) | ||
280 | { | ||
281 | struct nvme_ns *ns = q->queuedata; | ||
282 | struct nvme_dev *dev = ns->dev; | ||
283 | struct nvme_nvm_command c = {}; | ||
284 | u32 len = queue_max_hw_sectors(q) << 9; | ||
285 | u32 nlb_pr_rq = len / sizeof(u64); | ||
286 | u64 cmd_slba = slba; | ||
287 | void *entries; | ||
288 | int ret = 0; | ||
289 | |||
290 | c.l2p.opcode = nvme_nvm_admin_get_l2p_tbl; | ||
291 | c.l2p.nsid = cpu_to_le32(ns->ns_id); | ||
292 | entries = kmalloc(len, GFP_KERNEL); | ||
293 | if (!entries) | ||
294 | return -ENOMEM; | ||
295 | |||
296 | while (nlb) { | ||
297 | u32 cmd_nlb = min(nlb_pr_rq, nlb); | ||
298 | |||
299 | c.l2p.slba = cpu_to_le64(cmd_slba); | ||
300 | c.l2p.nlb = cpu_to_le32(cmd_nlb); | ||
301 | |||
302 | ret = nvme_submit_sync_cmd(q, (struct nvme_command *)&c, | ||
303 | entries, len); | ||
304 | if (ret) { | ||
305 | dev_err(dev->dev, "L2P table transfer failed (%d)\n", | ||
306 | ret); | ||
307 | ret = -EIO; | ||
308 | goto out; | ||
309 | } | ||
310 | |||
311 | if (update_l2p(cmd_slba, cmd_nlb, entries, priv)) { | ||
312 | ret = -EINTR; | ||
313 | goto out; | ||
314 | } | ||
315 | |||
316 | cmd_slba += cmd_nlb; | ||
317 | nlb -= cmd_nlb; | ||
318 | } | ||
319 | |||
320 | out: | ||
321 | kfree(entries); | ||
322 | return ret; | ||
323 | } | ||
324 | |||
325 | static int nvme_nvm_get_bb_tbl(struct request_queue *q, int lunid, | ||
326 | unsigned int nr_blocks, | ||
327 | nvm_bb_update_fn *update_bbtbl, void *priv) | ||
328 | { | ||
329 | struct nvme_ns *ns = q->queuedata; | ||
330 | struct nvme_dev *dev = ns->dev; | ||
331 | struct nvme_nvm_command c = {}; | ||
332 | void *bb_bitmap; | ||
333 | u16 bb_bitmap_size; | ||
334 | int ret = 0; | ||
335 | |||
336 | c.get_bb.opcode = nvme_nvm_admin_get_bb_tbl; | ||
337 | c.get_bb.nsid = cpu_to_le32(ns->ns_id); | ||
338 | c.get_bb.lbb = cpu_to_le32(lunid); | ||
339 | bb_bitmap_size = ((nr_blocks >> 15) + 1) * PAGE_SIZE; | ||
340 | bb_bitmap = kmalloc(bb_bitmap_size, GFP_KERNEL); | ||
341 | if (!bb_bitmap) | ||
342 | return -ENOMEM; | ||
343 | |||
344 | bitmap_zero(bb_bitmap, nr_blocks); | ||
345 | |||
346 | ret = nvme_submit_sync_cmd(q, (struct nvme_command *)&c, bb_bitmap, | ||
347 | bb_bitmap_size); | ||
348 | if (ret) { | ||
349 | dev_err(dev->dev, "get bad block table failed (%d)\n", ret); | ||
350 | ret = -EIO; | ||
351 | goto out; | ||
352 | } | ||
353 | |||
354 | ret = update_bbtbl(lunid, bb_bitmap, nr_blocks, priv); | ||
355 | if (ret) { | ||
356 | ret = -EINTR; | ||
357 | goto out; | ||
358 | } | ||
359 | |||
360 | out: | ||
361 | kfree(bb_bitmap); | ||
362 | return ret; | ||
363 | } | ||
364 | |||
365 | static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd, | ||
366 | struct nvme_ns *ns, struct nvme_nvm_command *c) | ||
367 | { | ||
368 | c->ph_rw.opcode = rqd->opcode; | ||
369 | c->ph_rw.nsid = cpu_to_le32(ns->ns_id); | ||
370 | c->ph_rw.spba = cpu_to_le64(rqd->ppa_addr.ppa); | ||
371 | c->ph_rw.control = cpu_to_le16(rqd->flags); | ||
372 | c->ph_rw.length = cpu_to_le16(rqd->nr_pages - 1); | ||
373 | |||
374 | if (rqd->opcode == NVM_OP_HBWRITE || rqd->opcode == NVM_OP_HBREAD) | ||
375 | c->hb_rw.slba = cpu_to_le64(nvme_block_nr(ns, | ||
376 | rqd->bio->bi_iter.bi_sector)); | ||
377 | } | ||
378 | |||
379 | static void nvme_nvm_end_io(struct request *rq, int error) | ||
380 | { | ||
381 | struct nvm_rq *rqd = rq->end_io_data; | ||
382 | struct nvm_dev *dev = rqd->dev; | ||
383 | |||
384 | if (dev->mt->end_io(rqd, error)) | ||
385 | pr_err("nvme: err status: %x result: %lx\n", | ||
386 | rq->errors, (unsigned long)rq->special); | ||
387 | |||
388 | kfree(rq->cmd); | ||
389 | blk_mq_free_request(rq); | ||
390 | } | ||
391 | |||
392 | static int nvme_nvm_submit_io(struct request_queue *q, struct nvm_rq *rqd) | ||
393 | { | ||
394 | struct nvme_ns *ns = q->queuedata; | ||
395 | struct request *rq; | ||
396 | struct bio *bio = rqd->bio; | ||
397 | struct nvme_nvm_command *cmd; | ||
398 | |||
399 | rq = blk_mq_alloc_request(q, bio_rw(bio), GFP_KERNEL, 0); | ||
400 | if (IS_ERR(rq)) | ||
401 | return -ENOMEM; | ||
402 | |||
403 | cmd = kzalloc(sizeof(struct nvme_nvm_command), GFP_KERNEL); | ||
404 | if (!cmd) { | ||
405 | blk_mq_free_request(rq); | ||
406 | return -ENOMEM; | ||
407 | } | ||
408 | |||
409 | rq->cmd_type = REQ_TYPE_DRV_PRIV; | ||
410 | rq->ioprio = bio_prio(bio); | ||
411 | |||
412 | if (bio_has_data(bio)) | ||
413 | rq->nr_phys_segments = bio_phys_segments(q, bio); | ||
414 | |||
415 | rq->__data_len = bio->bi_iter.bi_size; | ||
416 | rq->bio = rq->biotail = bio; | ||
417 | |||
418 | nvme_nvm_rqtocmd(rq, rqd, ns, cmd); | ||
419 | |||
420 | rq->cmd = (unsigned char *)cmd; | ||
421 | rq->cmd_len = sizeof(struct nvme_nvm_command); | ||
422 | rq->special = (void *)0; | ||
423 | |||
424 | rq->end_io_data = rqd; | ||
425 | |||
426 | blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io); | ||
427 | |||
428 | return 0; | ||
429 | } | ||
430 | |||
431 | static int nvme_nvm_erase_block(struct request_queue *q, struct nvm_rq *rqd) | ||
432 | { | ||
433 | struct nvme_ns *ns = q->queuedata; | ||
434 | struct nvme_nvm_command c = {}; | ||
435 | |||
436 | c.erase.opcode = NVM_OP_ERASE; | ||
437 | c.erase.nsid = cpu_to_le32(ns->ns_id); | ||
438 | c.erase.spba = cpu_to_le64(rqd->ppa_addr.ppa); | ||
439 | c.erase.length = cpu_to_le16(rqd->nr_pages - 1); | ||
440 | |||
441 | return nvme_submit_sync_cmd(q, (struct nvme_command *)&c, NULL, 0); | ||
442 | } | ||
443 | |||
444 | static void *nvme_nvm_create_dma_pool(struct request_queue *q, char *name) | ||
445 | { | ||
446 | struct nvme_ns *ns = q->queuedata; | ||
447 | struct nvme_dev *dev = ns->dev; | ||
448 | |||
449 | return dma_pool_create(name, dev->dev, PAGE_SIZE, PAGE_SIZE, 0); | ||
450 | } | ||
451 | |||
452 | static void nvme_nvm_destroy_dma_pool(void *pool) | ||
453 | { | ||
454 | struct dma_pool *dma_pool = pool; | ||
455 | |||
456 | dma_pool_destroy(dma_pool); | ||
457 | } | ||
458 | |||
459 | static void *nvme_nvm_dev_dma_alloc(struct request_queue *q, void *pool, | ||
460 | gfp_t mem_flags, dma_addr_t *dma_handler) | ||
461 | { | ||
462 | return dma_pool_alloc(pool, mem_flags, dma_handler); | ||
463 | } | ||
464 | |||
465 | static void nvme_nvm_dev_dma_free(void *pool, void *ppa_list, | ||
466 | dma_addr_t dma_handler) | ||
467 | { | ||
468 | dma_pool_free(pool, ppa_list, dma_handler); | ||
469 | } | ||
470 | |||
471 | static struct nvm_dev_ops nvme_nvm_dev_ops = { | ||
472 | .identity = nvme_nvm_identity, | ||
473 | |||
474 | .get_l2p_tbl = nvme_nvm_get_l2p_tbl, | ||
475 | |||
476 | .get_bb_tbl = nvme_nvm_get_bb_tbl, | ||
477 | |||
478 | .submit_io = nvme_nvm_submit_io, | ||
479 | .erase_block = nvme_nvm_erase_block, | ||
480 | |||
481 | .create_dma_pool = nvme_nvm_create_dma_pool, | ||
482 | .destroy_dma_pool = nvme_nvm_destroy_dma_pool, | ||
483 | .dev_dma_alloc = nvme_nvm_dev_dma_alloc, | ||
484 | .dev_dma_free = nvme_nvm_dev_dma_free, | ||
485 | |||
486 | .max_phys_sect = 64, | ||
487 | }; | ||
488 | |||
489 | int nvme_nvm_register(struct request_queue *q, char *disk_name) | ||
490 | { | ||
491 | return nvm_register(q, disk_name, &nvme_nvm_dev_ops); | ||
492 | } | ||
493 | |||
494 | void nvme_nvm_unregister(struct request_queue *q, char *disk_name) | ||
495 | { | ||
496 | nvm_unregister(disk_name); | ||
497 | } | ||
498 | |||
499 | int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id) | ||
500 | { | ||
501 | struct nvme_dev *dev = ns->dev; | ||
502 | struct pci_dev *pdev = to_pci_dev(dev->dev); | ||
503 | |||
504 | /* QEMU NVMe simulator - PCI ID + Vendor specific bit */ | ||
505 | if (pdev->vendor == PCI_VENDOR_ID_INTEL && pdev->device == 0x5845 && | ||
506 | id->vs[0] == 0x1) | ||
507 | return 1; | ||
508 | |||
509 | /* CNEX Labs - PCI ID + Vendor specific bit */ | ||
510 | if (pdev->vendor == 0x1d1d && pdev->device == 0x2807 && | ||
511 | id->vs[0] == 0x1) | ||
512 | return 1; | ||
513 | |||
514 | return 0; | ||
515 | } | ||
516 | #else | ||
517 | int nvme_nvm_register(struct request_queue *q, char *disk_name) | ||
518 | { | ||
519 | return 0; | ||
520 | } | ||
521 | void nvme_nvm_unregister(struct request_queue *q, char *disk_name) {}; | ||
522 | int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id) | ||
523 | { | ||
524 | return 0; | ||
525 | } | ||
526 | #endif /* CONFIG_NVM */ | ||
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index c1f41bf3c0f2..fdb4e5bad9ac 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h | |||
@@ -22,6 +22,11 @@ | |||
22 | extern unsigned char nvme_io_timeout; | 22 | extern unsigned char nvme_io_timeout; |
23 | #define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) | 23 | #define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) |
24 | 24 | ||
25 | enum { | ||
26 | NVME_NS_LBA = 0, | ||
27 | NVME_NS_LIGHTNVM = 1, | ||
28 | }; | ||
29 | |||
25 | /* | 30 | /* |
26 | * Represents an NVM Express device. Each nvme_dev is a PCI function. | 31 | * Represents an NVM Express device. Each nvme_dev is a PCI function. |
27 | */ | 32 | */ |
@@ -84,6 +89,7 @@ struct nvme_ns { | |||
84 | u16 ms; | 89 | u16 ms; |
85 | bool ext; | 90 | bool ext; |
86 | u8 pi_type; | 91 | u8 pi_type; |
92 | int type; | ||
87 | u64 mode_select_num_blocks; | 93 | u64 mode_select_num_blocks; |
88 | u32 mode_select_block_len; | 94 | u32 mode_select_block_len; |
89 | }; | 95 | }; |
@@ -130,4 +136,8 @@ int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr); | |||
130 | int nvme_sg_io32(struct nvme_ns *ns, unsigned long arg); | 136 | int nvme_sg_io32(struct nvme_ns *ns, unsigned long arg); |
131 | int nvme_sg_get_version_num(int __user *ip); | 137 | int nvme_sg_get_version_num(int __user *ip); |
132 | 138 | ||
139 | int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id); | ||
140 | int nvme_nvm_register(struct request_queue *q, char *disk_name); | ||
141 | void nvme_nvm_unregister(struct request_queue *q, char *disk_name); | ||
142 | |||
133 | #endif /* _NVME_H */ | 143 | #endif /* _NVME_H */ |
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 0a179ed9ddef..6c0d0fecaeb3 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c | |||
@@ -1952,6 +1952,9 @@ static void nvme_free_ns(struct kref *kref) | |||
1952 | { | 1952 | { |
1953 | struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref); | 1953 | struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref); |
1954 | 1954 | ||
1955 | if (ns->type == NVME_NS_LIGHTNVM) | ||
1956 | nvme_nvm_unregister(ns->queue, ns->disk->disk_name); | ||
1957 | |||
1955 | spin_lock(&dev_list_lock); | 1958 | spin_lock(&dev_list_lock); |
1956 | ns->disk->private_data = NULL; | 1959 | ns->disk->private_data = NULL; |
1957 | spin_unlock(&dev_list_lock); | 1960 | spin_unlock(&dev_list_lock); |
@@ -2021,6 +2024,16 @@ static int nvme_revalidate_disk(struct gendisk *disk) | |||
2021 | return -ENODEV; | 2024 | return -ENODEV; |
2022 | } | 2025 | } |
2023 | 2026 | ||
2027 | if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) { | ||
2028 | if (nvme_nvm_register(ns->queue, disk->disk_name)) { | ||
2029 | dev_warn(dev->dev, | ||
2030 | "%s: LightNVM init failure\n", __func__); | ||
2031 | kfree(id); | ||
2032 | return -ENODEV; | ||
2033 | } | ||
2034 | ns->type = NVME_NS_LIGHTNVM; | ||
2035 | } | ||
2036 | |||
2024 | old_ms = ns->ms; | 2037 | old_ms = ns->ms; |
2025 | lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK; | 2038 | lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK; |
2026 | ns->lba_shift = id->lbaf[lbaf].ds; | 2039 | ns->lba_shift = id->lbaf[lbaf].ds; |
@@ -2052,7 +2065,9 @@ static int nvme_revalidate_disk(struct gendisk *disk) | |||
2052 | !ns->ext) | 2065 | !ns->ext) |
2053 | nvme_init_integrity(ns); | 2066 | nvme_init_integrity(ns); |
2054 | 2067 | ||
2055 | if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk)) | 2068 | if ((ns->ms && !(ns->ms == 8 && ns->pi_type) && |
2069 | !blk_get_integrity(disk)) || | ||
2070 | ns->type == NVME_NS_LIGHTNVM) | ||
2056 | set_capacity(disk, 0); | 2071 | set_capacity(disk, 0); |
2057 | else | 2072 | else |
2058 | set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); | 2073 | set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); |
@@ -2175,17 +2190,19 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid) | |||
2175 | goto out_free_disk; | 2190 | goto out_free_disk; |
2176 | 2191 | ||
2177 | kref_get(&dev->kref); | 2192 | kref_get(&dev->kref); |
2178 | add_disk(ns->disk); | 2193 | if (ns->type != NVME_NS_LIGHTNVM) { |
2179 | if (ns->ms) { | 2194 | add_disk(ns->disk); |
2180 | struct block_device *bd = bdget_disk(ns->disk, 0); | 2195 | if (ns->ms) { |
2181 | if (!bd) | 2196 | struct block_device *bd = bdget_disk(ns->disk, 0); |
2182 | return; | 2197 | if (!bd) |
2183 | if (blkdev_get(bd, FMODE_READ, NULL)) { | 2198 | return; |
2184 | bdput(bd); | 2199 | if (blkdev_get(bd, FMODE_READ, NULL)) { |
2185 | return; | 2200 | bdput(bd); |
2201 | return; | ||
2202 | } | ||
2203 | blkdev_reread_part(bd); | ||
2204 | blkdev_put(bd, FMODE_READ); | ||
2186 | } | 2205 | } |
2187 | blkdev_reread_part(bd); | ||
2188 | blkdev_put(bd, FMODE_READ); | ||
2189 | } | 2206 | } |
2190 | return; | 2207 | return; |
2191 | out_free_disk: | 2208 | out_free_disk: |
diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h new file mode 100644 index 000000000000..5ebd70d12f35 --- /dev/null +++ b/include/linux/lightnvm.h | |||
@@ -0,0 +1,522 @@ | |||
1 | #ifndef NVM_H | ||
2 | #define NVM_H | ||
3 | |||
4 | enum { | ||
5 | NVM_IO_OK = 0, | ||
6 | NVM_IO_REQUEUE = 1, | ||
7 | NVM_IO_DONE = 2, | ||
8 | NVM_IO_ERR = 3, | ||
9 | |||
10 | NVM_IOTYPE_NONE = 0, | ||
11 | NVM_IOTYPE_GC = 1, | ||
12 | }; | ||
13 | |||
14 | #ifdef CONFIG_NVM | ||
15 | |||
16 | #include <linux/blkdev.h> | ||
17 | #include <linux/types.h> | ||
18 | #include <linux/file.h> | ||
19 | #include <linux/dmapool.h> | ||
20 | |||
21 | enum { | ||
22 | /* HW Responsibilities */ | ||
23 | NVM_RSP_L2P = 1 << 0, | ||
24 | NVM_RSP_ECC = 1 << 1, | ||
25 | |||
26 | /* Physical Adressing Mode */ | ||
27 | NVM_ADDRMODE_LINEAR = 0, | ||
28 | NVM_ADDRMODE_CHANNEL = 1, | ||
29 | |||
30 | /* Plane programming mode for LUN */ | ||
31 | NVM_PLANE_SINGLE = 0, | ||
32 | NVM_PLANE_DOUBLE = 1, | ||
33 | NVM_PLANE_QUAD = 2, | ||
34 | |||
35 | /* Status codes */ | ||
36 | NVM_RSP_SUCCESS = 0x0, | ||
37 | NVM_RSP_NOT_CHANGEABLE = 0x1, | ||
38 | NVM_RSP_ERR_FAILWRITE = 0x40ff, | ||
39 | NVM_RSP_ERR_EMPTYPAGE = 0x42ff, | ||
40 | |||
41 | /* Device opcodes */ | ||
42 | NVM_OP_HBREAD = 0x02, | ||
43 | NVM_OP_HBWRITE = 0x81, | ||
44 | NVM_OP_PWRITE = 0x91, | ||
45 | NVM_OP_PREAD = 0x92, | ||
46 | NVM_OP_ERASE = 0x90, | ||
47 | |||
48 | /* PPA Command Flags */ | ||
49 | NVM_IO_SNGL_ACCESS = 0x0, | ||
50 | NVM_IO_DUAL_ACCESS = 0x1, | ||
51 | NVM_IO_QUAD_ACCESS = 0x2, | ||
52 | |||
53 | NVM_IO_SUSPEND = 0x80, | ||
54 | NVM_IO_SLC_MODE = 0x100, | ||
55 | NVM_IO_SCRAMBLE_DISABLE = 0x200, | ||
56 | }; | ||
57 | |||
58 | struct nvm_id_group { | ||
59 | u8 mtype; | ||
60 | u8 fmtype; | ||
61 | u16 res16; | ||
62 | u8 num_ch; | ||
63 | u8 num_lun; | ||
64 | u8 num_pln; | ||
65 | u16 num_blk; | ||
66 | u16 num_pg; | ||
67 | u16 fpg_sz; | ||
68 | u16 csecs; | ||
69 | u16 sos; | ||
70 | u32 trdt; | ||
71 | u32 trdm; | ||
72 | u32 tprt; | ||
73 | u32 tprm; | ||
74 | u32 tbet; | ||
75 | u32 tbem; | ||
76 | u32 mpos; | ||
77 | u16 cpar; | ||
78 | u8 res[913]; | ||
79 | } __packed; | ||
80 | |||
81 | struct nvm_addr_format { | ||
82 | u8 ch_offset; | ||
83 | u8 ch_len; | ||
84 | u8 lun_offset; | ||
85 | u8 lun_len; | ||
86 | u8 pln_offset; | ||
87 | u8 pln_len; | ||
88 | u8 blk_offset; | ||
89 | u8 blk_len; | ||
90 | u8 pg_offset; | ||
91 | u8 pg_len; | ||
92 | u8 sect_offset; | ||
93 | u8 sect_len; | ||
94 | u8 res[4]; | ||
95 | }; | ||
96 | |||
97 | struct nvm_id { | ||
98 | u8 ver_id; | ||
99 | u8 vmnt; | ||
100 | u8 cgrps; | ||
101 | u8 res[5]; | ||
102 | u32 cap; | ||
103 | u32 dom; | ||
104 | struct nvm_addr_format ppaf; | ||
105 | u8 ppat; | ||
106 | u8 resv[224]; | ||
107 | struct nvm_id_group groups[4]; | ||
108 | } __packed; | ||
109 | |||
110 | struct nvm_target { | ||
111 | struct list_head list; | ||
112 | struct nvm_tgt_type *type; | ||
113 | struct gendisk *disk; | ||
114 | }; | ||
115 | |||
116 | struct nvm_tgt_instance { | ||
117 | struct nvm_tgt_type *tt; | ||
118 | }; | ||
119 | |||
120 | #define ADDR_EMPTY (~0ULL) | ||
121 | |||
122 | #define NVM_VERSION_MAJOR 1 | ||
123 | #define NVM_VERSION_MINOR 0 | ||
124 | #define NVM_VERSION_PATCH 0 | ||
125 | |||
126 | #define NVM_SEC_BITS (8) | ||
127 | #define NVM_PL_BITS (6) | ||
128 | #define NVM_PG_BITS (16) | ||
129 | #define NVM_BLK_BITS (16) | ||
130 | #define NVM_LUN_BITS (10) | ||
131 | #define NVM_CH_BITS (8) | ||
132 | |||
133 | struct ppa_addr { | ||
134 | union { | ||
135 | /* Channel-based PPA format in nand 4x2x2x2x8x10 */ | ||
136 | struct { | ||
137 | u64 ch : 4; | ||
138 | u64 sec : 2; /* 4 sectors per page */ | ||
139 | u64 pl : 2; /* 4 planes per LUN */ | ||
140 | u64 lun : 2; /* 4 LUNs per channel */ | ||
141 | u64 pg : 8; /* 256 pages per block */ | ||
142 | u64 blk : 10;/* 1024 blocks per plane */ | ||
143 | u64 resved : 36; | ||
144 | } chnl; | ||
145 | |||
146 | /* Generic structure for all addresses */ | ||
147 | struct { | ||
148 | u64 sec : NVM_SEC_BITS; | ||
149 | u64 pl : NVM_PL_BITS; | ||
150 | u64 pg : NVM_PG_BITS; | ||
151 | u64 blk : NVM_BLK_BITS; | ||
152 | u64 lun : NVM_LUN_BITS; | ||
153 | u64 ch : NVM_CH_BITS; | ||
154 | } g; | ||
155 | |||
156 | u64 ppa; | ||
157 | }; | ||
158 | } __packed; | ||
159 | |||
160 | struct nvm_rq { | ||
161 | struct nvm_tgt_instance *ins; | ||
162 | struct nvm_dev *dev; | ||
163 | |||
164 | struct bio *bio; | ||
165 | |||
166 | union { | ||
167 | struct ppa_addr ppa_addr; | ||
168 | dma_addr_t dma_ppa_list; | ||
169 | }; | ||
170 | |||
171 | struct ppa_addr *ppa_list; | ||
172 | |||
173 | void *metadata; | ||
174 | dma_addr_t dma_metadata; | ||
175 | |||
176 | uint8_t opcode; | ||
177 | uint16_t nr_pages; | ||
178 | uint16_t flags; | ||
179 | }; | ||
180 | |||
181 | static inline struct nvm_rq *nvm_rq_from_pdu(void *pdu) | ||
182 | { | ||
183 | return pdu - sizeof(struct nvm_rq); | ||
184 | } | ||
185 | |||
186 | static inline void *nvm_rq_to_pdu(struct nvm_rq *rqdata) | ||
187 | { | ||
188 | return rqdata + 1; | ||
189 | } | ||
190 | |||
191 | struct nvm_block; | ||
192 | |||
193 | typedef int (nvm_l2p_update_fn)(u64, u32, __le64 *, void *); | ||
194 | typedef int (nvm_bb_update_fn)(u32, void *, unsigned int, void *); | ||
195 | typedef int (nvm_id_fn)(struct request_queue *, struct nvm_id *); | ||
196 | typedef int (nvm_get_l2p_tbl_fn)(struct request_queue *, u64, u32, | ||
197 | nvm_l2p_update_fn *, void *); | ||
198 | typedef int (nvm_op_bb_tbl_fn)(struct request_queue *, int, unsigned int, | ||
199 | nvm_bb_update_fn *, void *); | ||
200 | typedef int (nvm_op_set_bb_fn)(struct request_queue *, struct nvm_rq *, int); | ||
201 | typedef int (nvm_submit_io_fn)(struct request_queue *, struct nvm_rq *); | ||
202 | typedef int (nvm_erase_blk_fn)(struct request_queue *, struct nvm_rq *); | ||
203 | typedef void *(nvm_create_dma_pool_fn)(struct request_queue *, char *); | ||
204 | typedef void (nvm_destroy_dma_pool_fn)(void *); | ||
205 | typedef void *(nvm_dev_dma_alloc_fn)(struct request_queue *, void *, gfp_t, | ||
206 | dma_addr_t *); | ||
207 | typedef void (nvm_dev_dma_free_fn)(void *, void*, dma_addr_t); | ||
208 | |||
209 | struct nvm_dev_ops { | ||
210 | nvm_id_fn *identity; | ||
211 | nvm_get_l2p_tbl_fn *get_l2p_tbl; | ||
212 | nvm_op_bb_tbl_fn *get_bb_tbl; | ||
213 | nvm_op_set_bb_fn *set_bb; | ||
214 | |||
215 | nvm_submit_io_fn *submit_io; | ||
216 | nvm_erase_blk_fn *erase_block; | ||
217 | |||
218 | nvm_create_dma_pool_fn *create_dma_pool; | ||
219 | nvm_destroy_dma_pool_fn *destroy_dma_pool; | ||
220 | nvm_dev_dma_alloc_fn *dev_dma_alloc; | ||
221 | nvm_dev_dma_free_fn *dev_dma_free; | ||
222 | |||
223 | uint8_t max_phys_sect; | ||
224 | }; | ||
225 | |||
226 | struct nvm_lun { | ||
227 | int id; | ||
228 | |||
229 | int lun_id; | ||
230 | int chnl_id; | ||
231 | |||
232 | unsigned int nr_free_blocks; /* Number of unused blocks */ | ||
233 | struct nvm_block *blocks; | ||
234 | |||
235 | spinlock_t lock; | ||
236 | }; | ||
237 | |||
238 | struct nvm_block { | ||
239 | struct list_head list; | ||
240 | struct nvm_lun *lun; | ||
241 | unsigned long id; | ||
242 | |||
243 | void *priv; | ||
244 | int type; | ||
245 | }; | ||
246 | |||
247 | struct nvm_dev { | ||
248 | struct nvm_dev_ops *ops; | ||
249 | |||
250 | struct list_head devices; | ||
251 | struct list_head online_targets; | ||
252 | |||
253 | /* Media manager */ | ||
254 | struct nvmm_type *mt; | ||
255 | void *mp; | ||
256 | |||
257 | /* Device information */ | ||
258 | int nr_chnls; | ||
259 | int nr_planes; | ||
260 | int luns_per_chnl; | ||
261 | int sec_per_pg; /* only sectors for a single page */ | ||
262 | int pgs_per_blk; | ||
263 | int blks_per_lun; | ||
264 | int sec_size; | ||
265 | int oob_size; | ||
266 | int addr_mode; | ||
267 | struct nvm_addr_format addr_format; | ||
268 | |||
269 | /* Calculated/Cached values. These do not reflect the actual usable | ||
270 | * blocks at run-time. | ||
271 | */ | ||
272 | int max_rq_size; | ||
273 | int plane_mode; /* drive device in single, double or quad mode */ | ||
274 | |||
275 | int sec_per_pl; /* all sectors across planes */ | ||
276 | int sec_per_blk; | ||
277 | int sec_per_lun; | ||
278 | |||
279 | unsigned long total_pages; | ||
280 | unsigned long total_blocks; | ||
281 | int nr_luns; | ||
282 | unsigned max_pages_per_blk; | ||
283 | |||
284 | void *ppalist_pool; | ||
285 | |||
286 | struct nvm_id identity; | ||
287 | |||
288 | /* Backend device */ | ||
289 | struct request_queue *q; | ||
290 | char name[DISK_NAME_LEN]; | ||
291 | }; | ||
292 | |||
293 | /* fallback conversion */ | ||
294 | static struct ppa_addr __generic_to_linear_addr(struct nvm_dev *dev, | ||
295 | struct ppa_addr r) | ||
296 | { | ||
297 | struct ppa_addr l; | ||
298 | |||
299 | l.ppa = r.g.sec + | ||
300 | r.g.pg * dev->sec_per_pg + | ||
301 | r.g.blk * (dev->pgs_per_blk * | ||
302 | dev->sec_per_pg) + | ||
303 | r.g.lun * (dev->blks_per_lun * | ||
304 | dev->pgs_per_blk * | ||
305 | dev->sec_per_pg) + | ||
306 | r.g.ch * (dev->blks_per_lun * | ||
307 | dev->pgs_per_blk * | ||
308 | dev->luns_per_chnl * | ||
309 | dev->sec_per_pg); | ||
310 | |||
311 | return l; | ||
312 | } | ||
313 | |||
314 | /* fallback conversion */ | ||
315 | static struct ppa_addr __linear_to_generic_addr(struct nvm_dev *dev, | ||
316 | struct ppa_addr r) | ||
317 | { | ||
318 | struct ppa_addr l; | ||
319 | int secs, pgs, blks, luns; | ||
320 | sector_t ppa = r.ppa; | ||
321 | |||
322 | l.ppa = 0; | ||
323 | |||
324 | div_u64_rem(ppa, dev->sec_per_pg, &secs); | ||
325 | l.g.sec = secs; | ||
326 | |||
327 | sector_div(ppa, dev->sec_per_pg); | ||
328 | div_u64_rem(ppa, dev->sec_per_blk, &pgs); | ||
329 | l.g.pg = pgs; | ||
330 | |||
331 | sector_div(ppa, dev->pgs_per_blk); | ||
332 | div_u64_rem(ppa, dev->blks_per_lun, &blks); | ||
333 | l.g.blk = blks; | ||
334 | |||
335 | sector_div(ppa, dev->blks_per_lun); | ||
336 | div_u64_rem(ppa, dev->luns_per_chnl, &luns); | ||
337 | l.g.lun = luns; | ||
338 | |||
339 | sector_div(ppa, dev->luns_per_chnl); | ||
340 | l.g.ch = ppa; | ||
341 | |||
342 | return l; | ||
343 | } | ||
344 | |||
345 | static struct ppa_addr __generic_to_chnl_addr(struct ppa_addr r) | ||
346 | { | ||
347 | struct ppa_addr l; | ||
348 | |||
349 | l.ppa = 0; | ||
350 | |||
351 | l.chnl.sec = r.g.sec; | ||
352 | l.chnl.pl = r.g.pl; | ||
353 | l.chnl.pg = r.g.pg; | ||
354 | l.chnl.blk = r.g.blk; | ||
355 | l.chnl.lun = r.g.lun; | ||
356 | l.chnl.ch = r.g.ch; | ||
357 | |||
358 | return l; | ||
359 | } | ||
360 | |||
361 | static struct ppa_addr __chnl_to_generic_addr(struct ppa_addr r) | ||
362 | { | ||
363 | struct ppa_addr l; | ||
364 | |||
365 | l.ppa = 0; | ||
366 | |||
367 | l.g.sec = r.chnl.sec; | ||
368 | l.g.pl = r.chnl.pl; | ||
369 | l.g.pg = r.chnl.pg; | ||
370 | l.g.blk = r.chnl.blk; | ||
371 | l.g.lun = r.chnl.lun; | ||
372 | l.g.ch = r.chnl.ch; | ||
373 | |||
374 | return l; | ||
375 | } | ||
376 | |||
377 | static inline struct ppa_addr addr_to_generic_mode(struct nvm_dev *dev, | ||
378 | struct ppa_addr gppa) | ||
379 | { | ||
380 | switch (dev->addr_mode) { | ||
381 | case NVM_ADDRMODE_LINEAR: | ||
382 | return __linear_to_generic_addr(dev, gppa); | ||
383 | case NVM_ADDRMODE_CHANNEL: | ||
384 | return __chnl_to_generic_addr(gppa); | ||
385 | default: | ||
386 | BUG(); | ||
387 | } | ||
388 | return gppa; | ||
389 | } | ||
390 | |||
391 | static inline struct ppa_addr generic_to_addr_mode(struct nvm_dev *dev, | ||
392 | struct ppa_addr gppa) | ||
393 | { | ||
394 | switch (dev->addr_mode) { | ||
395 | case NVM_ADDRMODE_LINEAR: | ||
396 | return __generic_to_linear_addr(dev, gppa); | ||
397 | case NVM_ADDRMODE_CHANNEL: | ||
398 | return __generic_to_chnl_addr(gppa); | ||
399 | default: | ||
400 | BUG(); | ||
401 | } | ||
402 | return gppa; | ||
403 | } | ||
404 | |||
405 | static inline int ppa_empty(struct ppa_addr ppa_addr) | ||
406 | { | ||
407 | return (ppa_addr.ppa == ADDR_EMPTY); | ||
408 | } | ||
409 | |||
410 | static inline void ppa_set_empty(struct ppa_addr *ppa_addr) | ||
411 | { | ||
412 | ppa_addr->ppa = ADDR_EMPTY; | ||
413 | } | ||
414 | |||
415 | static inline struct ppa_addr block_to_ppa(struct nvm_dev *dev, | ||
416 | struct nvm_block *blk) | ||
417 | { | ||
418 | struct ppa_addr ppa; | ||
419 | struct nvm_lun *lun = blk->lun; | ||
420 | |||
421 | ppa.ppa = 0; | ||
422 | ppa.g.blk = blk->id % dev->blks_per_lun; | ||
423 | ppa.g.lun = lun->lun_id; | ||
424 | ppa.g.ch = lun->chnl_id; | ||
425 | |||
426 | return ppa; | ||
427 | } | ||
428 | |||
429 | typedef void (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *); | ||
430 | typedef sector_t (nvm_tgt_capacity_fn)(void *); | ||
431 | typedef int (nvm_tgt_end_io_fn)(struct nvm_rq *, int); | ||
432 | typedef void *(nvm_tgt_init_fn)(struct nvm_dev *, struct gendisk *, int, int); | ||
433 | typedef void (nvm_tgt_exit_fn)(void *); | ||
434 | |||
435 | struct nvm_tgt_type { | ||
436 | const char *name; | ||
437 | unsigned int version[3]; | ||
438 | |||
439 | /* target entry points */ | ||
440 | nvm_tgt_make_rq_fn *make_rq; | ||
441 | nvm_tgt_capacity_fn *capacity; | ||
442 | nvm_tgt_end_io_fn *end_io; | ||
443 | |||
444 | /* module-specific init/teardown */ | ||
445 | nvm_tgt_init_fn *init; | ||
446 | nvm_tgt_exit_fn *exit; | ||
447 | |||
448 | /* For internal use */ | ||
449 | struct list_head list; | ||
450 | }; | ||
451 | |||
452 | extern int nvm_register_target(struct nvm_tgt_type *); | ||
453 | extern void nvm_unregister_target(struct nvm_tgt_type *); | ||
454 | |||
455 | extern void *nvm_dev_dma_alloc(struct nvm_dev *, gfp_t, dma_addr_t *); | ||
456 | extern void nvm_dev_dma_free(struct nvm_dev *, void *, dma_addr_t); | ||
457 | |||
458 | typedef int (nvmm_register_fn)(struct nvm_dev *); | ||
459 | typedef void (nvmm_unregister_fn)(struct nvm_dev *); | ||
460 | typedef struct nvm_block *(nvmm_get_blk_fn)(struct nvm_dev *, | ||
461 | struct nvm_lun *, unsigned long); | ||
462 | typedef void (nvmm_put_blk_fn)(struct nvm_dev *, struct nvm_block *); | ||
463 | typedef int (nvmm_open_blk_fn)(struct nvm_dev *, struct nvm_block *); | ||
464 | typedef int (nvmm_close_blk_fn)(struct nvm_dev *, struct nvm_block *); | ||
465 | typedef void (nvmm_flush_blk_fn)(struct nvm_dev *, struct nvm_block *); | ||
466 | typedef int (nvmm_submit_io_fn)(struct nvm_dev *, struct nvm_rq *); | ||
467 | typedef int (nvmm_end_io_fn)(struct nvm_rq *, int); | ||
468 | typedef int (nvmm_erase_blk_fn)(struct nvm_dev *, struct nvm_block *, | ||
469 | unsigned long); | ||
470 | typedef struct nvm_lun *(nvmm_get_lun_fn)(struct nvm_dev *, int); | ||
471 | typedef void (nvmm_free_blocks_print_fn)(struct nvm_dev *); | ||
472 | |||
473 | struct nvmm_type { | ||
474 | const char *name; | ||
475 | unsigned int version[3]; | ||
476 | |||
477 | nvmm_register_fn *register_mgr; | ||
478 | nvmm_unregister_fn *unregister_mgr; | ||
479 | |||
480 | /* Block administration callbacks */ | ||
481 | nvmm_get_blk_fn *get_blk; | ||
482 | nvmm_put_blk_fn *put_blk; | ||
483 | nvmm_open_blk_fn *open_blk; | ||
484 | nvmm_close_blk_fn *close_blk; | ||
485 | nvmm_flush_blk_fn *flush_blk; | ||
486 | |||
487 | nvmm_submit_io_fn *submit_io; | ||
488 | nvmm_end_io_fn *end_io; | ||
489 | nvmm_erase_blk_fn *erase_blk; | ||
490 | |||
491 | /* Configuration management */ | ||
492 | nvmm_get_lun_fn *get_lun; | ||
493 | |||
494 | /* Statistics */ | ||
495 | nvmm_free_blocks_print_fn *free_blocks_print; | ||
496 | struct list_head list; | ||
497 | }; | ||
498 | |||
499 | extern int nvm_register_mgr(struct nvmm_type *); | ||
500 | extern void nvm_unregister_mgr(struct nvmm_type *); | ||
501 | |||
502 | extern struct nvm_block *nvm_get_blk(struct nvm_dev *, struct nvm_lun *, | ||
503 | unsigned long); | ||
504 | extern void nvm_put_blk(struct nvm_dev *, struct nvm_block *); | ||
505 | |||
506 | extern int nvm_register(struct request_queue *, char *, | ||
507 | struct nvm_dev_ops *); | ||
508 | extern void nvm_unregister(char *); | ||
509 | |||
510 | extern int nvm_submit_io(struct nvm_dev *, struct nvm_rq *); | ||
511 | extern int nvm_erase_blk(struct nvm_dev *, struct nvm_block *); | ||
512 | #else /* CONFIG_NVM */ | ||
513 | struct nvm_dev_ops; | ||
514 | |||
515 | static inline int nvm_register(struct request_queue *q, char *disk_name, | ||
516 | struct nvm_dev_ops *ops) | ||
517 | { | ||
518 | return -EINVAL; | ||
519 | } | ||
520 | static inline void nvm_unregister(char *disk_name) {} | ||
521 | #endif /* CONFIG_NVM */ | ||
522 | #endif /* LIGHTNVM.H */ | ||
diff --git a/include/uapi/linux/lightnvm.h b/include/uapi/linux/lightnvm.h new file mode 100644 index 000000000000..928f98997d8a --- /dev/null +++ b/include/uapi/linux/lightnvm.h | |||
@@ -0,0 +1,130 @@ | |||
1 | /* | ||
2 | * Copyright (C) 2015 CNEX Labs. All rights reserved. | ||
3 | * | ||
4 | * This program is free software; you can redistribute it and/or | ||
5 | * modify it under the terms of the GNU General Public License version | ||
6 | * 2 as published by the Free Software Foundation. | ||
7 | * | ||
8 | * This program is distributed in the hope that it will be useful, but | ||
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | ||
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
11 | * General Public License for more details. | ||
12 | * | ||
13 | * You should have received a copy of the GNU General Public License | ||
14 | * along with this program; see the file COPYING. If not, write to | ||
15 | * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, | ||
16 | * USA. | ||
17 | */ | ||
18 | |||
19 | #ifndef _UAPI_LINUX_LIGHTNVM_H | ||
20 | #define _UAPI_LINUX_LIGHTNVM_H | ||
21 | |||
22 | #ifdef __KERNEL__ | ||
23 | #include <linux/kernel.h> | ||
24 | #include <linux/ioctl.h> | ||
25 | #else /* __KERNEL__ */ | ||
26 | #include <stdio.h> | ||
27 | #include <sys/ioctl.h> | ||
28 | #define DISK_NAME_LEN 32 | ||
29 | #endif /* __KERNEL__ */ | ||
30 | |||
31 | #include <linux/types.h> | ||
32 | #include <linux/ioctl.h> | ||
33 | |||
34 | #define NVM_TTYPE_NAME_MAX 48 | ||
35 | #define NVM_TTYPE_MAX 63 | ||
36 | |||
37 | #define NVM_CTRL_FILE "/dev/lightnvm/control" | ||
38 | |||
39 | struct nvm_ioctl_info_tgt { | ||
40 | __u32 version[3]; | ||
41 | __u32 reserved; | ||
42 | char tgtname[NVM_TTYPE_NAME_MAX]; | ||
43 | }; | ||
44 | |||
45 | struct nvm_ioctl_info { | ||
46 | __u32 version[3]; /* in/out - major, minor, patch */ | ||
47 | __u16 tgtsize; /* number of targets */ | ||
48 | __u16 reserved16; /* pad to 4K page */ | ||
49 | __u32 reserved[12]; | ||
50 | struct nvm_ioctl_info_tgt tgts[NVM_TTYPE_MAX]; | ||
51 | }; | ||
52 | |||
53 | enum { | ||
54 | NVM_DEVICE_ACTIVE = 1 << 0, | ||
55 | }; | ||
56 | |||
57 | struct nvm_ioctl_device_info { | ||
58 | char devname[DISK_NAME_LEN]; | ||
59 | char bmname[NVM_TTYPE_NAME_MAX]; | ||
60 | __u32 bmversion[3]; | ||
61 | __u32 flags; | ||
62 | __u32 reserved[8]; | ||
63 | }; | ||
64 | |||
65 | struct nvm_ioctl_get_devices { | ||
66 | __u32 nr_devices; | ||
67 | __u32 reserved[31]; | ||
68 | struct nvm_ioctl_device_info info[31]; | ||
69 | }; | ||
70 | |||
71 | struct nvm_ioctl_create_simple { | ||
72 | __u32 lun_begin; | ||
73 | __u32 lun_end; | ||
74 | }; | ||
75 | |||
76 | enum { | ||
77 | NVM_CONFIG_TYPE_SIMPLE = 0, | ||
78 | }; | ||
79 | |||
80 | struct nvm_ioctl_create_conf { | ||
81 | __u32 type; | ||
82 | union { | ||
83 | struct nvm_ioctl_create_simple s; | ||
84 | }; | ||
85 | }; | ||
86 | |||
87 | struct nvm_ioctl_create { | ||
88 | char dev[DISK_NAME_LEN]; /* open-channel SSD device */ | ||
89 | char tgttype[NVM_TTYPE_NAME_MAX]; /* target type name */ | ||
90 | char tgtname[DISK_NAME_LEN]; /* dev to expose target as */ | ||
91 | |||
92 | __u32 flags; | ||
93 | |||
94 | struct nvm_ioctl_create_conf conf; | ||
95 | }; | ||
96 | |||
97 | struct nvm_ioctl_remove { | ||
98 | char tgtname[DISK_NAME_LEN]; | ||
99 | |||
100 | __u32 flags; | ||
101 | }; | ||
102 | |||
103 | |||
104 | /* The ioctl type, 'L', 0x20 - 0x2F documented in ioctl-number.txt */ | ||
105 | enum { | ||
106 | /* top level cmds */ | ||
107 | NVM_INFO_CMD = 0x20, | ||
108 | NVM_GET_DEVICES_CMD, | ||
109 | |||
110 | /* device level cmds */ | ||
111 | NVM_DEV_CREATE_CMD, | ||
112 | NVM_DEV_REMOVE_CMD, | ||
113 | }; | ||
114 | |||
115 | #define NVM_IOCTL 'L' /* 0x4c */ | ||
116 | |||
117 | #define NVM_INFO _IOWR(NVM_IOCTL, NVM_INFO_CMD, \ | ||
118 | struct nvm_ioctl_info) | ||
119 | #define NVM_GET_DEVICES _IOR(NVM_IOCTL, NVM_GET_DEVICES_CMD, \ | ||
120 | struct nvm_ioctl_get_devices) | ||
121 | #define NVM_DEV_CREATE _IOW(NVM_IOCTL, NVM_DEV_CREATE_CMD, \ | ||
122 | struct nvm_ioctl_create) | ||
123 | #define NVM_DEV_REMOVE _IOW(NVM_IOCTL, NVM_DEV_REMOVE_CMD, \ | ||
124 | struct nvm_ioctl_remove) | ||
125 | |||
126 | #define NVM_VERSION_MAJOR 1 | ||
127 | #define NVM_VERSION_MINOR 0 | ||
128 | #define NVM_VERSION_PATCHLEVEL 0 | ||
129 | |||
130 | #endif | ||