aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-11-04 23:46:08 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2015-11-04 23:46:08 -0500
commiteffa04cc5a31b3f12cda6025ab93460f1f0e454e (patch)
tree4abea24fe619c2f7250d373af014c9693f350100
parenta9aa31cdc2a7be4a70b0ea24a451dfeb00ce0024 (diff)
parent5f436e5ef170e5d3301bf5777a3c7c048295db1c (diff)
Merge branch 'for-4.4/lightnvm' of git://git.kernel.dk/linux-block
Pull lightnvm support from Jens Axboe: "This adds support for lightnvm, and adds support to NVMe as well. This is pretty exciting, in that it enables new and interesting use cases for compatible flash devices. There's a LWN writeup about an earlier posting here: https://lwn.net/Articles/641247/ This has been underway for a while, and should be ready for merging at this point" * 'for-4.4/lightnvm' of git://git.kernel.dk/linux-block: nvme: lightnvm: clean up a data type lightnvm: refactor phys addrs type to u64 nvme: LightNVM support rrpc: Round-robin sector target with cost-based gc gennvm: Generic NVM manager lightnvm: Support for Open-Channel SSDs
-rw-r--r--Documentation/ioctl/ioctl-number.txt1
-rw-r--r--MAINTAINERS8
-rw-r--r--drivers/Kconfig2
-rw-r--r--drivers/Makefile1
-rw-r--r--drivers/lightnvm/Kconfig42
-rw-r--r--drivers/lightnvm/Makefile7
-rw-r--r--drivers/lightnvm/core.c826
-rw-r--r--drivers/lightnvm/gennvm.c485
-rw-r--r--drivers/lightnvm/gennvm.h46
-rw-r--r--drivers/lightnvm/rrpc.c1323
-rw-r--r--drivers/lightnvm/rrpc.h239
-rw-r--r--drivers/nvme/host/Makefile2
-rw-r--r--drivers/nvme/host/lightnvm.c526
-rw-r--r--drivers/nvme/host/nvme.h10
-rw-r--r--drivers/nvme/host/pci.c39
-rw-r--r--include/linux/lightnvm.h522
-rw-r--r--include/uapi/linux/lightnvm.h130
17 files changed, 4197 insertions, 12 deletions
diff --git a/Documentation/ioctl/ioctl-number.txt b/Documentation/ioctl/ioctl-number.txt
index df1b25eb8382..8a44d44cf901 100644
--- a/Documentation/ioctl/ioctl-number.txt
+++ b/Documentation/ioctl/ioctl-number.txt
@@ -149,6 +149,7 @@ Code Seq#(hex) Include File Comments
149'K' all linux/kd.h 149'K' all linux/kd.h
150'L' 00-1F linux/loop.h conflict! 150'L' 00-1F linux/loop.h conflict!
151'L' 10-1F drivers/scsi/mpt2sas/mpt2sas_ctl.h conflict! 151'L' 10-1F drivers/scsi/mpt2sas/mpt2sas_ctl.h conflict!
152'L' 20-2F linux/lightnvm.h
152'L' E0-FF linux/ppdd.h encrypted disk device driver 153'L' E0-FF linux/ppdd.h encrypted disk device driver
153 <http://linux01.gwdg.de/~alatham/ppdd.html> 154 <http://linux01.gwdg.de/~alatham/ppdd.html>
154'M' all linux/soundcard.h conflict! 155'M' all linux/soundcard.h conflict!
diff --git a/MAINTAINERS b/MAINTAINERS
index 249f96923642..ed6dccf2bfa0 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -6279,6 +6279,14 @@ F: drivers/nvdimm/pmem.c
6279F: include/linux/pmem.h 6279F: include/linux/pmem.h
6280F: arch/*/include/asm/pmem.h 6280F: arch/*/include/asm/pmem.h
6281 6281
6282LIGHTNVM PLATFORM SUPPORT
6283M: Matias Bjorling <mb@lightnvm.io>
6284W: http://github/OpenChannelSSD
6285S: Maintained
6286F: drivers/lightnvm/
6287F: include/linux/lightnvm.h
6288F: include/uapi/linux/lightnvm.h
6289
6282LINUX FOR IBM pSERIES (RS/6000) 6290LINUX FOR IBM pSERIES (RS/6000)
6283M: Paul Mackerras <paulus@au.ibm.com> 6291M: Paul Mackerras <paulus@au.ibm.com>
6284W: http://www.ibm.com/linux/ltc/projects/ppc 6292W: http://www.ibm.com/linux/ltc/projects/ppc
diff --git a/drivers/Kconfig b/drivers/Kconfig
index e69ec82ac80a..3a5ab4d5873d 100644
--- a/drivers/Kconfig
+++ b/drivers/Kconfig
@@ -44,6 +44,8 @@ source "drivers/net/Kconfig"
44 44
45source "drivers/isdn/Kconfig" 45source "drivers/isdn/Kconfig"
46 46
47source "drivers/lightnvm/Kconfig"
48
47# input before char - char/joystick depends on it. As does USB. 49# input before char - char/joystick depends on it. As does USB.
48 50
49source "drivers/input/Kconfig" 51source "drivers/input/Kconfig"
diff --git a/drivers/Makefile b/drivers/Makefile
index 42f9dd5f07c8..7f1b7c5a1cfd 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -70,6 +70,7 @@ obj-$(CONFIG_NUBUS) += nubus/
70obj-y += macintosh/ 70obj-y += macintosh/
71obj-$(CONFIG_IDE) += ide/ 71obj-$(CONFIG_IDE) += ide/
72obj-$(CONFIG_SCSI) += scsi/ 72obj-$(CONFIG_SCSI) += scsi/
73obj-$(CONFIG_NVM) += lightnvm/
73obj-y += nvme/ 74obj-y += nvme/
74obj-$(CONFIG_ATA) += ata/ 75obj-$(CONFIG_ATA) += ata/
75obj-$(CONFIG_TARGET_CORE) += target/ 76obj-$(CONFIG_TARGET_CORE) += target/
diff --git a/drivers/lightnvm/Kconfig b/drivers/lightnvm/Kconfig
new file mode 100644
index 000000000000..a16bf56d3f28
--- /dev/null
+++ b/drivers/lightnvm/Kconfig
@@ -0,0 +1,42 @@
1#
2# Open-Channel SSD NVM configuration
3#
4
5menuconfig NVM
6 bool "Open-Channel SSD target support"
7 depends on BLOCK
8 help
9 Say Y here to get to enable Open-channel SSDs.
10
11 Open-Channel SSDs implement a set of extension to SSDs, that
12 exposes direct access to the underlying non-volatile memory.
13
14 If you say N, all options in this submenu will be skipped and disabled
15 only do this if you know what you are doing.
16
17if NVM
18
19config NVM_DEBUG
20 bool "Open-Channel SSD debugging support"
21 ---help---
22 Exposes a debug management interface to create/remove targets at:
23
24 /sys/module/lnvm/parameters/configure_debug
25
26 It is required to create/remove targets without IOCTLs.
27
28config NVM_GENNVM
29 tristate "Generic NVM manager for Open-Channel SSDs"
30 ---help---
31 NVM media manager for Open-Channel SSDs that offload management
32 functionality to device, while keeping data placement and garbage
33 collection decisions on the host.
34
35config NVM_RRPC
36 tristate "Round-robin Hybrid Open-Channel SSD target"
37 ---help---
38 Allows an open-channel SSD to be exposed as a block device to the
39 host. The target is implemented using a linear mapping table and
40 cost-based garbage collection. It is optimized for 4K IO sizes.
41
42endif # NVM
diff --git a/drivers/lightnvm/Makefile b/drivers/lightnvm/Makefile
new file mode 100644
index 000000000000..7e0f42acb737
--- /dev/null
+++ b/drivers/lightnvm/Makefile
@@ -0,0 +1,7 @@
1#
2# Makefile for Open-Channel SSDs.
3#
4
5obj-$(CONFIG_NVM) := core.o
6obj-$(CONFIG_NVM_GENNVM) += gennvm.o
7obj-$(CONFIG_NVM_RRPC) += rrpc.o
diff --git a/drivers/lightnvm/core.c b/drivers/lightnvm/core.c
new file mode 100644
index 000000000000..f659e605a406
--- /dev/null
+++ b/drivers/lightnvm/core.c
@@ -0,0 +1,826 @@
1/*
2 * Copyright (C) 2015 IT University of Copenhagen. All rights reserved.
3 * Initial release: Matias Bjorling <m@bjorling.me>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version
7 * 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; see the file COPYING. If not, write to
16 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
17 * USA.
18 *
19 */
20
21#include <linux/blkdev.h>
22#include <linux/blk-mq.h>
23#include <linux/list.h>
24#include <linux/types.h>
25#include <linux/sem.h>
26#include <linux/bitmap.h>
27#include <linux/module.h>
28#include <linux/miscdevice.h>
29#include <linux/lightnvm.h>
30#include <uapi/linux/lightnvm.h>
31
32static LIST_HEAD(nvm_targets);
33static LIST_HEAD(nvm_mgrs);
34static LIST_HEAD(nvm_devices);
35static DECLARE_RWSEM(nvm_lock);
36
37static struct nvm_tgt_type *nvm_find_target_type(const char *name)
38{
39 struct nvm_tgt_type *tt;
40
41 list_for_each_entry(tt, &nvm_targets, list)
42 if (!strcmp(name, tt->name))
43 return tt;
44
45 return NULL;
46}
47
48int nvm_register_target(struct nvm_tgt_type *tt)
49{
50 int ret = 0;
51
52 down_write(&nvm_lock);
53 if (nvm_find_target_type(tt->name))
54 ret = -EEXIST;
55 else
56 list_add(&tt->list, &nvm_targets);
57 up_write(&nvm_lock);
58
59 return ret;
60}
61EXPORT_SYMBOL(nvm_register_target);
62
63void nvm_unregister_target(struct nvm_tgt_type *tt)
64{
65 if (!tt)
66 return;
67
68 down_write(&nvm_lock);
69 list_del(&tt->list);
70 up_write(&nvm_lock);
71}
72EXPORT_SYMBOL(nvm_unregister_target);
73
74void *nvm_dev_dma_alloc(struct nvm_dev *dev, gfp_t mem_flags,
75 dma_addr_t *dma_handler)
76{
77 return dev->ops->dev_dma_alloc(dev->q, dev->ppalist_pool, mem_flags,
78 dma_handler);
79}
80EXPORT_SYMBOL(nvm_dev_dma_alloc);
81
82void nvm_dev_dma_free(struct nvm_dev *dev, void *ppa_list,
83 dma_addr_t dma_handler)
84{
85 dev->ops->dev_dma_free(dev->ppalist_pool, ppa_list, dma_handler);
86}
87EXPORT_SYMBOL(nvm_dev_dma_free);
88
89static struct nvmm_type *nvm_find_mgr_type(const char *name)
90{
91 struct nvmm_type *mt;
92
93 list_for_each_entry(mt, &nvm_mgrs, list)
94 if (!strcmp(name, mt->name))
95 return mt;
96
97 return NULL;
98}
99
100int nvm_register_mgr(struct nvmm_type *mt)
101{
102 int ret = 0;
103
104 down_write(&nvm_lock);
105 if (nvm_find_mgr_type(mt->name))
106 ret = -EEXIST;
107 else
108 list_add(&mt->list, &nvm_mgrs);
109 up_write(&nvm_lock);
110
111 return ret;
112}
113EXPORT_SYMBOL(nvm_register_mgr);
114
115void nvm_unregister_mgr(struct nvmm_type *mt)
116{
117 if (!mt)
118 return;
119
120 down_write(&nvm_lock);
121 list_del(&mt->list);
122 up_write(&nvm_lock);
123}
124EXPORT_SYMBOL(nvm_unregister_mgr);
125
126static struct nvm_dev *nvm_find_nvm_dev(const char *name)
127{
128 struct nvm_dev *dev;
129
130 list_for_each_entry(dev, &nvm_devices, devices)
131 if (!strcmp(name, dev->name))
132 return dev;
133
134 return NULL;
135}
136
137struct nvm_block *nvm_get_blk(struct nvm_dev *dev, struct nvm_lun *lun,
138 unsigned long flags)
139{
140 return dev->mt->get_blk(dev, lun, flags);
141}
142EXPORT_SYMBOL(nvm_get_blk);
143
144/* Assumes that all valid pages have already been moved on release to bm */
145void nvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
146{
147 return dev->mt->put_blk(dev, blk);
148}
149EXPORT_SYMBOL(nvm_put_blk);
150
151int nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
152{
153 return dev->mt->submit_io(dev, rqd);
154}
155EXPORT_SYMBOL(nvm_submit_io);
156
157int nvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk)
158{
159 return dev->mt->erase_blk(dev, blk, 0);
160}
161EXPORT_SYMBOL(nvm_erase_blk);
162
163static void nvm_core_free(struct nvm_dev *dev)
164{
165 kfree(dev);
166}
167
168static int nvm_core_init(struct nvm_dev *dev)
169{
170 struct nvm_id *id = &dev->identity;
171 struct nvm_id_group *grp = &id->groups[0];
172
173 /* device values */
174 dev->nr_chnls = grp->num_ch;
175 dev->luns_per_chnl = grp->num_lun;
176 dev->pgs_per_blk = grp->num_pg;
177 dev->blks_per_lun = grp->num_blk;
178 dev->nr_planes = grp->num_pln;
179 dev->sec_size = grp->csecs;
180 dev->oob_size = grp->sos;
181 dev->sec_per_pg = grp->fpg_sz / grp->csecs;
182 dev->addr_mode = id->ppat;
183 dev->addr_format = id->ppaf;
184
185 dev->plane_mode = NVM_PLANE_SINGLE;
186 dev->max_rq_size = dev->ops->max_phys_sect * dev->sec_size;
187
188 if (grp->mpos & 0x020202)
189 dev->plane_mode = NVM_PLANE_DOUBLE;
190 if (grp->mpos & 0x040404)
191 dev->plane_mode = NVM_PLANE_QUAD;
192
193 /* calculated values */
194 dev->sec_per_pl = dev->sec_per_pg * dev->nr_planes;
195 dev->sec_per_blk = dev->sec_per_pl * dev->pgs_per_blk;
196 dev->sec_per_lun = dev->sec_per_blk * dev->blks_per_lun;
197 dev->nr_luns = dev->luns_per_chnl * dev->nr_chnls;
198
199 dev->total_blocks = dev->nr_planes *
200 dev->blks_per_lun *
201 dev->luns_per_chnl *
202 dev->nr_chnls;
203 dev->total_pages = dev->total_blocks * dev->pgs_per_blk;
204 INIT_LIST_HEAD(&dev->online_targets);
205
206 return 0;
207}
208
209static void nvm_free(struct nvm_dev *dev)
210{
211 if (!dev)
212 return;
213
214 if (dev->mt)
215 dev->mt->unregister_mgr(dev);
216
217 nvm_core_free(dev);
218}
219
220static int nvm_init(struct nvm_dev *dev)
221{
222 struct nvmm_type *mt;
223 int ret = 0;
224
225 if (!dev->q || !dev->ops)
226 return -EINVAL;
227
228 if (dev->ops->identity(dev->q, &dev->identity)) {
229 pr_err("nvm: device could not be identified\n");
230 ret = -EINVAL;
231 goto err;
232 }
233
234 pr_debug("nvm: ver:%x nvm_vendor:%x groups:%u\n",
235 dev->identity.ver_id, dev->identity.vmnt,
236 dev->identity.cgrps);
237
238 if (dev->identity.ver_id != 1) {
239 pr_err("nvm: device not supported by kernel.");
240 goto err;
241 }
242
243 if (dev->identity.cgrps != 1) {
244 pr_err("nvm: only one group configuration supported.");
245 goto err;
246 }
247
248 ret = nvm_core_init(dev);
249 if (ret) {
250 pr_err("nvm: could not initialize core structures.\n");
251 goto err;
252 }
253
254 /* register with device with a supported manager */
255 list_for_each_entry(mt, &nvm_mgrs, list) {
256 ret = mt->register_mgr(dev);
257 if (ret < 0)
258 goto err; /* initialization failed */
259 if (ret > 0) {
260 dev->mt = mt;
261 break; /* successfully initialized */
262 }
263 }
264
265 if (!ret) {
266 pr_info("nvm: no compatible manager found.\n");
267 return 0;
268 }
269
270 pr_info("nvm: registered %s [%u/%u/%u/%u/%u/%u]\n",
271 dev->name, dev->sec_per_pg, dev->nr_planes,
272 dev->pgs_per_blk, dev->blks_per_lun, dev->nr_luns,
273 dev->nr_chnls);
274 return 0;
275err:
276 nvm_free(dev);
277 pr_err("nvm: failed to initialize nvm\n");
278 return ret;
279}
280
281static void nvm_exit(struct nvm_dev *dev)
282{
283 if (dev->ppalist_pool)
284 dev->ops->destroy_dma_pool(dev->ppalist_pool);
285 nvm_free(dev);
286
287 pr_info("nvm: successfully unloaded\n");
288}
289
290int nvm_register(struct request_queue *q, char *disk_name,
291 struct nvm_dev_ops *ops)
292{
293 struct nvm_dev *dev;
294 int ret;
295
296 if (!ops->identity)
297 return -EINVAL;
298
299 dev = kzalloc(sizeof(struct nvm_dev), GFP_KERNEL);
300 if (!dev)
301 return -ENOMEM;
302
303 dev->q = q;
304 dev->ops = ops;
305 strncpy(dev->name, disk_name, DISK_NAME_LEN);
306
307 ret = nvm_init(dev);
308 if (ret)
309 goto err_init;
310
311 down_write(&nvm_lock);
312 list_add(&dev->devices, &nvm_devices);
313 up_write(&nvm_lock);
314
315 if (dev->ops->max_phys_sect > 1) {
316 dev->ppalist_pool = dev->ops->create_dma_pool(dev->q,
317 "ppalist");
318 if (!dev->ppalist_pool) {
319 pr_err("nvm: could not create ppa pool\n");
320 return -ENOMEM;
321 }
322 } else if (dev->ops->max_phys_sect > 256) {
323 pr_info("nvm: max sectors supported is 256.\n");
324 return -EINVAL;
325 }
326
327 return 0;
328err_init:
329 kfree(dev);
330 return ret;
331}
332EXPORT_SYMBOL(nvm_register);
333
334void nvm_unregister(char *disk_name)
335{
336 struct nvm_dev *dev = nvm_find_nvm_dev(disk_name);
337
338 if (!dev) {
339 pr_err("nvm: could not find device %s to unregister\n",
340 disk_name);
341 return;
342 }
343
344 nvm_exit(dev);
345
346 down_write(&nvm_lock);
347 list_del(&dev->devices);
348 up_write(&nvm_lock);
349}
350EXPORT_SYMBOL(nvm_unregister);
351
352static const struct block_device_operations nvm_fops = {
353 .owner = THIS_MODULE,
354};
355
356static int nvm_create_target(struct nvm_dev *dev,
357 struct nvm_ioctl_create *create)
358{
359 struct nvm_ioctl_create_simple *s = &create->conf.s;
360 struct request_queue *tqueue;
361 struct nvmm_type *mt;
362 struct gendisk *tdisk;
363 struct nvm_tgt_type *tt;
364 struct nvm_target *t;
365 void *targetdata;
366 int ret = 0;
367
368 if (!dev->mt) {
369 /* register with device with a supported NVM manager */
370 list_for_each_entry(mt, &nvm_mgrs, list) {
371 ret = mt->register_mgr(dev);
372 if (ret < 0)
373 return ret; /* initialization failed */
374 if (ret > 0) {
375 dev->mt = mt;
376 break; /* successfully initialized */
377 }
378 }
379
380 if (!ret) {
381 pr_info("nvm: no compatible nvm manager found.\n");
382 return -ENODEV;
383 }
384 }
385
386 tt = nvm_find_target_type(create->tgttype);
387 if (!tt) {
388 pr_err("nvm: target type %s not found\n", create->tgttype);
389 return -EINVAL;
390 }
391
392 down_write(&nvm_lock);
393 list_for_each_entry(t, &dev->online_targets, list) {
394 if (!strcmp(create->tgtname, t->disk->disk_name)) {
395 pr_err("nvm: target name already exists.\n");
396 up_write(&nvm_lock);
397 return -EINVAL;
398 }
399 }
400 up_write(&nvm_lock);
401
402 t = kmalloc(sizeof(struct nvm_target), GFP_KERNEL);
403 if (!t)
404 return -ENOMEM;
405
406 tqueue = blk_alloc_queue_node(GFP_KERNEL, dev->q->node);
407 if (!tqueue)
408 goto err_t;
409 blk_queue_make_request(tqueue, tt->make_rq);
410
411 tdisk = alloc_disk(0);
412 if (!tdisk)
413 goto err_queue;
414
415 sprintf(tdisk->disk_name, "%s", create->tgtname);
416 tdisk->flags = GENHD_FL_EXT_DEVT;
417 tdisk->major = 0;
418 tdisk->first_minor = 0;
419 tdisk->fops = &nvm_fops;
420 tdisk->queue = tqueue;
421
422 targetdata = tt->init(dev, tdisk, s->lun_begin, s->lun_end);
423 if (IS_ERR(targetdata))
424 goto err_init;
425
426 tdisk->private_data = targetdata;
427 tqueue->queuedata = targetdata;
428
429 blk_queue_max_hw_sectors(tqueue, 8 * dev->ops->max_phys_sect);
430
431 set_capacity(tdisk, tt->capacity(targetdata));
432 add_disk(tdisk);
433
434 t->type = tt;
435 t->disk = tdisk;
436
437 down_write(&nvm_lock);
438 list_add_tail(&t->list, &dev->online_targets);
439 up_write(&nvm_lock);
440
441 return 0;
442err_init:
443 put_disk(tdisk);
444err_queue:
445 blk_cleanup_queue(tqueue);
446err_t:
447 kfree(t);
448 return -ENOMEM;
449}
450
451static void nvm_remove_target(struct nvm_target *t)
452{
453 struct nvm_tgt_type *tt = t->type;
454 struct gendisk *tdisk = t->disk;
455 struct request_queue *q = tdisk->queue;
456
457 lockdep_assert_held(&nvm_lock);
458
459 del_gendisk(tdisk);
460 if (tt->exit)
461 tt->exit(tdisk->private_data);
462
463 blk_cleanup_queue(q);
464
465 put_disk(tdisk);
466
467 list_del(&t->list);
468 kfree(t);
469}
470
471static int __nvm_configure_create(struct nvm_ioctl_create *create)
472{
473 struct nvm_dev *dev;
474 struct nvm_ioctl_create_simple *s;
475
476 dev = nvm_find_nvm_dev(create->dev);
477 if (!dev) {
478 pr_err("nvm: device not found\n");
479 return -EINVAL;
480 }
481
482 if (create->conf.type != NVM_CONFIG_TYPE_SIMPLE) {
483 pr_err("nvm: config type not valid\n");
484 return -EINVAL;
485 }
486 s = &create->conf.s;
487
488 if (s->lun_begin > s->lun_end || s->lun_end > dev->nr_luns) {
489 pr_err("nvm: lun out of bound (%u:%u > %u)\n",
490 s->lun_begin, s->lun_end, dev->nr_luns);
491 return -EINVAL;
492 }
493
494 return nvm_create_target(dev, create);
495}
496
497static int __nvm_configure_remove(struct nvm_ioctl_remove *remove)
498{
499 struct nvm_target *t = NULL;
500 struct nvm_dev *dev;
501 int ret = -1;
502
503 down_write(&nvm_lock);
504 list_for_each_entry(dev, &nvm_devices, devices)
505 list_for_each_entry(t, &dev->online_targets, list) {
506 if (!strcmp(remove->tgtname, t->disk->disk_name)) {
507 nvm_remove_target(t);
508 ret = 0;
509 break;
510 }
511 }
512 up_write(&nvm_lock);
513
514 if (ret) {
515 pr_err("nvm: target \"%s\" doesn't exist.\n", remove->tgtname);
516 return -EINVAL;
517 }
518
519 return 0;
520}
521
522#ifdef CONFIG_NVM_DEBUG
523static int nvm_configure_show(const char *val)
524{
525 struct nvm_dev *dev;
526 char opcode, devname[DISK_NAME_LEN];
527 int ret;
528
529 ret = sscanf(val, "%c %32s", &opcode, devname);
530 if (ret != 2) {
531 pr_err("nvm: invalid command. Use \"opcode devicename\".\n");
532 return -EINVAL;
533 }
534
535 dev = nvm_find_nvm_dev(devname);
536 if (!dev) {
537 pr_err("nvm: device not found\n");
538 return -EINVAL;
539 }
540
541 if (!dev->mt)
542 return 0;
543
544 dev->mt->free_blocks_print(dev);
545
546 return 0;
547}
548
549static int nvm_configure_remove(const char *val)
550{
551 struct nvm_ioctl_remove remove;
552 char opcode;
553 int ret;
554
555 ret = sscanf(val, "%c %256s", &opcode, remove.tgtname);
556 if (ret != 2) {
557 pr_err("nvm: invalid command. Use \"d targetname\".\n");
558 return -EINVAL;
559 }
560
561 remove.flags = 0;
562
563 return __nvm_configure_remove(&remove);
564}
565
566static int nvm_configure_create(const char *val)
567{
568 struct nvm_ioctl_create create;
569 char opcode;
570 int lun_begin, lun_end, ret;
571
572 ret = sscanf(val, "%c %256s %256s %48s %u:%u", &opcode, create.dev,
573 create.tgtname, create.tgttype,
574 &lun_begin, &lun_end);
575 if (ret != 6) {
576 pr_err("nvm: invalid command. Use \"opcode device name tgttype lun_begin:lun_end\".\n");
577 return -EINVAL;
578 }
579
580 create.flags = 0;
581 create.conf.type = NVM_CONFIG_TYPE_SIMPLE;
582 create.conf.s.lun_begin = lun_begin;
583 create.conf.s.lun_end = lun_end;
584
585 return __nvm_configure_create(&create);
586}
587
588
589/* Exposes administrative interface through /sys/module/lnvm/configure_by_str */
590static int nvm_configure_by_str_event(const char *val,
591 const struct kernel_param *kp)
592{
593 char opcode;
594 int ret;
595
596 ret = sscanf(val, "%c", &opcode);
597 if (ret != 1) {
598 pr_err("nvm: string must have the format of \"cmd ...\"\n");
599 return -EINVAL;
600 }
601
602 switch (opcode) {
603 case 'a':
604 return nvm_configure_create(val);
605 case 'd':
606 return nvm_configure_remove(val);
607 case 's':
608 return nvm_configure_show(val);
609 default:
610 pr_err("nvm: invalid command\n");
611 return -EINVAL;
612 }
613
614 return 0;
615}
616
617static int nvm_configure_get(char *buf, const struct kernel_param *kp)
618{
619 int sz = 0;
620 char *buf_start = buf;
621 struct nvm_dev *dev;
622
623 buf += sprintf(buf, "available devices:\n");
624 down_write(&nvm_lock);
625 list_for_each_entry(dev, &nvm_devices, devices) {
626 if (sz > 4095 - DISK_NAME_LEN)
627 break;
628 buf += sprintf(buf, " %32s\n", dev->name);
629 }
630 up_write(&nvm_lock);
631
632 return buf - buf_start - 1;
633}
634
635static const struct kernel_param_ops nvm_configure_by_str_event_param_ops = {
636 .set = nvm_configure_by_str_event,
637 .get = nvm_configure_get,
638};
639
640#undef MODULE_PARAM_PREFIX
641#define MODULE_PARAM_PREFIX "lnvm."
642
643module_param_cb(configure_debug, &nvm_configure_by_str_event_param_ops, NULL,
644 0644);
645
646#endif /* CONFIG_NVM_DEBUG */
647
648static long nvm_ioctl_info(struct file *file, void __user *arg)
649{
650 struct nvm_ioctl_info *info;
651 struct nvm_tgt_type *tt;
652 int tgt_iter = 0;
653
654 if (!capable(CAP_SYS_ADMIN))
655 return -EPERM;
656
657 info = memdup_user(arg, sizeof(struct nvm_ioctl_info));
658 if (IS_ERR(info))
659 return -EFAULT;
660
661 info->version[0] = NVM_VERSION_MAJOR;
662 info->version[1] = NVM_VERSION_MINOR;
663 info->version[2] = NVM_VERSION_PATCH;
664
665 down_write(&nvm_lock);
666 list_for_each_entry(tt, &nvm_targets, list) {
667 struct nvm_ioctl_info_tgt *tgt = &info->tgts[tgt_iter];
668
669 tgt->version[0] = tt->version[0];
670 tgt->version[1] = tt->version[1];
671 tgt->version[2] = tt->version[2];
672 strncpy(tgt->tgtname, tt->name, NVM_TTYPE_NAME_MAX);
673
674 tgt_iter++;
675 }
676
677 info->tgtsize = tgt_iter;
678 up_write(&nvm_lock);
679
680 if (copy_to_user(arg, info, sizeof(struct nvm_ioctl_info)))
681 return -EFAULT;
682
683 kfree(info);
684 return 0;
685}
686
687static long nvm_ioctl_get_devices(struct file *file, void __user *arg)
688{
689 struct nvm_ioctl_get_devices *devices;
690 struct nvm_dev *dev;
691 int i = 0;
692
693 if (!capable(CAP_SYS_ADMIN))
694 return -EPERM;
695
696 devices = kzalloc(sizeof(struct nvm_ioctl_get_devices), GFP_KERNEL);
697 if (!devices)
698 return -ENOMEM;
699
700 down_write(&nvm_lock);
701 list_for_each_entry(dev, &nvm_devices, devices) {
702 struct nvm_ioctl_device_info *info = &devices->info[i];
703
704 sprintf(info->devname, "%s", dev->name);
705 if (dev->mt) {
706 info->bmversion[0] = dev->mt->version[0];
707 info->bmversion[1] = dev->mt->version[1];
708 info->bmversion[2] = dev->mt->version[2];
709 sprintf(info->bmname, "%s", dev->mt->name);
710 } else {
711 sprintf(info->bmname, "none");
712 }
713
714 i++;
715 if (i > 31) {
716 pr_err("nvm: max 31 devices can be reported.\n");
717 break;
718 }
719 }
720 up_write(&nvm_lock);
721
722 devices->nr_devices = i;
723
724 if (copy_to_user(arg, devices, sizeof(struct nvm_ioctl_get_devices)))
725 return -EFAULT;
726
727 kfree(devices);
728 return 0;
729}
730
731static long nvm_ioctl_dev_create(struct file *file, void __user *arg)
732{
733 struct nvm_ioctl_create create;
734
735 if (!capable(CAP_SYS_ADMIN))
736 return -EPERM;
737
738 if (copy_from_user(&create, arg, sizeof(struct nvm_ioctl_create)))
739 return -EFAULT;
740
741 create.dev[DISK_NAME_LEN - 1] = '\0';
742 create.tgttype[NVM_TTYPE_NAME_MAX - 1] = '\0';
743 create.tgtname[DISK_NAME_LEN - 1] = '\0';
744
745 if (create.flags != 0) {
746 pr_err("nvm: no flags supported\n");
747 return -EINVAL;
748 }
749
750 return __nvm_configure_create(&create);
751}
752
753static long nvm_ioctl_dev_remove(struct file *file, void __user *arg)
754{
755 struct nvm_ioctl_remove remove;
756
757 if (!capable(CAP_SYS_ADMIN))
758 return -EPERM;
759
760 if (copy_from_user(&remove, arg, sizeof(struct nvm_ioctl_remove)))
761 return -EFAULT;
762
763 remove.tgtname[DISK_NAME_LEN - 1] = '\0';
764
765 if (remove.flags != 0) {
766 pr_err("nvm: no flags supported\n");
767 return -EINVAL;
768 }
769
770 return __nvm_configure_remove(&remove);
771}
772
773static long nvm_ctl_ioctl(struct file *file, uint cmd, unsigned long arg)
774{
775 void __user *argp = (void __user *)arg;
776
777 switch (cmd) {
778 case NVM_INFO:
779 return nvm_ioctl_info(file, argp);
780 case NVM_GET_DEVICES:
781 return nvm_ioctl_get_devices(file, argp);
782 case NVM_DEV_CREATE:
783 return nvm_ioctl_dev_create(file, argp);
784 case NVM_DEV_REMOVE:
785 return nvm_ioctl_dev_remove(file, argp);
786 }
787 return 0;
788}
789
790static const struct file_operations _ctl_fops = {
791 .open = nonseekable_open,
792 .unlocked_ioctl = nvm_ctl_ioctl,
793 .owner = THIS_MODULE,
794 .llseek = noop_llseek,
795};
796
797static struct miscdevice _nvm_misc = {
798 .minor = MISC_DYNAMIC_MINOR,
799 .name = "lightnvm",
800 .nodename = "lightnvm/control",
801 .fops = &_ctl_fops,
802};
803
804MODULE_ALIAS_MISCDEV(MISC_DYNAMIC_MINOR);
805
806static int __init nvm_mod_init(void)
807{
808 int ret;
809
810 ret = misc_register(&_nvm_misc);
811 if (ret)
812 pr_err("nvm: misc_register failed for control device");
813
814 return ret;
815}
816
817static void __exit nvm_mod_exit(void)
818{
819 misc_deregister(&_nvm_misc);
820}
821
822MODULE_AUTHOR("Matias Bjorling <m@bjorling.me>");
823MODULE_LICENSE("GPL v2");
824MODULE_VERSION("0.1");
825module_init(nvm_mod_init);
826module_exit(nvm_mod_exit);
diff --git a/drivers/lightnvm/gennvm.c b/drivers/lightnvm/gennvm.c
new file mode 100644
index 000000000000..ae1fb2bdc5f4
--- /dev/null
+++ b/drivers/lightnvm/gennvm.c
@@ -0,0 +1,485 @@
1/*
2 * Copyright (C) 2015 Matias Bjorling <m@bjorling.me>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version
6 * 2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; see the file COPYING. If not, write to
15 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
16 * USA.
17 *
18 * Implementation of a generic nvm manager for Open-Channel SSDs.
19 */
20
21#include "gennvm.h"
22
23static void gennvm_blocks_free(struct nvm_dev *dev)
24{
25 struct gen_nvm *gn = dev->mp;
26 struct gen_lun *lun;
27 int i;
28
29 gennvm_for_each_lun(gn, lun, i) {
30 if (!lun->vlun.blocks)
31 break;
32 vfree(lun->vlun.blocks);
33 }
34}
35
36static void gennvm_luns_free(struct nvm_dev *dev)
37{
38 struct gen_nvm *gn = dev->mp;
39
40 kfree(gn->luns);
41}
42
43static int gennvm_luns_init(struct nvm_dev *dev, struct gen_nvm *gn)
44{
45 struct gen_lun *lun;
46 int i;
47
48 gn->luns = kcalloc(dev->nr_luns, sizeof(struct gen_lun), GFP_KERNEL);
49 if (!gn->luns)
50 return -ENOMEM;
51
52 gennvm_for_each_lun(gn, lun, i) {
53 spin_lock_init(&lun->vlun.lock);
54 INIT_LIST_HEAD(&lun->free_list);
55 INIT_LIST_HEAD(&lun->used_list);
56 INIT_LIST_HEAD(&lun->bb_list);
57
58 lun->reserved_blocks = 2; /* for GC only */
59 lun->vlun.id = i;
60 lun->vlun.lun_id = i % dev->luns_per_chnl;
61 lun->vlun.chnl_id = i / dev->luns_per_chnl;
62 lun->vlun.nr_free_blocks = dev->blks_per_lun;
63 }
64 return 0;
65}
66
67static int gennvm_block_bb(u32 lun_id, void *bb_bitmap, unsigned int nr_blocks,
68 void *private)
69{
70 struct gen_nvm *gn = private;
71 struct gen_lun *lun = &gn->luns[lun_id];
72 struct nvm_block *blk;
73 int i;
74
75 if (unlikely(bitmap_empty(bb_bitmap, nr_blocks)))
76 return 0;
77
78 i = -1;
79 while ((i = find_next_bit(bb_bitmap, nr_blocks, i + 1)) < nr_blocks) {
80 blk = &lun->vlun.blocks[i];
81 if (!blk) {
82 pr_err("gennvm: BB data is out of bounds.\n");
83 return -EINVAL;
84 }
85
86 list_move_tail(&blk->list, &lun->bb_list);
87 }
88
89 return 0;
90}
91
92static int gennvm_block_map(u64 slba, u32 nlb, __le64 *entries, void *private)
93{
94 struct nvm_dev *dev = private;
95 struct gen_nvm *gn = dev->mp;
96 sector_t max_pages = dev->total_pages * (dev->sec_size >> 9);
97 u64 elba = slba + nlb;
98 struct gen_lun *lun;
99 struct nvm_block *blk;
100 u64 i;
101 int lun_id;
102
103 if (unlikely(elba > dev->total_pages)) {
104 pr_err("gennvm: L2P data from device is out of bounds!\n");
105 return -EINVAL;
106 }
107
108 for (i = 0; i < nlb; i++) {
109 u64 pba = le64_to_cpu(entries[i]);
110
111 if (unlikely(pba >= max_pages && pba != U64_MAX)) {
112 pr_err("gennvm: L2P data entry is out of bounds!\n");
113 return -EINVAL;
114 }
115
116 /* Address zero is a special one. The first page on a disk is
117 * protected. It often holds internal device boot
118 * information.
119 */
120 if (!pba)
121 continue;
122
123 /* resolve block from physical address */
124 lun_id = div_u64(pba, dev->sec_per_lun);
125 lun = &gn->luns[lun_id];
126
127 /* Calculate block offset into lun */
128 pba = pba - (dev->sec_per_lun * lun_id);
129 blk = &lun->vlun.blocks[div_u64(pba, dev->sec_per_blk)];
130
131 if (!blk->type) {
132 /* at this point, we don't know anything about the
133 * block. It's up to the FTL on top to re-etablish the
134 * block state
135 */
136 list_move_tail(&blk->list, &lun->used_list);
137 blk->type = 1;
138 lun->vlun.nr_free_blocks--;
139 }
140 }
141
142 return 0;
143}
144
145static int gennvm_blocks_init(struct nvm_dev *dev, struct gen_nvm *gn)
146{
147 struct gen_lun *lun;
148 struct nvm_block *block;
149 sector_t lun_iter, blk_iter, cur_block_id = 0;
150 int ret;
151
152 gennvm_for_each_lun(gn, lun, lun_iter) {
153 lun->vlun.blocks = vzalloc(sizeof(struct nvm_block) *
154 dev->blks_per_lun);
155 if (!lun->vlun.blocks)
156 return -ENOMEM;
157
158 for (blk_iter = 0; blk_iter < dev->blks_per_lun; blk_iter++) {
159 block = &lun->vlun.blocks[blk_iter];
160
161 INIT_LIST_HEAD(&block->list);
162
163 block->lun = &lun->vlun;
164 block->id = cur_block_id++;
165
166 /* First block is reserved for device */
167 if (unlikely(lun_iter == 0 && blk_iter == 0))
168 continue;
169
170 list_add_tail(&block->list, &lun->free_list);
171 }
172
173 if (dev->ops->get_bb_tbl) {
174 ret = dev->ops->get_bb_tbl(dev->q, lun->vlun.id,
175 dev->blks_per_lun, gennvm_block_bb, gn);
176 if (ret)
177 pr_err("gennvm: could not read BB table\n");
178 }
179 }
180
181 if (dev->ops->get_l2p_tbl) {
182 ret = dev->ops->get_l2p_tbl(dev->q, 0, dev->total_pages,
183 gennvm_block_map, dev);
184 if (ret) {
185 pr_err("gennvm: could not read L2P table.\n");
186 pr_warn("gennvm: default block initialization");
187 }
188 }
189
190 return 0;
191}
192
193static int gennvm_register(struct nvm_dev *dev)
194{
195 struct gen_nvm *gn;
196 int ret;
197
198 gn = kzalloc(sizeof(struct gen_nvm), GFP_KERNEL);
199 if (!gn)
200 return -ENOMEM;
201
202 gn->nr_luns = dev->nr_luns;
203 dev->mp = gn;
204
205 ret = gennvm_luns_init(dev, gn);
206 if (ret) {
207 pr_err("gennvm: could not initialize luns\n");
208 goto err;
209 }
210
211 ret = gennvm_blocks_init(dev, gn);
212 if (ret) {
213 pr_err("gennvm: could not initialize blocks\n");
214 goto err;
215 }
216
217 return 1;
218err:
219 kfree(gn);
220 return ret;
221}
222
223static void gennvm_unregister(struct nvm_dev *dev)
224{
225 gennvm_blocks_free(dev);
226 gennvm_luns_free(dev);
227 kfree(dev->mp);
228 dev->mp = NULL;
229}
230
231static struct nvm_block *gennvm_get_blk(struct nvm_dev *dev,
232 struct nvm_lun *vlun, unsigned long flags)
233{
234 struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
235 struct nvm_block *blk = NULL;
236 int is_gc = flags & NVM_IOTYPE_GC;
237
238 spin_lock(&vlun->lock);
239
240 if (list_empty(&lun->free_list)) {
241 pr_err_ratelimited("gennvm: lun %u have no free pages available",
242 lun->vlun.id);
243 spin_unlock(&vlun->lock);
244 goto out;
245 }
246
247 while (!is_gc && lun->vlun.nr_free_blocks < lun->reserved_blocks) {
248 spin_unlock(&vlun->lock);
249 goto out;
250 }
251
252 blk = list_first_entry(&lun->free_list, struct nvm_block, list);
253 list_move_tail(&blk->list, &lun->used_list);
254 blk->type = 1;
255
256 lun->vlun.nr_free_blocks--;
257
258 spin_unlock(&vlun->lock);
259out:
260 return blk;
261}
262
263static void gennvm_put_blk(struct nvm_dev *dev, struct nvm_block *blk)
264{
265 struct nvm_lun *vlun = blk->lun;
266 struct gen_lun *lun = container_of(vlun, struct gen_lun, vlun);
267
268 spin_lock(&vlun->lock);
269
270 switch (blk->type) {
271 case 1:
272 list_move_tail(&blk->list, &lun->free_list);
273 lun->vlun.nr_free_blocks++;
274 blk->type = 0;
275 break;
276 case 2:
277 list_move_tail(&blk->list, &lun->bb_list);
278 break;
279 default:
280 WARN_ON_ONCE(1);
281 pr_err("gennvm: erroneous block type (%lu -> %u)\n",
282 blk->id, blk->type);
283 list_move_tail(&blk->list, &lun->bb_list);
284 }
285
286 spin_unlock(&vlun->lock);
287}
288
289static void gennvm_addr_to_generic_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
290{
291 int i;
292
293 if (rqd->nr_pages > 1) {
294 for (i = 0; i < rqd->nr_pages; i++)
295 rqd->ppa_list[i] = addr_to_generic_mode(dev,
296 rqd->ppa_list[i]);
297 } else {
298 rqd->ppa_addr = addr_to_generic_mode(dev, rqd->ppa_addr);
299 }
300}
301
302static void gennvm_generic_to_addr_mode(struct nvm_dev *dev, struct nvm_rq *rqd)
303{
304 int i;
305
306 if (rqd->nr_pages > 1) {
307 for (i = 0; i < rqd->nr_pages; i++)
308 rqd->ppa_list[i] = generic_to_addr_mode(dev,
309 rqd->ppa_list[i]);
310 } else {
311 rqd->ppa_addr = generic_to_addr_mode(dev, rqd->ppa_addr);
312 }
313}
314
315static int gennvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
316{
317 if (!dev->ops->submit_io)
318 return 0;
319
320 /* Convert address space */
321 gennvm_generic_to_addr_mode(dev, rqd);
322
323 rqd->dev = dev;
324 return dev->ops->submit_io(dev->q, rqd);
325}
326
327static void gennvm_blk_set_type(struct nvm_dev *dev, struct ppa_addr *ppa,
328 int type)
329{
330 struct gen_nvm *gn = dev->mp;
331 struct gen_lun *lun;
332 struct nvm_block *blk;
333
334 if (unlikely(ppa->g.ch > dev->nr_chnls ||
335 ppa->g.lun > dev->luns_per_chnl ||
336 ppa->g.blk > dev->blks_per_lun)) {
337 WARN_ON_ONCE(1);
338 pr_err("gennvm: ppa broken (ch: %u > %u lun: %u > %u blk: %u > %u",
339 ppa->g.ch, dev->nr_chnls,
340 ppa->g.lun, dev->luns_per_chnl,
341 ppa->g.blk, dev->blks_per_lun);
342 return;
343 }
344
345 lun = &gn->luns[ppa->g.lun * ppa->g.ch];
346 blk = &lun->vlun.blocks[ppa->g.blk];
347
348 /* will be moved to bb list on put_blk from target */
349 blk->type = type;
350}
351
352/* mark block bad. It is expected the target recover from the error. */
353static void gennvm_mark_blk_bad(struct nvm_dev *dev, struct nvm_rq *rqd)
354{
355 int i;
356
357 if (!dev->ops->set_bb)
358 return;
359
360 if (dev->ops->set_bb(dev->q, rqd, 1))
361 return;
362
363 gennvm_addr_to_generic_mode(dev, rqd);
364
365 /* look up blocks and mark them as bad */
366 if (rqd->nr_pages > 1)
367 for (i = 0; i < rqd->nr_pages; i++)
368 gennvm_blk_set_type(dev, &rqd->ppa_list[i], 2);
369 else
370 gennvm_blk_set_type(dev, &rqd->ppa_addr, 2);
371}
372
373static int gennvm_end_io(struct nvm_rq *rqd, int error)
374{
375 struct nvm_tgt_instance *ins = rqd->ins;
376 int ret = 0;
377
378 switch (error) {
379 case NVM_RSP_SUCCESS:
380 break;
381 case NVM_RSP_ERR_EMPTYPAGE:
382 break;
383 case NVM_RSP_ERR_FAILWRITE:
384 gennvm_mark_blk_bad(rqd->dev, rqd);
385 default:
386 ret++;
387 }
388
389 ret += ins->tt->end_io(rqd, error);
390
391 return ret;
392}
393
394static int gennvm_erase_blk(struct nvm_dev *dev, struct nvm_block *blk,
395 unsigned long flags)
396{
397 int plane_cnt = 0, pl_idx, ret;
398 struct ppa_addr addr;
399 struct nvm_rq rqd;
400
401 if (!dev->ops->erase_block)
402 return 0;
403
404 addr = block_to_ppa(dev, blk);
405
406 if (dev->plane_mode == NVM_PLANE_SINGLE) {
407 rqd.nr_pages = 1;
408 rqd.ppa_addr = addr;
409 } else {
410 plane_cnt = (1 << dev->plane_mode);
411 rqd.nr_pages = plane_cnt;
412
413 rqd.ppa_list = nvm_dev_dma_alloc(dev, GFP_KERNEL,
414 &rqd.dma_ppa_list);
415 if (!rqd.ppa_list) {
416 pr_err("gennvm: failed to allocate dma memory\n");
417 return -ENOMEM;
418 }
419
420 for (pl_idx = 0; pl_idx < plane_cnt; pl_idx++) {
421 addr.g.pl = pl_idx;
422 rqd.ppa_list[pl_idx] = addr;
423 }
424 }
425
426 gennvm_generic_to_addr_mode(dev, &rqd);
427
428 ret = dev->ops->erase_block(dev->q, &rqd);
429
430 if (plane_cnt)
431 nvm_dev_dma_free(dev, rqd.ppa_list, rqd.dma_ppa_list);
432
433 return ret;
434}
435
436static struct nvm_lun *gennvm_get_lun(struct nvm_dev *dev, int lunid)
437{
438 struct gen_nvm *gn = dev->mp;
439
440 return &gn->luns[lunid].vlun;
441}
442
443static void gennvm_free_blocks_print(struct nvm_dev *dev)
444{
445 struct gen_nvm *gn = dev->mp;
446 struct gen_lun *lun;
447 unsigned int i;
448
449 gennvm_for_each_lun(gn, lun, i)
450 pr_info("%s: lun%8u\t%u\n",
451 dev->name, i, lun->vlun.nr_free_blocks);
452}
453
454static struct nvmm_type gennvm = {
455 .name = "gennvm",
456 .version = {0, 1, 0},
457
458 .register_mgr = gennvm_register,
459 .unregister_mgr = gennvm_unregister,
460
461 .get_blk = gennvm_get_blk,
462 .put_blk = gennvm_put_blk,
463
464 .submit_io = gennvm_submit_io,
465 .end_io = gennvm_end_io,
466 .erase_blk = gennvm_erase_blk,
467
468 .get_lun = gennvm_get_lun,
469 .free_blocks_print = gennvm_free_blocks_print,
470};
471
472static int __init gennvm_module_init(void)
473{
474 return nvm_register_mgr(&gennvm);
475}
476
477static void gennvm_module_exit(void)
478{
479 nvm_unregister_mgr(&gennvm);
480}
481
482module_init(gennvm_module_init);
483module_exit(gennvm_module_exit);
484MODULE_LICENSE("GPL v2");
485MODULE_DESCRIPTION("Generic media manager for Open-Channel SSDs");
diff --git a/drivers/lightnvm/gennvm.h b/drivers/lightnvm/gennvm.h
new file mode 100644
index 000000000000..d23bd3501ddc
--- /dev/null
+++ b/drivers/lightnvm/gennvm.h
@@ -0,0 +1,46 @@
1/*
2 * Copyright: Matias Bjorling <mb@bjorling.me>
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version
6 * 2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 */
14
15#ifndef GENNVM_H_
16#define GENNVM_H_
17
18#include <linux/module.h>
19#include <linux/vmalloc.h>
20
21#include <linux/lightnvm.h>
22
23struct gen_lun {
24 struct nvm_lun vlun;
25
26 int reserved_blocks;
27 /* lun block lists */
28 struct list_head used_list; /* In-use blocks */
29 struct list_head free_list; /* Not used blocks i.e. released
30 * and ready for use
31 */
32 struct list_head bb_list; /* Bad blocks. Mutually exclusive with
33 * free_list and used_list
34 */
35};
36
37struct gen_nvm {
38 int nr_luns;
39 struct gen_lun *luns;
40};
41
42#define gennvm_for_each_lun(bm, lun, i) \
43 for ((i) = 0, lun = &(bm)->luns[0]; \
44 (i) < (bm)->nr_luns; (i)++, lun = &(bm)->luns[(i)])
45
46#endif /* GENNVM_H_ */
diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
new file mode 100644
index 000000000000..64a888a5e9b3
--- /dev/null
+++ b/drivers/lightnvm/rrpc.c
@@ -0,0 +1,1323 @@
1/*
2 * Copyright (C) 2015 IT University of Copenhagen
3 * Initial release: Matias Bjorling <m@bjorling.me>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version
7 * 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs.
15 */
16
17#include "rrpc.h"
18
19static struct kmem_cache *rrpc_gcb_cache, *rrpc_rq_cache;
20static DECLARE_RWSEM(rrpc_lock);
21
22static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
23 struct nvm_rq *rqd, unsigned long flags);
24
25#define rrpc_for_each_lun(rrpc, rlun, i) \
26 for ((i) = 0, rlun = &(rrpc)->luns[0]; \
27 (i) < (rrpc)->nr_luns; (i)++, rlun = &(rrpc)->luns[(i)])
28
29static void rrpc_page_invalidate(struct rrpc *rrpc, struct rrpc_addr *a)
30{
31 struct rrpc_block *rblk = a->rblk;
32 unsigned int pg_offset;
33
34 lockdep_assert_held(&rrpc->rev_lock);
35
36 if (a->addr == ADDR_EMPTY || !rblk)
37 return;
38
39 spin_lock(&rblk->lock);
40
41 div_u64_rem(a->addr, rrpc->dev->pgs_per_blk, &pg_offset);
42 WARN_ON(test_and_set_bit(pg_offset, rblk->invalid_pages));
43 rblk->nr_invalid_pages++;
44
45 spin_unlock(&rblk->lock);
46
47 rrpc->rev_trans_map[a->addr - rrpc->poffset].addr = ADDR_EMPTY;
48}
49
50static void rrpc_invalidate_range(struct rrpc *rrpc, sector_t slba,
51 unsigned len)
52{
53 sector_t i;
54
55 spin_lock(&rrpc->rev_lock);
56 for (i = slba; i < slba + len; i++) {
57 struct rrpc_addr *gp = &rrpc->trans_map[i];
58
59 rrpc_page_invalidate(rrpc, gp);
60 gp->rblk = NULL;
61 }
62 spin_unlock(&rrpc->rev_lock);
63}
64
65static struct nvm_rq *rrpc_inflight_laddr_acquire(struct rrpc *rrpc,
66 sector_t laddr, unsigned int pages)
67{
68 struct nvm_rq *rqd;
69 struct rrpc_inflight_rq *inf;
70
71 rqd = mempool_alloc(rrpc->rq_pool, GFP_ATOMIC);
72 if (!rqd)
73 return ERR_PTR(-ENOMEM);
74
75 inf = rrpc_get_inflight_rq(rqd);
76 if (rrpc_lock_laddr(rrpc, laddr, pages, inf)) {
77 mempool_free(rqd, rrpc->rq_pool);
78 return NULL;
79 }
80
81 return rqd;
82}
83
84static void rrpc_inflight_laddr_release(struct rrpc *rrpc, struct nvm_rq *rqd)
85{
86 struct rrpc_inflight_rq *inf = rrpc_get_inflight_rq(rqd);
87
88 rrpc_unlock_laddr(rrpc, inf);
89
90 mempool_free(rqd, rrpc->rq_pool);
91}
92
93static void rrpc_discard(struct rrpc *rrpc, struct bio *bio)
94{
95 sector_t slba = bio->bi_iter.bi_sector / NR_PHY_IN_LOG;
96 sector_t len = bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE;
97 struct nvm_rq *rqd;
98
99 do {
100 rqd = rrpc_inflight_laddr_acquire(rrpc, slba, len);
101 schedule();
102 } while (!rqd);
103
104 if (IS_ERR(rqd)) {
105 pr_err("rrpc: unable to acquire inflight IO\n");
106 bio_io_error(bio);
107 return;
108 }
109
110 rrpc_invalidate_range(rrpc, slba, len);
111 rrpc_inflight_laddr_release(rrpc, rqd);
112}
113
114static int block_is_full(struct rrpc *rrpc, struct rrpc_block *rblk)
115{
116 return (rblk->next_page == rrpc->dev->pgs_per_blk);
117}
118
119static u64 block_to_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
120{
121 struct nvm_block *blk = rblk->parent;
122
123 return blk->id * rrpc->dev->pgs_per_blk;
124}
125
126static struct ppa_addr rrpc_ppa_to_gaddr(struct nvm_dev *dev, u64 addr)
127{
128 struct ppa_addr paddr;
129
130 paddr.ppa = addr;
131 return __linear_to_generic_addr(dev, paddr);
132}
133
134/* requires lun->lock taken */
135static void rrpc_set_lun_cur(struct rrpc_lun *rlun, struct rrpc_block *rblk)
136{
137 struct rrpc *rrpc = rlun->rrpc;
138
139 BUG_ON(!rblk);
140
141 if (rlun->cur) {
142 spin_lock(&rlun->cur->lock);
143 WARN_ON(!block_is_full(rrpc, rlun->cur));
144 spin_unlock(&rlun->cur->lock);
145 }
146 rlun->cur = rblk;
147}
148
149static struct rrpc_block *rrpc_get_blk(struct rrpc *rrpc, struct rrpc_lun *rlun,
150 unsigned long flags)
151{
152 struct nvm_block *blk;
153 struct rrpc_block *rblk;
154
155 blk = nvm_get_blk(rrpc->dev, rlun->parent, 0);
156 if (!blk)
157 return NULL;
158
159 rblk = &rlun->blocks[blk->id];
160 blk->priv = rblk;
161
162 bitmap_zero(rblk->invalid_pages, rrpc->dev->pgs_per_blk);
163 rblk->next_page = 0;
164 rblk->nr_invalid_pages = 0;
165 atomic_set(&rblk->data_cmnt_size, 0);
166
167 return rblk;
168}
169
170static void rrpc_put_blk(struct rrpc *rrpc, struct rrpc_block *rblk)
171{
172 nvm_put_blk(rrpc->dev, rblk->parent);
173}
174
175static struct rrpc_lun *get_next_lun(struct rrpc *rrpc)
176{
177 int next = atomic_inc_return(&rrpc->next_lun);
178
179 return &rrpc->luns[next % rrpc->nr_luns];
180}
181
182static void rrpc_gc_kick(struct rrpc *rrpc)
183{
184 struct rrpc_lun *rlun;
185 unsigned int i;
186
187 for (i = 0; i < rrpc->nr_luns; i++) {
188 rlun = &rrpc->luns[i];
189 queue_work(rrpc->krqd_wq, &rlun->ws_gc);
190 }
191}
192
193/*
194 * timed GC every interval.
195 */
196static void rrpc_gc_timer(unsigned long data)
197{
198 struct rrpc *rrpc = (struct rrpc *)data;
199
200 rrpc_gc_kick(rrpc);
201 mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10));
202}
203
204static void rrpc_end_sync_bio(struct bio *bio)
205{
206 struct completion *waiting = bio->bi_private;
207
208 if (bio->bi_error)
209 pr_err("nvm: gc request failed (%u).\n", bio->bi_error);
210
211 complete(waiting);
212}
213
214/*
215 * rrpc_move_valid_pages -- migrate live data off the block
216 * @rrpc: the 'rrpc' structure
217 * @block: the block from which to migrate live pages
218 *
219 * Description:
220 * GC algorithms may call this function to migrate remaining live
221 * pages off the block prior to erasing it. This function blocks
222 * further execution until the operation is complete.
223 */
224static int rrpc_move_valid_pages(struct rrpc *rrpc, struct rrpc_block *rblk)
225{
226 struct request_queue *q = rrpc->dev->q;
227 struct rrpc_rev_addr *rev;
228 struct nvm_rq *rqd;
229 struct bio *bio;
230 struct page *page;
231 int slot;
232 int nr_pgs_per_blk = rrpc->dev->pgs_per_blk;
233 u64 phys_addr;
234 DECLARE_COMPLETION_ONSTACK(wait);
235
236 if (bitmap_full(rblk->invalid_pages, nr_pgs_per_blk))
237 return 0;
238
239 bio = bio_alloc(GFP_NOIO, 1);
240 if (!bio) {
241 pr_err("nvm: could not alloc bio to gc\n");
242 return -ENOMEM;
243 }
244
245 page = mempool_alloc(rrpc->page_pool, GFP_NOIO);
246
247 while ((slot = find_first_zero_bit(rblk->invalid_pages,
248 nr_pgs_per_blk)) < nr_pgs_per_blk) {
249
250 /* Lock laddr */
251 phys_addr = (rblk->parent->id * nr_pgs_per_blk) + slot;
252
253try:
254 spin_lock(&rrpc->rev_lock);
255 /* Get logical address from physical to logical table */
256 rev = &rrpc->rev_trans_map[phys_addr - rrpc->poffset];
257 /* already updated by previous regular write */
258 if (rev->addr == ADDR_EMPTY) {
259 spin_unlock(&rrpc->rev_lock);
260 continue;
261 }
262
263 rqd = rrpc_inflight_laddr_acquire(rrpc, rev->addr, 1);
264 if (IS_ERR_OR_NULL(rqd)) {
265 spin_unlock(&rrpc->rev_lock);
266 schedule();
267 goto try;
268 }
269
270 spin_unlock(&rrpc->rev_lock);
271
272 /* Perform read to do GC */
273 bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);
274 bio->bi_rw = READ;
275 bio->bi_private = &wait;
276 bio->bi_end_io = rrpc_end_sync_bio;
277
278 /* TODO: may fail when EXP_PG_SIZE > PAGE_SIZE */
279 bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
280
281 if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {
282 pr_err("rrpc: gc read failed.\n");
283 rrpc_inflight_laddr_release(rrpc, rqd);
284 goto finished;
285 }
286 wait_for_completion_io(&wait);
287
288 bio_reset(bio);
289 reinit_completion(&wait);
290
291 bio->bi_iter.bi_sector = rrpc_get_sector(rev->addr);
292 bio->bi_rw = WRITE;
293 bio->bi_private = &wait;
294 bio->bi_end_io = rrpc_end_sync_bio;
295
296 bio_add_pc_page(q, bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
297
298 /* turn the command around and write the data back to a new
299 * address
300 */
301 if (rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_GC)) {
302 pr_err("rrpc: gc write failed.\n");
303 rrpc_inflight_laddr_release(rrpc, rqd);
304 goto finished;
305 }
306 wait_for_completion_io(&wait);
307
308 rrpc_inflight_laddr_release(rrpc, rqd);
309
310 bio_reset(bio);
311 }
312
313finished:
314 mempool_free(page, rrpc->page_pool);
315 bio_put(bio);
316
317 if (!bitmap_full(rblk->invalid_pages, nr_pgs_per_blk)) {
318 pr_err("nvm: failed to garbage collect block\n");
319 return -EIO;
320 }
321
322 return 0;
323}
324
325static void rrpc_block_gc(struct work_struct *work)
326{
327 struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc,
328 ws_gc);
329 struct rrpc *rrpc = gcb->rrpc;
330 struct rrpc_block *rblk = gcb->rblk;
331 struct nvm_dev *dev = rrpc->dev;
332
333 pr_debug("nvm: block '%lu' being reclaimed\n", rblk->parent->id);
334
335 if (rrpc_move_valid_pages(rrpc, rblk))
336 goto done;
337
338 nvm_erase_blk(dev, rblk->parent);
339 rrpc_put_blk(rrpc, rblk);
340done:
341 mempool_free(gcb, rrpc->gcb_pool);
342}
343
344/* the block with highest number of invalid pages, will be in the beginning
345 * of the list
346 */
347static struct rrpc_block *rblock_max_invalid(struct rrpc_block *ra,
348 struct rrpc_block *rb)
349{
350 if (ra->nr_invalid_pages == rb->nr_invalid_pages)
351 return ra;
352
353 return (ra->nr_invalid_pages < rb->nr_invalid_pages) ? rb : ra;
354}
355
356/* linearly find the block with highest number of invalid pages
357 * requires lun->lock
358 */
359static struct rrpc_block *block_prio_find_max(struct rrpc_lun *rlun)
360{
361 struct list_head *prio_list = &rlun->prio_list;
362 struct rrpc_block *rblock, *max;
363
364 BUG_ON(list_empty(prio_list));
365
366 max = list_first_entry(prio_list, struct rrpc_block, prio);
367 list_for_each_entry(rblock, prio_list, prio)
368 max = rblock_max_invalid(max, rblock);
369
370 return max;
371}
372
373static void rrpc_lun_gc(struct work_struct *work)
374{
375 struct rrpc_lun *rlun = container_of(work, struct rrpc_lun, ws_gc);
376 struct rrpc *rrpc = rlun->rrpc;
377 struct nvm_lun *lun = rlun->parent;
378 struct rrpc_block_gc *gcb;
379 unsigned int nr_blocks_need;
380
381 nr_blocks_need = rrpc->dev->blks_per_lun / GC_LIMIT_INVERSE;
382
383 if (nr_blocks_need < rrpc->nr_luns)
384 nr_blocks_need = rrpc->nr_luns;
385
386 spin_lock(&lun->lock);
387 while (nr_blocks_need > lun->nr_free_blocks &&
388 !list_empty(&rlun->prio_list)) {
389 struct rrpc_block *rblock = block_prio_find_max(rlun);
390 struct nvm_block *block = rblock->parent;
391
392 if (!rblock->nr_invalid_pages)
393 break;
394
395 list_del_init(&rblock->prio);
396
397 BUG_ON(!block_is_full(rrpc, rblock));
398
399 pr_debug("rrpc: selected block '%lu' for GC\n", block->id);
400
401 gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
402 if (!gcb)
403 break;
404
405 gcb->rrpc = rrpc;
406 gcb->rblk = rblock;
407 INIT_WORK(&gcb->ws_gc, rrpc_block_gc);
408
409 queue_work(rrpc->kgc_wq, &gcb->ws_gc);
410
411 nr_blocks_need--;
412 }
413 spin_unlock(&lun->lock);
414
415 /* TODO: Hint that request queue can be started again */
416}
417
418static void rrpc_gc_queue(struct work_struct *work)
419{
420 struct rrpc_block_gc *gcb = container_of(work, struct rrpc_block_gc,
421 ws_gc);
422 struct rrpc *rrpc = gcb->rrpc;
423 struct rrpc_block *rblk = gcb->rblk;
424 struct nvm_lun *lun = rblk->parent->lun;
425 struct rrpc_lun *rlun = &rrpc->luns[lun->id - rrpc->lun_offset];
426
427 spin_lock(&rlun->lock);
428 list_add_tail(&rblk->prio, &rlun->prio_list);
429 spin_unlock(&rlun->lock);
430
431 mempool_free(gcb, rrpc->gcb_pool);
432 pr_debug("nvm: block '%lu' is full, allow GC (sched)\n",
433 rblk->parent->id);
434}
435
436static const struct block_device_operations rrpc_fops = {
437 .owner = THIS_MODULE,
438};
439
440static struct rrpc_lun *rrpc_get_lun_rr(struct rrpc *rrpc, int is_gc)
441{
442 unsigned int i;
443 struct rrpc_lun *rlun, *max_free;
444
445 if (!is_gc)
446 return get_next_lun(rrpc);
447
448 /* during GC, we don't care about RR, instead we want to make
449 * sure that we maintain evenness between the block luns.
450 */
451 max_free = &rrpc->luns[0];
452 /* prevent GC-ing lun from devouring pages of a lun with
453 * little free blocks. We don't take the lock as we only need an
454 * estimate.
455 */
456 rrpc_for_each_lun(rrpc, rlun, i) {
457 if (rlun->parent->nr_free_blocks >
458 max_free->parent->nr_free_blocks)
459 max_free = rlun;
460 }
461
462 return max_free;
463}
464
465static struct rrpc_addr *rrpc_update_map(struct rrpc *rrpc, sector_t laddr,
466 struct rrpc_block *rblk, u64 paddr)
467{
468 struct rrpc_addr *gp;
469 struct rrpc_rev_addr *rev;
470
471 BUG_ON(laddr >= rrpc->nr_pages);
472
473 gp = &rrpc->trans_map[laddr];
474 spin_lock(&rrpc->rev_lock);
475 if (gp->rblk)
476 rrpc_page_invalidate(rrpc, gp);
477
478 gp->addr = paddr;
479 gp->rblk = rblk;
480
481 rev = &rrpc->rev_trans_map[gp->addr - rrpc->poffset];
482 rev->addr = laddr;
483 spin_unlock(&rrpc->rev_lock);
484
485 return gp;
486}
487
488static u64 rrpc_alloc_addr(struct rrpc *rrpc, struct rrpc_block *rblk)
489{
490 u64 addr = ADDR_EMPTY;
491
492 spin_lock(&rblk->lock);
493 if (block_is_full(rrpc, rblk))
494 goto out;
495
496 addr = block_to_addr(rrpc, rblk) + rblk->next_page;
497
498 rblk->next_page++;
499out:
500 spin_unlock(&rblk->lock);
501 return addr;
502}
503
504/* Simple round-robin Logical to physical address translation.
505 *
506 * Retrieve the mapping using the active append point. Then update the ap for
507 * the next write to the disk.
508 *
509 * Returns rrpc_addr with the physical address and block. Remember to return to
510 * rrpc->addr_cache when request is finished.
511 */
512static struct rrpc_addr *rrpc_map_page(struct rrpc *rrpc, sector_t laddr,
513 int is_gc)
514{
515 struct rrpc_lun *rlun;
516 struct rrpc_block *rblk;
517 struct nvm_lun *lun;
518 u64 paddr;
519
520 rlun = rrpc_get_lun_rr(rrpc, is_gc);
521 lun = rlun->parent;
522
523 if (!is_gc && lun->nr_free_blocks < rrpc->nr_luns * 4)
524 return NULL;
525
526 spin_lock(&rlun->lock);
527
528 rblk = rlun->cur;
529retry:
530 paddr = rrpc_alloc_addr(rrpc, rblk);
531
532 if (paddr == ADDR_EMPTY) {
533 rblk = rrpc_get_blk(rrpc, rlun, 0);
534 if (rblk) {
535 rrpc_set_lun_cur(rlun, rblk);
536 goto retry;
537 }
538
539 if (is_gc) {
540 /* retry from emergency gc block */
541 paddr = rrpc_alloc_addr(rrpc, rlun->gc_cur);
542 if (paddr == ADDR_EMPTY) {
543 rblk = rrpc_get_blk(rrpc, rlun, 1);
544 if (!rblk) {
545 pr_err("rrpc: no more blocks");
546 goto err;
547 }
548
549 rlun->gc_cur = rblk;
550 paddr = rrpc_alloc_addr(rrpc, rlun->gc_cur);
551 }
552 rblk = rlun->gc_cur;
553 }
554 }
555
556 spin_unlock(&rlun->lock);
557 return rrpc_update_map(rrpc, laddr, rblk, paddr);
558err:
559 spin_unlock(&rlun->lock);
560 return NULL;
561}
562
563static void rrpc_run_gc(struct rrpc *rrpc, struct rrpc_block *rblk)
564{
565 struct rrpc_block_gc *gcb;
566
567 gcb = mempool_alloc(rrpc->gcb_pool, GFP_ATOMIC);
568 if (!gcb) {
569 pr_err("rrpc: unable to queue block for gc.");
570 return;
571 }
572
573 gcb->rrpc = rrpc;
574 gcb->rblk = rblk;
575
576 INIT_WORK(&gcb->ws_gc, rrpc_gc_queue);
577 queue_work(rrpc->kgc_wq, &gcb->ws_gc);
578}
579
580static void rrpc_end_io_write(struct rrpc *rrpc, struct rrpc_rq *rrqd,
581 sector_t laddr, uint8_t npages)
582{
583 struct rrpc_addr *p;
584 struct rrpc_block *rblk;
585 struct nvm_lun *lun;
586 int cmnt_size, i;
587
588 for (i = 0; i < npages; i++) {
589 p = &rrpc->trans_map[laddr + i];
590 rblk = p->rblk;
591 lun = rblk->parent->lun;
592
593 cmnt_size = atomic_inc_return(&rblk->data_cmnt_size);
594 if (unlikely(cmnt_size == rrpc->dev->pgs_per_blk))
595 rrpc_run_gc(rrpc, rblk);
596 }
597}
598
599static int rrpc_end_io(struct nvm_rq *rqd, int error)
600{
601 struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance);
602 struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
603 uint8_t npages = rqd->nr_pages;
604 sector_t laddr = rrpc_get_laddr(rqd->bio) - npages;
605
606 if (bio_data_dir(rqd->bio) == WRITE)
607 rrpc_end_io_write(rrpc, rrqd, laddr, npages);
608
609 if (rrqd->flags & NVM_IOTYPE_GC)
610 return 0;
611
612 rrpc_unlock_rq(rrpc, rqd);
613 bio_put(rqd->bio);
614
615 if (npages > 1)
616 nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
617 if (rqd->metadata)
618 nvm_dev_dma_free(rrpc->dev, rqd->metadata, rqd->dma_metadata);
619
620 mempool_free(rqd, rrpc->rq_pool);
621
622 return 0;
623}
624
625static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
626 struct nvm_rq *rqd, unsigned long flags, int npages)
627{
628 struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
629 struct rrpc_addr *gp;
630 sector_t laddr = rrpc_get_laddr(bio);
631 int is_gc = flags & NVM_IOTYPE_GC;
632 int i;
633
634 if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {
635 nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
636 return NVM_IO_REQUEUE;
637 }
638
639 for (i = 0; i < npages; i++) {
640 /* We assume that mapping occurs at 4KB granularity */
641 BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_pages));
642 gp = &rrpc->trans_map[laddr + i];
643
644 if (gp->rblk) {
645 rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
646 gp->addr);
647 } else {
648 BUG_ON(is_gc);
649 rrpc_unlock_laddr(rrpc, r);
650 nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
651 rqd->dma_ppa_list);
652 return NVM_IO_DONE;
653 }
654 }
655
656 rqd->opcode = NVM_OP_HBREAD;
657
658 return NVM_IO_OK;
659}
660
661static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
662 unsigned long flags)
663{
664 struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
665 int is_gc = flags & NVM_IOTYPE_GC;
666 sector_t laddr = rrpc_get_laddr(bio);
667 struct rrpc_addr *gp;
668
669 if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd))
670 return NVM_IO_REQUEUE;
671
672 BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_pages));
673 gp = &rrpc->trans_map[laddr];
674
675 if (gp->rblk) {
676 rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
677 } else {
678 BUG_ON(is_gc);
679 rrpc_unlock_rq(rrpc, rqd);
680 return NVM_IO_DONE;
681 }
682
683 rqd->opcode = NVM_OP_HBREAD;
684 rrqd->addr = gp;
685
686 return NVM_IO_OK;
687}
688
689static int rrpc_write_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
690 struct nvm_rq *rqd, unsigned long flags, int npages)
691{
692 struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
693 struct rrpc_addr *p;
694 sector_t laddr = rrpc_get_laddr(bio);
695 int is_gc = flags & NVM_IOTYPE_GC;
696 int i;
697
698 if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd)) {
699 nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
700 return NVM_IO_REQUEUE;
701 }
702
703 for (i = 0; i < npages; i++) {
704 /* We assume that mapping occurs at 4KB granularity */
705 p = rrpc_map_page(rrpc, laddr + i, is_gc);
706 if (!p) {
707 BUG_ON(is_gc);
708 rrpc_unlock_laddr(rrpc, r);
709 nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
710 rqd->dma_ppa_list);
711 rrpc_gc_kick(rrpc);
712 return NVM_IO_REQUEUE;
713 }
714
715 rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
716 p->addr);
717 }
718
719 rqd->opcode = NVM_OP_HBWRITE;
720
721 return NVM_IO_OK;
722}
723
724static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
725 struct nvm_rq *rqd, unsigned long flags)
726{
727 struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
728 struct rrpc_addr *p;
729 int is_gc = flags & NVM_IOTYPE_GC;
730 sector_t laddr = rrpc_get_laddr(bio);
731
732 if (!is_gc && rrpc_lock_rq(rrpc, bio, rqd))
733 return NVM_IO_REQUEUE;
734
735 p = rrpc_map_page(rrpc, laddr, is_gc);
736 if (!p) {
737 BUG_ON(is_gc);
738 rrpc_unlock_rq(rrpc, rqd);
739 rrpc_gc_kick(rrpc);
740 return NVM_IO_REQUEUE;
741 }
742
743 rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, p->addr);
744 rqd->opcode = NVM_OP_HBWRITE;
745 rrqd->addr = p;
746
747 return NVM_IO_OK;
748}
749
750static int rrpc_setup_rq(struct rrpc *rrpc, struct bio *bio,
751 struct nvm_rq *rqd, unsigned long flags, uint8_t npages)
752{
753 if (npages > 1) {
754 rqd->ppa_list = nvm_dev_dma_alloc(rrpc->dev, GFP_KERNEL,
755 &rqd->dma_ppa_list);
756 if (!rqd->ppa_list) {
757 pr_err("rrpc: not able to allocate ppa list\n");
758 return NVM_IO_ERR;
759 }
760
761 if (bio_rw(bio) == WRITE)
762 return rrpc_write_ppalist_rq(rrpc, bio, rqd, flags,
763 npages);
764
765 return rrpc_read_ppalist_rq(rrpc, bio, rqd, flags, npages);
766 }
767
768 if (bio_rw(bio) == WRITE)
769 return rrpc_write_rq(rrpc, bio, rqd, flags);
770
771 return rrpc_read_rq(rrpc, bio, rqd, flags);
772}
773
774static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
775 struct nvm_rq *rqd, unsigned long flags)
776{
777 int err;
778 struct rrpc_rq *rrq = nvm_rq_to_pdu(rqd);
779 uint8_t nr_pages = rrpc_get_pages(bio);
780 int bio_size = bio_sectors(bio) << 9;
781
782 if (bio_size < rrpc->dev->sec_size)
783 return NVM_IO_ERR;
784 else if (bio_size > rrpc->dev->max_rq_size)
785 return NVM_IO_ERR;
786
787 err = rrpc_setup_rq(rrpc, bio, rqd, flags, nr_pages);
788 if (err)
789 return err;
790
791 bio_get(bio);
792 rqd->bio = bio;
793 rqd->ins = &rrpc->instance;
794 rqd->nr_pages = nr_pages;
795 rrq->flags = flags;
796
797 err = nvm_submit_io(rrpc->dev, rqd);
798 if (err) {
799 pr_err("rrpc: I/O submission failed: %d\n", err);
800 return NVM_IO_ERR;
801 }
802
803 return NVM_IO_OK;
804}
805
806static void rrpc_make_rq(struct request_queue *q, struct bio *bio)
807{
808 struct rrpc *rrpc = q->queuedata;
809 struct nvm_rq *rqd;
810 int err;
811
812 if (bio->bi_rw & REQ_DISCARD) {
813 rrpc_discard(rrpc, bio);
814 return;
815 }
816
817 rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL);
818 if (!rqd) {
819 pr_err_ratelimited("rrpc: not able to queue bio.");
820 bio_io_error(bio);
821 return;
822 }
823 memset(rqd, 0, sizeof(struct nvm_rq));
824
825 err = rrpc_submit_io(rrpc, bio, rqd, NVM_IOTYPE_NONE);
826 switch (err) {
827 case NVM_IO_OK:
828 return;
829 case NVM_IO_ERR:
830 bio_io_error(bio);
831 break;
832 case NVM_IO_DONE:
833 bio_endio(bio);
834 break;
835 case NVM_IO_REQUEUE:
836 spin_lock(&rrpc->bio_lock);
837 bio_list_add(&rrpc->requeue_bios, bio);
838 spin_unlock(&rrpc->bio_lock);
839 queue_work(rrpc->kgc_wq, &rrpc->ws_requeue);
840 break;
841 }
842
843 mempool_free(rqd, rrpc->rq_pool);
844}
845
846static void rrpc_requeue(struct work_struct *work)
847{
848 struct rrpc *rrpc = container_of(work, struct rrpc, ws_requeue);
849 struct bio_list bios;
850 struct bio *bio;
851
852 bio_list_init(&bios);
853
854 spin_lock(&rrpc->bio_lock);
855 bio_list_merge(&bios, &rrpc->requeue_bios);
856 bio_list_init(&rrpc->requeue_bios);
857 spin_unlock(&rrpc->bio_lock);
858
859 while ((bio = bio_list_pop(&bios)))
860 rrpc_make_rq(rrpc->disk->queue, bio);
861}
862
863static void rrpc_gc_free(struct rrpc *rrpc)
864{
865 struct rrpc_lun *rlun;
866 int i;
867
868 if (rrpc->krqd_wq)
869 destroy_workqueue(rrpc->krqd_wq);
870
871 if (rrpc->kgc_wq)
872 destroy_workqueue(rrpc->kgc_wq);
873
874 if (!rrpc->luns)
875 return;
876
877 for (i = 0; i < rrpc->nr_luns; i++) {
878 rlun = &rrpc->luns[i];
879
880 if (!rlun->blocks)
881 break;
882 vfree(rlun->blocks);
883 }
884}
885
886static int rrpc_gc_init(struct rrpc *rrpc)
887{
888 rrpc->krqd_wq = alloc_workqueue("rrpc-lun", WQ_MEM_RECLAIM|WQ_UNBOUND,
889 rrpc->nr_luns);
890 if (!rrpc->krqd_wq)
891 return -ENOMEM;
892
893 rrpc->kgc_wq = alloc_workqueue("rrpc-bg", WQ_MEM_RECLAIM, 1);
894 if (!rrpc->kgc_wq)
895 return -ENOMEM;
896
897 setup_timer(&rrpc->gc_timer, rrpc_gc_timer, (unsigned long)rrpc);
898
899 return 0;
900}
901
902static void rrpc_map_free(struct rrpc *rrpc)
903{
904 vfree(rrpc->rev_trans_map);
905 vfree(rrpc->trans_map);
906}
907
908static int rrpc_l2p_update(u64 slba, u32 nlb, __le64 *entries, void *private)
909{
910 struct rrpc *rrpc = (struct rrpc *)private;
911 struct nvm_dev *dev = rrpc->dev;
912 struct rrpc_addr *addr = rrpc->trans_map + slba;
913 struct rrpc_rev_addr *raddr = rrpc->rev_trans_map;
914 sector_t max_pages = dev->total_pages * (dev->sec_size >> 9);
915 u64 elba = slba + nlb;
916 u64 i;
917
918 if (unlikely(elba > dev->total_pages)) {
919 pr_err("nvm: L2P data from device is out of bounds!\n");
920 return -EINVAL;
921 }
922
923 for (i = 0; i < nlb; i++) {
924 u64 pba = le64_to_cpu(entries[i]);
925 /* LNVM treats address-spaces as silos, LBA and PBA are
926 * equally large and zero-indexed.
927 */
928 if (unlikely(pba >= max_pages && pba != U64_MAX)) {
929 pr_err("nvm: L2P data entry is out of bounds!\n");
930 return -EINVAL;
931 }
932
933 /* Address zero is a special one. The first page on a disk is
934 * protected. As it often holds internal device boot
935 * information.
936 */
937 if (!pba)
938 continue;
939
940 addr[i].addr = pba;
941 raddr[pba].addr = slba + i;
942 }
943
944 return 0;
945}
946
947static int rrpc_map_init(struct rrpc *rrpc)
948{
949 struct nvm_dev *dev = rrpc->dev;
950 sector_t i;
951 int ret;
952
953 rrpc->trans_map = vzalloc(sizeof(struct rrpc_addr) * rrpc->nr_pages);
954 if (!rrpc->trans_map)
955 return -ENOMEM;
956
957 rrpc->rev_trans_map = vmalloc(sizeof(struct rrpc_rev_addr)
958 * rrpc->nr_pages);
959 if (!rrpc->rev_trans_map)
960 return -ENOMEM;
961
962 for (i = 0; i < rrpc->nr_pages; i++) {
963 struct rrpc_addr *p = &rrpc->trans_map[i];
964 struct rrpc_rev_addr *r = &rrpc->rev_trans_map[i];
965
966 p->addr = ADDR_EMPTY;
967 r->addr = ADDR_EMPTY;
968 }
969
970 if (!dev->ops->get_l2p_tbl)
971 return 0;
972
973 /* Bring up the mapping table from device */
974 ret = dev->ops->get_l2p_tbl(dev->q, 0, dev->total_pages,
975 rrpc_l2p_update, rrpc);
976 if (ret) {
977 pr_err("nvm: rrpc: could not read L2P table.\n");
978 return -EINVAL;
979 }
980
981 return 0;
982}
983
984
985/* Minimum pages needed within a lun */
986#define PAGE_POOL_SIZE 16
987#define ADDR_POOL_SIZE 64
988
989static int rrpc_core_init(struct rrpc *rrpc)
990{
991 down_write(&rrpc_lock);
992 if (!rrpc_gcb_cache) {
993 rrpc_gcb_cache = kmem_cache_create("rrpc_gcb",
994 sizeof(struct rrpc_block_gc), 0, 0, NULL);
995 if (!rrpc_gcb_cache) {
996 up_write(&rrpc_lock);
997 return -ENOMEM;
998 }
999
1000 rrpc_rq_cache = kmem_cache_create("rrpc_rq",
1001 sizeof(struct nvm_rq) + sizeof(struct rrpc_rq),
1002 0, 0, NULL);
1003 if (!rrpc_rq_cache) {
1004 kmem_cache_destroy(rrpc_gcb_cache);
1005 up_write(&rrpc_lock);
1006 return -ENOMEM;
1007 }
1008 }
1009 up_write(&rrpc_lock);
1010
1011 rrpc->page_pool = mempool_create_page_pool(PAGE_POOL_SIZE, 0);
1012 if (!rrpc->page_pool)
1013 return -ENOMEM;
1014
1015 rrpc->gcb_pool = mempool_create_slab_pool(rrpc->dev->nr_luns,
1016 rrpc_gcb_cache);
1017 if (!rrpc->gcb_pool)
1018 return -ENOMEM;
1019
1020 rrpc->rq_pool = mempool_create_slab_pool(64, rrpc_rq_cache);
1021 if (!rrpc->rq_pool)
1022 return -ENOMEM;
1023
1024 spin_lock_init(&rrpc->inflights.lock);
1025 INIT_LIST_HEAD(&rrpc->inflights.reqs);
1026
1027 return 0;
1028}
1029
1030static void rrpc_core_free(struct rrpc *rrpc)
1031{
1032 mempool_destroy(rrpc->page_pool);
1033 mempool_destroy(rrpc->gcb_pool);
1034 mempool_destroy(rrpc->rq_pool);
1035}
1036
1037static void rrpc_luns_free(struct rrpc *rrpc)
1038{
1039 kfree(rrpc->luns);
1040}
1041
1042static int rrpc_luns_init(struct rrpc *rrpc, int lun_begin, int lun_end)
1043{
1044 struct nvm_dev *dev = rrpc->dev;
1045 struct rrpc_lun *rlun;
1046 int i, j;
1047
1048 spin_lock_init(&rrpc->rev_lock);
1049
1050 rrpc->luns = kcalloc(rrpc->nr_luns, sizeof(struct rrpc_lun),
1051 GFP_KERNEL);
1052 if (!rrpc->luns)
1053 return -ENOMEM;
1054
1055 /* 1:1 mapping */
1056 for (i = 0; i < rrpc->nr_luns; i++) {
1057 struct nvm_lun *lun = dev->mt->get_lun(dev, lun_begin + i);
1058
1059 if (dev->pgs_per_blk >
1060 MAX_INVALID_PAGES_STORAGE * BITS_PER_LONG) {
1061 pr_err("rrpc: number of pages per block too high.");
1062 goto err;
1063 }
1064
1065 rlun = &rrpc->luns[i];
1066 rlun->rrpc = rrpc;
1067 rlun->parent = lun;
1068 INIT_LIST_HEAD(&rlun->prio_list);
1069 INIT_WORK(&rlun->ws_gc, rrpc_lun_gc);
1070 spin_lock_init(&rlun->lock);
1071
1072 rrpc->total_blocks += dev->blks_per_lun;
1073 rrpc->nr_pages += dev->sec_per_lun;
1074
1075 rlun->blocks = vzalloc(sizeof(struct rrpc_block) *
1076 rrpc->dev->blks_per_lun);
1077 if (!rlun->blocks)
1078 goto err;
1079
1080 for (j = 0; j < rrpc->dev->blks_per_lun; j++) {
1081 struct rrpc_block *rblk = &rlun->blocks[j];
1082 struct nvm_block *blk = &lun->blocks[j];
1083
1084 rblk->parent = blk;
1085 INIT_LIST_HEAD(&rblk->prio);
1086 spin_lock_init(&rblk->lock);
1087 }
1088 }
1089
1090 return 0;
1091err:
1092 return -ENOMEM;
1093}
1094
1095static void rrpc_free(struct rrpc *rrpc)
1096{
1097 rrpc_gc_free(rrpc);
1098 rrpc_map_free(rrpc);
1099 rrpc_core_free(rrpc);
1100 rrpc_luns_free(rrpc);
1101
1102 kfree(rrpc);
1103}
1104
1105static void rrpc_exit(void *private)
1106{
1107 struct rrpc *rrpc = private;
1108
1109 del_timer(&rrpc->gc_timer);
1110
1111 flush_workqueue(rrpc->krqd_wq);
1112 flush_workqueue(rrpc->kgc_wq);
1113
1114 rrpc_free(rrpc);
1115}
1116
1117static sector_t rrpc_capacity(void *private)
1118{
1119 struct rrpc *rrpc = private;
1120 struct nvm_dev *dev = rrpc->dev;
1121 sector_t reserved, provisioned;
1122
1123 /* cur, gc, and two emergency blocks for each lun */
1124 reserved = rrpc->nr_luns * dev->max_pages_per_blk * 4;
1125 provisioned = rrpc->nr_pages - reserved;
1126
1127 if (reserved > rrpc->nr_pages) {
1128 pr_err("rrpc: not enough space available to expose storage.\n");
1129 return 0;
1130 }
1131
1132 sector_div(provisioned, 10);
1133 return provisioned * 9 * NR_PHY_IN_LOG;
1134}
1135
1136/*
1137 * Looks up the logical address from reverse trans map and check if its valid by
1138 * comparing the logical to physical address with the physical address.
1139 * Returns 0 on free, otherwise 1 if in use
1140 */
1141static void rrpc_block_map_update(struct rrpc *rrpc, struct rrpc_block *rblk)
1142{
1143 struct nvm_dev *dev = rrpc->dev;
1144 int offset;
1145 struct rrpc_addr *laddr;
1146 u64 paddr, pladdr;
1147
1148 for (offset = 0; offset < dev->pgs_per_blk; offset++) {
1149 paddr = block_to_addr(rrpc, rblk) + offset;
1150
1151 pladdr = rrpc->rev_trans_map[paddr].addr;
1152 if (pladdr == ADDR_EMPTY)
1153 continue;
1154
1155 laddr = &rrpc->trans_map[pladdr];
1156
1157 if (paddr == laddr->addr) {
1158 laddr->rblk = rblk;
1159 } else {
1160 set_bit(offset, rblk->invalid_pages);
1161 rblk->nr_invalid_pages++;
1162 }
1163 }
1164}
1165
1166static int rrpc_blocks_init(struct rrpc *rrpc)
1167{
1168 struct rrpc_lun *rlun;
1169 struct rrpc_block *rblk;
1170 int lun_iter, blk_iter;
1171
1172 for (lun_iter = 0; lun_iter < rrpc->nr_luns; lun_iter++) {
1173 rlun = &rrpc->luns[lun_iter];
1174
1175 for (blk_iter = 0; blk_iter < rrpc->dev->blks_per_lun;
1176 blk_iter++) {
1177 rblk = &rlun->blocks[blk_iter];
1178 rrpc_block_map_update(rrpc, rblk);
1179 }
1180 }
1181
1182 return 0;
1183}
1184
1185static int rrpc_luns_configure(struct rrpc *rrpc)
1186{
1187 struct rrpc_lun *rlun;
1188 struct rrpc_block *rblk;
1189 int i;
1190
1191 for (i = 0; i < rrpc->nr_luns; i++) {
1192 rlun = &rrpc->luns[i];
1193
1194 rblk = rrpc_get_blk(rrpc, rlun, 0);
1195 if (!rblk)
1196 return -EINVAL;
1197
1198 rrpc_set_lun_cur(rlun, rblk);
1199
1200 /* Emergency gc block */
1201 rblk = rrpc_get_blk(rrpc, rlun, 1);
1202 if (!rblk)
1203 return -EINVAL;
1204 rlun->gc_cur = rblk;
1205 }
1206
1207 return 0;
1208}
1209
1210static struct nvm_tgt_type tt_rrpc;
1211
1212static void *rrpc_init(struct nvm_dev *dev, struct gendisk *tdisk,
1213 int lun_begin, int lun_end)
1214{
1215 struct request_queue *bqueue = dev->q;
1216 struct request_queue *tqueue = tdisk->queue;
1217 struct rrpc *rrpc;
1218 int ret;
1219
1220 if (!(dev->identity.dom & NVM_RSP_L2P)) {
1221 pr_err("nvm: rrpc: device does not support l2p (%x)\n",
1222 dev->identity.dom);
1223 return ERR_PTR(-EINVAL);
1224 }
1225
1226 rrpc = kzalloc(sizeof(struct rrpc), GFP_KERNEL);
1227 if (!rrpc)
1228 return ERR_PTR(-ENOMEM);
1229
1230 rrpc->instance.tt = &tt_rrpc;
1231 rrpc->dev = dev;
1232 rrpc->disk = tdisk;
1233
1234 bio_list_init(&rrpc->requeue_bios);
1235 spin_lock_init(&rrpc->bio_lock);
1236 INIT_WORK(&rrpc->ws_requeue, rrpc_requeue);
1237
1238 rrpc->nr_luns = lun_end - lun_begin + 1;
1239
1240 /* simple round-robin strategy */
1241 atomic_set(&rrpc->next_lun, -1);
1242
1243 ret = rrpc_luns_init(rrpc, lun_begin, lun_end);
1244 if (ret) {
1245 pr_err("nvm: rrpc: could not initialize luns\n");
1246 goto err;
1247 }
1248
1249 rrpc->poffset = dev->sec_per_lun * lun_begin;
1250 rrpc->lun_offset = lun_begin;
1251
1252 ret = rrpc_core_init(rrpc);
1253 if (ret) {
1254 pr_err("nvm: rrpc: could not initialize core\n");
1255 goto err;
1256 }
1257
1258 ret = rrpc_map_init(rrpc);
1259 if (ret) {
1260 pr_err("nvm: rrpc: could not initialize maps\n");
1261 goto err;
1262 }
1263
1264 ret = rrpc_blocks_init(rrpc);
1265 if (ret) {
1266 pr_err("nvm: rrpc: could not initialize state for blocks\n");
1267 goto err;
1268 }
1269
1270 ret = rrpc_luns_configure(rrpc);
1271 if (ret) {
1272 pr_err("nvm: rrpc: not enough blocks available in LUNs.\n");
1273 goto err;
1274 }
1275
1276 ret = rrpc_gc_init(rrpc);
1277 if (ret) {
1278 pr_err("nvm: rrpc: could not initialize gc\n");
1279 goto err;
1280 }
1281
1282 /* inherit the size from the underlying device */
1283 blk_queue_logical_block_size(tqueue, queue_physical_block_size(bqueue));
1284 blk_queue_max_hw_sectors(tqueue, queue_max_hw_sectors(bqueue));
1285
1286 pr_info("nvm: rrpc initialized with %u luns and %llu pages.\n",
1287 rrpc->nr_luns, (unsigned long long)rrpc->nr_pages);
1288
1289 mod_timer(&rrpc->gc_timer, jiffies + msecs_to_jiffies(10));
1290
1291 return rrpc;
1292err:
1293 rrpc_free(rrpc);
1294 return ERR_PTR(ret);
1295}
1296
1297/* round robin, page-based FTL, and cost-based GC */
1298static struct nvm_tgt_type tt_rrpc = {
1299 .name = "rrpc",
1300 .version = {1, 0, 0},
1301
1302 .make_rq = rrpc_make_rq,
1303 .capacity = rrpc_capacity,
1304 .end_io = rrpc_end_io,
1305
1306 .init = rrpc_init,
1307 .exit = rrpc_exit,
1308};
1309
1310static int __init rrpc_module_init(void)
1311{
1312 return nvm_register_target(&tt_rrpc);
1313}
1314
1315static void rrpc_module_exit(void)
1316{
1317 nvm_unregister_target(&tt_rrpc);
1318}
1319
1320module_init(rrpc_module_init);
1321module_exit(rrpc_module_exit);
1322MODULE_LICENSE("GPL v2");
1323MODULE_DESCRIPTION("Block-Device Target for Open-Channel SSDs");
diff --git a/drivers/lightnvm/rrpc.h b/drivers/lightnvm/rrpc.h
new file mode 100644
index 000000000000..a9696a06c38c
--- /dev/null
+++ b/drivers/lightnvm/rrpc.h
@@ -0,0 +1,239 @@
1/*
2 * Copyright (C) 2015 IT University of Copenhagen
3 * Initial release: Matias Bjorling <m@bjorling.me>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License version
7 * 2 as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * General Public License for more details.
13 *
14 * Implementation of a Round-robin page-based Hybrid FTL for Open-channel SSDs.
15 */
16
17#ifndef RRPC_H_
18#define RRPC_H_
19
20#include <linux/blkdev.h>
21#include <linux/blk-mq.h>
22#include <linux/bio.h>
23#include <linux/module.h>
24#include <linux/kthread.h>
25#include <linux/vmalloc.h>
26
27#include <linux/lightnvm.h>
28
29/* Run only GC if less than 1/X blocks are free */
30#define GC_LIMIT_INVERSE 10
31#define GC_TIME_SECS 100
32
33#define RRPC_SECTOR (512)
34#define RRPC_EXPOSED_PAGE_SIZE (4096)
35
36#define NR_PHY_IN_LOG (RRPC_EXPOSED_PAGE_SIZE / RRPC_SECTOR)
37
38struct rrpc_inflight {
39 struct list_head reqs;
40 spinlock_t lock;
41};
42
43struct rrpc_inflight_rq {
44 struct list_head list;
45 sector_t l_start;
46 sector_t l_end;
47};
48
49struct rrpc_rq {
50 struct rrpc_inflight_rq inflight_rq;
51 struct rrpc_addr *addr;
52 unsigned long flags;
53};
54
55struct rrpc_block {
56 struct nvm_block *parent;
57 struct list_head prio;
58
59#define MAX_INVALID_PAGES_STORAGE 8
60 /* Bitmap for invalid page intries */
61 unsigned long invalid_pages[MAX_INVALID_PAGES_STORAGE];
62 /* points to the next writable page within a block */
63 unsigned int next_page;
64 /* number of pages that are invalid, wrt host page size */
65 unsigned int nr_invalid_pages;
66
67 spinlock_t lock;
68 atomic_t data_cmnt_size; /* data pages committed to stable storage */
69};
70
71struct rrpc_lun {
72 struct rrpc *rrpc;
73 struct nvm_lun *parent;
74 struct rrpc_block *cur, *gc_cur;
75 struct rrpc_block *blocks; /* Reference to block allocation */
76 struct list_head prio_list; /* Blocks that may be GC'ed */
77 struct work_struct ws_gc;
78
79 spinlock_t lock;
80};
81
82struct rrpc {
83 /* instance must be kept in top to resolve rrpc in unprep */
84 struct nvm_tgt_instance instance;
85
86 struct nvm_dev *dev;
87 struct gendisk *disk;
88
89 u64 poffset; /* physical page offset */
90 int lun_offset;
91
92 int nr_luns;
93 struct rrpc_lun *luns;
94
95 /* calculated values */
96 unsigned long long nr_pages;
97 unsigned long total_blocks;
98
99 /* Write strategy variables. Move these into each for structure for each
100 * strategy
101 */
102 atomic_t next_lun; /* Whenever a page is written, this is updated
103 * to point to the next write lun
104 */
105
106 spinlock_t bio_lock;
107 struct bio_list requeue_bios;
108 struct work_struct ws_requeue;
109
110 /* Simple translation map of logical addresses to physical addresses.
111 * The logical addresses is known by the host system, while the physical
112 * addresses are used when writing to the disk block device.
113 */
114 struct rrpc_addr *trans_map;
115 /* also store a reverse map for garbage collection */
116 struct rrpc_rev_addr *rev_trans_map;
117 spinlock_t rev_lock;
118
119 struct rrpc_inflight inflights;
120
121 mempool_t *addr_pool;
122 mempool_t *page_pool;
123 mempool_t *gcb_pool;
124 mempool_t *rq_pool;
125
126 struct timer_list gc_timer;
127 struct workqueue_struct *krqd_wq;
128 struct workqueue_struct *kgc_wq;
129};
130
131struct rrpc_block_gc {
132 struct rrpc *rrpc;
133 struct rrpc_block *rblk;
134 struct work_struct ws_gc;
135};
136
137/* Logical to physical mapping */
138struct rrpc_addr {
139 u64 addr;
140 struct rrpc_block *rblk;
141};
142
143/* Physical to logical mapping */
144struct rrpc_rev_addr {
145 u64 addr;
146};
147
148static inline sector_t rrpc_get_laddr(struct bio *bio)
149{
150 return bio->bi_iter.bi_sector / NR_PHY_IN_LOG;
151}
152
153static inline unsigned int rrpc_get_pages(struct bio *bio)
154{
155 return bio->bi_iter.bi_size / RRPC_EXPOSED_PAGE_SIZE;
156}
157
158static inline sector_t rrpc_get_sector(sector_t laddr)
159{
160 return laddr * NR_PHY_IN_LOG;
161}
162
163static inline int request_intersects(struct rrpc_inflight_rq *r,
164 sector_t laddr_start, sector_t laddr_end)
165{
166 return (laddr_end >= r->l_start && laddr_end <= r->l_end) &&
167 (laddr_start >= r->l_start && laddr_start <= r->l_end);
168}
169
170static int __rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr,
171 unsigned pages, struct rrpc_inflight_rq *r)
172{
173 sector_t laddr_end = laddr + pages - 1;
174 struct rrpc_inflight_rq *rtmp;
175
176 spin_lock_irq(&rrpc->inflights.lock);
177 list_for_each_entry(rtmp, &rrpc->inflights.reqs, list) {
178 if (unlikely(request_intersects(rtmp, laddr, laddr_end))) {
179 /* existing, overlapping request, come back later */
180 spin_unlock_irq(&rrpc->inflights.lock);
181 return 1;
182 }
183 }
184
185 r->l_start = laddr;
186 r->l_end = laddr_end;
187
188 list_add_tail(&r->list, &rrpc->inflights.reqs);
189 spin_unlock_irq(&rrpc->inflights.lock);
190 return 0;
191}
192
193static inline int rrpc_lock_laddr(struct rrpc *rrpc, sector_t laddr,
194 unsigned pages,
195 struct rrpc_inflight_rq *r)
196{
197 BUG_ON((laddr + pages) > rrpc->nr_pages);
198
199 return __rrpc_lock_laddr(rrpc, laddr, pages, r);
200}
201
202static inline struct rrpc_inflight_rq *rrpc_get_inflight_rq(struct nvm_rq *rqd)
203{
204 struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
205
206 return &rrqd->inflight_rq;
207}
208
209static inline int rrpc_lock_rq(struct rrpc *rrpc, struct bio *bio,
210 struct nvm_rq *rqd)
211{
212 sector_t laddr = rrpc_get_laddr(bio);
213 unsigned int pages = rrpc_get_pages(bio);
214 struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
215
216 return rrpc_lock_laddr(rrpc, laddr, pages, r);
217}
218
219static inline void rrpc_unlock_laddr(struct rrpc *rrpc,
220 struct rrpc_inflight_rq *r)
221{
222 unsigned long flags;
223
224 spin_lock_irqsave(&rrpc->inflights.lock, flags);
225 list_del_init(&r->list);
226 spin_unlock_irqrestore(&rrpc->inflights.lock, flags);
227}
228
229static inline void rrpc_unlock_rq(struct rrpc *rrpc, struct nvm_rq *rqd)
230{
231 struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rqd);
232 uint8_t pages = rqd->nr_pages;
233
234 BUG_ON((r->l_start + pages) > rrpc->nr_pages);
235
236 rrpc_unlock_laddr(rrpc, r);
237}
238
239#endif /* RRPC_H_ */
diff --git a/drivers/nvme/host/Makefile b/drivers/nvme/host/Makefile
index cfb6679ec245..219dc206fa5f 100644
--- a/drivers/nvme/host/Makefile
+++ b/drivers/nvme/host/Makefile
@@ -1,4 +1,4 @@
1 1
2obj-$(CONFIG_BLK_DEV_NVME) += nvme.o 2obj-$(CONFIG_BLK_DEV_NVME) += nvme.o
3 3
4nvme-y += pci.o scsi.o 4nvme-y += pci.o scsi.o lightnvm.o
diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c
new file mode 100644
index 000000000000..e0b7b95813bc
--- /dev/null
+++ b/drivers/nvme/host/lightnvm.c
@@ -0,0 +1,526 @@
1/*
2 * nvme-lightnvm.c - LightNVM NVMe device
3 *
4 * Copyright (C) 2014-2015 IT University of Copenhagen
5 * Initial release: Matias Bjorling <mb@lightnvm.io>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License version
9 * 2 as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; see the file COPYING. If not, write to
18 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
19 * USA.
20 *
21 */
22
23#include "nvme.h"
24
25#ifdef CONFIG_NVM
26
27#include <linux/nvme.h>
28#include <linux/bitops.h>
29#include <linux/lightnvm.h>
30#include <linux/vmalloc.h>
31
32enum nvme_nvm_admin_opcode {
33 nvme_nvm_admin_identity = 0xe2,
34 nvme_nvm_admin_get_l2p_tbl = 0xea,
35 nvme_nvm_admin_get_bb_tbl = 0xf2,
36 nvme_nvm_admin_set_bb_tbl = 0xf1,
37};
38
39struct nvme_nvm_hb_rw {
40 __u8 opcode;
41 __u8 flags;
42 __u16 command_id;
43 __le32 nsid;
44 __u64 rsvd2;
45 __le64 metadata;
46 __le64 prp1;
47 __le64 prp2;
48 __le64 spba;
49 __le16 length;
50 __le16 control;
51 __le32 dsmgmt;
52 __le64 slba;
53};
54
55struct nvme_nvm_ph_rw {
56 __u8 opcode;
57 __u8 flags;
58 __u16 command_id;
59 __le32 nsid;
60 __u64 rsvd2;
61 __le64 metadata;
62 __le64 prp1;
63 __le64 prp2;
64 __le64 spba;
65 __le16 length;
66 __le16 control;
67 __le32 dsmgmt;
68 __le64 resv;
69};
70
71struct nvme_nvm_identity {
72 __u8 opcode;
73 __u8 flags;
74 __u16 command_id;
75 __le32 nsid;
76 __u64 rsvd[2];
77 __le64 prp1;
78 __le64 prp2;
79 __le32 chnl_off;
80 __u32 rsvd11[5];
81};
82
83struct nvme_nvm_l2ptbl {
84 __u8 opcode;
85 __u8 flags;
86 __u16 command_id;
87 __le32 nsid;
88 __le32 cdw2[4];
89 __le64 prp1;
90 __le64 prp2;
91 __le64 slba;
92 __le32 nlb;
93 __le16 cdw14[6];
94};
95
96struct nvme_nvm_bbtbl {
97 __u8 opcode;
98 __u8 flags;
99 __u16 command_id;
100 __le32 nsid;
101 __u64 rsvd[2];
102 __le64 prp1;
103 __le64 prp2;
104 __le32 prp1_len;
105 __le32 prp2_len;
106 __le32 lbb;
107 __u32 rsvd11[3];
108};
109
110struct nvme_nvm_erase_blk {
111 __u8 opcode;
112 __u8 flags;
113 __u16 command_id;
114 __le32 nsid;
115 __u64 rsvd[2];
116 __le64 prp1;
117 __le64 prp2;
118 __le64 spba;
119 __le16 length;
120 __le16 control;
121 __le32 dsmgmt;
122 __le64 resv;
123};
124
125struct nvme_nvm_command {
126 union {
127 struct nvme_common_command common;
128 struct nvme_nvm_identity identity;
129 struct nvme_nvm_hb_rw hb_rw;
130 struct nvme_nvm_ph_rw ph_rw;
131 struct nvme_nvm_l2ptbl l2p;
132 struct nvme_nvm_bbtbl get_bb;
133 struct nvme_nvm_bbtbl set_bb;
134 struct nvme_nvm_erase_blk erase;
135 };
136};
137
138struct nvme_nvm_id_group {
139 __u8 mtype;
140 __u8 fmtype;
141 __le16 res16;
142 __u8 num_ch;
143 __u8 num_lun;
144 __u8 num_pln;
145 __le16 num_blk;
146 __le16 num_pg;
147 __le16 fpg_sz;
148 __le16 csecs;
149 __le16 sos;
150 __le32 trdt;
151 __le32 trdm;
152 __le32 tprt;
153 __le32 tprm;
154 __le32 tbet;
155 __le32 tbem;
156 __le32 mpos;
157 __le16 cpar;
158 __u8 reserved[913];
159} __packed;
160
161struct nvme_nvm_addr_format {
162 __u8 ch_offset;
163 __u8 ch_len;
164 __u8 lun_offset;
165 __u8 lun_len;
166 __u8 pln_offset;
167 __u8 pln_len;
168 __u8 blk_offset;
169 __u8 blk_len;
170 __u8 pg_offset;
171 __u8 pg_len;
172 __u8 sect_offset;
173 __u8 sect_len;
174 __u8 res[4];
175} __packed;
176
177struct nvme_nvm_id {
178 __u8 ver_id;
179 __u8 vmnt;
180 __u8 cgrps;
181 __u8 res[5];
182 __le32 cap;
183 __le32 dom;
184 struct nvme_nvm_addr_format ppaf;
185 __u8 ppat;
186 __u8 resv[223];
187 struct nvme_nvm_id_group groups[4];
188} __packed;
189
190/*
191 * Check we didn't inadvertently grow the command struct
192 */
193static inline void _nvme_nvm_check_size(void)
194{
195 BUILD_BUG_ON(sizeof(struct nvme_nvm_identity) != 64);
196 BUILD_BUG_ON(sizeof(struct nvme_nvm_hb_rw) != 64);
197 BUILD_BUG_ON(sizeof(struct nvme_nvm_ph_rw) != 64);
198 BUILD_BUG_ON(sizeof(struct nvme_nvm_bbtbl) != 64);
199 BUILD_BUG_ON(sizeof(struct nvme_nvm_l2ptbl) != 64);
200 BUILD_BUG_ON(sizeof(struct nvme_nvm_erase_blk) != 64);
201 BUILD_BUG_ON(sizeof(struct nvme_nvm_id_group) != 960);
202 BUILD_BUG_ON(sizeof(struct nvme_nvm_addr_format) != 128);
203 BUILD_BUG_ON(sizeof(struct nvme_nvm_id) != 4096);
204}
205
206static int init_grps(struct nvm_id *nvm_id, struct nvme_nvm_id *nvme_nvm_id)
207{
208 struct nvme_nvm_id_group *src;
209 struct nvm_id_group *dst;
210 int i, end;
211
212 end = min_t(u32, 4, nvm_id->cgrps);
213
214 for (i = 0; i < end; i++) {
215 src = &nvme_nvm_id->groups[i];
216 dst = &nvm_id->groups[i];
217
218 dst->mtype = src->mtype;
219 dst->fmtype = src->fmtype;
220 dst->num_ch = src->num_ch;
221 dst->num_lun = src->num_lun;
222 dst->num_pln = src->num_pln;
223
224 dst->num_pg = le16_to_cpu(src->num_pg);
225 dst->num_blk = le16_to_cpu(src->num_blk);
226 dst->fpg_sz = le16_to_cpu(src->fpg_sz);
227 dst->csecs = le16_to_cpu(src->csecs);
228 dst->sos = le16_to_cpu(src->sos);
229
230 dst->trdt = le32_to_cpu(src->trdt);
231 dst->trdm = le32_to_cpu(src->trdm);
232 dst->tprt = le32_to_cpu(src->tprt);
233 dst->tprm = le32_to_cpu(src->tprm);
234 dst->tbet = le32_to_cpu(src->tbet);
235 dst->tbem = le32_to_cpu(src->tbem);
236 dst->mpos = le32_to_cpu(src->mpos);
237
238 dst->cpar = le16_to_cpu(src->cpar);
239 }
240
241 return 0;
242}
243
244static int nvme_nvm_identity(struct request_queue *q, struct nvm_id *nvm_id)
245{
246 struct nvme_ns *ns = q->queuedata;
247 struct nvme_nvm_id *nvme_nvm_id;
248 struct nvme_nvm_command c = {};
249 int ret;
250
251 c.identity.opcode = nvme_nvm_admin_identity;
252 c.identity.nsid = cpu_to_le32(ns->ns_id);
253 c.identity.chnl_off = 0;
254
255 nvme_nvm_id = kmalloc(sizeof(struct nvme_nvm_id), GFP_KERNEL);
256 if (!nvme_nvm_id)
257 return -ENOMEM;
258
259 ret = nvme_submit_sync_cmd(q, (struct nvme_command *)&c, nvme_nvm_id,
260 sizeof(struct nvme_nvm_id));
261 if (ret) {
262 ret = -EIO;
263 goto out;
264 }
265
266 nvm_id->ver_id = nvme_nvm_id->ver_id;
267 nvm_id->vmnt = nvme_nvm_id->vmnt;
268 nvm_id->cgrps = nvme_nvm_id->cgrps;
269 nvm_id->cap = le32_to_cpu(nvme_nvm_id->cap);
270 nvm_id->dom = le32_to_cpu(nvme_nvm_id->dom);
271
272 ret = init_grps(nvm_id, nvme_nvm_id);
273out:
274 kfree(nvme_nvm_id);
275 return ret;
276}
277
278static int nvme_nvm_get_l2p_tbl(struct request_queue *q, u64 slba, u32 nlb,
279 nvm_l2p_update_fn *update_l2p, void *priv)
280{
281 struct nvme_ns *ns = q->queuedata;
282 struct nvme_dev *dev = ns->dev;
283 struct nvme_nvm_command c = {};
284 u32 len = queue_max_hw_sectors(q) << 9;
285 u32 nlb_pr_rq = len / sizeof(u64);
286 u64 cmd_slba = slba;
287 void *entries;
288 int ret = 0;
289
290 c.l2p.opcode = nvme_nvm_admin_get_l2p_tbl;
291 c.l2p.nsid = cpu_to_le32(ns->ns_id);
292 entries = kmalloc(len, GFP_KERNEL);
293 if (!entries)
294 return -ENOMEM;
295
296 while (nlb) {
297 u32 cmd_nlb = min(nlb_pr_rq, nlb);
298
299 c.l2p.slba = cpu_to_le64(cmd_slba);
300 c.l2p.nlb = cpu_to_le32(cmd_nlb);
301
302 ret = nvme_submit_sync_cmd(q, (struct nvme_command *)&c,
303 entries, len);
304 if (ret) {
305 dev_err(dev->dev, "L2P table transfer failed (%d)\n",
306 ret);
307 ret = -EIO;
308 goto out;
309 }
310
311 if (update_l2p(cmd_slba, cmd_nlb, entries, priv)) {
312 ret = -EINTR;
313 goto out;
314 }
315
316 cmd_slba += cmd_nlb;
317 nlb -= cmd_nlb;
318 }
319
320out:
321 kfree(entries);
322 return ret;
323}
324
325static int nvme_nvm_get_bb_tbl(struct request_queue *q, int lunid,
326 unsigned int nr_blocks,
327 nvm_bb_update_fn *update_bbtbl, void *priv)
328{
329 struct nvme_ns *ns = q->queuedata;
330 struct nvme_dev *dev = ns->dev;
331 struct nvme_nvm_command c = {};
332 void *bb_bitmap;
333 u16 bb_bitmap_size;
334 int ret = 0;
335
336 c.get_bb.opcode = nvme_nvm_admin_get_bb_tbl;
337 c.get_bb.nsid = cpu_to_le32(ns->ns_id);
338 c.get_bb.lbb = cpu_to_le32(lunid);
339 bb_bitmap_size = ((nr_blocks >> 15) + 1) * PAGE_SIZE;
340 bb_bitmap = kmalloc(bb_bitmap_size, GFP_KERNEL);
341 if (!bb_bitmap)
342 return -ENOMEM;
343
344 bitmap_zero(bb_bitmap, nr_blocks);
345
346 ret = nvme_submit_sync_cmd(q, (struct nvme_command *)&c, bb_bitmap,
347 bb_bitmap_size);
348 if (ret) {
349 dev_err(dev->dev, "get bad block table failed (%d)\n", ret);
350 ret = -EIO;
351 goto out;
352 }
353
354 ret = update_bbtbl(lunid, bb_bitmap, nr_blocks, priv);
355 if (ret) {
356 ret = -EINTR;
357 goto out;
358 }
359
360out:
361 kfree(bb_bitmap);
362 return ret;
363}
364
365static inline void nvme_nvm_rqtocmd(struct request *rq, struct nvm_rq *rqd,
366 struct nvme_ns *ns, struct nvme_nvm_command *c)
367{
368 c->ph_rw.opcode = rqd->opcode;
369 c->ph_rw.nsid = cpu_to_le32(ns->ns_id);
370 c->ph_rw.spba = cpu_to_le64(rqd->ppa_addr.ppa);
371 c->ph_rw.control = cpu_to_le16(rqd->flags);
372 c->ph_rw.length = cpu_to_le16(rqd->nr_pages - 1);
373
374 if (rqd->opcode == NVM_OP_HBWRITE || rqd->opcode == NVM_OP_HBREAD)
375 c->hb_rw.slba = cpu_to_le64(nvme_block_nr(ns,
376 rqd->bio->bi_iter.bi_sector));
377}
378
379static void nvme_nvm_end_io(struct request *rq, int error)
380{
381 struct nvm_rq *rqd = rq->end_io_data;
382 struct nvm_dev *dev = rqd->dev;
383
384 if (dev->mt->end_io(rqd, error))
385 pr_err("nvme: err status: %x result: %lx\n",
386 rq->errors, (unsigned long)rq->special);
387
388 kfree(rq->cmd);
389 blk_mq_free_request(rq);
390}
391
392static int nvme_nvm_submit_io(struct request_queue *q, struct nvm_rq *rqd)
393{
394 struct nvme_ns *ns = q->queuedata;
395 struct request *rq;
396 struct bio *bio = rqd->bio;
397 struct nvme_nvm_command *cmd;
398
399 rq = blk_mq_alloc_request(q, bio_rw(bio), GFP_KERNEL, 0);
400 if (IS_ERR(rq))
401 return -ENOMEM;
402
403 cmd = kzalloc(sizeof(struct nvme_nvm_command), GFP_KERNEL);
404 if (!cmd) {
405 blk_mq_free_request(rq);
406 return -ENOMEM;
407 }
408
409 rq->cmd_type = REQ_TYPE_DRV_PRIV;
410 rq->ioprio = bio_prio(bio);
411
412 if (bio_has_data(bio))
413 rq->nr_phys_segments = bio_phys_segments(q, bio);
414
415 rq->__data_len = bio->bi_iter.bi_size;
416 rq->bio = rq->biotail = bio;
417
418 nvme_nvm_rqtocmd(rq, rqd, ns, cmd);
419
420 rq->cmd = (unsigned char *)cmd;
421 rq->cmd_len = sizeof(struct nvme_nvm_command);
422 rq->special = (void *)0;
423
424 rq->end_io_data = rqd;
425
426 blk_execute_rq_nowait(q, NULL, rq, 0, nvme_nvm_end_io);
427
428 return 0;
429}
430
431static int nvme_nvm_erase_block(struct request_queue *q, struct nvm_rq *rqd)
432{
433 struct nvme_ns *ns = q->queuedata;
434 struct nvme_nvm_command c = {};
435
436 c.erase.opcode = NVM_OP_ERASE;
437 c.erase.nsid = cpu_to_le32(ns->ns_id);
438 c.erase.spba = cpu_to_le64(rqd->ppa_addr.ppa);
439 c.erase.length = cpu_to_le16(rqd->nr_pages - 1);
440
441 return nvme_submit_sync_cmd(q, (struct nvme_command *)&c, NULL, 0);
442}
443
444static void *nvme_nvm_create_dma_pool(struct request_queue *q, char *name)
445{
446 struct nvme_ns *ns = q->queuedata;
447 struct nvme_dev *dev = ns->dev;
448
449 return dma_pool_create(name, dev->dev, PAGE_SIZE, PAGE_SIZE, 0);
450}
451
452static void nvme_nvm_destroy_dma_pool(void *pool)
453{
454 struct dma_pool *dma_pool = pool;
455
456 dma_pool_destroy(dma_pool);
457}
458
459static void *nvme_nvm_dev_dma_alloc(struct request_queue *q, void *pool,
460 gfp_t mem_flags, dma_addr_t *dma_handler)
461{
462 return dma_pool_alloc(pool, mem_flags, dma_handler);
463}
464
465static void nvme_nvm_dev_dma_free(void *pool, void *ppa_list,
466 dma_addr_t dma_handler)
467{
468 dma_pool_free(pool, ppa_list, dma_handler);
469}
470
471static struct nvm_dev_ops nvme_nvm_dev_ops = {
472 .identity = nvme_nvm_identity,
473
474 .get_l2p_tbl = nvme_nvm_get_l2p_tbl,
475
476 .get_bb_tbl = nvme_nvm_get_bb_tbl,
477
478 .submit_io = nvme_nvm_submit_io,
479 .erase_block = nvme_nvm_erase_block,
480
481 .create_dma_pool = nvme_nvm_create_dma_pool,
482 .destroy_dma_pool = nvme_nvm_destroy_dma_pool,
483 .dev_dma_alloc = nvme_nvm_dev_dma_alloc,
484 .dev_dma_free = nvme_nvm_dev_dma_free,
485
486 .max_phys_sect = 64,
487};
488
489int nvme_nvm_register(struct request_queue *q, char *disk_name)
490{
491 return nvm_register(q, disk_name, &nvme_nvm_dev_ops);
492}
493
494void nvme_nvm_unregister(struct request_queue *q, char *disk_name)
495{
496 nvm_unregister(disk_name);
497}
498
499int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
500{
501 struct nvme_dev *dev = ns->dev;
502 struct pci_dev *pdev = to_pci_dev(dev->dev);
503
504 /* QEMU NVMe simulator - PCI ID + Vendor specific bit */
505 if (pdev->vendor == PCI_VENDOR_ID_INTEL && pdev->device == 0x5845 &&
506 id->vs[0] == 0x1)
507 return 1;
508
509 /* CNEX Labs - PCI ID + Vendor specific bit */
510 if (pdev->vendor == 0x1d1d && pdev->device == 0x2807 &&
511 id->vs[0] == 0x1)
512 return 1;
513
514 return 0;
515}
516#else
517int nvme_nvm_register(struct request_queue *q, char *disk_name)
518{
519 return 0;
520}
521void nvme_nvm_unregister(struct request_queue *q, char *disk_name) {};
522int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id)
523{
524 return 0;
525}
526#endif /* CONFIG_NVM */
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index c1f41bf3c0f2..fdb4e5bad9ac 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -22,6 +22,11 @@
22extern unsigned char nvme_io_timeout; 22extern unsigned char nvme_io_timeout;
23#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ) 23#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ)
24 24
25enum {
26 NVME_NS_LBA = 0,
27 NVME_NS_LIGHTNVM = 1,
28};
29
25/* 30/*
26 * Represents an NVM Express device. Each nvme_dev is a PCI function. 31 * Represents an NVM Express device. Each nvme_dev is a PCI function.
27 */ 32 */
@@ -84,6 +89,7 @@ struct nvme_ns {
84 u16 ms; 89 u16 ms;
85 bool ext; 90 bool ext;
86 u8 pi_type; 91 u8 pi_type;
92 int type;
87 u64 mode_select_num_blocks; 93 u64 mode_select_num_blocks;
88 u32 mode_select_block_len; 94 u32 mode_select_block_len;
89}; 95};
@@ -130,4 +136,8 @@ int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr);
130int nvme_sg_io32(struct nvme_ns *ns, unsigned long arg); 136int nvme_sg_io32(struct nvme_ns *ns, unsigned long arg);
131int nvme_sg_get_version_num(int __user *ip); 137int nvme_sg_get_version_num(int __user *ip);
132 138
139int nvme_nvm_ns_supported(struct nvme_ns *ns, struct nvme_id_ns *id);
140int nvme_nvm_register(struct request_queue *q, char *disk_name);
141void nvme_nvm_unregister(struct request_queue *q, char *disk_name);
142
133#endif /* _NVME_H */ 143#endif /* _NVME_H */
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 0a179ed9ddef..6c0d0fecaeb3 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1952,6 +1952,9 @@ static void nvme_free_ns(struct kref *kref)
1952{ 1952{
1953 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref); 1953 struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
1954 1954
1955 if (ns->type == NVME_NS_LIGHTNVM)
1956 nvme_nvm_unregister(ns->queue, ns->disk->disk_name);
1957
1955 spin_lock(&dev_list_lock); 1958 spin_lock(&dev_list_lock);
1956 ns->disk->private_data = NULL; 1959 ns->disk->private_data = NULL;
1957 spin_unlock(&dev_list_lock); 1960 spin_unlock(&dev_list_lock);
@@ -2021,6 +2024,16 @@ static int nvme_revalidate_disk(struct gendisk *disk)
2021 return -ENODEV; 2024 return -ENODEV;
2022 } 2025 }
2023 2026
2027 if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
2028 if (nvme_nvm_register(ns->queue, disk->disk_name)) {
2029 dev_warn(dev->dev,
2030 "%s: LightNVM init failure\n", __func__);
2031 kfree(id);
2032 return -ENODEV;
2033 }
2034 ns->type = NVME_NS_LIGHTNVM;
2035 }
2036
2024 old_ms = ns->ms; 2037 old_ms = ns->ms;
2025 lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK; 2038 lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
2026 ns->lba_shift = id->lbaf[lbaf].ds; 2039 ns->lba_shift = id->lbaf[lbaf].ds;
@@ -2052,7 +2065,9 @@ static int nvme_revalidate_disk(struct gendisk *disk)
2052 !ns->ext) 2065 !ns->ext)
2053 nvme_init_integrity(ns); 2066 nvme_init_integrity(ns);
2054 2067
2055 if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk)) 2068 if ((ns->ms && !(ns->ms == 8 && ns->pi_type) &&
2069 !blk_get_integrity(disk)) ||
2070 ns->type == NVME_NS_LIGHTNVM)
2056 set_capacity(disk, 0); 2071 set_capacity(disk, 0);
2057 else 2072 else
2058 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9)); 2073 set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));
@@ -2175,17 +2190,19 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
2175 goto out_free_disk; 2190 goto out_free_disk;
2176 2191
2177 kref_get(&dev->kref); 2192 kref_get(&dev->kref);
2178 add_disk(ns->disk); 2193 if (ns->type != NVME_NS_LIGHTNVM) {
2179 if (ns->ms) { 2194 add_disk(ns->disk);
2180 struct block_device *bd = bdget_disk(ns->disk, 0); 2195 if (ns->ms) {
2181 if (!bd) 2196 struct block_device *bd = bdget_disk(ns->disk, 0);
2182 return; 2197 if (!bd)
2183 if (blkdev_get(bd, FMODE_READ, NULL)) { 2198 return;
2184 bdput(bd); 2199 if (blkdev_get(bd, FMODE_READ, NULL)) {
2185 return; 2200 bdput(bd);
2201 return;
2202 }
2203 blkdev_reread_part(bd);
2204 blkdev_put(bd, FMODE_READ);
2186 } 2205 }
2187 blkdev_reread_part(bd);
2188 blkdev_put(bd, FMODE_READ);
2189 } 2206 }
2190 return; 2207 return;
2191 out_free_disk: 2208 out_free_disk:
diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h
new file mode 100644
index 000000000000..5ebd70d12f35
--- /dev/null
+++ b/include/linux/lightnvm.h
@@ -0,0 +1,522 @@
1#ifndef NVM_H
2#define NVM_H
3
4enum {
5 NVM_IO_OK = 0,
6 NVM_IO_REQUEUE = 1,
7 NVM_IO_DONE = 2,
8 NVM_IO_ERR = 3,
9
10 NVM_IOTYPE_NONE = 0,
11 NVM_IOTYPE_GC = 1,
12};
13
14#ifdef CONFIG_NVM
15
16#include <linux/blkdev.h>
17#include <linux/types.h>
18#include <linux/file.h>
19#include <linux/dmapool.h>
20
21enum {
22 /* HW Responsibilities */
23 NVM_RSP_L2P = 1 << 0,
24 NVM_RSP_ECC = 1 << 1,
25
26 /* Physical Adressing Mode */
27 NVM_ADDRMODE_LINEAR = 0,
28 NVM_ADDRMODE_CHANNEL = 1,
29
30 /* Plane programming mode for LUN */
31 NVM_PLANE_SINGLE = 0,
32 NVM_PLANE_DOUBLE = 1,
33 NVM_PLANE_QUAD = 2,
34
35 /* Status codes */
36 NVM_RSP_SUCCESS = 0x0,
37 NVM_RSP_NOT_CHANGEABLE = 0x1,
38 NVM_RSP_ERR_FAILWRITE = 0x40ff,
39 NVM_RSP_ERR_EMPTYPAGE = 0x42ff,
40
41 /* Device opcodes */
42 NVM_OP_HBREAD = 0x02,
43 NVM_OP_HBWRITE = 0x81,
44 NVM_OP_PWRITE = 0x91,
45 NVM_OP_PREAD = 0x92,
46 NVM_OP_ERASE = 0x90,
47
48 /* PPA Command Flags */
49 NVM_IO_SNGL_ACCESS = 0x0,
50 NVM_IO_DUAL_ACCESS = 0x1,
51 NVM_IO_QUAD_ACCESS = 0x2,
52
53 NVM_IO_SUSPEND = 0x80,
54 NVM_IO_SLC_MODE = 0x100,
55 NVM_IO_SCRAMBLE_DISABLE = 0x200,
56};
57
58struct nvm_id_group {
59 u8 mtype;
60 u8 fmtype;
61 u16 res16;
62 u8 num_ch;
63 u8 num_lun;
64 u8 num_pln;
65 u16 num_blk;
66 u16 num_pg;
67 u16 fpg_sz;
68 u16 csecs;
69 u16 sos;
70 u32 trdt;
71 u32 trdm;
72 u32 tprt;
73 u32 tprm;
74 u32 tbet;
75 u32 tbem;
76 u32 mpos;
77 u16 cpar;
78 u8 res[913];
79} __packed;
80
81struct nvm_addr_format {
82 u8 ch_offset;
83 u8 ch_len;
84 u8 lun_offset;
85 u8 lun_len;
86 u8 pln_offset;
87 u8 pln_len;
88 u8 blk_offset;
89 u8 blk_len;
90 u8 pg_offset;
91 u8 pg_len;
92 u8 sect_offset;
93 u8 sect_len;
94 u8 res[4];
95};
96
97struct nvm_id {
98 u8 ver_id;
99 u8 vmnt;
100 u8 cgrps;
101 u8 res[5];
102 u32 cap;
103 u32 dom;
104 struct nvm_addr_format ppaf;
105 u8 ppat;
106 u8 resv[224];
107 struct nvm_id_group groups[4];
108} __packed;
109
110struct nvm_target {
111 struct list_head list;
112 struct nvm_tgt_type *type;
113 struct gendisk *disk;
114};
115
116struct nvm_tgt_instance {
117 struct nvm_tgt_type *tt;
118};
119
120#define ADDR_EMPTY (~0ULL)
121
122#define NVM_VERSION_MAJOR 1
123#define NVM_VERSION_MINOR 0
124#define NVM_VERSION_PATCH 0
125
126#define NVM_SEC_BITS (8)
127#define NVM_PL_BITS (6)
128#define NVM_PG_BITS (16)
129#define NVM_BLK_BITS (16)
130#define NVM_LUN_BITS (10)
131#define NVM_CH_BITS (8)
132
133struct ppa_addr {
134 union {
135 /* Channel-based PPA format in nand 4x2x2x2x8x10 */
136 struct {
137 u64 ch : 4;
138 u64 sec : 2; /* 4 sectors per page */
139 u64 pl : 2; /* 4 planes per LUN */
140 u64 lun : 2; /* 4 LUNs per channel */
141 u64 pg : 8; /* 256 pages per block */
142 u64 blk : 10;/* 1024 blocks per plane */
143 u64 resved : 36;
144 } chnl;
145
146 /* Generic structure for all addresses */
147 struct {
148 u64 sec : NVM_SEC_BITS;
149 u64 pl : NVM_PL_BITS;
150 u64 pg : NVM_PG_BITS;
151 u64 blk : NVM_BLK_BITS;
152 u64 lun : NVM_LUN_BITS;
153 u64 ch : NVM_CH_BITS;
154 } g;
155
156 u64 ppa;
157 };
158} __packed;
159
160struct nvm_rq {
161 struct nvm_tgt_instance *ins;
162 struct nvm_dev *dev;
163
164 struct bio *bio;
165
166 union {
167 struct ppa_addr ppa_addr;
168 dma_addr_t dma_ppa_list;
169 };
170
171 struct ppa_addr *ppa_list;
172
173 void *metadata;
174 dma_addr_t dma_metadata;
175
176 uint8_t opcode;
177 uint16_t nr_pages;
178 uint16_t flags;
179};
180
181static inline struct nvm_rq *nvm_rq_from_pdu(void *pdu)
182{
183 return pdu - sizeof(struct nvm_rq);
184}
185
186static inline void *nvm_rq_to_pdu(struct nvm_rq *rqdata)
187{
188 return rqdata + 1;
189}
190
191struct nvm_block;
192
193typedef int (nvm_l2p_update_fn)(u64, u32, __le64 *, void *);
194typedef int (nvm_bb_update_fn)(u32, void *, unsigned int, void *);
195typedef int (nvm_id_fn)(struct request_queue *, struct nvm_id *);
196typedef int (nvm_get_l2p_tbl_fn)(struct request_queue *, u64, u32,
197 nvm_l2p_update_fn *, void *);
198typedef int (nvm_op_bb_tbl_fn)(struct request_queue *, int, unsigned int,
199 nvm_bb_update_fn *, void *);
200typedef int (nvm_op_set_bb_fn)(struct request_queue *, struct nvm_rq *, int);
201typedef int (nvm_submit_io_fn)(struct request_queue *, struct nvm_rq *);
202typedef int (nvm_erase_blk_fn)(struct request_queue *, struct nvm_rq *);
203typedef void *(nvm_create_dma_pool_fn)(struct request_queue *, char *);
204typedef void (nvm_destroy_dma_pool_fn)(void *);
205typedef void *(nvm_dev_dma_alloc_fn)(struct request_queue *, void *, gfp_t,
206 dma_addr_t *);
207typedef void (nvm_dev_dma_free_fn)(void *, void*, dma_addr_t);
208
209struct nvm_dev_ops {
210 nvm_id_fn *identity;
211 nvm_get_l2p_tbl_fn *get_l2p_tbl;
212 nvm_op_bb_tbl_fn *get_bb_tbl;
213 nvm_op_set_bb_fn *set_bb;
214
215 nvm_submit_io_fn *submit_io;
216 nvm_erase_blk_fn *erase_block;
217
218 nvm_create_dma_pool_fn *create_dma_pool;
219 nvm_destroy_dma_pool_fn *destroy_dma_pool;
220 nvm_dev_dma_alloc_fn *dev_dma_alloc;
221 nvm_dev_dma_free_fn *dev_dma_free;
222
223 uint8_t max_phys_sect;
224};
225
226struct nvm_lun {
227 int id;
228
229 int lun_id;
230 int chnl_id;
231
232 unsigned int nr_free_blocks; /* Number of unused blocks */
233 struct nvm_block *blocks;
234
235 spinlock_t lock;
236};
237
238struct nvm_block {
239 struct list_head list;
240 struct nvm_lun *lun;
241 unsigned long id;
242
243 void *priv;
244 int type;
245};
246
247struct nvm_dev {
248 struct nvm_dev_ops *ops;
249
250 struct list_head devices;
251 struct list_head online_targets;
252
253 /* Media manager */
254 struct nvmm_type *mt;
255 void *mp;
256
257 /* Device information */
258 int nr_chnls;
259 int nr_planes;
260 int luns_per_chnl;
261 int sec_per_pg; /* only sectors for a single page */
262 int pgs_per_blk;
263 int blks_per_lun;
264 int sec_size;
265 int oob_size;
266 int addr_mode;
267 struct nvm_addr_format addr_format;
268
269 /* Calculated/Cached values. These do not reflect the actual usable
270 * blocks at run-time.
271 */
272 int max_rq_size;
273 int plane_mode; /* drive device in single, double or quad mode */
274
275 int sec_per_pl; /* all sectors across planes */
276 int sec_per_blk;
277 int sec_per_lun;
278
279 unsigned long total_pages;
280 unsigned long total_blocks;
281 int nr_luns;
282 unsigned max_pages_per_blk;
283
284 void *ppalist_pool;
285
286 struct nvm_id identity;
287
288 /* Backend device */
289 struct request_queue *q;
290 char name[DISK_NAME_LEN];
291};
292
293/* fallback conversion */
294static struct ppa_addr __generic_to_linear_addr(struct nvm_dev *dev,
295 struct ppa_addr r)
296{
297 struct ppa_addr l;
298
299 l.ppa = r.g.sec +
300 r.g.pg * dev->sec_per_pg +
301 r.g.blk * (dev->pgs_per_blk *
302 dev->sec_per_pg) +
303 r.g.lun * (dev->blks_per_lun *
304 dev->pgs_per_blk *
305 dev->sec_per_pg) +
306 r.g.ch * (dev->blks_per_lun *
307 dev->pgs_per_blk *
308 dev->luns_per_chnl *
309 dev->sec_per_pg);
310
311 return l;
312}
313
314/* fallback conversion */
315static struct ppa_addr __linear_to_generic_addr(struct nvm_dev *dev,
316 struct ppa_addr r)
317{
318 struct ppa_addr l;
319 int secs, pgs, blks, luns;
320 sector_t ppa = r.ppa;
321
322 l.ppa = 0;
323
324 div_u64_rem(ppa, dev->sec_per_pg, &secs);
325 l.g.sec = secs;
326
327 sector_div(ppa, dev->sec_per_pg);
328 div_u64_rem(ppa, dev->sec_per_blk, &pgs);
329 l.g.pg = pgs;
330
331 sector_div(ppa, dev->pgs_per_blk);
332 div_u64_rem(ppa, dev->blks_per_lun, &blks);
333 l.g.blk = blks;
334
335 sector_div(ppa, dev->blks_per_lun);
336 div_u64_rem(ppa, dev->luns_per_chnl, &luns);
337 l.g.lun = luns;
338
339 sector_div(ppa, dev->luns_per_chnl);
340 l.g.ch = ppa;
341
342 return l;
343}
344
345static struct ppa_addr __generic_to_chnl_addr(struct ppa_addr r)
346{
347 struct ppa_addr l;
348
349 l.ppa = 0;
350
351 l.chnl.sec = r.g.sec;
352 l.chnl.pl = r.g.pl;
353 l.chnl.pg = r.g.pg;
354 l.chnl.blk = r.g.blk;
355 l.chnl.lun = r.g.lun;
356 l.chnl.ch = r.g.ch;
357
358 return l;
359}
360
361static struct ppa_addr __chnl_to_generic_addr(struct ppa_addr r)
362{
363 struct ppa_addr l;
364
365 l.ppa = 0;
366
367 l.g.sec = r.chnl.sec;
368 l.g.pl = r.chnl.pl;
369 l.g.pg = r.chnl.pg;
370 l.g.blk = r.chnl.blk;
371 l.g.lun = r.chnl.lun;
372 l.g.ch = r.chnl.ch;
373
374 return l;
375}
376
377static inline struct ppa_addr addr_to_generic_mode(struct nvm_dev *dev,
378 struct ppa_addr gppa)
379{
380 switch (dev->addr_mode) {
381 case NVM_ADDRMODE_LINEAR:
382 return __linear_to_generic_addr(dev, gppa);
383 case NVM_ADDRMODE_CHANNEL:
384 return __chnl_to_generic_addr(gppa);
385 default:
386 BUG();
387 }
388 return gppa;
389}
390
391static inline struct ppa_addr generic_to_addr_mode(struct nvm_dev *dev,
392 struct ppa_addr gppa)
393{
394 switch (dev->addr_mode) {
395 case NVM_ADDRMODE_LINEAR:
396 return __generic_to_linear_addr(dev, gppa);
397 case NVM_ADDRMODE_CHANNEL:
398 return __generic_to_chnl_addr(gppa);
399 default:
400 BUG();
401 }
402 return gppa;
403}
404
405static inline int ppa_empty(struct ppa_addr ppa_addr)
406{
407 return (ppa_addr.ppa == ADDR_EMPTY);
408}
409
410static inline void ppa_set_empty(struct ppa_addr *ppa_addr)
411{
412 ppa_addr->ppa = ADDR_EMPTY;
413}
414
415static inline struct ppa_addr block_to_ppa(struct nvm_dev *dev,
416 struct nvm_block *blk)
417{
418 struct ppa_addr ppa;
419 struct nvm_lun *lun = blk->lun;
420
421 ppa.ppa = 0;
422 ppa.g.blk = blk->id % dev->blks_per_lun;
423 ppa.g.lun = lun->lun_id;
424 ppa.g.ch = lun->chnl_id;
425
426 return ppa;
427}
428
429typedef void (nvm_tgt_make_rq_fn)(struct request_queue *, struct bio *);
430typedef sector_t (nvm_tgt_capacity_fn)(void *);
431typedef int (nvm_tgt_end_io_fn)(struct nvm_rq *, int);
432typedef void *(nvm_tgt_init_fn)(struct nvm_dev *, struct gendisk *, int, int);
433typedef void (nvm_tgt_exit_fn)(void *);
434
435struct nvm_tgt_type {
436 const char *name;
437 unsigned int version[3];
438
439 /* target entry points */
440 nvm_tgt_make_rq_fn *make_rq;
441 nvm_tgt_capacity_fn *capacity;
442 nvm_tgt_end_io_fn *end_io;
443
444 /* module-specific init/teardown */
445 nvm_tgt_init_fn *init;
446 nvm_tgt_exit_fn *exit;
447
448 /* For internal use */
449 struct list_head list;
450};
451
452extern int nvm_register_target(struct nvm_tgt_type *);
453extern void nvm_unregister_target(struct nvm_tgt_type *);
454
455extern void *nvm_dev_dma_alloc(struct nvm_dev *, gfp_t, dma_addr_t *);
456extern void nvm_dev_dma_free(struct nvm_dev *, void *, dma_addr_t);
457
458typedef int (nvmm_register_fn)(struct nvm_dev *);
459typedef void (nvmm_unregister_fn)(struct nvm_dev *);
460typedef struct nvm_block *(nvmm_get_blk_fn)(struct nvm_dev *,
461 struct nvm_lun *, unsigned long);
462typedef void (nvmm_put_blk_fn)(struct nvm_dev *, struct nvm_block *);
463typedef int (nvmm_open_blk_fn)(struct nvm_dev *, struct nvm_block *);
464typedef int (nvmm_close_blk_fn)(struct nvm_dev *, struct nvm_block *);
465typedef void (nvmm_flush_blk_fn)(struct nvm_dev *, struct nvm_block *);
466typedef int (nvmm_submit_io_fn)(struct nvm_dev *, struct nvm_rq *);
467typedef int (nvmm_end_io_fn)(struct nvm_rq *, int);
468typedef int (nvmm_erase_blk_fn)(struct nvm_dev *, struct nvm_block *,
469 unsigned long);
470typedef struct nvm_lun *(nvmm_get_lun_fn)(struct nvm_dev *, int);
471typedef void (nvmm_free_blocks_print_fn)(struct nvm_dev *);
472
473struct nvmm_type {
474 const char *name;
475 unsigned int version[3];
476
477 nvmm_register_fn *register_mgr;
478 nvmm_unregister_fn *unregister_mgr;
479
480 /* Block administration callbacks */
481 nvmm_get_blk_fn *get_blk;
482 nvmm_put_blk_fn *put_blk;
483 nvmm_open_blk_fn *open_blk;
484 nvmm_close_blk_fn *close_blk;
485 nvmm_flush_blk_fn *flush_blk;
486
487 nvmm_submit_io_fn *submit_io;
488 nvmm_end_io_fn *end_io;
489 nvmm_erase_blk_fn *erase_blk;
490
491 /* Configuration management */
492 nvmm_get_lun_fn *get_lun;
493
494 /* Statistics */
495 nvmm_free_blocks_print_fn *free_blocks_print;
496 struct list_head list;
497};
498
499extern int nvm_register_mgr(struct nvmm_type *);
500extern void nvm_unregister_mgr(struct nvmm_type *);
501
502extern struct nvm_block *nvm_get_blk(struct nvm_dev *, struct nvm_lun *,
503 unsigned long);
504extern void nvm_put_blk(struct nvm_dev *, struct nvm_block *);
505
506extern int nvm_register(struct request_queue *, char *,
507 struct nvm_dev_ops *);
508extern void nvm_unregister(char *);
509
510extern int nvm_submit_io(struct nvm_dev *, struct nvm_rq *);
511extern int nvm_erase_blk(struct nvm_dev *, struct nvm_block *);
512#else /* CONFIG_NVM */
513struct nvm_dev_ops;
514
515static inline int nvm_register(struct request_queue *q, char *disk_name,
516 struct nvm_dev_ops *ops)
517{
518 return -EINVAL;
519}
520static inline void nvm_unregister(char *disk_name) {}
521#endif /* CONFIG_NVM */
522#endif /* LIGHTNVM.H */
diff --git a/include/uapi/linux/lightnvm.h b/include/uapi/linux/lightnvm.h
new file mode 100644
index 000000000000..928f98997d8a
--- /dev/null
+++ b/include/uapi/linux/lightnvm.h
@@ -0,0 +1,130 @@
1/*
2 * Copyright (C) 2015 CNEX Labs. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version
6 * 2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; see the file COPYING. If not, write to
15 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139,
16 * USA.
17 */
18
19#ifndef _UAPI_LINUX_LIGHTNVM_H
20#define _UAPI_LINUX_LIGHTNVM_H
21
22#ifdef __KERNEL__
23#include <linux/kernel.h>
24#include <linux/ioctl.h>
25#else /* __KERNEL__ */
26#include <stdio.h>
27#include <sys/ioctl.h>
28#define DISK_NAME_LEN 32
29#endif /* __KERNEL__ */
30
31#include <linux/types.h>
32#include <linux/ioctl.h>
33
34#define NVM_TTYPE_NAME_MAX 48
35#define NVM_TTYPE_MAX 63
36
37#define NVM_CTRL_FILE "/dev/lightnvm/control"
38
39struct nvm_ioctl_info_tgt {
40 __u32 version[3];
41 __u32 reserved;
42 char tgtname[NVM_TTYPE_NAME_MAX];
43};
44
45struct nvm_ioctl_info {
46 __u32 version[3]; /* in/out - major, minor, patch */
47 __u16 tgtsize; /* number of targets */
48 __u16 reserved16; /* pad to 4K page */
49 __u32 reserved[12];
50 struct nvm_ioctl_info_tgt tgts[NVM_TTYPE_MAX];
51};
52
53enum {
54 NVM_DEVICE_ACTIVE = 1 << 0,
55};
56
57struct nvm_ioctl_device_info {
58 char devname[DISK_NAME_LEN];
59 char bmname[NVM_TTYPE_NAME_MAX];
60 __u32 bmversion[3];
61 __u32 flags;
62 __u32 reserved[8];
63};
64
65struct nvm_ioctl_get_devices {
66 __u32 nr_devices;
67 __u32 reserved[31];
68 struct nvm_ioctl_device_info info[31];
69};
70
71struct nvm_ioctl_create_simple {
72 __u32 lun_begin;
73 __u32 lun_end;
74};
75
76enum {
77 NVM_CONFIG_TYPE_SIMPLE = 0,
78};
79
80struct nvm_ioctl_create_conf {
81 __u32 type;
82 union {
83 struct nvm_ioctl_create_simple s;
84 };
85};
86
87struct nvm_ioctl_create {
88 char dev[DISK_NAME_LEN]; /* open-channel SSD device */
89 char tgttype[NVM_TTYPE_NAME_MAX]; /* target type name */
90 char tgtname[DISK_NAME_LEN]; /* dev to expose target as */
91
92 __u32 flags;
93
94 struct nvm_ioctl_create_conf conf;
95};
96
97struct nvm_ioctl_remove {
98 char tgtname[DISK_NAME_LEN];
99
100 __u32 flags;
101};
102
103
104/* The ioctl type, 'L', 0x20 - 0x2F documented in ioctl-number.txt */
105enum {
106 /* top level cmds */
107 NVM_INFO_CMD = 0x20,
108 NVM_GET_DEVICES_CMD,
109
110 /* device level cmds */
111 NVM_DEV_CREATE_CMD,
112 NVM_DEV_REMOVE_CMD,
113};
114
115#define NVM_IOCTL 'L' /* 0x4c */
116
117#define NVM_INFO _IOWR(NVM_IOCTL, NVM_INFO_CMD, \
118 struct nvm_ioctl_info)
119#define NVM_GET_DEVICES _IOR(NVM_IOCTL, NVM_GET_DEVICES_CMD, \
120 struct nvm_ioctl_get_devices)
121#define NVM_DEV_CREATE _IOW(NVM_IOCTL, NVM_DEV_CREATE_CMD, \
122 struct nvm_ioctl_create)
123#define NVM_DEV_REMOVE _IOW(NVM_IOCTL, NVM_DEV_REMOVE_CMD, \
124 struct nvm_ioctl_remove)
125
126#define NVM_VERSION_MAJOR 1
127#define NVM_VERSION_MINOR 0
128#define NVM_VERSION_PATCHLEVEL 0
129
130#endif