aboutsummaryrefslogtreecommitdiffstats
path: root/fs/nfs/objlayout/objio_osd.c
diff options
context:
space:
mode:
authorBoaz Harrosh <bharrosh@panasas.com>2011-10-31 18:15:38 -0400
committerTrond Myklebust <Trond.Myklebust@netapp.com>2011-11-02 23:56:08 -0400
commiteecfc6312a24e6d0d2883de0a9a6ccf8e993f472 (patch)
tree83b7b6d8895a007e698e59f065924527e8870d90 /fs/nfs/objlayout/objio_osd.c
parentaf4f5b54bcf0379089d01518e818f37258708fb7 (diff)
pnfs-obj: move to ore 02: move to ORE
In this patch we are actually moving to the ORE. (Object Raid Engine). objio_state holds a pointer to an ore_io_state. Once we have an ore_io_state at hand we can call the ore for reading/writing. We register on the done path to kick off the nfs io_done mechanism. Again for Ease of reviewing the old code is "#if 0" but is not removed so the diff command works better. The old code will be removed in the next patch. fs/exofs/Kconfig::ORE is modified to also be auto-included if PNFS_OBJLAYOUT is set. Since we now depend on ORE. (See comments in fs/exofs/Kconfig) Signed-off-by: Boaz Harrosh <bharrosh@panasas.com> Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
Diffstat (limited to 'fs/nfs/objlayout/objio_osd.c')
-rw-r--r--fs/nfs/objlayout/objio_osd.c133
1 files changed, 59 insertions, 74 deletions
diff --git a/fs/nfs/objlayout/objio_osd.c b/fs/nfs/objlayout/objio_osd.c
index bd7ec26e2840..00b384934c32 100644
--- a/fs/nfs/objlayout/objio_osd.c
+++ b/fs/nfs/objlayout/objio_osd.c
@@ -44,12 +44,6 @@
44 44
45#define NFSDBG_FACILITY NFSDBG_PNFS_LD 45#define NFSDBG_FACILITY NFSDBG_PNFS_LD
46 46
47#define _LLU(x) ((unsigned long long)x)
48
49enum { BIO_MAX_PAGES_KMALLOC =
50 (PAGE_SIZE - sizeof(struct bio)) / sizeof(struct bio_vec),
51};
52
53struct objio_dev_ent { 47struct objio_dev_ent {
54 struct nfs4_deviceid_node id_node; 48 struct nfs4_deviceid_node id_node;
55 struct ore_dev od; 49 struct ore_dev od;
@@ -124,37 +118,13 @@ OBJIO_LSEG(struct pnfs_layout_segment *lseg)
124 return container_of(lseg, struct objio_segment, lseg); 118 return container_of(lseg, struct objio_segment, lseg);
125} 119}
126 120
127struct objio_state;
128typedef int (*objio_done_fn)(struct objio_state *ios);
129
130struct objio_state { 121struct objio_state {
131 /* Generic layer */ 122 /* Generic layer */
132 struct objlayout_io_res oir; 123 struct objlayout_io_res oir;
133 124
134 struct page **pages;
135 unsigned pgbase;
136 unsigned nr_pages;
137 unsigned long count;
138 loff_t offset;
139 bool sync; 125 bool sync;
140 126 /*FIXME: Support for extra_bytes at ore_get_rw_state() */
141 struct ore_layout *layout; 127 struct ore_io_state *ios;
142 struct ore_components *oc;
143
144 struct kref kref;
145 objio_done_fn done;
146 void *private;
147
148 unsigned long length;
149 unsigned numdevs; /* Actually used devs in this IO */
150 /* A per-device variable array of size numdevs */
151 struct _objio_per_comp {
152 struct bio *bio;
153 struct osd_request *or;
154 unsigned long length;
155 u64 offset;
156 unsigned dev;
157 } per_dev[];
158}; 128};
159 129
160/* Send and wait for a get_device_info of devices in the layout, 130/* Send and wait for a get_device_info of devices in the layout,
@@ -374,16 +344,16 @@ void objio_free_lseg(struct pnfs_layout_segment *lseg)
374} 344}
375 345
376static int 346static int
377objio_alloc_io_state(struct pnfs_layout_hdr *pnfs_layout_type, 347objio_alloc_io_state(struct pnfs_layout_hdr *pnfs_layout_type, bool is_reading,
378 struct pnfs_layout_segment *lseg, struct page **pages, unsigned pgbase, 348 struct pnfs_layout_segment *lseg, struct page **pages, unsigned pgbase,
379 loff_t offset, size_t count, void *rpcdata, gfp_t gfp_flags, 349 loff_t offset, size_t count, void *rpcdata, gfp_t gfp_flags,
380 struct objio_state **outp) 350 struct objio_state **outp)
381{ 351{
382 struct objio_segment *objio_seg = OBJIO_LSEG(lseg); 352 struct objio_segment *objio_seg = OBJIO_LSEG(lseg);
383 struct objio_state *ios; 353 struct ore_io_state *ios;
354 int ret;
384 struct __alloc_objio_state { 355 struct __alloc_objio_state {
385 struct objio_state objios; 356 struct objio_state objios;
386 struct _objio_per_comp per_dev[objio_seg->oc.numdevs];
387 struct pnfs_osd_ioerr ioerrs[objio_seg->oc.numdevs]; 357 struct pnfs_osd_ioerr ioerrs[objio_seg->oc.numdevs];
388 } *aos; 358 } *aos;
389 359
@@ -391,30 +361,33 @@ objio_alloc_io_state(struct pnfs_layout_hdr *pnfs_layout_type,
391 if (unlikely(!aos)) 361 if (unlikely(!aos))
392 return -ENOMEM; 362 return -ENOMEM;
393 363
394 ios = &aos->objios;
395
396 ios->layout = &objio_seg->layout;
397 ios->oc = &objio_seg->oc;
398 objlayout_init_ioerrs(&aos->objios.oir, objio_seg->oc.numdevs, 364 objlayout_init_ioerrs(&aos->objios.oir, objio_seg->oc.numdevs,
399 aos->ioerrs, rpcdata, pnfs_layout_type); 365 aos->ioerrs, rpcdata, pnfs_layout_type);
400 366
367 ret = ore_get_rw_state(&objio_seg->layout, &objio_seg->oc, is_reading,
368 offset, count, &ios);
369 if (unlikely(ret)) {
370 kfree(aos);
371 return ret;
372 }
373
401 ios->pages = pages; 374 ios->pages = pages;
402 ios->pgbase = pgbase; 375 ios->pgbase = pgbase;
403 ios->nr_pages = (pgbase + count + PAGE_SIZE - 1) >> PAGE_SHIFT; 376 ios->private = aos;
404 ios->offset = offset;
405 ios->count = count;
406 ios->sync = 0;
407 BUG_ON(ios->nr_pages > (pgbase + count + PAGE_SIZE - 1) >> PAGE_SHIFT); 377 BUG_ON(ios->nr_pages > (pgbase + count + PAGE_SIZE - 1) >> PAGE_SHIFT);
408 378
409 *outp = ios; 379 aos->objios.sync = 0;
380 aos->objios.ios = ios;
381 *outp = &aos->objios;
410 return 0; 382 return 0;
411} 383}
412 384
413void objio_free_result(struct objlayout_io_res *oir) 385void objio_free_result(struct objlayout_io_res *oir)
414{ 386{
415 struct objio_state *ios = container_of(oir, struct objio_state, oir); 387 struct objio_state *objios = container_of(oir, struct objio_state, oir);
416 388
417 kfree(ios); 389 ore_put_io_state(objios->ios);
390 kfree(objios);
418} 391}
419 392
420enum pnfs_osd_errno osd_pri_2_pnfs_err(enum osd_err_priority oep) 393enum pnfs_osd_errno osd_pri_2_pnfs_err(enum osd_err_priority oep)
@@ -447,7 +420,7 @@ enum pnfs_osd_errno osd_pri_2_pnfs_err(enum osd_err_priority oep)
447 } 420 }
448} 421}
449 422
450static void __on_dev_error(struct objio_state *ios, bool is_write, 423static void __on_dev_error(struct ore_io_state *ios,
451 struct ore_dev *od, unsigned dev_index, enum osd_err_priority oep, 424 struct ore_dev *od, unsigned dev_index, enum osd_err_priority oep,
452 u64 dev_offset, u64 dev_len) 425 u64 dev_offset, u64 dev_len)
453{ 426{
@@ -465,9 +438,10 @@ static void __on_dev_error(struct objio_state *ios, bool is_write,
465 438
466 objlayout_io_set_result(&objios->oir, comp, 439 objlayout_io_set_result(&objios->oir, comp,
467 &pooid, osd_pri_2_pnfs_err(oep), 440 &pooid, osd_pri_2_pnfs_err(oep),
468 dev_offset, dev_len, is_write); 441 dev_offset, dev_len, !ios->reading);
469} 442}
470 443
444#if 0
471static void _clear_bio(struct bio *bio) 445static void _clear_bio(struct bio *bio)
472{ 446{
473 struct bio_vec *bv; 447 struct bio_vec *bv;
@@ -786,26 +760,28 @@ static int _io_exec(struct objio_state *ios)
786 760
787 return ret; 761 return ret;
788} 762}
763#endif
789 764
790/* 765/*
791 * read 766 * read
792 */ 767 */
793static int _read_done(struct objio_state *ios) 768static void _read_done(struct ore_io_state *ios, void *private)
794{ 769{
770 struct objio_state *objios = private;
795 ssize_t status; 771 ssize_t status;
796 int ret = _io_check(ios, false); 772 int ret = ore_check_io(ios, &__on_dev_error);
797 773
798 _io_free(ios); 774 /* FIXME: _io_free(ios) can we dealocate the libosd resources; */
799 775
800 if (likely(!ret)) 776 if (likely(!ret))
801 status = ios->length; 777 status = ios->length;
802 else 778 else
803 status = ret; 779 status = ret;
804 780
805 objlayout_read_done(&ios->oir, status, ios->sync); 781 objlayout_read_done(&objios->oir, status, objios->sync);
806 return ret;
807} 782}
808 783
784#if 0
809static int _read_mirrors(struct objio_state *ios, unsigned cur_comp) 785static int _read_mirrors(struct objio_state *ios, unsigned cur_comp)
810{ 786{
811 struct osd_request *or = NULL; 787 struct osd_request *or = NULL;
@@ -860,49 +836,50 @@ err:
860 _io_free(ios); 836 _io_free(ios);
861 return ret; 837 return ret;
862} 838}
839#endif
863 840
864int objio_read_pagelist(struct nfs_read_data *rdata) 841int objio_read_pagelist(struct nfs_read_data *rdata)
865{ 842{
866 struct objio_state *ios; 843 struct objio_state *objios;
867 int ret; 844 int ret;
868 845
869 ret = objio_alloc_io_state(NFS_I(rdata->inode)->layout, 846 ret = objio_alloc_io_state(NFS_I(rdata->inode)->layout, true,
870 rdata->lseg, rdata->args.pages, rdata->args.pgbase, 847 rdata->lseg, rdata->args.pages, rdata->args.pgbase,
871 rdata->args.offset, rdata->args.count, rdata, 848 rdata->args.offset, rdata->args.count, rdata,
872 GFP_KERNEL, &ios); 849 GFP_KERNEL, &objios);
873 if (unlikely(ret))
874 return ret;
875
876 ret = _io_rw_pagelist(ios, GFP_KERNEL);
877 if (unlikely(ret)) 850 if (unlikely(ret))
878 return ret; 851 return ret;
879 852
880 return _read_exec(ios); 853 objios->ios->done = _read_done;
854 dprintk("%s: offset=0x%llx length=0x%x\n", __func__,
855 rdata->args.offset, rdata->args.count);
856 return ore_read(objios->ios);
881} 857}
882 858
883/* 859/*
884 * write 860 * write
885 */ 861 */
886static int _write_done(struct objio_state *ios) 862static void _write_done(struct ore_io_state *ios, void *private)
887{ 863{
864 struct objio_state *objios = private;
888 ssize_t status; 865 ssize_t status;
889 int ret = _io_check(ios, true); 866 int ret = ore_check_io(ios, &__on_dev_error);
890 867
891 _io_free(ios); 868 /* FIXME: _io_free(ios) can we dealocate the libosd resources; */
892 869
893 if (likely(!ret)) { 870 if (likely(!ret)) {
894 /* FIXME: should be based on the OSD's persistence model 871 /* FIXME: should be based on the OSD's persistence model
895 * See OSD2r05 Section 4.13 Data persistence model */ 872 * See OSD2r05 Section 4.13 Data persistence model */
896 ios->oir.committed = NFS_FILE_SYNC; 873 objios->oir.committed = NFS_FILE_SYNC;
897 status = ios->length; 874 status = ios->length;
898 } else { 875 } else {
899 status = ret; 876 status = ret;
900 } 877 }
901 878
902 objlayout_write_done(&ios->oir, status, ios->sync); 879 objlayout_write_done(&objios->oir, status, objios->sync);
903 return ret;
904} 880}
905 881
882#if 0
906static int _write_mirrors(struct objio_state *ios, unsigned cur_comp) 883static int _write_mirrors(struct objio_state *ios, unsigned cur_comp)
907{ 884{
908 struct _objio_per_comp *master_dev = &ios->per_dev[cur_comp]; 885 struct _objio_per_comp *master_dev = &ios->per_dev[cur_comp];
@@ -984,27 +961,35 @@ err:
984 _io_free(ios); 961 _io_free(ios);
985 return ret; 962 return ret;
986} 963}
964#endif
987 965
988int objio_write_pagelist(struct nfs_write_data *wdata, int how) 966int objio_write_pagelist(struct nfs_write_data *wdata, int how)
989{ 967{
990 struct objio_state *ios; 968 struct objio_state *objios;
991 int ret; 969 int ret;
992 970
993 ret = objio_alloc_io_state(NFS_I(wdata->inode)->layout, 971 ret = objio_alloc_io_state(NFS_I(wdata->inode)->layout, false,
994 wdata->lseg, wdata->args.pages, wdata->args.pgbase, 972 wdata->lseg, wdata->args.pages, wdata->args.pgbase,
995 wdata->args.offset, wdata->args.count, wdata, GFP_NOFS, 973 wdata->args.offset, wdata->args.count, wdata, GFP_NOFS,
996 &ios); 974 &objios);
997 if (unlikely(ret)) 975 if (unlikely(ret))
998 return ret; 976 return ret;
999 977
1000 ios->sync = 0 != (how & FLUSH_SYNC); 978 objios->sync = 0 != (how & FLUSH_SYNC);
1001 979
1002 /* TODO: ios->stable = stable; */ 980 if (!objios->sync)
1003 ret = _io_rw_pagelist(ios, GFP_NOFS); 981 objios->ios->done = _write_done;
982
983 dprintk("%s: offset=0x%llx length=0x%x\n", __func__,
984 wdata->args.offset, wdata->args.count);
985 ret = ore_write(objios->ios);
1004 if (unlikely(ret)) 986 if (unlikely(ret))
1005 return ret; 987 return ret;
1006 988
1007 return _write_exec(ios); 989 if (objios->sync)
990 _write_done(objios->ios, objios);
991
992 return 0;
1008} 993}
1009 994
1010static bool objio_pg_test(struct nfs_pageio_descriptor *pgio, 995static bool objio_pg_test(struct nfs_pageio_descriptor *pgio,