litmus-rt.git - The LITMUS^RT kernel.

diff options

author	Ewan D. Milne <emilne@redhat.com>	2014-12-04 11:49:26 -0500
committer	Christoph Hellwig <hch@lst.de>	2014-12-15 07:34:26 -0500
commit	4bc6b63482a069baa2eeaba3e4a0b5df75263dc5 (patch)
tree	91459de2c97b39f534c43fddd19ffdcf421656be /tools/perf/scripts/python/net_dropmonitor.py
parent	6d6f3807ccb7ac884ff0d0dcb2510a0ca4dd3ace (diff)

scsi_debug: take sdebug_host_list_lock when changing capacity

All other traversals of the sdebug_host_list take the lock. Signed-off-by: Ewan D. Milne <emilne@redhat.com> Acked-by: Douglas Gilbert <dgilbert@interlog.com> Signed-off-by: Christoph Hellwig <hch@lst.de>

Diffstat (limited to 'tools/perf/scripts/python/net_dropmonitor.py')

0 files changed, 0 insertions, 0 deletions

/*[A[A * Copyright (C) 2005-2009 Red Hat, Inc. All rights reserved. * * Module Author: Heinz Mauelshagen <heinzm@redhat.com> * * This file is released under the GPL. * * * Linux 2.6 Device Mapper RAID4 and RAID5 target. * * Supports: * o RAID4 with dedicated and selectable parity device * o RAID5 with rotating parity (left+right, symmetric+asymmetric) * o recovery of out of sync device for initial * RAID set creation or after dead drive replacement * o run time optimization of xor algorithm used to calculate parity * * * Thanks to MD for: * o the raid address calculation algorithm * o the base of the biovec <-> page list copier. * * * Uses region hash to keep track of how many writes are in flight to * regions in order to use dirty log to keep state of regions to recover: * * o clean regions (those which are synchronized * and don't have write io in flight) * o dirty regions (those with write io in flight) * * * On startup, any dirty regions are migrated to the * 'nosync' state and are subject to recovery by the daemon. * * See raid_ctr() for table definition. * * FIXME: recovery bandwidth */ static const char *version = "v0.2594b"; #include "dm.h" #include "dm-memcache.h" #include "dm-message.h" #include "dm-raid45.h" #include <linux/kernel.h> #include <linux/vmalloc.h> #include <linux/raid/xor.h> #include <linux/bio.h> #include <linux/dm-io.h> #include <linux/dm-dirty-log.h> #include "dm-region-hash.h" /* * Configurable parameters */ /* Minimum/maximum and default # of selectable stripes. */ #define STRIPES_MIN 8 #define STRIPES_MAX 16384 #define STRIPES_DEFAULT 80 /* Maximum and default chunk size in sectors if not set in constructor. */ #define CHUNK_SIZE_MIN 8 #define CHUNK_SIZE_MAX 16384 #define CHUNK_SIZE_DEFAULT 64 /* Default io size in sectors if not set in constructor. */ #define IO_SIZE_MIN CHUNK_SIZE_MIN #define IO_SIZE_DEFAULT IO_SIZE_MIN /* Recover io size default in sectors. */ #define RECOVER_IO_SIZE_MIN 64 #define RECOVER_IO_SIZE_DEFAULT 256 /* Default, minimum and maximum percentage of recover io bandwidth. */ #define BANDWIDTH_DEFAULT 10 #define BANDWIDTH_MIN 1 #define BANDWIDTH_MAX 100 /* # of parallel recovered regions */ #define RECOVERY_STRIPES_MIN 1 #define RECOVERY_STRIPES_MAX 64 #define RECOVERY_STRIPES_DEFAULT RECOVERY_STRIPES_MIN /* * END Configurable parameters */ #define TARGET "dm-raid45" #define DAEMON "kraid45d" #define DM_MSG_PREFIX TARGET #define SECTORS_PER_PAGE (PAGE_SIZE >> SECTOR_SHIFT) /* Amount/size for __xor(). */ #define XOR_SIZE PAGE_SIZE /* Check value in range. */ #define range_ok(i, min, max) (i >= min && i <= max) /* Check argument is power of 2. */ #define POWER_OF_2(a) (!(a & (a - 1))) /* Structure access macros. */ /* Derive raid_set from stripe_cache pointer. */ #define RS(x) container_of(x, struct raid_set, sc) /* Page reference. */ #define PAGE(stripe, p) ((stripe)->obj[p].pl->page) /* Stripe chunk reference. */ #define CHUNK(stripe, p) ((stripe)->chunk + p) /* Bio list reference. */ #define BL(stripe, p, rw) (stripe->chunk[p].bl + rw) #define BL_CHUNK(chunk, rw) (chunk->bl + rw) /* Page list reference. */ #define PL(stripe, p) (stripe->obj[p].pl) /* END: structure access macros. */ /* Factor out to dm-bio-list.h */ static inline void bio_list_push(struct bio_list *bl, struct bio *bio) { bio->bi_next = bl->head; bl->head = bio; if (!bl->tail) bl->tail = bio; } /* Factor out to dm.h */ #define TI_ERR_RET(str, ret) \ do { ti->error = str; return ret; } while (0); #define TI_ERR(str) TI_ERR_RET(str, -EINVAL) /* Macro to define access IO flags access inline functions. */ #define BITOPS(name, what, var, flag) \ static inline int TestClear ## name ## what(struct var *v) \ { return test_and_clear_bit(flag, &v->io.flags); } \ static inline int TestSet ## name ## what(struct var *v) \ { return test_and_set_bit(flag, &v->io.flags); } \ static inline void Clear ## name ## what(struct var *v) \ { clear_bit(flag, &v->io.flags); } \ static inline void Set ## name ## what(struct var *v) \ { set_bit(flag, &v->io.flags); } \ static inline int name ## what(struct var *v) \ { return test_bit(flag, &v->io.flags); } /*----------------------------------------------------------------- * Stripe cache * * Cache for all reads and writes to raid sets (operational or degraded) * * We need to run all data to and from a RAID set through this cache, * because parity chunks need to get calculated from data chunks * or, in the degraded/resynchronization case, missing chunks need * to be reconstructed using the other chunks of the stripe. *---------------------------------------------------------------*/ /* A chunk within a stripe (holds bios hanging off). */ /* IO status flags for chunks of a stripe. */ enum chunk_flags { CHUNK_DIRTY, /* Pages of chunk dirty; need writing. */ CHUNK_ERROR, /* IO error on any chunk page. */ CHUNK_IO, /* Allow/prohibit IO on chunk pages. */ CHUNK_LOCKED, /* Chunk pages locked during IO. */ CHUNK_MUST_IO, /* Chunk must io. */ CHUNK_UNLOCK, /* Enforce chunk unlock. */ CHUNK_UPTODATE, /* Chunk pages are uptodate. */ }; #if READ != 0 || WRITE != 1 #error dm-raid45: READ/WRITE != 0/1 used as index!!! #endif enum bl_type { WRITE_QUEUED = WRITE + 1, WRITE_MERGED, NR_BL_TYPES, /* Must be last one! */ }; struct stripe_chunk { atomic_t cnt; /* Reference count. */ struct stripe *stripe; /* Backpointer to stripe for endio(). */ /* Bio lists for reads, writes, and writes merged. */ struct bio_list bl[NR_BL_TYPES]; struct { unsigned long flags; /* IO status flags. */ } io; }; /* Define chunk bit operations. */ BITOPS(Chunk, Dirty, stripe_chunk, CHUNK_DIRTY) BITOPS(Chunk, Error, stripe_chunk, CHUNK_ERROR) BITOPS(Chunk, Io, stripe_chunk, CHUNK_IO) BITOPS(Chunk, Locked, stripe_chunk, CHUNK_LOCKED) BITOPS(Chunk, MustIo, stripe_chunk, CHUNK_MUST_IO) BITOPS(Chunk, Unlock, stripe_chunk, CHUNK_UNLOCK) BITOPS(Chunk, Uptodate, stripe_chunk, CHUNK_UPTODATE) /* * Stripe linked list indexes. Keep order, because the stripe * and the stripe cache rely on the first 3! */ enum list_types { LIST_FLUSH, /* Stripes to flush for io. */ LIST_ENDIO, /* Stripes to endio. */ LIST_LRU, /* Least recently used stripes. */ SC_NR_LISTS, /* # of lists in stripe cache. */ LIST_HASH = SC_NR_LISTS, /* Hashed stripes. */ LIST_RECOVER = LIST_HASH, /* For recovery type stripes only. */ STRIPE_NR_LISTS,/* To size array in struct stripe. */ }; /* Adressing region recovery. */ struct recover_addr { struct dm_region *reg; /* Actual region to recover. */ sector_t pos; /* Position within region to recover. */ sector_t end; /* End of region to recover. */ }; /* A stripe: the io object to handle all reads and writes to a RAID set. */ struct stripe { atomic_t cnt; /* Reference count. */ struct stripe_cache *sc; /* Backpointer to stripe cache. */ /* * 4 linked lists: * o io list to flush io * o endio list * o LRU list to put stripes w/o reference count on * o stripe cache hash */ struct list_head lists[STRIPE_NR_LISTS]; sector_t key; /* Hash key. */ region_t region; /* Region stripe is mapped to. */ struct { unsigned long flags; /* Stripe state flags (see below). */ /* * Pending ios in flight: * * used to control move of stripe to endio list */ atomic_t pending; /* Sectors to read and write for multi page stripe sets. */ unsigned size; } io; /* Address region recovery. */ struct recover_addr *recover; /* Lock on stripe (Future: for clustering). */ void *lock; struct { unsigned short parity; /* Parity chunk index. */ short recover; /* Recovery chunk index. */ } idx; /* * This stripe's memory cache object (dm-mem-cache); * i.e. the io chunk pages. */ struct dm_mem_cache_object *obj; /* Array of stripe sets (dynamically allocated). */ struct stripe_chunk chunk[0]; }; /* States stripes can be in (flags field). */ enum stripe_states { STRIPE_ERROR, /* io error on stripe. */ STRIPE_MERGED, /* Writes got merged to be written. */ STRIPE_RBW, /* Read-before-write stripe. */ STRIPE_RECONSTRUCT, /* Reconstruct of a missing chunk required. */ STRIPE_RECONSTRUCTED, /* Reconstructed of a missing chunk. */ STRIPE_RECOVER, /* Stripe used for RAID set recovery. */ }; /* Define stripe bit operations. */ BITOPS(Stripe, Error, stripe, STRIPE_ERROR) BITOPS(Stripe, Merged, stripe, STRIPE_MERGED) BITOPS(Stripe, RBW, stripe, STRIPE_RBW) BITOPS(Stripe, Reconstruct, stripe, STRIPE_RECONSTRUCT) BITOPS(Stripe, Reconstructed, stripe, STRIPE_RECONSTRUCTED) BITOPS(Stripe, Recover, stripe, STRIPE_RECOVER) /* A stripe hash. */ struct stripe_hash { struct list_head *hash; unsigned buckets; unsigned mask; unsigned prime; unsigned shift; }; enum sc_lock_types { LOCK_ENDIO, /* Protect endio list. */ LOCK_LRU, /* Protect LRU list. */ NR_LOCKS, /* To size array in struct stripe_cache. */ }; /* A stripe cache. */ struct stripe_cache { /* Stripe hash. */ struct stripe_hash hash; spinlock_t locks[NR_LOCKS]; /* Locks to protect lists. */ /* Stripes with io to flush, stripes to endio and LRU lists. */ struct list_head lists[SC_NR_LISTS]; /* Slab cache to allocate stripes from. */ struct { struct kmem_cache *cache; /* Cache itself. */ char name[32]; /* Unique name. */ } kc; struct dm_io_client *dm_io_client; /* dm-io client resource context. */ /* dm-mem-cache client resource context. */ struct dm_mem_cache_client *mem_cache_client; int stripes_parm; /* # stripes parameter from constructor. */ atomic_t stripes; /* actual # of stripes in cache. */ atomic_t stripes_to_set; /* # of stripes to resize cache to. */ atomic_t stripes_last; /* last # of stripes in cache. */ atomic_t active_stripes; /* actual # of active stripes in cache. */ /* REMOVEME: */ atomic_t active_stripes_max; /* actual # of active stripes in cache. */ }; /* Flag specs for raid_dev */ ; enum raid_dev_flags { DEV_FAILED, /* Device failed. */ DEV_IO_QUEUED, /* Io got queued to device. */ }; /* The raid device in a set. */ struct raid_dev { struct dm_dev *dev; sector_t start; /* Offset to map to. */ struct { /* Using struct to be able to BITOPS(). */ unsigned long flags; /* raid_dev_flags. */ } io; }; BITOPS(Dev, Failed, raid_dev, DEV_FAILED) BITOPS(Dev, IoQueued, raid_dev, DEV_IO_QUEUED) /* Flags spec for raid_set. */ enum raid_set_flags { RS_CHECK_OVERWRITE, /* Check for chunk overwrites. */ RS_DEAD, /* RAID set inoperational. */ RS_DEGRADED, /* Io errors on RAID device. */ RS_DEVEL_STATS, /* REMOVEME: display status information. */ RS_RECOVER, /* Do recovery. */ RS_RECOVERY_BANDWIDTH, /* Allow recovery bandwidth (delayed bios). */ RS_SC_BUSY, /* Stripe cache busy -> send an event. */ RS_SUSPEND, /* Suspend RAID set. */ }; /* REMOVEME: devel stats counters. */ enum stats_types { S_BIOS_READ, S_BIOS_ADDED_READ, S_BIOS_ENDIO_READ, S_BIOS_WRITE, S_BIOS_ADDED_WRITE, S_BIOS_ENDIO_WRITE, S_CAN_MERGE, S_CANT_MERGE, S_CONGESTED, S_DM_IO_READ, S_DM_IO_WRITE, S_BANDWIDTH, S_BARRIER, S_BIO_COPY_PL_NEXT, S_DEGRADED, S_DELAYED_BIOS, S_FLUSHS, S_HITS_1ST, S_IOS_POST, S_INSCACHE, S_MAX_LOOKUP, S_CHUNK_LOCKED, S_NO_BANDWIDTH, S_NOT_CONGESTED, S_NO_RW, S_NOSYNC, S_OVERWRITE, S_PROHIBITCHUNKIO, S_RECONSTRUCT_EI, S_RECONSTRUCT_DEV, S_RECONSTRUCT_SET, S_RECONSTRUCTED, S_REQUEUE, S_STRIPE_ERROR, S_SUM_DELAYED_BIOS, S_XORS, S_NR_STATS, /* # of stats counters. Must be last! */ }; /* Status type -> string mappings. */ struct stats_map { const enum stats_types type; const char *str; }; static struct stats_map stats_map[] = { { S_BIOS_READ, "r=" }, { S_BIOS_ADDED_READ, "/" }, { S_BIOS_ENDIO_READ, "/" }, { S_BIOS_WRITE, " w=" }, { S_BIOS_ADDED_WRITE, "/" }, { S_BIOS_ENDIO_WRITE, "/" }, { S_DM_IO_READ, " rc=" }, { S_DM_IO_WRITE, " wc=" }, { S_BANDWIDTH, "\nbw=" }, { S_NO_BANDWIDTH, " no_bw=" }, { S_BARRIER, "\nbarrier=" }, { S_BIO_COPY_PL_NEXT, "\nbio_cp_next=" }, { S_CAN_MERGE, "\nmerge=" }, { S_CANT_MERGE, "/no_merge=" }, { S_CHUNK_LOCKED, "\nchunk_locked=" }, { S_CONGESTED, "\ncgst=" }, { S_NOT_CONGESTED, "/not_cgst=" }, { S_DEGRADED, "\ndegraded=" }, { S_DELAYED_BIOS, "\ndel_bios=" }, { S_SUM_DELAYED_BIOS, "/sum_del_bios=" }, { S_FLUSHS, "\nflushs=" }, { S_HITS_1ST, "\nhits_1st=" }, { S_IOS_POST, " ios_post=" }, { S_INSCACHE, " inscache=" }, { S_MAX_LOOKUP, " maxlookup=" }, { S_NO_RW, "\nno_rw=" }, { S_NOSYNC, " nosync=" }, { S_OVERWRITE, " ovr=" }, { S_PROHIBITCHUNKIO, " prhbt_io=" }, { S_RECONSTRUCT_EI, "\nrec_ei=" }, { S_RECONSTRUCT_DEV, " rec_dev=" }, { S_RECONSTRUCT_SET, " rec_set=" }, { S_RECONSTRUCTED, " rec=" }, { S_REQUEUE, " requeue=" }, { S_STRIPE_ERROR, " stripe_err=" }, { S_XORS, " xors=" }, }; /* * A RAID set. */ #define dm_rh_client dm_region_hash enum count_type { IO_WORK = 0, IO_RECOVER, IO_NR_COUNT }; typedef void (*xor_function_t)(unsigned count, unsigned long **data); struct raid_set { struct dm_target *ti; /* Target pointer. */ struct { unsigned long flags; /* State flags. */ struct mutex in_lock; /* Protects central input list below. */ struct bio_list in; /* Pending ios (central input list). */ struct bio_list work; /* ios work set. */ wait_queue_head_t suspendq; /* suspend synchronization. */ atomic_t in_process; /* counter of queued bios (suspendq). */ atomic_t in_process_max;/* counter of queued bios max. */ /* io work. */ struct workqueue_struct *wq; struct delayed_work dws_do_raid; /* For main worker. */ struct work_struct ws_do_table_event; /* For event worker. */ } io; /* Stripe locking abstraction. */ struct dm_raid45_locking_type *locking; struct stripe_cache sc; /* Stripe cache for this set. */ /* Xor optimization. */ struct { struct xor_func *f; unsigned chunks; unsigned speed; } xor; /* Recovery parameters. */ struct recover { struct dm_dirty_log *dl; /* Dirty log. */ struct dm_rh_client *rh; /* Region hash. */ struct dm_io_client *dm_io_client; /* recovery dm-io client. */ /* dm-mem-cache client resource context for recovery stripes. */ struct dm_mem_cache_client *mem_cache_client; struct list_head stripes; /* List of recovery stripes. */ region_t nr_regions; region_t nr_regions_to_recover; region_t nr_regions_recovered; unsigned long start_jiffies; unsigned long end_jiffies; unsigned bandwidth; /* Recovery bandwidth [%]. */ unsigned bandwidth_work; /* Recovery bandwidth [factor]. */ unsigned bandwidth_parm; /* " constructor parm. */ unsigned io_size; /* recovery io size <= region size. */ unsigned io_size_parm; /* recovery io size ctr parameter. */ unsigned recovery; /* Recovery allowed/prohibited. */ unsigned recovery_stripes; /* # of parallel recovery stripes. */ /* recovery io throttling. */ atomic_t io_count[IO_NR_COUNT]; /* counter recover/regular io.*/ unsigned long last_jiffies; } recover; /* RAID set parameters. */ struct { struct raid_type *raid_type; /* RAID type (eg, RAID4). */ unsigned raid_parms; /* # variable raid parameters. */ unsigned chunk_size; /* Sectors per chunk. */ unsigned chunk_size_parm; unsigned chunk_shift; /* rsector chunk size shift. */ unsigned io_size; /* Sectors per io. */ unsigned io_size_parm; unsigned io_mask; /* Mask for bio_copy_page_list(). */ unsigned io_inv_mask; /* Mask for raid_address(). */ sector_t sectors_per_dev; /* Sectors per device. */ atomic_t failed_devs; /* Amount of devices failed. */ /* Index of device to initialize. */ int dev_to_init; int dev_to_init_parm; /* Raid devices dynamically allocated. */ unsigned raid_devs; /* # of RAID devices below. */ unsigned data_devs; /* # of RAID data devices. */ int ei; /* index of failed RAID device. */ /* Index of dedicated parity device (i.e. RAID4). */ int pi; int pi_parm; /* constructor parm for status output. */ } set; /* REMOVEME: devel stats counters. */ atomic_t stats[S_NR_STATS]; /* Dynamically allocated temporary pointers for xor(). */ unsigned long **data; /* Dynamically allocated RAID devices. Alignment? */ struct raid_dev dev[0]; }; /* Define RAID set bit operations. */ BITOPS(RS, Bandwidth, raid_set, RS_RECOVERY_BANDWIDTH) BITOPS(RS, CheckOverwrite, raid_set, RS_CHECK_OVERWRITE) BITOPS(RS, Dead, raid_set, RS_DEAD) BITOPS(RS, Degraded, raid_set, RS_DEGRADED) BITOPS(RS, DevelStats, raid_set, RS_DEVEL_STATS) BITOPS(RS, Recover, raid_set, RS_RECOVER) BITOPS(RS, ScBusy, raid_set, RS_SC_BUSY) BITOPS(RS, Suspend, raid_set, RS_SUSPEND) #undef BITOPS /*----------------------------------------------------------------- * Raid-4/5 set structures. *---------------------------------------------------------------*/ /* RAID level definitions. */ enum raid_level { raid4, raid5, }; /* Symmetric/Asymmetric, Left/Right parity rotating algorithms. */ enum raid_algorithm { none, left_asym, right_asym, left_sym, right_sym, }; struct raid_type { const char *name; /* RAID algorithm. */ const char *descr; /* Descriptor text for logging. */ const unsigned parity_devs; /* # of parity devices. */ const unsigned minimal_devs; /* minimal # of devices in set. */ const enum raid_level level; /* RAID level. */ const enum raid_algorithm algorithm; /* RAID algorithm. */ }; /* Supported raid types and properties. */ static struct raid_type raid_types[] = { {"raid4", "RAID4 (dedicated parity disk)", 1, 3, raid4, none}, {"raid5_la", "RAID5 (left asymmetric)", 1, 3, raid5, left_asym}, {"raid5_ra", "RAID5 (right asymmetric)", 1, 3, raid5, right_asym}, {"raid5_ls", "RAID5 (left symmetric)", 1, 3, raid5, left_sym}, {"raid5_rs", "RAID5 (right symmetric)", 1, 3, raid5, right_sym}, }; /* Address as calculated by raid_address(). */ struct raid_address { sector_t key; /* Hash key (address of stripe % chunk_size). */ unsigned di, pi; /* Data and parity disks index. */ }; /* REMOVEME: reset statistics counters. */ static void stats_reset(struct raid_set *rs) { unsigned s = S_NR_STATS; while (s--) atomic_set(rs->stats + s, 0); } /*---------------------------------------------------------------- * RAID set management routines. *--------------------------------------------------------------*/ /* * Begin small helper functions. */ /* No need to be called from region hash indirectly at dm_rh_dec(). */ static void wake_dummy(void *context) {} /* Return # of io reference. */ static int io_ref(struct raid_set *rs) { return atomic_read(&rs->io.in_process); } /* Get an io reference. */ static void io_get(struct raid_set *rs) { int p = atomic_inc_return(&rs->io.in_process); if (p > atomic_read(&rs->io.in_process_max)) atomic_set(&rs->io.in_process_max, p); /* REMOVEME: max. */ } /* Put the io reference and conditionally wake io waiters. */ static void io_put(struct raid_set *rs) { /* Intel: rebuild data corrupter? */ if (atomic_dec_and_test(&rs->io.in_process)) wake_up(&rs->io.suspendq); else BUG_ON(io_ref(rs) < 0); } /* Wait until all io has been processed. */ static void wait_ios(struct raid_set *rs) { wait_event(rs->io.suspendq, !io_ref(rs)); } /* Queue (optionally delayed) io work. */ static void wake_do_raid_delayed(struct raid_set *rs, unsigned long delay) { queue_delayed_work(rs->io.wq, &rs->io.dws_do_raid, delay); } /* Queue io work immediately (called from region hash too). */ static void wake_do_raid(void *context) { struct raid_set *rs = context; queue_work(rs->io.wq, &rs->io.dws_do_raid.work); } /* Calculate device sector offset. */ static sector_t _sector(struct raid_set *rs, struct bio *bio) { sector_t sector = bio->bi_sector; sector_div(sector, rs->set.data_devs); return sector; } /* Return # of active stripes in stripe cache. */ static int sc_active(struct stripe_cache *sc) { return atomic_read(&sc->active_stripes); } /* Stripe cache busy indicator. */ static int sc_busy(struct raid_set *rs) { return sc_active(&rs->sc) > atomic_read(&rs->sc.stripes) - (STRIPES_MIN / 2); } /* Set chunks states. */ enum chunk_dirty_type { CLEAN, DIRTY, ERROR }; static void chunk_set(struct stripe_chunk *chunk, enum chunk_dirty_type type) { switch (type) { case CLEAN: ClearChunkDirty(chunk); break; case DIRTY: SetChunkDirty(chunk); break; case ERROR: SetChunkError(chunk); SetStripeError(chunk->stripe); return; default: BUG(); } SetChunkUptodate(chunk); SetChunkIo(chunk); ClearChunkError(chunk); } /* Return region state for a sector. */ static int region_state(struct raid_set *rs, sector_t sector, enum dm_rh_region_states state) { struct dm_rh_client *rh = rs->recover.rh; region_t region = dm_rh_sector_to_region(rh, sector); return !!(dm_rh_get_state(rh, region, 1) & state); } /* * Return true in case a chunk should be read/written * * Conditions to read/write: * o chunk not uptodate * o chunk dirty * * Conditios to avoid io: * o io already ongoing on chunk * o io explitely prohibited */ static int chunk_io(struct stripe_chunk *chunk) { /* 2nd run optimization (flag set below on first run). */ if (TestClearChunkMustIo(chunk)) return 1; /* Avoid io if prohibited or a locked chunk. */ if (!ChunkIo(chunk) || ChunkLocked(chunk)) return 0; if (!ChunkUptodate(chunk) || ChunkDirty(chunk)) { SetChunkMustIo(chunk); /* 2nd run optimization. */ return 1; } return 0; } /* Call a function on each chunk needing io unless device failed. */ static unsigned for_each_io_dev(struct stripe *stripe, void (*f_io)(struct stripe *stripe, unsigned p)) { struct raid_set *rs = RS(stripe->sc); unsigned p, r = 0; for (p = 0; p < rs->set.raid_devs; p++) { if (chunk_io(CHUNK(stripe, p)) && !DevFailed(rs->dev + p)) { f_io(stripe, p); r++; } } return r; } /* * Index of device to calculate parity on. * * Either the parity device index *or* the selected * device to init after a spare replacement. */ static int dev_for_parity(struct stripe *stripe, int *sync) { struct raid_set *rs = RS(stripe->sc); int r = region_state(rs, stripe->key, DM_RH_NOSYNC | DM_RH_RECOVERING); *sync = !r; /* Reconstruct a particular device ?. */ if (r && rs->set.dev_to_init > -1) return rs->set.dev_to_init; else if (rs->set.raid_type->level == raid4) return rs->set.pi; else if (!StripeRecover(stripe)) return stripe->idx.parity; else return -1; } /* RAID set congested function. */ static int rs_congested(void *congested_data, int bdi_bits) { int r; unsigned p; struct raid_set *rs = congested_data; if (sc_busy(rs) || RSSuspend(rs)) r = 1; else for (r = 0, p = rs->set.raid_devs; !r && p--; ) { /* If any of our component devices are overloaded. */ struct request_queue *q = bdev_get_queue(rs->dev[p].dev->bdev); r |= bdi_congested(&q->backing_dev_info, bdi_bits); } /* REMOVEME: statistics. */ atomic_inc(rs->stats + (r ? S_CONGESTED : S_NOT_CONGESTED)); return r; } /* RAID device degrade check. */ static void rs_check_degrade_dev(struct raid_set *rs, struct stripe *stripe, unsigned p) { if (TestSetDevFailed(rs->dev + p)) return; /* Through an event in case of member device errors. */ if (atomic_inc_return(&rs->set.failed_devs) > rs->set.raid_type->parity_devs && !TestSetRSDead(rs)) { /* Display RAID set dead message once. */ unsigned p; char buf[BDEVNAME_SIZE]; DMERR("FATAL: too many devices failed -> RAID set broken"); for (p = 0; p < rs->set.raid_devs; p++) { if (DevFailed(rs->dev + p)) DMERR("device /dev/%s failed", bdevname(rs->dev[p].dev->bdev, buf)); } } /* Only log the first member error. */ if (!TestSetRSDegraded(rs)) { char buf[BDEVNAME_SIZE]; /* Store index for recovery. */ rs->set.ei = p; DMERR("CRITICAL: %sio error on device /dev/%s " "in region=%llu; DEGRADING RAID set\n", stripe ? "" : "FAKED ", bdevname(rs->dev[p].dev->bdev, buf), (unsigned long long) (stripe ? stripe->key : 0)); DMERR("further device error messages suppressed"); } schedule_work(&rs->io.ws_do_table_event); } /* RAID set degrade check. */ static void rs_check_degrade(struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); unsigned p = rs->set.raid_devs; while (p--) { if (ChunkError(CHUNK(stripe, p))) rs_check_degrade_dev(rs, stripe, p); } } /* Lookup a RAID device by name or by major:minor number. */ static int raid_dev_lookup(struct raid_set *rs, struct raid_dev *dev_lookup) { unsigned p; struct raid_dev *dev; /* * Must be an incremental loop, because the device array * can have empty slots still on calls from raid_ctr() */ for (dev = rs->dev, p = 0; dev->dev && p < rs->set.raid_devs; dev++, p++) { if (dev_lookup->dev->bdev->bd_dev == dev->dev->bdev->bd_dev) return p; } return -ENODEV; } /* * End small helper functions. */ /* * Stripe hash functions */ /* Initialize/destroy stripe hash. */ static int hash_init(struct stripe_hash *hash, unsigned stripes) { unsigned buckets = 2, max_buckets = stripes >> 1; static unsigned hash_primes[] = { /* Table of primes for hash_fn/table size optimization. */ 1, 2, 3, 7, 13, 27, 53, 97, 193, 389, 769, 1543, 3079, 6151, 12289, 24593, 49157, 98317, }; /* Calculate number of buckets (2^^n <= stripes / 2). */ while (buckets < max_buckets) buckets <<= 1; /* Allocate stripe hash buckets. */ hash->hash = vmalloc(buckets * sizeof(*hash->hash)); if (!hash->hash) return -ENOMEM; hash->buckets = buckets; hash->mask = buckets - 1; hash->shift = ffs(buckets); if (hash->shift > ARRAY_SIZE(hash_primes)) hash->shift = ARRAY_SIZE(hash_primes) - 1; BUG_ON(hash->shift < 2); hash->prime = hash_primes[hash->shift]; /* Initialize buckets. */ while (buckets--) INIT_LIST_HEAD(hash->hash + buckets); return 0; } static void hash_exit(struct stripe_hash *hash) { if (hash->hash) { vfree(hash->hash); hash->hash = NULL; } } static unsigned hash_fn(struct stripe_hash *hash, sector_t key) { return (unsigned) (((key * hash->prime) >> hash->shift) & hash->mask); } static struct list_head *hash_bucket(struct stripe_hash *hash, sector_t key) { return hash->hash + hash_fn(hash, key); } /* Insert an entry into a hash. */ static void stripe_insert(struct stripe_hash *hash, struct stripe *stripe) { list_add(stripe->lists + LIST_HASH, hash_bucket(hash, stripe->key)); } /* Lookup an entry in the stripe hash. */ static struct stripe *stripe_lookup(struct stripe_cache *sc, sector_t key) { unsigned look = 0; struct stripe *stripe; struct list_head *bucket = hash_bucket(&sc->hash, key); list_for_each_entry(stripe, bucket, lists[LIST_HASH]) { look++; if (stripe->key == key) { /* REMOVEME: statisics. */ if (look > atomic_read(RS(sc)->stats + S_MAX_LOOKUP)) atomic_set(RS(sc)->stats + S_MAX_LOOKUP, look); return stripe; } } return NULL; } /* Resize the stripe cache hash on size changes. */ static int sc_hash_resize(struct stripe_cache *sc) { /* Resize indicated ? */ if (atomic_read(&sc->stripes) != atomic_read(&sc->stripes_last)) { int r; struct stripe_hash hash; r = hash_init(&hash, atomic_read(&sc->stripes)); if (r) return r; if (sc->hash.hash) { unsigned b = sc->hash.buckets; struct list_head *pos, *tmp; /* Walk old buckets and insert into new. */ while (b--) { list_for_each_safe(pos, tmp, sc->hash.hash + b) stripe_insert(&hash, list_entry(pos, struct stripe, lists[LIST_HASH])); } } hash_exit(&sc->hash); memcpy(&sc->hash, &hash, sizeof(sc->hash)); atomic_set(&sc->stripes_last, atomic_read(&sc->stripes)); } return 0; } /* End hash stripe hash function. */ /* List add, delete, push and pop functions. */ /* Add stripe to flush list. */ #define DEL_LIST(lh) \ if (!list_empty(lh)) \ list_del_init(lh); /* Delete stripe from hash. */ static void stripe_hash_del(struct stripe *stripe) { DEL_LIST(stripe->lists + LIST_HASH); } /* Return stripe reference count. */ static inline int stripe_ref(struct stripe *stripe) { return atomic_read(&stripe->cnt); } static void stripe_flush_add(struct stripe *stripe) { struct stripe_cache *sc = stripe->sc; struct list_head *lh = stripe->lists + LIST_FLUSH; if (!StripeReconstruct(stripe) && list_empty(lh)) list_add_tail(lh, sc->lists + LIST_FLUSH); } /* * Add stripe to LRU (inactive) list. * * Need lock, because of concurrent access from message interface. */ static void stripe_lru_add(struct stripe *stripe) { if (!StripeRecover(stripe)) { unsigned long flags; struct list_head *lh = stripe->lists + LIST_LRU; spinlock_t *lock = stripe->sc->locks + LOCK_LRU; spin_lock_irqsave(lock, flags); if (list_empty(lh)) list_add_tail(lh, stripe->sc->lists + LIST_LRU); spin_unlock_irqrestore(lock, flags); } } #define POP_LIST(list) \ do { \ if (list_empty(sc->lists + (list))) \ stripe = NULL; \ else { \ stripe = list_first_entry(sc->lists + (list), \ struct stripe, \ lists[(list)]); \ list_del_init(stripe->lists + (list)); \ } \ } while (0); /* Pop an available stripe off the LRU list. */ static struct stripe *stripe_lru_pop(struct stripe_cache *sc) { struct stripe *stripe; spinlock_t *lock = sc->locks + LOCK_LRU; spin_lock_irq(lock); POP_LIST(LIST_LRU); spin_unlock_irq(lock); return stripe; } /* Pop an available stripe off the io list. */ static struct stripe *stripe_io_pop(struct stripe_cache *sc) { struct stripe *stripe; POP_LIST(LIST_FLUSH); return stripe; } /* Push a stripe safely onto the endio list to be handled by do_endios(). */ static void stripe_endio_push(struct stripe *stripe) { unsigned long flags; struct stripe_cache *sc = stripe->sc; struct list_head *stripe_list = stripe->lists + LIST_ENDIO, *sc_list = sc->lists + LIST_ENDIO; spinlock_t *lock = sc->locks + LOCK_ENDIO; /* This runs in parallel with do_endios(). */ spin_lock_irqsave(lock, flags); if (list_empty(stripe_list)) list_add_tail(stripe_list, sc_list); spin_unlock_irqrestore(lock, flags); wake_do_raid(RS(sc)); /* Wake myself. */ } /* Pop a stripe off safely off the endio list. */ static struct stripe *stripe_endio_pop(struct stripe_cache *sc) { struct stripe *stripe; spinlock_t *lock = sc->locks + LOCK_ENDIO; /* This runs in parallel with endio(). */ spin_lock_irq(lock); POP_LIST(LIST_ENDIO) spin_unlock_irq(lock); return stripe; } #undef POP_LIST /* * Stripe cache locking functions */ /* Dummy lock function for single host RAID4+5. */ static void *no_lock(sector_t key, enum dm_lock_type type) { return &no_lock; } /* Dummy unlock function for single host RAID4+5. */ static void no_unlock(void *lock_handle) { } /* No locking (for single host RAID 4+5). */ static struct dm_raid45_locking_type locking_none = { .lock = no_lock, .unlock = no_unlock, }; /* Lock a stripe (for clustering). */ static int stripe_lock(struct stripe *stripe, int rw, sector_t key) { stripe->lock = RS(stripe->sc)->locking->lock(key, rw == READ ? DM_RAID45_SHARED : DM_RAID45_EX); return stripe->lock ? 0 : -EPERM; } /* Unlock a stripe (for clustering). */ static void stripe_unlock(struct stripe *stripe) { RS(stripe->sc)->locking->unlock(stripe->lock); stripe->lock = NULL; } /* Test io pending on stripe. */ static int stripe_io_ref(struct stripe *stripe) { return atomic_read(&stripe->io.pending); } static void stripe_io_get(struct stripe *stripe) { if (atomic_inc_return(&stripe->io.pending) == 1) /* REMOVEME: statistics */ atomic_inc(&stripe->sc->active_stripes); else BUG_ON(stripe_io_ref(stripe) < 0); } static void stripe_io_put(struct stripe *stripe) { if (atomic_dec_and_test(&stripe->io.pending)) { if (unlikely(StripeRecover(stripe))) /* Don't put recovery stripe on endio list. */ wake_do_raid(RS(stripe->sc)); else /* Add regular stripe to endio list and wake daemon. */ stripe_endio_push(stripe); /* REMOVEME: statistics */ atomic_dec(&stripe->sc->active_stripes); } else BUG_ON(stripe_io_ref(stripe) < 0); } /* Take stripe reference out. */ static int stripe_get(struct stripe *stripe) { int r; struct list_head *lh = stripe->lists + LIST_LRU; spinlock_t *lock = stripe->sc->locks + LOCK_LRU; /* Delete stripe from LRU (inactive) list if on. */ spin_lock_irq(lock); DEL_LIST(lh); spin_unlock_irq(lock); BUG_ON(stripe_ref(stripe) < 0); /* Lock stripe on first reference */ r = (atomic_inc_return(&stripe->cnt) == 1) ? stripe_lock(stripe, WRITE, stripe->key) : 0; return r; } #undef DEL_LIST /* Return references on a chunk. */ static int chunk_ref(struct stripe_chunk *chunk) { return atomic_read(&chunk->cnt); } /* Take out reference on a chunk. */ static int chunk_get(struct stripe_chunk *chunk) { return atomic_inc_return(&chunk->cnt); } /* Drop reference on a chunk. */ static void chunk_put(struct stripe_chunk *chunk) { BUG_ON(atomic_dec_return(&chunk->cnt) < 0); } /* * Drop reference on a stripe. * * Move it to list of LRU stripes if zero. */ static void stripe_put(struct stripe *stripe) { if (atomic_dec_and_test(&stripe->cnt)) { BUG_ON(stripe_io_ref(stripe)); stripe_unlock(stripe); } else BUG_ON(stripe_ref(stripe) < 0); } /* Helper needed by for_each_io_dev(). */ static void stripe_get_references(struct stripe *stripe, unsigned p) { /* * Another one to reference the stripe in * order to protect vs. LRU list moves. */ io_get(RS(stripe->sc)); /* Global io references. */ stripe_get(stripe); stripe_io_get(stripe); /* One for each chunk io. */ } /* Helper for endio() to put all take references. */ static void stripe_put_references(struct stripe *stripe) { stripe_io_put(stripe); /* One for each chunk io. */ stripe_put(stripe); io_put(RS(stripe->sc)); } /* * Stripe cache functions. */ /* * Invalidate all chunks (i.e. their pages) of a stripe. * * I only keep state for the whole chunk. */ static inline void stripe_chunk_invalidate(struct stripe_chunk *chunk) { chunk->io.flags = 0; } static void stripe_chunks_invalidate(struct stripe *stripe) { unsigned p = RS(stripe->sc)->set.raid_devs; while (p--) stripe_chunk_invalidate(CHUNK(stripe, p)); } /* Prepare stripe for (re)use. */ static void stripe_invalidate(struct stripe *stripe) { stripe->io.flags = 0; stripe->idx.parity = stripe->idx.recover = -1; stripe_chunks_invalidate(stripe); } /* * Allow io on all chunks of a stripe. * If not set, IO will not occur; i.e. it's prohibited. * * Actual IO submission for allowed chunks depends * on their !uptodate or dirty state. */ static void stripe_allow_io(struct stripe *stripe) { unsigned p = RS(stripe->sc)->set.raid_devs; while (p--) SetChunkIo(CHUNK(stripe, p)); } /* Initialize a stripe. */ static void stripe_init(struct stripe_cache *sc, struct stripe *stripe) { unsigned i, p = RS(sc)->set.raid_devs; /* Work all io chunks. */ while (p--) { struct stripe_chunk *chunk = CHUNK(stripe, p); atomic_set(&chunk->cnt, 0); chunk->stripe = stripe; i = ARRAY_SIZE(chunk->bl); while (i--) bio_list_init(chunk->bl + i); } stripe->sc = sc; i = ARRAY_SIZE(stripe->lists); while (i--) INIT_LIST_HEAD(stripe->lists + i); stripe->io.size = RS(sc)->set.io_size; atomic_set(&stripe->cnt, 0); atomic_set(&stripe->io.pending, 0); stripe_invalidate(stripe); } /* Number of pages per chunk. */ static inline unsigned chunk_pages(unsigned sectors) { return dm_div_up(sectors, SECTORS_PER_PAGE); } /* Number of pages per stripe. */ static inline unsigned stripe_pages(struct raid_set *rs, unsigned io_size) { return chunk_pages(io_size) * rs->set.raid_devs; } /* Initialize part of page_list (recovery). */ static void stripe_zero_pl_part(struct stripe *stripe, int p, unsigned start, unsigned count) { unsigned o = start / SECTORS_PER_PAGE, pages = chunk_pages(count); /* Get offset into the page_list. */ struct page_list *pl = pl_elem(PL(stripe, p), o); BUG_ON(!pl); while (pl && pages--) { BUG_ON(!pl->page); memset(page_address(pl->page), 0, PAGE_SIZE); pl = pl->next; } } /* Initialize parity chunk of stripe. */ static void stripe_zero_chunk(struct stripe *stripe, int p) { if (p > -1) stripe_zero_pl_part(stripe, p, 0, stripe->io.size); } /* Return dynamic stripe structure size. */ static size_t stripe_size(struct raid_set *rs) { return sizeof(struct stripe) + rs->set.raid_devs * sizeof(struct stripe_chunk); } /* Allocate a stripe and its memory object. */ /* XXX adjust to cope with stripe cache and recovery stripe caches. */ enum grow { SC_GROW, SC_KEEP }; static struct stripe *stripe_alloc(struct stripe_cache *sc, struct dm_mem_cache_client *mc, enum grow grow) { int r; struct stripe *stripe; stripe = kmem_cache_zalloc(sc->kc.cache, GFP_KERNEL); if (stripe) { /* Grow the dm-mem-cache by one object. */ if (grow == SC_GROW) { r = dm_mem_cache_grow(mc, 1); if (r) goto err_free; } stripe->obj = dm_mem_cache_alloc(mc); if (!stripe->obj) goto err_shrink; stripe_init(sc, stripe); } return stripe; err_shrink: if (grow == SC_GROW) dm_mem_cache_shrink(mc, 1); err_free: kmem_cache_free(sc->kc.cache, stripe); return NULL; } /* * Free a stripes memory object, shrink the * memory cache and free the stripe itself. */ static void stripe_free(struct stripe *stripe, struct dm_mem_cache_client *mc) { dm_mem_cache_free(mc, stripe->obj); dm_mem_cache_shrink(mc, 1); kmem_cache_free(stripe->sc->kc.cache, stripe); } /* Free the recovery stripe. */ static void stripe_recover_free(struct raid_set *rs) { struct recover *rec = &rs->recover; struct dm_mem_cache_client *mc; mc = rec->mem_cache_client; rec->mem_cache_client = NULL; if (mc) { struct stripe *stripe; while (!list_empty(&rec->stripes)) { stripe = list_first_entry(&rec->stripes, struct stripe, lists[LIST_RECOVER]); list_del(stripe->lists + LIST_RECOVER); kfree(stripe->recover); stripe_free(stripe, mc); } dm_mem_cache_client_destroy(mc); dm_io_client_destroy(rec->dm_io_client); rec->dm_io_client = NULL; } } /* Grow stripe cache. */ static int sc_grow(struct stripe_cache *sc, unsigned stripes, enum grow grow) { int r = 0; /* Try to allocate this many (additional) stripes. */ while (stripes--) { struct stripe *stripe = stripe_alloc(sc, sc->mem_cache_client, grow); if (likely(stripe)) { stripe_lru_add(stripe); atomic_inc(&sc->stripes); } else { r = -ENOMEM; break; } } return r ? r : sc_hash_resize(sc); } /* Shrink stripe cache. */ static int sc_shrink(struct stripe_cache *sc, unsigned stripes) { int r = 0; /* Try to get unused stripe from LRU list. */ while (stripes--) { struct stripe *stripe; stripe = stripe_lru_pop(sc); if (stripe) { /* An LRU stripe may never have ios pending! */ BUG_ON(stripe_io_ref(stripe)); BUG_ON(stripe_ref(stripe)); atomic_dec(&sc->stripes); /* Remove from hash if on before deletion. */ stripe_hash_del(stripe); stripe_free(stripe, sc->mem_cache_client); } else { r = -ENOENT; break; } } /* Check if stats are still sane. */ if (atomic_read(&sc->active_stripes_max) > atomic_read(&sc->stripes)) atomic_set(&sc->active_stripes_max, 0); if (r) return r; return atomic_read(&sc->stripes) ? sc_hash_resize(sc) : 0; } /* Create stripe cache and recovery. */ static int sc_init(struct raid_set *rs, unsigned stripes) { unsigned i, r, rstripes; struct stripe_cache *sc = &rs->sc; struct stripe *stripe; struct recover *rec = &rs->recover; struct mapped_device *md; struct gendisk *disk; /* Initialize lists and locks. */ i = ARRAY_SIZE(sc->lists); while (i--) INIT_LIST_HEAD(sc->lists + i); INIT_LIST_HEAD(&rec->stripes); /* Initialize endio and LRU list locks. */ i = NR_LOCKS; while (i--) spin_lock_init(sc->locks + i); /* Initialize atomic variables. */ atomic_set(&sc->stripes, 0); atomic_set(&sc->stripes_to_set, 0); atomic_set(&sc->active_stripes, 0); atomic_set(&sc->active_stripes_max, 0); /* REMOVEME: statistics. */ /* * We need a runtime unique # to suffix the kmem cache name * because we'll have one for each active RAID set. */ md = dm_table_get_md(rs->ti->table); disk = dm_disk(md); sprintf(sc->kc.name, "%s-%d", TARGET, disk->first_minor); dm_put(md); sc->kc.cache = kmem_cache_create(sc->kc.name, stripe_size(rs), 0, 0, NULL); if (!sc->kc.cache) return -ENOMEM; /* Create memory cache client context for RAID stripe cache. */ sc->mem_cache_client = dm_mem_cache_client_create(stripes, rs->set.raid_devs, chunk_pages(rs->set.io_size)); if (IS_ERR(sc->mem_cache_client)) return PTR_ERR(sc->mem_cache_client); /* Create memory cache client context for RAID recovery stripe(s). */ rstripes = rec->recovery_stripes; rec->mem_cache_client = dm_mem_cache_client_create(rstripes, rs->set.raid_devs, chunk_pages(rec->io_size)); if (IS_ERR(rec->mem_cache_client)) return PTR_ERR(rec->mem_cache_client); /* Create dm-io client context for IO stripes. */ sc->dm_io_client = dm_io_client_create((stripes > 32 ? 32 : stripes) * rs->set.raid_devs * chunk_pages(rs->set.io_size)); if (IS_ERR(sc->dm_io_client)) return PTR_ERR(sc->dm_io_client); /* FIXME: intermingeled with stripe cache initialization. */ /* Create dm-io client context for recovery stripes. */ rec->dm_io_client = dm_io_client_create(rstripes * rs->set.raid_devs * chunk_pages(rec->io_size)); if (IS_ERR(rec->dm_io_client)) return PTR_ERR(rec->dm_io_client); /* Allocate stripes for set recovery. */ while (rstripes--) { stripe = stripe_alloc(sc, rec->mem_cache_client, SC_KEEP); if (!stripe) return -ENOMEM; stripe->recover = kzalloc(sizeof(*stripe->recover), GFP_KERNEL); if (!stripe->recover) { stripe_free(stripe, rec->mem_cache_client); return -ENOMEM; } SetStripeRecover(stripe); stripe->io.size = rec->io_size; list_add_tail(stripe->lists + LIST_RECOVER, &rec->stripes); /* Don't add recovery stripes to LRU list! */ } /* * Allocate the stripe objetcs from the * cache and add them to the LRU list. */ r = sc_grow(sc, stripes, SC_KEEP); if (!r) atomic_set(&sc->stripes_last, stripes); return r; } /* Destroy the stripe cache. */ static void sc_exit(struct stripe_cache *sc) { struct raid_set *rs = RS(sc); if (sc->kc.cache) { stripe_recover_free(rs); BUG_ON(sc_shrink(sc, atomic_read(&sc->stripes))); kmem_cache_destroy(sc->kc.cache); sc->kc.cache = NULL; if (sc->mem_cache_client && !IS_ERR(sc->mem_cache_client)) dm_mem_cache_client_destroy(sc->mem_cache_client); if (sc->dm_io_client && !IS_ERR(sc->dm_io_client)) dm_io_client_destroy(sc->dm_io_client); hash_exit(&sc->hash); } } /* * Calculate RAID address * * Delivers tuple with the index of the data disk holding the chunk * in the set, the parity disks index and the start of the stripe * within the address space of the set (used as the stripe cache hash key). */ /* thx MD. */ static struct raid_address *raid_address(struct raid_set *rs, sector_t sector, struct raid_address *addr) { sector_t stripe, tmp; /* * chunk_number = sector / chunk_size * stripe_number = chunk_number / data_devs * di = stripe % data_devs; */ stripe = sector >> rs->set.chunk_shift; addr->di = sector_div(stripe, rs->set.data_devs); switch (rs->set.raid_type->level) { case raid4: addr->pi = rs->set.pi; goto check_shift_di; case raid5: tmp = stripe; addr->pi = sector_div(tmp, rs->set.raid_devs); switch (rs->set.raid_type->algorithm) { case left_asym: /* Left asymmetric. */ addr->pi = rs->set.data_devs - addr->pi; case right_asym: /* Right asymmetric. */ check_shift_di: if (addr->di >= addr->pi) addr->di++; break; case left_sym: /* Left symmetric. */ addr->pi = rs->set.data_devs - addr->pi; case right_sym: /* Right symmetric. */ addr->di = (addr->pi + addr->di + 1) % rs->set.raid_devs; break; case none: /* Ain't happen: RAID4 algorithm placeholder. */ BUG(); } } /* * Start offset of the stripes chunk on any single device of the RAID * set, adjusted in case io size differs from chunk size. */ addr->key = (stripe << rs->set.chunk_shift) + (sector & rs->set.io_inv_mask); return addr; } /* * Copy data across between stripe pages and bio vectors. * * Pay attention to data alignment in stripe and bio pages. */ static void bio_copy_page_list(int rw, struct stripe *stripe, struct page_list *pl, struct bio *bio) { unsigned i, page_offset; void *page_addr; struct raid_set *rs = RS(stripe->sc); struct bio_vec *bv; /* Get start page in page list for this sector. */ i = (bio->bi_sector & rs->set.io_mask) / SECTORS_PER_PAGE; pl = pl_elem(pl, i); BUG_ON(!pl); BUG_ON(!pl->page); page_addr = page_address(pl->page); page_offset = to_bytes(bio->bi_sector & (SECTORS_PER_PAGE - 1)); /* Walk all segments and copy data across between bio_vecs and pages. */ bio_for_each_segment(bv, bio, i) { int len = bv->bv_len, size; unsigned bio_offset = 0; void *bio_addr = __bio_kmap_atomic(bio, i, KM_USER0); redo: size = (page_offset + len > PAGE_SIZE) ? PAGE_SIZE - page_offset : len; if (rw == READ) memcpy(bio_addr + bio_offset, page_addr + page_offset, size); else memcpy(page_addr + page_offset, bio_addr + bio_offset, size); page_offset += size; if (page_offset == PAGE_SIZE) { /* * We reached the end of the chunk page -> * need to refer to the next one to copy more data. */ len -= size; if (len) { /* Get next page. */ pl = pl->next; BUG_ON(!pl); BUG_ON(!pl->page); page_addr = page_address(pl->page); page_offset = 0; bio_offset += size; /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_BIO_COPY_PL_NEXT); goto redo; } } __bio_kunmap_atomic(bio_addr, KM_USER0); } } /* * Xor optimization macros. */ /* Xor data pointer declaration and initialization macros. */ #define DECLARE_2 unsigned long *d0 = data[0], *d1 = data[1] #define DECLARE_3 DECLARE_2, *d2 = data[2] #define DECLARE_4 DECLARE_3, *d3 = data[3] #define DECLARE_5 DECLARE_4, *d4 = data[4] #define DECLARE_6 DECLARE_5, *d5 = data[5] #define DECLARE_7 DECLARE_6, *d6 = data[6] #define DECLARE_8 DECLARE_7, *d7 = data[7] /* Xor unrole macros. */ #define D2(n) d0[n] = d0[n] ^ d1[n] #define D3(n) D2(n) ^ d2[n] #define D4(n) D3(n) ^ d3[n] #define D5(n) D4(n) ^ d4[n] #define D6(n) D5(n) ^ d5[n] #define D7(n) D6(n) ^ d6[n] #define D8(n) D7(n) ^ d7[n] #define X_2(macro, offset) macro(offset); macro(offset + 1); #define X_4(macro, offset) X_2(macro, offset); X_2(macro, offset + 2); #define X_8(macro, offset) X_4(macro, offset); X_4(macro, offset + 4); #define X_16(macro, offset) X_8(macro, offset); X_8(macro, offset + 8); #define X_32(macro, offset) X_16(macro, offset); X_16(macro, offset + 16); #define X_64(macro, offset) X_32(macro, offset); X_32(macro, offset + 32); /* Define a _xor_#chunks_#xors_per_run() function. */ #define _XOR(chunks, xors_per_run) \ static void _xor ## chunks ## _ ## xors_per_run(unsigned long **data) \ { \ unsigned end = XOR_SIZE / sizeof(data[0]), i; \ DECLARE_ ## chunks; \ \ for (i = 0; i < end; i += xors_per_run) { \ X_ ## xors_per_run(D ## chunks, i); \ } \ } /* Define xor functions for 2 - 8 chunks and xors per run. */ #define MAKE_XOR_PER_RUN(xors_per_run) \ _XOR(2, xors_per_run); _XOR(3, xors_per_run); \ _XOR(4, xors_per_run); _XOR(5, xors_per_run); \ _XOR(6, xors_per_run); _XOR(7, xors_per_run); \ _XOR(8, xors_per_run); MAKE_XOR_PER_RUN(8) /* Define _xor_*_8() functions. */ MAKE_XOR_PER_RUN(16) /* Define _xor_*_16() functions. */ MAKE_XOR_PER_RUN(32) /* Define _xor_*_32() functions. */ MAKE_XOR_PER_RUN(64) /* Define _xor_*_64() functions. */ #define MAKE_XOR(xors_per_run) \ struct { \ void (*f)(unsigned long **); \ } static xor_funcs ## xors_per_run[] = { \ { NULL }, /* NULL pointers to optimize indexing in xor(). */ \ { NULL }, \ { _xor2_ ## xors_per_run }, \ { _xor3_ ## xors_per_run }, \ { _xor4_ ## xors_per_run }, \ { _xor5_ ## xors_per_run }, \ { _xor6_ ## xors_per_run }, \ { _xor7_ ## xors_per_run }, \ { _xor8_ ## xors_per_run }, \ }; \ \ static void xor_ ## xors_per_run(unsigned n, unsigned long **data) \ { \ /* Call respective function for amount of chunks. */ \ xor_funcs ## xors_per_run[n].f(data); \ } /* Define xor_8() - xor_64 functions. */ MAKE_XOR(8) MAKE_XOR(16) MAKE_XOR(32) MAKE_XOR(64) /* Maximum number of chunks, which can be xor'ed in one go. */ #define XOR_CHUNKS_MAX (ARRAY_SIZE(xor_funcs8) - 1) static void xor_blocks_wrapper(unsigned n, unsigned long **data) { BUG_ON(n < 2 || n > MAX_XOR_BLOCKS + 1); xor_blocks(n - 1, XOR_SIZE, (void *) data[0], (void **) data + 1); } struct xor_func { xor_function_t f; const char *name; } static xor_funcs[] = { { xor_8, "xor_8" }, { xor_16, "xor_16" }, { xor_32, "xor_32" }, { xor_64, "xor_64" }, { xor_blocks_wrapper, "xor_blocks" }, }; /* * Check, if chunk has to be xored in/out: * * o if writes are queued * o if writes are merged * o if stripe is to be reconstructed * o if recovery stripe */ static inline int chunk_must_xor(struct stripe_chunk *chunk) { if (ChunkUptodate(chunk)) { BUG_ON(!bio_list_empty(BL_CHUNK(chunk, WRITE_QUEUED)) && !bio_list_empty(BL_CHUNK(chunk, WRITE_MERGED))); if (!bio_list_empty(BL_CHUNK(chunk, WRITE_QUEUED)) || !bio_list_empty(BL_CHUNK(chunk, WRITE_MERGED))) return 1; if (StripeReconstruct(chunk->stripe) || StripeRecover(chunk->stripe)) return 1; } return 0; } /* * Calculate crc. * * This indexes into the chunks of a stripe and their pages. * * All chunks will be xored into the indexed (@pi) * chunk in maximum groups of xor.chunks. * */ static void xor(struct stripe *stripe, unsigned pi, unsigned sector) { struct raid_set *rs = RS(stripe->sc); unsigned max_chunks = rs->xor.chunks, n = 1, o = sector / SECTORS_PER_PAGE, /* Offset into the page_list. */ p = rs->set.raid_devs; unsigned long **d = rs->data; xor_function_t xor_f = rs->xor.f->f; BUG_ON(sector > stripe->io.size); /* Address of parity page to xor into. */ d[0] = page_address(pl_elem(PL(stripe, pi), o)->page); while (p--) { /* Preset pointers to data pages. */ if (p != pi && chunk_must_xor(CHUNK(stripe, p))) d[n++] = page_address(pl_elem(PL(stripe, p), o)->page); /* If max chunks -> xor. */ if (n == max_chunks) { xor_f(n, d); n = 1; } } /* If chunks -> xor. */ if (n > 1) xor_f(n, d); } /* Common xor loop through all stripe page lists. */ static void common_xor(struct stripe *stripe, sector_t count, unsigned off, unsigned pi) { unsigned sector; BUG_ON(!count); for (sector = off; sector < count; sector += SECTORS_PER_PAGE) xor(stripe, pi, sector); /* Set parity page uptodate and clean. */ chunk_set(CHUNK(stripe, pi), CLEAN); atomic_inc(RS(stripe->sc)->stats + S_XORS); /* REMOVEME: statistics. */ } /* * Calculate parity sectors on intact stripes. * * Need to calculate raid address for recover stripe, because its * chunk sizes differs and is typically larger than io chunk size. */ static void parity_xor(struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); unsigned chunk_size = rs->set.chunk_size, io_size = stripe->io.size, xor_size = chunk_size > io_size ? io_size : chunk_size; sector_t off; /* This can be the recover stripe with a larger io size. */ for (off = 0; off < io_size; off += xor_size) { /* * Recover stripe is likely bigger than regular io * ones and has no precalculated parity disk index -> * need to calculate RAID address. */ if (unlikely(StripeRecover(stripe))) { struct raid_address addr; raid_address(rs, (stripe->key + off) * rs->set.data_devs, &addr); stripe->idx.parity = addr.pi; stripe_zero_pl_part(stripe, addr.pi, off, xor_size); } common_xor(stripe, xor_size, off, stripe->idx.parity); chunk_set(CHUNK(stripe, stripe->idx.parity), DIRTY); } } /* Reconstruct missing chunk. */ static void stripe_reconstruct(struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); int p = rs->set.raid_devs, pr = stripe->idx.recover; BUG_ON(pr < 0); /* Check if all but the chunk to be reconstructed are uptodate. */ while (p--) BUG_ON(p != pr && !ChunkUptodate(CHUNK(stripe, p))); /* REMOVEME: statistics. */ atomic_inc(rs->stats + (RSDegraded(rs) ? S_RECONSTRUCT_EI : S_RECONSTRUCT_DEV)); /* Zero chunk to be reconstructed. */ stripe_zero_chunk(stripe, pr); common_xor(stripe, stripe->io.size, 0, pr); stripe->idx.recover = -1; } /* * Recovery io throttling */ /* Conditionally reset io counters. */ static int recover_io_reset(struct raid_set *rs) { unsigned long j = jiffies; /* Pay attention to jiffies overflows. */ if (j > rs->recover.last_jiffies + HZ / 20 || j < rs->recover.last_jiffies) { atomic_set(rs->recover.io_count + IO_WORK, 0); atomic_set(rs->recover.io_count + IO_RECOVER, 0); rs->recover.last_jiffies = j; return 1; } return 0; } /* Count ios. */ static void recover_io_count(struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); recover_io_reset(rs); atomic_inc(rs->recover.io_count + (StripeRecover(stripe) ? IO_RECOVER : IO_WORK)); } /* Try getting a stripe either from the hash or from the LRU list. */ static struct stripe *stripe_find(struct raid_set *rs, struct raid_address *addr) { int r; struct stripe_cache *sc = &rs->sc; struct stripe *stripe; /* Try stripe from hash. */ stripe = stripe_lookup(sc, addr->key); if (stripe) { r = stripe_get(stripe); if (r) goto get_lock_failed; atomic_inc(rs->stats + S_HITS_1ST); /* REMOVEME: statistics. */ } else { /* Not in hash -> try to get an LRU stripe. */ stripe = stripe_lru_pop(sc); if (stripe) { /* * An LRU stripe may not be referenced * and may never have ios pending! */ BUG_ON(stripe_ref(stripe)); BUG_ON(stripe_io_ref(stripe)); /* Remove from hash if on before reuse. */ stripe_hash_del(stripe); /* Invalidate before reinserting with changed key. */ stripe_invalidate(stripe); stripe->key = addr->key; stripe->region = dm_rh_sector_to_region(rs->recover.rh, addr->key); stripe->idx.parity = addr->pi; r = stripe_get(stripe); if (r) goto get_lock_failed; /* Insert stripe into the stripe hash. */ stripe_insert(&sc->hash, stripe); /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_INSCACHE); } } return stripe; get_lock_failed: stripe_put(stripe); return NULL; } /* * Process end io * * I need to do it here because I can't in interrupt */ /* End io all bios on a bio list. */ static void bio_list_endio(struct stripe *stripe, struct bio_list *bl, int p, int error) { struct raid_set *rs = RS(stripe->sc); struct bio *bio; struct page_list *pl = PL(stripe, p); struct stripe_chunk *chunk = CHUNK(stripe, p); /* Update region counters. */ while ((bio = bio_list_pop(bl))) { if (bio_data_dir(bio) == WRITE) /* Drop io pending count for any writes. */ dm_rh_dec(rs->recover.rh, stripe->region); else if (!error) /* Copy data accross. */ bio_copy_page_list(READ, stripe, pl, bio); bio_endio(bio, error); /* REMOVEME: statistics. */ atomic_inc(rs->stats + (bio_data_dir(bio) == READ ? S_BIOS_ENDIO_READ : S_BIOS_ENDIO_WRITE)); chunk_put(chunk); stripe_put(stripe); io_put(rs); /* Wake any suspend waiters on last bio. */ } } /* * End io all reads/writes on a stripe copying * read data accross from stripe to bios and * decrementing region counters for writes. * * Processing of ios depeding on state: * o no chunk error -> endio ok * o degraded: * - chunk error and read -> ignore to be requeued * - chunk error and write -> endio ok * o dead (more than parity_devs failed) and chunk_error-> endio failed */ static void stripe_endio(int rw, struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); unsigned p = rs->set.raid_devs; int write = (rw != READ); while (p--) { struct stripe_chunk *chunk = CHUNK(stripe, p); struct bio_list *bl; BUG_ON(ChunkLocked(chunk)); bl = BL_CHUNK(chunk, rw); if (bio_list_empty(bl)) continue; if (unlikely(ChunkError(chunk) || !ChunkUptodate(chunk))) { /* RAID set dead. */ if (unlikely(RSDead(rs))) bio_list_endio(stripe, bl, p, -EIO); /* RAID set degraded. */ else if (write) bio_list_endio(stripe, bl, p, 0); } else { BUG_ON(!RSDegraded(rs) && ChunkDirty(chunk)); bio_list_endio(stripe, bl, p, 0); } } } /* Fail all ios hanging off all bio lists of a stripe. */ static void stripe_fail_io(struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); unsigned p = rs->set.raid_devs; while (p--) { struct stripe_chunk *chunk = CHUNK(stripe, p); int i = ARRAY_SIZE(chunk->bl); /* Fail all bios on all bio lists of the stripe. */ while (i--) { struct bio_list *bl = chunk->bl + i; if (!bio_list_empty(bl)) bio_list_endio(stripe, bl, p, -EIO); } } /* Put stripe on LRU list. */ BUG_ON(stripe_io_ref(stripe)); BUG_ON(stripe_ref(stripe)); } /* Unlock all required chunks. */ static void stripe_chunks_unlock(struct stripe *stripe) { unsigned p = RS(stripe->sc)->set.raid_devs; struct stripe_chunk *chunk; while (p--) { chunk = CHUNK(stripe, p); if (TestClearChunkUnlock(chunk)) ClearChunkLocked(chunk); } } /* * Queue reads and writes to a stripe by hanging * their bios off the stripesets read/write lists. */ static int stripe_queue_bio(struct raid_set *rs, struct bio *bio, struct bio_list *reject) { struct raid_address addr; struct stripe *stripe; stripe = stripe_find(rs, raid_address(rs, bio->bi_sector, &addr)); if (stripe) { int r = 0, rw = bio_data_dir(bio); /* Distinguish reads and writes. */ bio_list_add(BL(stripe, addr.di, rw), bio); if (rw == READ) /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_BIOS_ADDED_READ); else { /* Inrement pending write count on region. */ dm_rh_inc(rs->recover.rh, stripe->region); r = 1; /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_BIOS_ADDED_WRITE); } /* * Put on io (flush) list in case of * initial bio queued to chunk. */ if (chunk_get(CHUNK(stripe, addr.di)) == 1) stripe_flush_add(stripe); return r; } /* Got no stripe from cache or failed to lock it -> reject bio. */ bio_list_add(reject, bio); atomic_inc(rs->stats + S_IOS_POST); /* REMOVEME: statistics. */ return 0; } /* * Handle all stripes by handing them to the daemon, because we can't * map their chunk pages to copy the data in interrupt context. * * We don't want to handle them here either, while interrupts are disabled. */ /* Read/write endio function for dm-io (interrupt context). */ static void endio(unsigned long error, void *context) { struct stripe_chunk *chunk = context; if (unlikely(error)) { chunk_set(chunk, ERROR); /* REMOVEME: statistics. */ atomic_inc(RS(chunk->stripe->sc)->stats + S_STRIPE_ERROR); } else chunk_set(chunk, CLEAN); /* * For recovery stripes, I need to reset locked locked * here, because those aren't processed in do_endios(). */ if (unlikely(StripeRecover(chunk->stripe))) ClearChunkLocked(chunk); else SetChunkUnlock(chunk); /* Indirectly puts stripe on cache's endio list via stripe_io_put(). */ stripe_put_references(chunk->stripe); } /* Read/Write a chunk asynchronously. */ static void stripe_chunk_rw(struct stripe *stripe, unsigned p) { struct stripe_cache *sc = stripe->sc; struct raid_set *rs = RS(sc); struct dm_mem_cache_object *obj = stripe->obj + p; struct page_list *pl = obj->pl; struct stripe_chunk *chunk = CHUNK(stripe, p); struct raid_dev *dev = rs->dev + p; struct dm_io_region io = { .bdev = dev->dev->bdev, .sector = stripe->key, .count = stripe->io.size, }; struct dm_io_request control = { .bi_rw = ChunkDirty(chunk) ? WRITE : READ, .mem = { .type = DM_IO_PAGE_LIST, .ptr.pl = pl, .offset = 0, }, .notify = { .fn = endio, .context = chunk, }, .client = StripeRecover(stripe) ? rs->recover.dm_io_client : sc->dm_io_client, }; BUG_ON(ChunkLocked(chunk)); BUG_ON(!ChunkUptodate(chunk) && ChunkDirty(chunk)); BUG_ON(ChunkUptodate(chunk) && !ChunkDirty(chunk)); /* * Don't rw past end of device, which can happen, because * typically sectors_per_dev isn't divisible by io_size. */ if (unlikely(io.sector + io.count > rs->set.sectors_per_dev)) io.count = rs->set.sectors_per_dev - io.sector; BUG_ON(!io.count); io.sector += dev->start; /* Add <offset>. */ if (RSRecover(rs)) recover_io_count(stripe); /* Recovery io accounting. */ /* REMOVEME: statistics. */ atomic_inc(rs->stats + (ChunkDirty(chunk) ? S_DM_IO_WRITE : S_DM_IO_READ)); SetChunkLocked(chunk); SetDevIoQueued(dev); BUG_ON(dm_io(&control, 1, &io, NULL)); } /* * Write dirty or read not uptodate page lists of a stripe. */ static int stripe_chunks_rw(struct stripe *stripe) { int r; struct raid_set *rs = RS(stripe->sc); /* * Increment the pending count on the stripe * first, so that we don't race in endio(). * * An inc (IO) is needed for any chunk unless !ChunkIo(chunk): * * o not uptodate * o dirtied by writes merged * o dirtied by parity calculations */ r = for_each_io_dev(stripe, stripe_get_references); if (r) { /* Io needed: chunks are either not uptodate or dirty. */ int max; /* REMOVEME: */ struct stripe_cache *sc = &rs->sc; /* Submit actual io. */ for_each_io_dev(stripe, stripe_chunk_rw); /* REMOVEME: statistics */ max = sc_active(sc); if (atomic_read(&sc->active_stripes_max) < max) atomic_set(&sc->active_stripes_max, max); atomic_inc(rs->stats + S_FLUSHS); /* END REMOVEME: statistics */ } return r; } /* Merge in all writes hence dirtying respective chunks. */ static void stripe_merge_writes(struct stripe *stripe) { unsigned p = RS(stripe->sc)->set.raid_devs; while (p--) { struct stripe_chunk *chunk = CHUNK(stripe, p); struct bio_list *write = BL_CHUNK(chunk, WRITE_QUEUED); if (!bio_list_empty(write)) { struct bio *bio; struct page_list *pl = stripe->obj[p].pl; /* * We can play with the lists without holding a lock, * because it is just us accessing them anyway. */ bio_list_for_each(bio, write) bio_copy_page_list(WRITE, stripe, pl, bio); bio_list_merge(BL_CHUNK(chunk, WRITE_MERGED), write); bio_list_init(write); chunk_set(chunk, DIRTY); } } } /* Queue all writes to get merged. */ static int stripe_queue_writes(struct stripe *stripe) { int r = 0; unsigned p = RS(stripe->sc)->set.raid_devs; while (p--) { struct stripe_chunk *chunk = CHUNK(stripe, p); struct bio_list *write = BL_CHUNK(chunk, WRITE); if (!bio_list_empty(write)) { bio_list_merge(BL_CHUNK(chunk, WRITE_QUEUED), write); bio_list_init(write); SetChunkIo(chunk); r = 1; } } return r; } /* Check, if a chunk gets completely overwritten. */ static int stripe_check_chunk_overwrite(struct stripe *stripe, unsigned p) { unsigned sectors = 0; struct bio *bio; struct bio_list *bl = BL(stripe, p, WRITE_QUEUED); bio_list_for_each(bio, bl) sectors += bio_sectors(bio); BUG_ON(sectors > RS(stripe->sc)->set.io_size); return sectors == RS(stripe->sc)->set.io_size; } /* * Avoid io on broken/reconstructed drive in order to * reconstruct date on endio. * * (*1*) We set StripeReconstruct() in here, so that _do_endios() * will trigger a reconstruct call before resetting it. */ static int stripe_chunk_set_io_flags(struct stripe *stripe, int pr) { struct stripe_chunk *chunk = CHUNK(stripe, pr); /* * Allow io on all chunks but the indexed one, * because we're either degraded or prohibit it * on the one for later reconstruction. */ /* Includes ClearChunkIo(), ClearChunkUptodate(). */ stripe_chunk_invalidate(chunk); stripe->idx.recover = pr; SetStripeReconstruct(stripe); /* REMOVEME: statistics. */ atomic_inc(RS(stripe->sc)->stats + S_PROHIBITCHUNKIO); return -EPERM; } /* Chunk locked/uptodate and device failed tests. */ static struct stripe_chunk * stripe_chunk_check(struct stripe *stripe, unsigned p, unsigned *chunks_uptodate) { struct raid_set *rs = RS(stripe->sc); struct stripe_chunk *chunk = CHUNK(stripe, p); /* Can't access active chunks. */ if (ChunkLocked(chunk)) { /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_CHUNK_LOCKED); return NULL; } /* Can't access broken devive. */ if (ChunkError(chunk) || DevFailed(rs->dev + p)) return NULL; /* Can access uptodate chunks. */ if (ChunkUptodate(chunk)) { (*chunks_uptodate)++; return NULL; } return chunk; } /* * Degraded/reconstruction mode. * * Check stripe state to figure which chunks don't need IO. * * Returns 0 for fully operational, -EPERM for degraded/resynchronizing. */ static int stripe_check_reconstruct(struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); if (RSDead(rs)) { ClearStripeReconstruct(stripe); ClearStripeReconstructed(stripe); stripe_allow_io(stripe); return 0; } /* Avoid further reconstruction setting, when already set. */ if (StripeReconstruct(stripe)) { /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_RECONSTRUCT_SET); return -EBUSY; } /* Initially allow io on all chunks. */ stripe_allow_io(stripe); /* Return if stripe is already reconstructed. */ if (StripeReconstructed(stripe)) { atomic_inc(rs->stats + S_RECONSTRUCTED); return 0; } /* * Degraded/reconstruction mode (device failed) -> * avoid io on the failed device. */ if (unlikely(RSDegraded(rs))) { /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_DEGRADED); /* Allow IO on all devices but the dead one. */ BUG_ON(rs->set.ei < 0); return stripe_chunk_set_io_flags(stripe, rs->set.ei); } else { int sync, pi = dev_for_parity(stripe, &sync); /* * Reconstruction mode (ie. a particular (replaced) device or * some (rotating) parity chunk is being resynchronized) -> * o make sure all needed chunks are read in * o writes are allowed to go through */ if (!sync) { /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_NOSYNC); /* Allow IO on all devs but the one to reconstruct. */ return stripe_chunk_set_io_flags(stripe, pi); } } return 0; } /* * Check, if stripe is ready to merge writes. * I.e. if all chunks present to allow to merge bios. * * We prohibit io on: * * o chunks without bios * o chunks which get completely written over */ static int stripe_merge_possible(struct stripe *stripe, int nosync) { struct raid_set *rs = RS(stripe->sc); unsigned chunks_overwrite = 0, chunks_prohibited = 0, chunks_uptodate = 0, p = rs->set.raid_devs; /* Walk all chunks. */ while (p--) { struct stripe_chunk *chunk; /* Prohibit io on broken devices. */ if (DevFailed(rs->dev + p)) { chunk = CHUNK(stripe, p); goto prohibit_io; } /* We can't optimize any further if no chunk. */ chunk = stripe_chunk_check(stripe, p, &chunks_uptodate); if (!chunk || nosync) continue; /* * We have a chunk, which is not uptodate. * * If this is not parity and we don't have * reads queued, we can optimize further. */ if (p != stripe->idx.parity && bio_list_empty(BL_CHUNK(chunk, READ)) && bio_list_empty(BL_CHUNK(chunk, WRITE_MERGED))) { if (bio_list_empty(BL_CHUNK(chunk, WRITE_QUEUED))) goto prohibit_io; else if (RSCheckOverwrite(rs) && stripe_check_chunk_overwrite(stripe, p)) /* Completely overwritten chunk. */ chunks_overwrite++; } /* Allow io for chunks with bios and overwritten ones. */ SetChunkIo(chunk); continue; prohibit_io: /* No io for broken devices or for chunks w/o bios. */ ClearChunkIo(chunk); chunks_prohibited++; /* REMOVEME: statistics. */ atomic_inc(RS(stripe->sc)->stats + S_PROHIBITCHUNKIO); } /* All data chunks will get written over. */ if (chunks_overwrite == rs->set.data_devs) atomic_inc(rs->stats + S_OVERWRITE); /* REMOVEME: statistics.*/ else if (chunks_uptodate + chunks_prohibited < rs->set.raid_devs) { /* We don't have enough chunks to merge. */ atomic_inc(rs->stats + S_CANT_MERGE); /* REMOVEME: statistics.*/ return -EPERM; } /* * If we have all chunks up to date or overwrite them, we * just zero the parity chunk and let stripe_rw() recreate it. */ if (chunks_uptodate == rs->set.raid_devs || chunks_overwrite == rs->set.data_devs) { stripe_zero_chunk(stripe, stripe->idx.parity); BUG_ON(StripeReconstruct(stripe)); SetStripeReconstruct(stripe); /* Enforce xor in caller. */ } else { /* * With less chunks, we xor parity out. * * (*4*) We rely on !StripeReconstruct() in chunk_must_xor(), * so that only chunks with queued or merged writes * are being xored. */ parity_xor(stripe); } /* * We do have enough chunks to merge. * All chunks are uptodate or get written over. */ atomic_inc(rs->stats + S_CAN_MERGE); /* REMOVEME: statistics. */ return 0; } /* * Avoid reading chunks in case we're fully operational. * * We prohibit io on any chunks without bios but the parity chunk. */ static void stripe_avoid_reads(struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); unsigned dummy = 0, p = rs->set.raid_devs; /* Walk all chunks. */ while (p--) { struct stripe_chunk *chunk = stripe_chunk_check(stripe, p, &dummy); if (!chunk) continue; /* If parity or any bios pending -> allow io. */ if (chunk_ref(chunk) || p == stripe->idx.parity) SetChunkIo(chunk); else { ClearChunkIo(chunk); /* REMOVEME: statistics. */ atomic_inc(RS(stripe->sc)->stats + S_PROHIBITCHUNKIO); } } } /* * Read/write a stripe. * * All stripe read/write activity goes through this function * unless recovery, which has to call stripe_chunk_rw() directly. * * Make sure we don't try already merged stripes in order * to avoid data corruption. * * Check the state of the RAID set and if degraded (or * resynchronizing for reads), read in all other chunks but * the one on the dead/resynchronizing device in order to be * able to reconstruct the missing one in _do_endios(). * * Can be called on active stripes in order * to dispatch new io on inactive chunks. * * States to cover: * o stripe to read and/or write * o stripe with error to reconstruct */ static void stripe_rw(struct stripe *stripe) { int nosync, r; struct raid_set *rs = RS(stripe->sc); /* * Check, if a chunk needs to be reconstructed * because of a degraded set or a region out of sync. */ nosync = stripe_check_reconstruct(stripe); switch (nosync) { case -EBUSY: return; /* Wait for stripe reconstruction to finish. */ case -EPERM: goto io; } /* * If we don't have merged writes pending, we can schedule * queued writes to be merged next without corrupting data. */ if (!StripeMerged(stripe)) { r = stripe_queue_writes(stripe); if (r) /* Writes got queued -> flag RBW. */ SetStripeRBW(stripe); } /* * Merge all writes hanging off uptodate/overwritten * chunks of the stripe. */ if (StripeRBW(stripe)) { r = stripe_merge_possible(stripe, nosync); if (!r) { /* Merge possible. */ struct stripe_chunk *chunk; /* * I rely on valid parity in order * to xor a fraction of chunks out * of parity and back in. */ stripe_merge_writes(stripe); /* Merge writes in. */ parity_xor(stripe); /* Update parity. */ ClearStripeReconstruct(stripe); /* Reset xor enforce. */ SetStripeMerged(stripe); /* Writes merged. */ ClearStripeRBW(stripe); /* Disable RBW. */ /* * REMOVEME: sanity check on parity chunk * states after writes got merged. */ chunk = CHUNK(stripe, stripe->idx.parity); BUG_ON(ChunkLocked(chunk)); BUG_ON(!ChunkUptodate(chunk)); BUG_ON(!ChunkDirty(chunk)); BUG_ON(!ChunkIo(chunk)); } } else if (!nosync && !StripeMerged(stripe)) /* Read avoidance if not degraded/resynchronizing/merged. */ stripe_avoid_reads(stripe); io: /* Now submit any reads/writes for non-uptodate or dirty chunks. */ r = stripe_chunks_rw(stripe); if (!r) { /* * No io submitted because of chunk io * prohibited or locked chunks/failed devices * -> push to end io list for processing. */ stripe_endio_push(stripe); atomic_inc(rs->stats + S_NO_RW); /* REMOVEME: statistics. */ } } /* * Recovery functions */ /* Read a stripe off a raid set for recovery. */ static int stripe_recover_read(struct stripe *stripe, int pi) { BUG_ON(stripe_io_ref(stripe)); /* Invalidate all chunks so that they get read in. */ stripe_chunks_invalidate(stripe); stripe_allow_io(stripe); /* Allow io on all recovery chunks. */ /* * If we are reconstructing a perticular device, we can avoid * reading the respective chunk in, because we're going to * reconstruct it anyway. * * We can't do that for resynchronization of rotating parity, * because the recovery stripe chunk size is typically larger * than the sets chunk size. */ if (pi > -1) ClearChunkIo(CHUNK(stripe, pi)); return stripe_chunks_rw(stripe); } /* Write a stripe to a raid set for recovery. */ static int stripe_recover_write(struct stripe *stripe, int pi) { BUG_ON(stripe_io_ref(stripe)); /* * If this is a reconstruct of a particular device, then * reconstruct the respective chunk, else create parity chunk. */ if (pi > -1) { stripe_zero_chunk(stripe, pi); common_xor(stripe, stripe->io.size, 0, pi); chunk_set(CHUNK(stripe, pi), DIRTY); } else parity_xor(stripe); return stripe_chunks_rw(stripe); } /* Read/write a recovery stripe. */ static int stripe_recover_rw(struct stripe *stripe) { int r = 0, sync = 0; /* Read/write flip-flop. */ if (TestClearStripeRBW(stripe)) { SetStripeMerged(stripe); stripe->key = stripe->recover->pos; r = stripe_recover_read(stripe, dev_for_parity(stripe, &sync)); BUG_ON(!r); } else if (TestClearStripeMerged(stripe)) { r = stripe_recover_write(stripe, dev_for_parity(stripe, &sync)); BUG_ON(!r); } BUG_ON(sync); return r; } /* Recover bandwidth available ?. */ static int recover_bandwidth(struct raid_set *rs) { int r, work; /* On reset or when bios delayed -> allow recovery. */ r = recover_io_reset(rs); if (r || RSBandwidth(rs)) goto out; work = atomic_read(rs->recover.io_count + IO_WORK); if (work) { /* Pay attention to larger recover stripe size. */ int recover = atomic_read(rs->recover.io_count + IO_RECOVER) * rs->recover.io_size / rs->set.io_size; /* * Don't use more than given bandwidth * of the work io for recovery. */ if (recover > work / rs->recover.bandwidth_work) { /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_NO_BANDWIDTH); return 0; } } out: atomic_inc(rs->stats + S_BANDWIDTH); /* REMOVEME: statistics. */ return 1; } /* Try to get a region to recover. */ static int stripe_recover_get_region(struct stripe *stripe) { struct raid_set *rs = RS(stripe->sc); struct recover *rec = &rs->recover; struct recover_addr *addr = stripe->recover; struct dm_dirty_log *dl = rec->dl; struct dm_rh_client *rh = rec->rh; BUG_ON(!dl); BUG_ON(!rh); /* Return, that we have region first to finish it during suspension. */ if (addr->reg) return 1; if (RSSuspend(rs)) return -EPERM; if (dl->type->get_sync_count(dl) >= rec->nr_regions) return -ENOENT; /* If we don't have enough bandwidth, we don't proceed recovering. */ if (!recover_bandwidth(rs)) return -EAGAIN; /* Start quiescing a region. */ dm_rh_recovery_prepare(rh); addr->reg = dm_rh_recovery_start(rh); if (!addr->reg) return -EAGAIN; addr->pos = dm_rh_region_to_sector(rh, dm_rh_get_region_key(addr->reg)); addr->end = addr->pos + dm_rh_get_region_size(rh); /* * Take one global io reference out for the * whole region, which is going to be released * when the region is completely done with. */ io_get(rs); return 0; } /* Update region hash state. */ enum recover_type { REC_FAILURE = 0, REC_SUCCESS = 1 }; static void recover_rh_update(struct stripe *stripe, enum recover_type success) { struct recover_addr *addr = stripe->recover; struct raid_set *rs = RS(stripe->sc); struct recover *rec = &rs->recover; if (!addr->reg) { DMERR("%s- Called w/o region", __func__); return; } dm_rh_recovery_end(addr->reg, success); if (success) rec->nr_regions_recovered++; addr->reg = NULL; /* * Completely done with this region -> * release the 1st io reference. */ io_put(rs); } /* Set start of recovery state. */ static void set_start_recovery(struct raid_set *rs) { /* Initialize recovery. */ rs->recover.start_jiffies = jiffies; rs->recover.end_jiffies = 0; } /* Set end of recovery state. */ static void set_end_recovery(struct raid_set *rs) { ClearRSRecover(rs); rs->set.dev_to_init = -1; /* Check for jiffies overrun. */ rs->recover.end_jiffies = jiffies; if (rs->recover.end_jiffies < rs->recover.start_jiffies) rs->recover.end_jiffies = ~0; } /* Handle recovery on one recovery stripe. */ static int _do_recovery(struct stripe *stripe) { int r; struct raid_set *rs = RS(stripe->sc); struct recover_addr *addr = stripe->recover; /* If recovery is active -> return. */ if (stripe_io_ref(stripe)) return 1; /* IO error is fatal for recovery -> stop it. */ if (unlikely(StripeError(stripe))) goto err; /* Recovery end required. */ if (!RSRecover(rs)) goto err; /* Get a region to recover. */ r = stripe_recover_get_region(stripe); switch (r) { case 0: /* Got a new region: flag initial read before write. */ SetStripeRBW(stripe); case 1: /* Have a region in the works. */ break; case -EAGAIN: /* No bandwidth/quiesced region yet, try later. */ if (!io_ref(rs)) wake_do_raid_delayed(rs, HZ / 4); case -EPERM: /* Suspend. */ return 1; case -ENOENT: /* No more regions to recover. */ schedule_work(&rs->io.ws_do_table_event); return 0; default: BUG(); } /* Read/write a recover stripe. */ r = stripe_recover_rw(stripe); if (r) /* IO initiated. */ return 1; /* Read and write finished-> update recovery position within region. */ addr->pos += stripe->io.size; /* If we're at end of region, update region hash. */ if (addr->pos >= addr->end || addr->pos >= rs->set.sectors_per_dev) recover_rh_update(stripe, REC_SUCCESS); else /* Prepare to read next region segment. */ SetStripeRBW(stripe); /* Schedule myself for another round... */ wake_do_raid(rs); return 1; err: /* FIXME: rather try recovering other regions on error? */ rs_check_degrade(stripe); recover_rh_update(stripe, REC_FAILURE); /* Check state of partially recovered array. */ if (RSDegraded(rs) && !RSDead(rs) && rs->set.dev_to_init != -1 && rs->set.ei != rs->set.dev_to_init) /* Broken drive != drive to recover -> FATAL. */ SetRSDead(rs); if (StripeError(stripe)) { char buf[BDEVNAME_SIZE]; DMERR("stopping recovery due to " "ERROR on /dev/%s, stripe at offset %llu", bdevname(rs->dev[rs->set.ei].dev->bdev, buf), (unsigned long long) stripe->key); } /* Make sure, that all quiesced regions get released. */ while (addr->reg) { dm_rh_recovery_end(addr->reg, -EIO); addr->reg = dm_rh_recovery_start(rs->recover.rh); } return 0; } /* Called by main io daemon to recover regions. */ static void do_recovery(struct raid_set *rs) { if (RSRecover(rs)) { int r = 0; struct stripe *stripe; list_for_each_entry(stripe, &rs->recover.stripes, lists[LIST_RECOVER]) r += _do_recovery(stripe); if (!r) { set_end_recovery(rs); stripe_recover_free(rs); } } } /* * END recovery functions */ /* End io process all stripes handed in by endio() callback. */ static void _do_endios(struct raid_set *rs, struct stripe *stripe, struct list_head *flush_list) { /* First unlock all required chunks. */ stripe_chunks_unlock(stripe); /* * If an io error on a stripe occured, degrade the RAID set * and try to endio as many bios as possible. If any bios can't * be endio processed, requeue the stripe (stripe_ref() != 0). */ if (TestClearStripeError(stripe)) { /* * FIXME: if read, rewrite the failed chunk after reconstruction * in order to trigger disk bad sector relocation. */ rs_check_degrade(stripe); /* Resets ChunkError(). */ ClearStripeReconstruct(stripe); ClearStripeReconstructed(stripe); } /* Got to reconstruct a missing chunk. */ if (StripeReconstruct(stripe)) { /* * (*2*) We use StripeReconstruct() to allow for * all chunks to be xored into the reconstructed * one (see chunk_must_xor()). */ stripe_reconstruct(stripe); /* * (*3*) Now we reset StripeReconstruct() and flag * StripeReconstructed() to show to stripe_rw(), * that we have reconstructed a missing chunk. */ ClearStripeReconstruct(stripe); SetStripeReconstructed(stripe); /* FIXME: reschedule to be written in case of read. */ // if (!StripeRBW(stripe)) { // chunk_set(CHUNK(stripe, pr), DIRTY); // stripe_chunks_rw(stripe); // } } /* * Now that we eventually got a complete stripe, we * can process the rest of the end ios on reads. */ stripe_endio(READ, stripe); /* End io all merged writes. */ if (TestClearStripeMerged(stripe)) stripe_endio(WRITE_MERGED, stripe); /* If RAID set is dead -> fail any ios to dead drives. */ if (RSDead(rs)) { DMERR_LIMIT("RAID set dead: failing ios to dead devices"); stripe_fail_io(stripe); } /* * We have stripe references still, * beacuse of read befeore writes or IO errors -> * got to put on flush list for processing. */ if (stripe_ref(stripe)) { BUG_ON(!list_empty(stripe->lists + LIST_LRU)); list_add_tail(stripe->lists + LIST_FLUSH, flush_list); atomic_inc(rs->stats + S_REQUEUE); /* REMOVEME: statistics. */ } else stripe_lru_add(stripe); } /* Pop any endio stripes off of the endio list and belabour them. */ static void do_endios(struct raid_set *rs) { struct stripe_cache *sc = &rs->sc; struct stripe *stripe; /* IO flush list for sorted requeued stripes. */ struct list_head flush_list; INIT_LIST_HEAD(&flush_list); while ((stripe = stripe_endio_pop(sc))) { /* Avoid endio on stripes with newly io'ed chunks. */ if (!stripe_io_ref(stripe)) _do_endios(rs, stripe, &flush_list); } /* * Insert any requeued stripes in the proper * order at the beginning of the io (flush) list. */ list_splice(&flush_list, sc->lists + LIST_FLUSH); } /* Flush any stripes on the io list. */ static void do_flush(struct raid_set *rs) { struct stripe *stripe; while ((stripe = stripe_io_pop(&rs->sc))) stripe_rw(stripe); /* Read/write stripe. */ } /* Stripe cache resizing. */ static void do_sc_resize(struct raid_set *rs) { unsigned set = atomic_read(&rs->sc.stripes_to_set); if (set) { unsigned cur = atomic_read(&rs->sc.stripes); int r = (set > cur) ? sc_grow(&rs->sc, set - cur, SC_GROW) : sc_shrink(&rs->sc, cur - set); /* Flag end of resizeing if ok. */ if (!r) atomic_set(&rs->sc.stripes_to_set, 0); } } /* * Process all ios * * We do different things with the io depending * on the state of the region that it is in: * * o reads: hang off stripe cache or postpone if full * * o writes: * * CLEAN/DIRTY/NOSYNC: increment pending and hang io off stripe's stripe set. * In case stripe cache is full or busy, postpone the io. * * RECOVERING: delay the io until recovery of the region completes. * */ static void do_ios(struct raid_set *rs, struct bio_list *ios) { int r; unsigned flush = 0, delay = 0; sector_t sector; struct dm_rh_client *rh = rs->recover.rh; struct bio *bio; struct bio_list reject; bio_list_init(&reject); /* * Classify each io: * o delay writes to recovering regions (let reads go through) * o queue io to all other regions */ while ((bio = bio_list_pop(ios))) { /* * In case we get a barrier bio, push it back onto * the input queue unless all work queues are empty * and the stripe cache is inactive. */ if (unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER))) { /* REMOVEME: statistics. */ atomic_inc(rs->stats + S_BARRIER); if (delay || !list_empty(rs->sc.lists + LIST_FLUSH) || !bio_list_empty(&reject) || sc_active(&rs->sc)) { bio_list_push(ios, bio); break; } } /* Check for recovering regions. */ sector = _sector(rs, bio); r = region_state(rs, sector, DM_RH_RECOVERING); if (unlikely(r && bio_data_dir(bio) == WRITE)) { delay++; /* Wait writing to recovering regions. */ dm_rh_delay_by_region(rh, bio, dm_rh_sector_to_region(rh, sector)); /* REMOVEME: statistics.*/ atomic_inc(rs->stats + S_DELAYED_BIOS); atomic_inc(rs->stats + S_SUM_DELAYED_BIOS); /* Force bandwidth tests in recovery. */ SetRSBandwidth(rs); } else { /* * Process ios to non-recovering regions by queueing * them to stripes (does dm_rh_inc()) for writes). */ flush += stripe_queue_bio(rs, bio, &reject); } } if (flush) { /* FIXME: better error handling. */ r = dm_rh_flush(rh); /* Writes got queued -> flush dirty log. */ if (r) DMERR_LIMIT("dirty log flush"); } /* Merge any rejected bios back to the head of the input list. */ bio_list_merge_head(ios, &reject); } /* Unplug: let any queued io role on the sets devices. */ static void do_unplug(struct raid_set *rs) { struct raid_dev *dev = rs->dev + rs->set.raid_devs; while (dev-- > rs->dev) { /* Only call any device unplug function, if io got queued. */ if (TestClearDevIoQueued(dev)) blk_unplug(bdev_get_queue(dev->dev->bdev)); } } /* Send an event in case we're getting too busy. */ static void do_busy_event(struct raid_set *rs) { if (sc_busy(rs)) { if (!TestSetRSScBusy(rs)) schedule_work(&rs->io.ws_do_table_event); } ClearRSScBusy(rs); } /* Throw an event. */ static void do_table_event(struct work_struct *ws) { struct raid_set *rs = container_of(ws, struct raid_set, io.ws_do_table_event); dm_table_event(rs->ti->table); } /*----------------------------------------------------------------- * RAID daemon *---------------------------------------------------------------*/ /* * o belabour all end ios * o update the region hash states * o optionally shrink the stripe cache * o optionally do recovery * o unplug any component raid devices with queued bios * o grab the input queue * o work an all requeued or new ios and perform stripe cache flushs * o unplug any component raid devices with queued bios * o check, if the stripe cache gets too busy and throw an event if so */ static void do_raid(struct work_struct *ws) { struct raid_set *rs = container_of(ws, struct raid_set, io.dws_do_raid.work); struct bio_list *ios = &rs->io.work, *ios_in = &rs->io.in; /* * We always need to end io, so that ios can get errored in * case the set failed and the region counters get decremented * before we update region hash states and go any further. */ do_endios(rs); dm_rh_update_states(rs->recover.rh, 1); /* * Now that we've end io'd, which may have put stripes on the LRU list * to allow for shrinking, we resize the stripe cache if requested. */ do_sc_resize(rs); /* Try to recover regions. */ do_recovery(rs); do_unplug(rs); /* Unplug the sets device queues. */ /* Quickly grab all new ios queued and add them to the work list. */ mutex_lock(&rs->io.in_lock); bio_list_merge(ios, ios_in); bio_list_init(ios_in); mutex_unlock(&rs->io.in_lock); if (!bio_list_empty(ios)) do_ios(rs, ios); /* Got ios to work into the cache. */ do_flush(rs); /* Flush any stripes on io list. */ do_unplug(rs); /* Unplug the sets device queues. */ do_busy_event(rs); /* Check if we got too busy. */ } /* * Callback for region hash to dispatch * delayed bios queued to recovered regions * (gets called via dm_rh_update_states()). */ static void dispatch_delayed_bios(void *context, struct bio_list *bl) { struct raid_set *rs = context; struct bio *bio; /* REMOVEME: statistics; decrement pending delayed bios counter. */ bio_list_for_each(bio, bl) atomic_dec(rs->stats + S_DELAYED_BIOS); /* Merge region hash private list to work list. */ bio_list_merge_head(&rs->io.work, bl); bio_list_init(bl); ClearRSBandwidth(rs); } /************************************************************* * Constructor helpers *************************************************************/ /* Calculate MB/sec. */ static unsigned mbpers(struct raid_set *rs, unsigned speed) { return to_bytes(speed * rs->set.data_devs * rs->recover.io_size * HZ >> 10) >> 10; } /* * Discover fastest xor algorithm and # of chunks combination. */ /* Calculate speed for algorithm and # of chunks. */ static unsigned xor_speed(struct stripe *stripe) { unsigned r = 0; unsigned long j; /* Wait for next tick. */ for (j = jiffies; j == jiffies; ) ; /* Do xors for a full tick. */ for (j = jiffies; j == jiffies; ) { mb(); common_xor(stripe, stripe->io.size, 0, 0); mb(); r++; } return r; } /* Optimize xor algorithm for this RAID set. */ static unsigned xor_optimize(struct raid_set *rs) { unsigned chunks_max = 2, p = rs->set.raid_devs, speed_max = 0; struct xor_func *f = ARRAY_END(xor_funcs), *f_max = NULL; struct stripe *stripe; BUG_ON(list_empty(&rs->recover.stripes)); stripe = list_first_entry(&rs->recover.stripes, struct stripe, lists[LIST_RECOVER]); /* Must set uptodate so that xor() will belabour chunks. */ while (p--) SetChunkUptodate(CHUNK(stripe, p)); /* Try all xor functions. */ while (f-- > xor_funcs) { unsigned speed; /* Set actual xor function for common_xor(). */ rs->xor.f = f; rs->xor.chunks = (f->f == xor_blocks_wrapper ? (MAX_XOR_BLOCKS + 1) : XOR_CHUNKS_MAX) + 1; while (rs->xor.chunks-- > 2) { speed = xor_speed(stripe); if (speed > speed_max) { speed_max = speed; chunks_max = rs->xor.chunks; f_max = f; } } } /* Memorize optimum parameters. */ rs->xor.f = f_max; rs->xor.chunks = chunks_max; return speed_max; } /* * Allocate a RAID context (a RAID set) */ /* Structure for variable RAID parameters. */ struct variable_parms { int bandwidth; int bandwidth_parm; int chunk_size; int chunk_size_parm; int io_size; int io_size_parm; int stripes; int stripes_parm; int recover_io_size; int recover_io_size_parm; int raid_parms; int recovery; int recovery_stripes; int recovery_stripes_parm; }; static struct raid_set * context_alloc(struct raid_type *raid_type, struct variable_parms *p, unsigned raid_devs, sector_t sectors_per_dev, struct dm_target *ti, unsigned dl_parms, char **argv) { int r; size_t len; sector_t region_size, ti_len; struct raid_set *rs = NULL; struct dm_dirty_log *dl; struct recover *rec; /* * Create the dirty log * * We need to change length for the dirty log constructor, * because we want an amount of regions for all stripes derived * from the single device size, so that we can keep region * size = 2^^n independant of the number of devices */ ti_len = ti->len; ti->len = sectors_per_dev; dl = dm_dirty_log_create(argv[0], ti, NULL, dl_parms, argv + 2); ti->len = ti_len; if (!dl) goto bad_dirty_log; /* Chunk size *must* be smaller than region size. */ region_size = dl->type->get_region_size(dl); if (p->chunk_size > region_size) goto bad_chunk_size; /* Recover io size *must* be smaller than region size as well. */ if (p->recover_io_size > region_size) goto bad_recover_io_size; /* Size and allocate the RAID set structure. */ len = sizeof(*rs->data) + sizeof(*rs->dev); if (dm_array_too_big(sizeof(*rs), len, raid_devs)) goto bad_array; len = sizeof(*rs) + raid_devs * len; rs = kzalloc(len, GFP_KERNEL); if (!rs) goto bad_alloc; rec = &rs->recover; atomic_set(&rs->io.in_process, 0); atomic_set(&rs->io.in_process_max, 0); rec->io_size = p->recover_io_size; /* Pointer to data array. */ rs->data = (unsigned long **) ((void *) rs->dev + raid_devs * sizeof(*rs->dev)); rec->dl = dl; rs->set.raid_devs = raid_devs; rs->set.data_devs = raid_devs - raid_type->parity_devs; rs->set.raid_type = raid_type; rs->set.raid_parms = p->raid_parms; rs->set.chunk_size_parm = p->chunk_size_parm; rs->set.io_size_parm = p->io_size_parm; rs->sc.stripes_parm = p->stripes_parm; rec->io_size_parm = p->recover_io_size_parm; rec->bandwidth_parm = p->bandwidth_parm; rec->recovery = p->recovery; rec->recovery_stripes = p->recovery_stripes; /* * Set chunk and io size and respective shifts * (used to avoid divisions) */ rs->set.chunk_size = p->chunk_size; rs->set.chunk_shift = ffs(p->chunk_size) - 1; rs->set.io_size = p->io_size; rs->set.io_mask = p->io_size - 1; /* Mask to adjust address key in case io_size != chunk_size. */ rs->set.io_inv_mask = (p->chunk_size - 1) & ~rs->set.io_mask; rs->set.sectors_per_dev = sectors_per_dev; rs->set.ei = -1; /* Indicate no failed device. */ atomic_set(&rs->set.failed_devs, 0); rs->ti = ti; atomic_set(rec->io_count + IO_WORK, 0); atomic_set(rec->io_count + IO_RECOVER, 0); /* Initialize io lock and queues. */ mutex_init(&rs->io.in_lock); bio_list_init(&rs->io.in); bio_list_init(&rs->io.work); init_waitqueue_head(&rs->io.suspendq); /* Suspend waiters (dm-io). */ rec->nr_regions = dm_sector_div_up(sectors_per_dev, region_size); rec->rh = dm_region_hash_create(rs, dispatch_delayed_bios, wake_dummy, wake_do_raid, 0, p->recovery_stripes, dl, region_size, rec->nr_regions); if (IS_ERR(rec->rh)) goto bad_rh; /* Initialize stripe cache. */ r = sc_init(rs, p->stripes); if (r) goto bad_sc; /* REMOVEME: statistics. */ stats_reset(rs); ClearRSDevelStats(rs); /* Disnable development status. */ return rs; bad_dirty_log: TI_ERR_RET("Error creating dirty log", ERR_PTR(-ENOMEM)); bad_chunk_size: dm_dirty_log_destroy(dl); TI_ERR_RET("Chunk size larger than region size", ERR_PTR(-EINVAL)); bad_recover_io_size: dm_dirty_log_destroy(dl); TI_ERR_RET("Recover stripe io size larger than region size", ERR_PTR(-EINVAL)); bad_array: dm_dirty_log_destroy(dl); TI_ERR_RET("Arry too big", ERR_PTR(-EINVAL)); bad_alloc: dm_dirty_log_destroy(dl); TI_ERR_RET("Cannot allocate raid context", ERR_PTR(-ENOMEM)); bad_rh: dm_dirty_log_destroy(dl); ti->error = DM_MSG_PREFIX "Error creating dirty region hash"; goto free_rs; bad_sc: dm_region_hash_destroy(rec->rh); /* Destroys dirty log too. */ sc_exit(&rs->sc); ti->error = DM_MSG_PREFIX "Error creating stripe cache"; free_rs: kfree(rs); return ERR_PTR(-ENOMEM); } /* Free a RAID context (a RAID set). */ static void context_free(struct raid_set *rs, unsigned p) { while (p--) dm_put_device(rs->ti, rs->dev[p].dev); sc_exit(&rs->sc); dm_region_hash_destroy(rs->recover.rh); /* Destroys dirty log too. */ kfree(rs); } /* Create work queue and initialize delayed work. */ static int rs_workqueue_init(struct raid_set *rs) { struct dm_target *ti = rs->ti; rs->io.wq = create_singlethread_workqueue(DAEMON); if (!rs->io.wq) TI_ERR_RET("failed to create " DAEMON, -ENOMEM); INIT_DELAYED_WORK(&rs->io.dws_do_raid, do_raid); INIT_WORK(&rs->io.ws_do_table_event, do_table_event); return 0; } /* Return pointer to raid_type structure for raid name. */ static struct raid_type *get_raid_type(char *name) { struct raid_type *r = ARRAY_END(raid_types); while (r-- > raid_types) { if (!strcmp(r->name, name)) return r; } return NULL; } /* FIXME: factor out to dm core. */ static int multiple(sector_t a, sector_t b, sector_t *n) { sector_t r = a; sector_div(r, b); *n = r; return a == r * b; } /* Log RAID set information to kernel log. */ static void rs_log(struct raid_set *rs, unsigned speed) { unsigned p; char buf[BDEVNAME_SIZE]; for (p = 0; p < rs->set.raid_devs; p++) DMINFO("/dev/%s is raid disk %u%s", bdevname(rs->dev[p].dev->bdev, buf), p, (p == rs->set.pi) ? " (parity)" : ""); DMINFO("%d/%d/%d sectors chunk/io/recovery size, %u stripes\n" "algorithm \"%s\", %u chunks with %uMB/s\n" "%s set with net %u/%u devices", rs->set.chunk_size, rs->set.io_size, rs->recover.io_size, atomic_read(&rs->sc.stripes), rs->xor.f->name, rs->xor.chunks, mbpers(rs, speed), rs->set.raid_type->descr, rs->set.data_devs, rs->set.raid_devs); } /* Get all devices and offsets. */ static int dev_parms(struct raid_set *rs, char **argv, int *p) { struct dm_target *ti = rs->ti; for (*p = 0; *p < rs->set.raid_devs; (*p)++, argv += 2) { int r; unsigned long long tmp; struct raid_dev *dev = rs->dev + *p; /* Get offset and device. */ if (sscanf(argv[1], "%llu", &tmp) != 1 || tmp > rs->set.sectors_per_dev) TI_ERR("Invalid RAID device offset parameter"); dev->start = tmp; r = dm_get_device(ti, *argv, dev->start, rs->set.sectors_per_dev, dm_table_get_mode(ti->table), &dev->dev); if (r) TI_ERR_RET("RAID device lookup failure", r); r = raid_dev_lookup(rs, dev); if (r != -ENODEV && r < *p) { (*p)++; /* Ensure dm_put_device() on actual device. */ TI_ERR_RET("Duplicate RAID device", -ENXIO); } } return 0; } /* Set recovery bandwidth. */ static void recover_set_bandwidth(struct raid_set *rs, unsigned bandwidth) { rs->recover.bandwidth = bandwidth; rs->recover.bandwidth_work = 100 / bandwidth; } /* Handle variable number of RAID parameters. */ static int get_raid_variable_parms(struct dm_target *ti, char **argv, struct variable_parms *vp) { int p, value; struct { int action; /* -1: skip, 0: no pwer2 check, 1: power2 check */ char *errmsg; int min, max; int *var, *var2, *var3; } argctr[] = { { 1, "Invalid chunk size; must be -1 or 2^^n and <= 16384", IO_SIZE_MIN, CHUNK_SIZE_MAX, &vp->chunk_size_parm, &vp->chunk_size, &vp->io_size }, { 0, "Invalid number of stripes: must be -1 or >= 8 and <= 16384", STRIPES_MIN, STRIPES_MAX, &vp->stripes_parm, &vp->stripes, NULL }, { 1, "Invalid io size; must -1 or >= 8, 2^^n and less equal " "min(BIO_MAX_SECTORS/2, chunk size)", IO_SIZE_MIN, 0, /* Needs to be updated in loop below. */ &vp->io_size_parm, &vp->io_size, NULL }, { 1, "Invalid recovery io size; must be -1 or " "2^^n and less equal BIO_MAX_SECTORS/2", RECOVER_IO_SIZE_MIN, BIO_MAX_SECTORS / 2, &vp->recover_io_size_parm, &vp->recover_io_size, NULL }, { 0, "Invalid recovery bandwidth percentage; " "must be -1 or > 0 and <= 100", BANDWIDTH_MIN, BANDWIDTH_MAX, &vp->bandwidth_parm, &vp->bandwidth, NULL }, /* Handle sync argument seperately in loop. */ { -1, "Invalid recovery switch; must be \"sync\" or \"nosync\"" }, { 0, "Invalid number of recovery stripes;" "must be -1, > 0 and <= 16384", RECOVERY_STRIPES_MIN, RECOVERY_STRIPES_MAX, &vp->recovery_stripes_parm, &vp->recovery_stripes, NULL }, }, *varp; /* Fetch # of variable raid parameters. */ if (sscanf(*(argv++), "%d", &vp->raid_parms) != 1 || !range_ok(vp->raid_parms, 0, 7)) TI_ERR("Bad variable raid parameters number"); /* Preset variable RAID parameters. */ vp->chunk_size = CHUNK_SIZE_DEFAULT; vp->io_size = IO_SIZE_DEFAULT; vp->stripes = STRIPES_DEFAULT; vp->recover_io_size = RECOVER_IO_SIZE_DEFAULT; vp->bandwidth = BANDWIDTH_DEFAULT; vp->recovery = 1; vp->recovery_stripes = RECOVERY_STRIPES_DEFAULT; /* Walk the array of argument constraints for all given ones. */ for (p = 0, varp = argctr; p < vp->raid_parms; p++, varp++) { BUG_ON(varp >= ARRAY_END(argctr)); /* Special case for "[no]sync" string argument. */ if (varp->action < 0) { if (!strcmp(*argv, "sync")) ; else if (!strcmp(*argv, "nosync")) vp->recovery = 0; else TI_ERR(varp->errmsg); argv++; continue; } /* * Special case for io_size depending * on previously set chunk size. */ if (p == 2) varp->max = min(BIO_MAX_SECTORS / 2, vp->chunk_size); if (sscanf(*(argv++), "%d", &value) != 1 || (value != -1 && ((varp->action && !POWER_OF_2(value)) || !range_ok(value, varp->min, varp->max)))) TI_ERR(varp->errmsg); *varp->var = value; if (value != -1) { if (varp->var2) *varp->var2 = value; if (varp->var3) *varp->var3 = value; } } return 0; } /* Parse optional locking parameters. */ static int get_raid_locking_parms(struct dm_target *ti, char **argv, int *locking_parms, struct dm_raid45_locking_type **locking_type) { if (!strnicmp(argv[0], "locking", strlen(argv[0]))) { char *lckstr = argv[1]; size_t lcksz = strlen(lckstr); if (!strnicmp(lckstr, "none", lcksz)) { *locking_type = &locking_none; *locking_parms = 2; } else if (!strnicmp(lckstr, "cluster", lcksz)) { DMERR("locking type \"%s\" not yet implemented", lckstr); return -EINVAL; } else { DMERR("unknown locking type \"%s\"", lckstr); return -EINVAL; } } *locking_parms = 0; *locking_type = &locking_none; return 0; } /* Set backing device read ahead properties of RAID set. */ static void rs_set_read_ahead(struct raid_set *rs, unsigned sectors, unsigned stripes) { unsigned ra_pages = dm_div_up(sectors, SECTORS_PER_PAGE); struct mapped_device *md = dm_table_get_md(rs->ti->table); struct backing_dev_info *bdi = &dm_disk(md)->queue->backing_dev_info; /* Set read-ahead for the RAID set and the component devices. */ if (ra_pages) { unsigned p = rs->set.raid_devs; bdi->ra_pages = stripes * ra_pages * rs->set.data_devs; while (p--) { struct request_queue *q = bdev_get_queue(rs->dev[p].dev->bdev); q->backing_dev_info.ra_pages = ra_pages; } } dm_put(md); } /* Set congested function. */ static void rs_set_congested_fn(struct raid_set *rs) { struct mapped_device *md = dm_table_get_md(rs->ti->table); struct backing_dev_info *bdi = &dm_disk(md)->queue->backing_dev_info; /* Set congested function and data. */ bdi->congested_fn = rs_congested; bdi->congested_data = rs; dm_put(md); } /* * Construct a RAID4/5 mapping: * * log_type #log_params <log_params> \ * raid_type [#parity_dev] #raid_variable_params <raid_params> \ * [locking "none"/"cluster"] * #raid_devs #dev_to_initialize [<dev_path> <offset>]{3,} * * log_type = "core"/"disk", * #log_params = 1-3 (1-2 for core dirty log type, 3 for disk dirty log only) * log_params = [dirty_log_path] region_size [[no]sync]) * * raid_type = "raid4", "raid5_la", "raid5_ra", "raid5_ls", "raid5_rs" * * #parity_dev = N if raid_type = "raid4" * o N = -1: pick default = last device * o N >= 0 and < #raid_devs: parity device index * * #raid_variable_params = 0-7; raid_params (-1 = default): * [chunk_size [#stripes [io_size [recover_io_size \ * [%recovery_bandwidth [recovery_switch [#recovery_stripes]]]]]]] * o chunk_size (unit to calculate drive addresses; must be 2^^n, > 8 * and <= CHUNK_SIZE_MAX) * o #stripes is number of stripes allocated to stripe cache * (must be > 1 and < STRIPES_MAX) * o io_size (io unit size per device in sectors; must be 2^^n and > 8) * o recover_io_size (io unit size per device for recovery in sectors; must be 2^^n, > SECTORS_PER_PAGE and <= region_size) * o %recovery_bandwith is the maximum amount spend for recovery during * application io (1-100%) * o recovery switch = [sync|nosync] * o #recovery_stripes is the number of recovery stripes used for * parallel recovery of the RAID set * If raid_variable_params = 0, defaults will be used. * Any raid_variable_param can be set to -1 to apply a default * * #raid_devs = N (N >= 3) * * #dev_to_initialize = N * -1: initialize parity on all devices * >= 0 and < #raid_devs: initialize raid_path; used to force reconstruction * of a failed devices content after replacement * * <dev_path> = device_path (eg, /dev/sdd1) * <offset> = begin at offset on <dev_path> * */ #define MIN_PARMS 13 static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv) { int dev_to_init, dl_parms, i, locking_parms, parity_parm, pi = -1, r, raid_devs; unsigned speed; sector_t tmp, sectors_per_dev; struct dm_raid45_locking_type *locking; struct raid_set *rs; struct raid_type *raid_type; struct variable_parms parms; /* Ensure minimum number of parameters. */ if (argc < MIN_PARMS) TI_ERR("Not enough parameters"); /* Fetch # of dirty log parameters. */ if (sscanf(argv[1], "%d", &dl_parms) != 1 || !range_ok(dl_parms, 1, 4711)) /* ;-) */ TI_ERR("Bad dirty log parameters number"); /* Check raid_type. */ raid_type = get_raid_type(argv[dl_parms + 2]); if (!raid_type) TI_ERR("Bad raid type"); /* In case of RAID4, parity drive is selectable. */ parity_parm = !!(raid_type->level == raid4); /* Handle variable number of RAID parameters. */ r = get_raid_variable_parms(ti, argv + dl_parms + parity_parm + 3, &parms); if (r) return r; /* Handle any locking parameters. */ r = get_raid_locking_parms(ti, argv + dl_parms + parity_parm + parms.raid_parms + 4, &locking_parms, &locking); if (r) return r; /* # of raid devices. */ i = dl_parms + parity_parm + parms.raid_parms + locking_parms + 4; if (sscanf(argv[i], "%d", &raid_devs) != 1 || raid_devs < raid_type->minimal_devs) TI_ERR("Invalid number of raid devices"); /* In case of RAID4, check parity drive index is in limits. */ if (raid_type->level == raid4) { /* Fetch index of parity device. */ if (sscanf(argv[dl_parms + 3], "%d", &pi) != 1 || (pi != -1 && !range_ok(pi, 0, raid_devs - 1))) TI_ERR("Invalid RAID4 parity device index"); } /* * Index of device to initialize starts at 0 * * o -1 -> don't initialize a selected device; * initialize parity conforming to algorithm * o 0..raid_devs-1 -> initialize respective device * (used for reconstruction of a replaced device) */ if (sscanf(argv[dl_parms + parity_parm + parms.raid_parms + locking_parms + 5], "%d", &dev_to_init) != 1 || !range_ok(dev_to_init, -1, raid_devs - 1)) TI_ERR("Invalid number for raid device to initialize"); /* Check # of raid device arguments. */ if (argc - dl_parms - parity_parm - parms.raid_parms - 6 != 2 * raid_devs) TI_ERR("Wrong number of raid device/offset arguments"); /* * Check that the table length is devisable * w/o rest by (raid_devs - parity_devs) */ if (!multiple(ti->len, raid_devs - raid_type->parity_devs, &sectors_per_dev)) TI_ERR("Target length not divisible by number of data devices"); /* * Check that the device size is * devisable w/o rest by chunk size */ if (!multiple(sectors_per_dev, parms.chunk_size, &tmp)) TI_ERR("Device length not divisible by chunk_size"); /**************************************************************** * Now that we checked the constructor arguments -> * let's allocate the RAID set ****************************************************************/ rs = context_alloc(raid_type, &parms, raid_devs, sectors_per_dev, ti, dl_parms, argv); if (IS_ERR(rs)) return PTR_ERR(rs); rs->set.dev_to_init = rs->set.dev_to_init_parm = dev_to_init; rs->set.pi = rs->set.pi_parm = pi; /* Set RAID4 parity drive index. */ if (raid_type->level == raid4) rs->set.pi = (pi == -1) ? rs->set.data_devs : pi; recover_set_bandwidth(rs, parms.bandwidth); /* Use locking type to lock stripe access. */ rs->locking = locking; /* Get the device/offset tupels. */ argv += dl_parms + 6 + parity_parm + parms.raid_parms; r = dev_parms(rs, argv, &i); if (r) goto err; /* Set backing device information (eg. read ahead). */ rs_set_read_ahead(rs, 2 * rs->set.chunk_size, 4 /* stripes */); rs_set_congested_fn(rs); /* Set congested function. */ SetRSCheckOverwrite(rs); /* Allow chunk overwrite checks. */ speed = xor_optimize(rs); /* Select best xor algorithm. */ /* Set for recovery of any nosync regions. */ if (parms.recovery) SetRSRecover(rs); else { /* * Need to free recovery stripe(s) here in case * of nosync, because xor_optimize uses one. */ set_start_recovery(rs); set_end_recovery(rs); stripe_recover_free(rs); } /* * Make sure that dm core only hands maximum io size * length down and pays attention to io boundaries. */ ti->split_io = rs->set.io_size; ti->private = rs; /* Initialize work queue to handle this RAID set's io. */ r = rs_workqueue_init(rs); if (r) goto err; rs_log(rs, speed); /* Log information about RAID set. */ return 0; err: context_free(rs, i); return r; } /* * Destruct a raid mapping */ static void raid_dtr(struct dm_target *ti) { struct raid_set *rs = ti->private; destroy_workqueue(rs->io.wq); context_free(rs, rs->set.raid_devs); } /* Raid mapping function. */ static int raid_map(struct dm_target *ti, struct bio *bio, union map_info *map_context) { /* I don't want to waste stripe cache capacity. */ if (bio_rw(bio) == READA) return -EIO; else { struct raid_set *rs = ti->private; /* * Get io reference to be waiting for to drop * to zero on device suspension/destruction. */ io_get(rs); bio->bi_sector -= ti->begin; /* Remap sector. */ /* Queue io to RAID set. */ mutex_lock(&rs->io.in_lock); bio_list_add(&rs->io.in, bio); mutex_unlock(&rs->io.in_lock); /* Wake daemon to process input list. */ wake_do_raid(rs); /* REMOVEME: statistics. */ atomic_inc(rs->stats + (bio_data_dir(bio) == READ ? S_BIOS_READ : S_BIOS_WRITE)); return DM_MAPIO_SUBMITTED; /* Handle later. */ } } /* Device suspend. */ static void raid_presuspend(struct dm_target *ti) { struct raid_set *rs = ti->private; struct dm_dirty_log *dl = rs->recover.dl; SetRSSuspend(rs); if (RSRecover(rs)) dm_rh_stop_recovery(rs->recover.rh); cancel_delayed_work(&rs->io.dws_do_raid); flush_workqueue(rs->io.wq); wait_ios(rs); /* Wait for completion of all ios being processed. */ if (dl->type->presuspend && dl->type->presuspend(dl)) /* FIXME: need better error handling. */ DMWARN("log presuspend failed"); } static void raid_postsuspend(struct dm_target *ti) { struct raid_set *rs = ti->private; struct dm_dirty_log *dl = rs->recover.dl; if (dl->type->postsuspend && dl->type->postsuspend(dl)) /* FIXME: need better error handling. */ DMWARN("log postsuspend failed"); } /* Device resume. */ static void raid_resume(struct dm_target *ti) { struct raid_set *rs = ti->private; struct recover *rec = &rs->recover; struct dm_dirty_log *dl = rec->dl; if (dl->type->resume && dl->type->resume(dl)) /* Resume dirty log. */ /* FIXME: need better error handling. */ DMWARN("log resume failed"); rec->nr_regions_to_recover = rec->nr_regions - dl->type->get_sync_count(dl); /* Restart any unfinished recovery. */ if (RSRecover(rs)) { set_start_recovery(rs); dm_rh_start_recovery(rec->rh); } ClearRSSuspend(rs); wake_do_raid(rs); } /* Return stripe cache size. */ static unsigned sc_size(struct raid_set *rs) { return to_sector(atomic_read(&rs->sc.stripes) * (sizeof(struct stripe) + (sizeof(struct stripe_chunk) + (sizeof(struct page_list) + to_bytes(rs->set.io_size) * rs->set.raid_devs)) + (rs->recover.end_jiffies ? 0 : rs->recover.recovery_stripes * to_bytes(rs->set.raid_devs * rs->recover.io_size)))); } /* REMOVEME: status output for development. */ static void raid_devel_stats(struct dm_target *ti, char *result, unsigned *size, unsigned maxlen) { unsigned sz = *size; unsigned long j; char buf[BDEVNAME_SIZE], *p; struct stats_map *sm; struct raid_set *rs = ti->private; struct recover *rec = &rs->recover; struct timespec ts; DMEMIT("%s %s %u\n", version, rs->xor.f->name, rs->xor.chunks); DMEMIT("act_ios=%d ", io_ref(rs)); DMEMIT("act_ios_max=%d\n", atomic_read(&rs->io.in_process_max)); DMEMIT("act_stripes=%d ", sc_active(&rs->sc)); DMEMIT("act_stripes_max=%d\n", atomic_read(&rs->sc.active_stripes_max)); for (sm = stats_map; sm < ARRAY_END(stats_map); sm++) DMEMIT("%s%d", sm->str, atomic_read(rs->stats + sm->type)); DMEMIT(" checkovr=%s\n", RSCheckOverwrite(rs) ? "on" : "off"); DMEMIT("sc=%u/%u/%u/%u/%u/%u/%u\n", rs->set.chunk_size, atomic_read(&rs->sc.stripes), rs->set.io_size, rec->recovery_stripes, rec->io_size, rs->sc.hash.buckets, sc_size(rs)); j = (rec->end_jiffies ? rec->end_jiffies : jiffies) - rec->start_jiffies; jiffies_to_timespec(j, &ts); sprintf(buf, "%ld.%ld", ts.tv_sec, ts.tv_nsec); p = strchr(buf, '.'); p[3] = 0; DMEMIT("rg=%llu/%llu/%llu/%u %s\n", (unsigned long long) rec->nr_regions_recovered, (unsigned long long) rec->nr_regions_to_recover, (unsigned long long) rec->nr_regions, rec->bandwidth, buf); *size = sz; } static int raid_status(struct dm_target *ti, status_type_t type, char *result, unsigned maxlen) { unsigned p, sz = 0; char buf[BDEVNAME_SIZE]; struct raid_set *rs = ti->private; int raid_parms[] = { rs->set.chunk_size_parm, rs->sc.stripes_parm, rs->set.io_size_parm, rs->recover.io_size_parm, rs->recover.bandwidth_parm, -2, rs->recover.recovery_stripes, }; switch (type) { case STATUSTYPE_INFO: /* REMOVEME: statistics. */ if (RSDevelStats(rs)) raid_devel_stats(ti, result, &sz, maxlen); DMEMIT("%u ", rs->set.raid_devs); for (p = 0; p < rs->set.raid_devs; p++) DMEMIT("%s ", format_dev_t(buf, rs->dev[p].dev->bdev->bd_dev)); DMEMIT("1 "); for (p = 0; p < rs->set.raid_devs; p++) { DMEMIT("%c", !DevFailed(rs->dev + p) ? 'A' : 'D'); if (p == rs->set.pi) DMEMIT("p"); if (rs->set.dev_to_init == p) DMEMIT("i"); } break; case STATUSTYPE_TABLE: sz = rs->recover.dl->type->status(rs->recover.dl, type, result, maxlen); DMEMIT("%s %u ", rs->set.raid_type->name, rs->set.raid_parms); for (p = 0; p < rs->set.raid_parms; p++) { if (raid_parms[p] > -2) DMEMIT("%d ", raid_parms[p]); else DMEMIT("%s ", rs->recover.recovery ? "sync" : "nosync"); } DMEMIT("%u %d ", rs->set.raid_devs, rs->set.dev_to_init); for (p = 0; p < rs->set.raid_devs; p++) DMEMIT("%s %llu ", format_dev_t(buf, rs->dev[p].dev->bdev->bd_dev), (unsigned long long) rs->dev[p].start); } return 0; } /* * Message interface */ enum raid_msg_actions { act_bw, /* Recovery bandwidth switch. */ act_dev, /* Device failure switch. */ act_overwrite, /* Stripe overwrite check. */ act_stats, /* Development statistics switch. */ act_sc, /* Stripe cache switch. */ act_on, /* Set entity on. */ act_off, /* Set entity off. */ act_reset, /* Reset entity. */ act_set = act_on, /* Set # absolute. */ act_grow = act_off, /* Grow # by an amount. */ act_shrink = act_reset, /* Shrink # by an amount. */ }; /* Turn a delta into an absolute value. */ static int _absolute(unsigned long action, int act, int r) { /* Make delta absolute. */ if (test_bit(act_set, &action)) ; else if (test_bit(act_grow, &action)) r += act; else if (test_bit(act_shrink, &action)) r = act - r; else r = -EINVAL; return r; } /* Change recovery io bandwidth. */ static int bandwidth_change(struct dm_msg *msg, void *context) { struct raid_set *rs = context; int act = rs->recover.bandwidth; int bandwidth = DM_MSG_INT_ARG(msg); if (range_ok(bandwidth, BANDWIDTH_MIN, BANDWIDTH_MAX)) { /* Make delta bandwidth absolute. */ bandwidth = _absolute(msg->action, act, bandwidth); /* Check range. */ if (range_ok(bandwidth, BANDWIDTH_MIN, BANDWIDTH_MAX)) { recover_set_bandwidth(rs, bandwidth); return 0; } } set_bit(dm_msg_ret_arg, &msg->ret); set_bit(dm_msg_ret_inval, &msg->ret); return -EINVAL; } /* Set/reset development feature flags. */ static int devel_flags(struct dm_msg *msg, void *context) { struct raid_set *rs = context; if (test_bit(act_on, &msg->action)) return test_and_set_bit(msg->spec->parm, &rs->io.flags) ? -EPERM : 0; else if (test_bit(act_off, &msg->action)) return test_and_clear_bit(msg->spec->parm, &rs->io.flags) ? 0 : -EPERM; else if (test_bit(act_reset, &msg->action)) { if (test_bit(act_stats, &msg->action)) { stats_reset(rs); goto on; } else if (test_bit(act_overwrite, &msg->action)) { on: set_bit(msg->spec->parm, &rs->io.flags); return 0; } } return -EINVAL; } /* Resize the stripe cache. */ static int sc_resize(struct dm_msg *msg, void *context) { int act, stripes; struct raid_set *rs = context; /* Deny permission in case the daemon is still resizing!. */ if (atomic_read(&rs->sc.stripes_to_set)) return -EPERM; stripes = DM_MSG_INT_ARG(msg); if (stripes > 0) { act = atomic_read(&rs->sc.stripes); /* Make delta stripes absolute. */ stripes = _absolute(msg->action, act, stripes); /* * Check range and that the # of stripes changes. * We leave the resizing to the wroker. */ if (range_ok(stripes, STRIPES_MIN, STRIPES_MAX) && stripes != atomic_read(&rs->sc.stripes)) { atomic_set(&rs->sc.stripes_to_set, stripes); wake_do_raid(rs); return 0; } } set_bit(dm_msg_ret_arg, &msg->ret); set_bit(dm_msg_ret_inval, &msg->ret); return -EINVAL; } /* Parse the RAID message action. */ /* * 'ba[ndwidth] {se[t],g[row],sh[rink]} #' # e.g 'ba se 50' * "o[verwrite] {on,of[f],r[eset]}' # e.g. 'o of' * 'sta[tistics] {on,of[f],r[eset]}' # e.g. 'stat of' * 'str[ipecache] {se[t],g[row],sh[rink]} #' # e.g. 'stripe set 1024' * */ static int raid_message(struct dm_target *ti, unsigned argc, char **argv) { /* Variables to store the parsed parameters im. */ static int i[2]; static unsigned long *i_arg[] = { (unsigned long *) i + 0, (unsigned long *) i + 1, }; /* Declare all message option strings. */ static char *str_sgs[] = { "set", "grow", "shrink" }; static char *str_oor[] = { "on", "off", "reset" }; /* Declare all actions. */ static unsigned long act_sgs[] = { act_set, act_grow, act_shrink }; static unsigned long act_oor[] = { act_on, act_off, act_reset }; /* Bandwidth option. */ static struct dm_message_option bw_opt = { 3, str_sgs, act_sgs }; static struct dm_message_argument bw_args = { 1, i_arg, { dm_msg_int_t } }; static struct dm_message_argument null_args = { 0, NULL, { dm_msg_int_t } }; /* Overwrite and statistics option. */ static struct dm_message_option ovr_stats_opt = { 3, str_oor, act_oor }; /* Sripecache option. */ static struct dm_message_option stripe_opt = { 3, str_sgs, act_sgs }; /* Declare messages. */ static struct dm_msg_spec specs[] = { { "bandwidth", act_bw, &bw_opt, &bw_args, 0, bandwidth_change }, { "overwrite", act_overwrite, &ovr_stats_opt, &null_args, RS_CHECK_OVERWRITE, devel_flags }, { "statistics", act_stats, &ovr_stats_opt, &null_args, RS_DEVEL_STATS, devel_flags }, { "stripecache", act_sc, &stripe_opt, &bw_args, 0, sc_resize }, }; /* The message for the parser. */ struct dm_msg msg = { .num_specs = ARRAY_SIZE(specs), .specs = specs, }; return dm_message_parse(TARGET, &msg, ti->private, argc, argv); } /* * END message interface */ static struct target_type raid_target = { .name = "raid45", .version = {1, 0, 0}, .module = THIS_MODULE, .ctr = raid_ctr, .dtr = raid_dtr, .map = raid_map, .presuspend = raid_presuspend, .postsuspend = raid_postsuspend, .resume = raid_resume, .status = raid_status, .message = raid_message, }; static void init_exit(const char *bad_msg, const char *good_msg, int r) { if (r) DMERR("Failed to %sregister target [%d]", bad_msg, r); else DMINFO("%s %s", good_msg, version); } static int __init dm_raid_init(void) { int r = dm_register_target(&raid_target); init_exit("", "initialized", r); return r; } static void __exit dm_raid_exit(void) { dm_unregister_target(&raid_target); init_exit("un", "exit", 0); } /* Module hooks. */ module_init(dm_raid_init); module_exit(dm_raid_exit); MODULE_DESCRIPTION(DM_NAME " raid4/5 target"); MODULE_AUTHOR("Heinz Mauelshagen <hjm@redhat.com>"); MODULE_LICENSE("GPL"); MODULE_ALIAS("dm-raid4"); MODULE_ALIAS("dm-raid5");


context:
space:
mode: