/* * raid5.c : Multiple Devices driver for Linux * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman * Copyright (C) 1999, 2000 Ingo Molnar * Copyright (C) 2002, 2003 H. Peter Anvin * * RAID-4/5/6 management functions. * Thanks to Penguin Computing for making the RAID-6 development possible * by donating a test server! * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2, or (at your option) * any later version. * * You should have received a copy of the GNU General Public License * (for example /usr/src/linux/COPYING); if not, write to the Free * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ /* * BITMAP UNPLUGGING: * * The sequencing for updating the bitmap reliably is a little * subtle (and I got it wrong the first time) so it deserves some * explanation. * * We group bitmap updates into batches. Each batch has a number. * We may write out several batches at once, but that isn't very important. * conf->seq_write is the number of the last batch successfully written. * conf->seq_flush is the number of the last batch that was closed to * new additions. * When we discover that we will need to write to any block in a stripe * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq * the number of the batch it will be in. This is seq_flush+1. * When we are ready to do a write, if that batch hasn't been written yet, * we plug the array and queue the stripe for later. * When an unplug happens, we increment bm_flush, thus closing the current * batch. * When we notice that bm_flush > bm_write, we write out all pending updates * to the bitmap, and advance bm_write to where bm_flush was. * This may occasionally write a bit out twice, but is sure never to * miss any bits. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "md.h" #include "raid5.h" #include "raid0.h" #include "bitmap.h" #define cpu_to_group(cpu) cpu_to_node(cpu) #define ANY_GROUP NUMA_NO_NODE static struct workqueue_struct *raid5_wq; /* * Stripe cache */ #define NR_STRIPES 256 #define STRIPE_SIZE PAGE_SIZE #define STRIPE_SHIFT (PAGE_SHIFT - 9) #define STRIPE_SECTORS (STRIPE_SIZE>>9) #define IO_THRESHOLD 1 #define BYPASS_THRESHOLD 1 #define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) #define HASH_MASK (NR_HASH - 1) #define MAX_STRIPE_BATCH 8 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) { int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; return &conf->stripe_hashtbl[hash]; } static inline int stripe_hash_locks_hash(sector_t sect) { return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; } static inline void lock_device_hash_lock(struct r5conf *conf, int hash) { spin_lock_irq(conf->hash_locks + hash); spin_lock(&conf->device_lock); } static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) { spin_unlock(&conf->device_lock); spin_unlock_irq(conf->hash_locks + hash); } static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) { int i; local_irq_disable(); spin_lock(conf->hash_locks); for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); spin_lock(&conf->device_lock); } static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) { int i; spin_unlock(&conf->device_lock); for (i = NR_STRIPE_HASH_LOCKS; i; i--) spin_unlock(conf->hash_locks + i - 1); local_irq_enable(); } /* bio's attached to a stripe+device for I/O are linked together in bi_sector * order without overlap. There may be several bio's per stripe+device, and * a bio could span several devices. * When walking this list for a particular stripe+device, we must never proceed * beyond a bio that extends past this device, as the next bio might no longer * be valid. * This function is used to determine the 'next' bio in the list, given the sector * of the current stripe+device */ static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) { int sectors = bio_sectors(bio); if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS) return bio->bi_next; else return NULL; } /* * We maintain a biased count of active stripes in the bottom 16 bits of * bi_phys_segments, and a count of processed stripes in the upper 16 bits */ static inline int raid5_bi_processed_stripes(struct bio *bio) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; return (atomic_read(segments) >> 16) & 0xffff; } static inline int raid5_dec_bi_active_stripes(struct bio *bio) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; return atomic_sub_return(1, segments) & 0xffff; } static inline void raid5_inc_bi_active_stripes(struct bio *bio) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; atomic_inc(segments); } static inline void raid5_set_bi_processed_stripes(struct bio *bio, unsigned int cnt) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; int old, new; do { old = atomic_read(segments); new = (old & 0xffff) | (cnt << 16); } while (atomic_cmpxchg(segments, old, new) != old); } static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) { atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; atomic_set(segments, cnt); } /* Find first data disk in a raid6 stripe */ static inline int raid6_d0(struct stripe_head *sh) { if (sh->ddf_layout) /* ddf always start from first device */ return 0; /* md starts just after Q block */ if (sh->qd_idx == sh->disks - 1) return 0; else return sh->qd_idx + 1; } static inline int raid6_next_disk(int disk, int raid_disks) { disk++; return (disk < raid_disks) ? disk : 0; } /* When walking through the disks in a raid5, starting at raid6_d0, * We need to map each disk to a 'slot', where the data disks are slot * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk * is raid_disks-1. This help does that mapping. */ static int raid6_idx_to_slot(int idx, struct stripe_head *sh, int *count, int syndrome_disks) { int slot = *count; if (sh->ddf_layout) (*count)++; if (idx == sh->pd_idx) return syndrome_disks; if (idx == sh->qd_idx) return syndrome_disks + 1; if (!sh->ddf_layout) (*count)++; return slot; } static void return_io(struct bio *return_bi) { struct bio *bi = return_bi; while (bi) { return_bi = bi->bi_next; bi->bi_next = NULL; bi->bi_iter.bi_size = 0; trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), bi, 0); bio_endio(bi, 0); bi = return_bi; } } static void print_raid5_conf (struct r5conf *conf); static int stripe_operations_active(struct stripe_head *sh) { return sh->check_state || sh->reconstruct_state || test_bit(STRIPE_BIOFILL_RUN, &sh->state) || test_bit(STRIPE_COMPUTE_RUN, &sh->state); } static void raid5_wakeup_stripe_thread(struct stripe_head *sh) { struct r5conf *conf = sh->raid_conf; struct r5worker_group *group; int thread_cnt; int i, cpu = sh->cpu; if (!cpu_online(cpu)) { cpu = cpumask_any(cpu_online_mask); sh->cpu = cpu; } if (list_empty(&sh->lru)) { struct r5worker_group *group; group = conf->worker_groups + cpu_to_group(cpu); list_add_tail(&sh->lru, &group->handle_list); group->stripes_cnt++; sh->group = group; } if (conf->worker_cnt_per_group == 0) { md_wakeup_thread(conf->mddev->thread); return; } group = conf->worker_groups + cpu_to_group(sh->cpu); group->workers[0].working = true; /* at least one worker should run to avoid race */ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; /* wakeup more workers */ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { if (group->workers[i].working == false) { group->workers[i].working = true; queue_work_on(sh->cpu, raid5_wq, &group->workers[i].work); thread_cnt--; } } } static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list) { BUG_ON(!list_empty(&sh->lru)); BUG_ON(atomic_read(&conf->active_stripes)==0); if (test_bit(STRIPE_HANDLE, &sh->state)) { if (test_bit(STRIPE_DELAYED, &sh->state) && !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) list_add_tail(&sh->lru, &conf->delayed_list); else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && sh->bm_seq - conf->seq_write > 0) list_add_tail(&sh->lru, &conf->bitmap_list); else { clear_bit(STRIPE_DELAYED, &sh->state); clear_bit(STRIPE_BIT_DELAY, &sh->state); if (conf->worker_cnt_per_group == 0) { list_add_tail(&sh->lru, &conf->handle_list); } else { raid5_wakeup_stripe_thread(sh); return; } } md_wakeup_thread(conf->mddev->thread); } else { BUG_ON(stripe_operations_active(sh)); if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) if (atomic_dec_return(&conf->preread_active_stripes) < IO_THRESHOLD) md_wakeup_thread(conf->mddev->thread); atomic_dec(&conf->active_stripes); if (!test_bit(STRIPE_EXPANDING, &sh->state)) list_add_tail(&sh->lru, temp_inactive_list); } } static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, struct list_head *temp_inactive_list) { if (atomic_dec_and_test(&sh->count)) do_release_stripe(conf, sh, temp_inactive_list); } /* * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list * * Be careful: Only one task can add/delete stripes from temp_inactive_list at * given time. Adding stripes only takes device lock, while deleting stripes * only takes hash lock. */ static void release_inactive_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list, int hash) { int size; bool do_wakeup = false; unsigned long flags; if (hash == NR_STRIPE_HASH_LOCKS) { size = NR_STRIPE_HASH_LOCKS; hash = NR_STRIPE_HASH_LOCKS - 1; } else size = 1; while (size) { struct list_head *list = &temp_inactive_list[size - 1]; /* * We don't hold any lock here yet, get_active_stripe() might * remove stripes from the list */ if (!list_empty_careful(list)) { spin_lock_irqsave(conf->hash_locks + hash, flags); if (list_empty(conf->inactive_list + hash) && !list_empty(list)) atomic_dec(&conf->empty_inactive_list_nr); list_splice_tail_init(list, conf->inactive_list + hash); do_wakeup = true; spin_unlock_irqrestore(conf->hash_locks + hash, flags); } size--; hash--; } if (do_wakeup) { wake_up(&conf->wait_for_stripe); if (conf->retry_read_aligned) md_wakeup_thread(conf->mddev->thread); } } /* should hold conf->device_lock already */ static int release_stripe_list(struct r5conf *conf, struct list_head *temp_inactive_list) { struct stripe_head *sh; int count = 0; struct llist_node *head; head = llist_del_all(&conf->released_stripes); head = llist_reverse_order(head); while (head) { int hash; sh = llist_entry(head, struct stripe_head, release_list); head = llist_next(head); /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ smp_mb(); clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); /* * Don't worry the bit is set here, because if the bit is set * again, the count is always > 1. This is true for * STRIPE_ON_UNPLUG_LIST bit too. */ hash = sh->hash_lock_index; __release_stripe(conf, sh, &temp_inactive_list[hash]); count++; } return count; } static void release_stripe(struct stripe_head *sh) { struct r5conf *conf = sh->raid_conf; unsigned long flags; struct list_head list; int hash; bool wakeup; if (unlikely(!conf->mddev->thread) || test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) goto slow_path; wakeup = llist_add(&sh->release_list, &conf->released_stripes); if (wakeup) md_wakeup_thread(conf->mddev->thread); return; slow_path: local_irq_save(flags); /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { INIT_LIST_HEAD(&list); hash = sh->hash_lock_index; do_release_stripe(conf, sh, &list); spin_unlock(&conf->device_lock); release_inactive_stripe_list(conf, &list, hash); } local_irq_restore(flags); } static inline void remove_hash(struct stripe_head *sh) { pr_debug("remove_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_del_init(&sh->hash); } static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) { struct hlist_head *hp = stripe_hash(conf, sh->sector); pr_debug("insert_hash(), stripe %llu\n", (unsigned long long)sh->sector); hlist_add_head(&sh->hash, hp); } /* find an idle stripe, make sure it is unhashed, and return it. */ static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) { struct stripe_head *sh = NULL; struct list_head *first; if (list_empty(conf->inactive_list + hash)) goto out; first = (conf->inactive_list + hash)->next; sh = list_entry(first, struct stripe_head, lru); list_del_init(first); remove_hash(sh); atomic_inc(&conf->active_stripes); BUG_ON(hash != sh->hash_lock_index); if (list_empty(conf->inactive_list + hash)) atomic_inc(&conf->empty_inactive_list_nr); out: return sh; } static void shrink_buffers(struct stripe_head *sh) { struct page *p; int i; int num = sh->raid_conf->pool_size; for (i = 0; i < num ; i++) { p = sh->dev[i].page; if (!p) continue; sh->dev[i].page = NULL; put_page(p); } } static int grow_buffers(struct stripe_head *sh) { int i; int num = sh->raid_conf->pool_size; for (i = 0; i < num; i++) { struct page *page; if (!(page = alloc_page(GFP_KERNEL))) { return 1; } sh->dev[i].page = page; } return 0; } static void raid5_build_block(struct stripe_head *sh, int i, int previous); static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, struct stripe_head *sh); static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) { struct r5conf *conf = sh->raid_conf; int i, seq; BUG_ON(atomic_read(&sh->count) != 0); BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); BUG_ON(stripe_operations_active(sh)); pr_debug("init_stripe called, stripe %llu\n", (unsigned long long)sh->sector); remove_hash(sh); retry: seq = read_seqcount_begin(&conf->gen_lock); sh->generation = conf->generation - previous; sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; sh->sector = sector; stripe_set_idx(sector, conf, previous, sh); sh->state = 0; for (i = sh->disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (dev->toread || dev->read || dev->towrite || dev->written || test_bit(R5_LOCKED, &dev->flags)) { printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", (unsigned long long)sh->sector, i, dev->toread, dev->read, dev->towrite, dev->written, test_bit(R5_LOCKED, &dev->flags)); WARN_ON(1); } dev->flags = 0; raid5_build_block(sh, i, previous); } if (read_seqcount_retry(&conf->gen_lock, seq)) goto retry; insert_hash(conf, sh); sh->cpu = smp_processor_id(); } static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, short generation) { struct stripe_head *sh; pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) if (sh->sector == sector && sh->generation == generation) return sh; pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); return NULL; } /* * Need to check if array has failed when deciding whether to: * - start an array * - remove non-faulty devices * - add a spare * - allow a reshape * This determination is simple when no reshape is happening. * However if there is a reshape, we need to carefully check * both the before and after sections. * This is because some failed devices may only affect one * of the two sections, and some non-in_sync devices may * be insync in the section most affected by failed devices. */ static int calc_degraded(struct r5conf *conf) { int degraded, degraded2; int i; rcu_read_lock(); degraded = 0; for (i = 0; i < conf->previous_raid_disks; i++) { struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); if (rdev && test_bit(Faulty, &rdev->flags)) rdev = rcu_dereference(conf->disks[i].replacement); if (!rdev || test_bit(Faulty, &rdev->flags)) degraded++; else if (test_bit(In_sync, &rdev->flags)) ; else /* not in-sync or faulty. * If the reshape increases the number of devices, * this is being recovered by the reshape, so * this 'previous' section is not in_sync. * If the number of devices is being reduced however, * the device can only be part of the array if * we are reverting a reshape, so this section will * be in-sync. */ if (conf->raid_disks >= conf->previous_raid_disks) degraded++; } rcu_read_unlock(); if (conf->raid_disks == conf->previous_raid_disks) return degraded; rcu_read_lock(); degraded2 = 0; for (i = 0; i < conf->raid_disks; i++) { struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); if (rdev && test_bit(Faulty, &rdev->flags)) rdev = rcu_dereference(conf->disks[i].replacement); if (!rdev || test_bit(Faulty, &rdev->flags)) degraded2++; else if (test_bit(In_sync, &rdev->flags)) ; else /* not in-sync or faulty. * If reshape increases the number of devices, this * section has already been recovered, else it * almost certainly hasn't. */ if (conf->raid_disks <= conf->previous_raid_disks) degraded2++; } rcu_read_unlock(); if (degraded2 > degraded) return degraded2; return degraded; } static int has_failed(struct r5conf *conf) { int degraded; if (conf->mddev->reshape_position == MaxSector) return conf->mddev->degraded > conf->max_degraded; degraded = calc_degraded(conf); if (degraded > conf->max_degraded) return 1; return 0; } static struct stripe_head * get_active_stripe(struct r5conf *conf, sector_t sector, int previous, int noblock, int noquiesce) { struct stripe_head *sh; int hash = stripe_hash_locks_hash(sector); pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); spin_lock_irq(conf->hash_locks + hash); do { wait_event_lock_irq(conf->wait_for_stripe, conf->quiesce == 0 || noquiesce, *(conf->hash_locks + hash)); sh = __find_stripe(conf, sector, conf->generation - previous); if (!sh) { if (!conf->inactive_blocked) sh = get_free_stripe(conf, hash); if (noblock && sh == NULL) break; if (!sh) { conf->inactive_blocked = 1; wait_event_lock_irq( conf->wait_for_stripe, !list_empty(conf->inactive_list + hash) && (atomic_read(&conf->active_stripes) < (conf->max_nr_stripes * 3 / 4) || !conf->inactive_blocked), *(conf->hash_locks + hash)); conf->inactive_blocked = 0; } else { init_stripe(sh, sector, previous); atomic_inc(&sh->count); } } else { spin_lock(&conf->device_lock); if (atomic_read(&sh->count)) { BUG_ON(!list_empty(&sh->lru) && !test_bit(STRIPE_EXPANDING, &sh->state) && !test_bit(STRIPE_ON_UNPLUG_LIST, &sh->state) ); } else { if (!test_bit(STRIPE_HANDLE, &sh->state)) atomic_inc(&conf->active_stripes); BUG_ON(list_empty(&sh->lru) && !test_bit(STRIPE_EXPANDING, &sh->state)); list_del_init(&sh->lru); if (sh->group) { sh->group->stripes_cnt--; sh->group = NULL; } } atomic_inc(&sh->count); spin_unlock(&conf->device_lock); } } while (sh == NULL); spin_unlock_irq(conf->hash_locks + hash); return sh; } /* Determine if 'data_offset' or 'new_data_offset' should be used * in this stripe_head. */ static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) { sector_t progress = conf->reshape_progress; /* Need a memory barrier to make sure we see the value * of conf->generation, or ->data_offset that was set before * reshape_progress was updated. */ smp_rmb(); if (progress == MaxSector) return 0; if (sh->generation == conf->generation - 1) return 0; /* We are in a reshape, and this is a new-generation stripe, * so use new_data_offset. */ return 1; } static void raid5_end_read_request(struct bio *bi, int error); static void raid5_end_write_request(struct bio *bi, int error); static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) { struct r5conf *conf = sh->raid_conf; int i, disks = sh->disks; might_sleep(); for (i = disks; i--; ) { int rw; int replace_only = 0; struct bio *bi, *rbi; struct md_rdev *rdev, *rrdev = NULL; if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) rw = WRITE_FUA; else rw = WRITE; if (test_bit(R5_Discard, &sh->dev[i].flags)) rw |= REQ_DISCARD; } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) rw = READ; else if (test_and_clear_bit(R5_WantReplace, &sh->dev[i].flags)) { rw = WRITE; replace_only = 1; } else continue; if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) rw |= REQ_SYNC; bi = &sh->dev[i].req; rbi = &sh->dev[i].rreq; /* For writing to replacement */ rcu_read_lock(); rrdev = rcu_dereference(conf->disks[i].replacement); smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ rdev = rcu_dereference(conf->disks[i].rdev); if (!rdev) { rdev = rrdev; rrdev = NULL; } if (rw & WRITE) { if (replace_only) rdev = NULL; if (rdev == rrdev) /* We raced and saw duplicates */ rrdev = NULL; } else { if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev) rdev = rrdev; rrdev = NULL; } if (rdev && test_bit(Faulty, &rdev->flags)) rdev = NULL; if (rdev) atomic_inc(&rdev->nr_pending); if (rrdev && test_bit(Faulty, &rrdev->flags)) rrdev = NULL; if (rrdev) atomic_inc(&rrdev->nr_pending); rcu_read_unlock(); /* We have already checked bad blocks for reads. Now * need to check for writes. We never accept write errors * on the replacement, so we don't to check rrdev. */ while ((rw & WRITE) && rdev && test_bit(WriteErrorSeen, &rdev->flags)) { sector_t first_bad; int bad_sectors; int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, &first_bad, &bad_sectors); if (!bad) break; if (bad < 0) { set_bit(BlockedBadBlocks, &rdev->flags); if (!conf->mddev->external && conf->mddev->flags) { /* It is very unlikely, but we might * still need to write out the * bad block log - better give it * a chance*/ md_check_recovery(conf->mddev); } /* * Because md_wait_for_blocked_rdev * will dec nr_pending, we must * increment it first. */ atomic_inc(&rdev->nr_pending); md_wait_for_blocked_rdev(rdev, conf->mddev); } else { /* Acknowledged bad block - skip the write */ rdev_dec_pending(rdev, conf->mddev); rdev = NULL; } } if (rdev) { if (s->syncing || s->expanding || s->expanded || s->replacing) md_sync_acct(rdev->bdev, STRIPE_SECTORS); set_bit(STRIPE_IO_STARTED, &sh->state); bio_reset(bi); bi->bi_bdev = rdev->bdev; bi->bi_rw = rw; bi->bi_end_io = (rw & WRITE) ? raid5_end_write_request : raid5_end_read_request; bi->bi_private = sh; pr_debug("%s: for %llu schedule op %ld on disc %d\n", __func__, (unsigned long long)sh->sector, bi->bi_rw, i); atomic_inc(&sh->count); if (use_new_offset(conf, sh)) bi->bi_iter.bi_sector = (sh->sector + rdev->new_data_offset); else bi->bi_iter.bi_sector = (sh->sector + rdev->data_offset); if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) bi->bi_rw |= REQ_NOMERGE; bi->bi_vcnt = 1; bi->bi_io_vec[0].bv_len = STRIPE_SIZE; bi->bi_io_vec[0].bv_offset = 0; bi->bi_iter.bi_size = STRIPE_SIZE; /* * If this is discard request, set bi_vcnt 0. We don't * want to confuse SCSI because SCSI will replace payload */ if (rw & REQ_DISCARD) bi->bi_vcnt = 0; if (rrdev) set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); if (conf->mddev->gendisk) trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), bi, disk_devt(conf->mddev->gendisk), sh->dev[i].sector); generic_make_request(bi); } if (rrdev) { if (s->syncing || s->expanding || s->expanded || s->replacing) md_sync_acct(rrdev->bdev, STRIPE_SECTORS); set_bit(STRIPE_IO_STARTED, &sh->state); bio_reset(rbi); rbi->bi_bdev = rrdev->bdev; rbi->bi_rw = rw; BUG_ON(!(rw & WRITE)); rbi->bi_end_io = raid5_end_write_request; rbi->bi_private = sh; pr_debug("%s: for %llu schedule op %ld on " "replacement disc %d\n", __func__, (unsigned long long)sh->sector, rbi->bi_rw, i); atomic_inc(&sh->count); if (use_new_offset(conf, sh)) rbi->bi_iter.bi_sector = (sh->sector + rrdev->new_data_offset); else rbi->bi_iter.bi_sector = (sh->sector + rrdev->data_offset); rbi->bi_vcnt = 1; rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; rbi->bi_io_vec[0].bv_offset = 0; rbi->bi_iter.bi_size = STRIPE_SIZE; /* * If this is discard request, set bi_vcnt 0. We don't * want to confuse SCSI because SCSI will replace payload */ if (rw & REQ_DISCARD) rbi->bi_vcnt = 0; if (conf->mddev->gendisk) trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), rbi, disk_devt(conf->mddev->gendisk), sh->dev[i].sector); generic_make_request(rbi); } if (!rdev && !rrdev) { if (rw & WRITE) set_bit(STRIPE_DEGRADED, &sh->state); pr_debug("skip op %ld on disc %d for sector %llu\n", bi->bi_rw, i, (unsigned long long)sh->sector); clear_bit(R5_LOCKED, &sh->dev[i].flags); set_bit(STRIPE_HANDLE, &sh->state); } } } static struct dma_async_tx_descriptor * async_copy_data(int frombio, struct bio *bio, struct page *page, sector_t sector, struct dma_async_tx_descriptor *tx) { struct bio_vec bvl; struct bvec_iter iter; struct page *bio_page; int page_offset; struct async_submit_ctl submit; enum async_tx_flags flags = 0; if (bio->bi_iter.bi_sector >= sector) page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512; else page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512; if (frombio) flags |= ASYNC_TX_FENCE; init_async_submit(&submit, flags, tx, NULL, NULL, NULL); bio_for_each_segment(bvl, bio, iter) { int len = bvl.bv_len; int clen; int b_offset = 0; if (page_offset < 0) { b_offset = -page_offset; page_offset += b_offset; len -= b_offset; } if (len > 0 && page_offset + len > STRIPE_SIZE) clen = STRIPE_SIZE - page_offset; else clen = len; if (clen > 0) { b_offset += bvl.bv_offset; bio_page = bvl.bv_page; if (frombio) tx = async_memcpy(page, bio_page, page_offset, b_offset, clen, &submit); else tx = async_memcpy(bio_page, page, b_offset, page_offset, clen, &submit); } /* chain the operations */ submit.depend_tx = tx; if (clen < len) /* hit end of page */ break; page_offset += len; } return tx; } static void ops_complete_biofill(void *stripe_head_ref) { struct stripe_head *sh = stripe_head_ref; struct bio *return_bi = NULL; int i; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); /* clear completed biofills */ for (i = sh->disks; i--; ) { struct r5dev *dev = &sh->dev[i]; /* acknowledge completion of a biofill operation */ /* and check if we need to reply to a read request, * new R5_Wantfill requests are held off until * !STRIPE_BIOFILL_RUN */ if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { struct bio *rbi, *rbi2; BUG_ON(!dev->read); rbi = dev->read; dev->read = NULL; while (rbi && rbi->bi_iter.bi_sector < dev->sector + STRIPE_SECTORS) { rbi2 = r5_next_bio(rbi, dev->sector); if (!raid5_dec_bi_active_stripes(rbi)) { rbi->bi_next = return_bi; return_bi = rbi; } rbi = rbi2; } } } clear_bit(STRIPE_BIOFILL_RUN, &sh->state); return_io(return_bi); set_bit(STRIPE_HANDLE, &sh->state); release_stripe(sh); } static void ops_run_biofill(struct stripe_head *sh) { struct dma_async_tx_descriptor *tx = NULL; struct async_submit_ctl submit; int i; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); for (i = sh->disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (test_bit(R5_Wantfill, &dev->flags)) { struct bio *rbi; spin_lock_irq(&sh->stripe_lock); dev->read = rbi = dev->toread; dev->toread = NULL; spin_unlock_irq(&sh->stripe_lock); while (rbi && rbi->bi_iter.bi_sector < dev->sector + STRIPE_SECTORS) { tx = async_copy_data(0, rbi, dev->page, dev->sector, tx); rbi = r5_next_bio(rbi, dev->sector); } } } atomic_inc(&sh->count); init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); async_trigger_callback(&submit); } static void mark_target_uptodate(struct stripe_head *sh, int target) { struct r5dev *tgt; if (target < 0) return; tgt = &sh->dev[target]; set_bit(R5_UPTODATE, &tgt->flags); BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); clear_bit(R5_Wantcompute, &tgt->flags); } static void ops_complete_compute(void *stripe_head_ref) { struct stripe_head *sh = stripe_head_ref; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); /* mark the computed target(s) as uptodate */ mark_target_uptodate(sh, sh->ops.target); mark_target_uptodate(sh, sh->ops.target2); clear_bit(STRIPE_COMPUTE_RUN, &sh->state); if (sh->check_state == check_state_compute_run) sh->check_state = check_state_compute_result; set_bit(STRIPE_HANDLE, &sh->state); release_stripe(sh); } /* return a pointer to the address conversion region of the scribble buffer */ static addr_conv_t *to_addr_conv(struct stripe_head *sh, struct raid5_percpu *percpu) { return percpu->scribble + sizeof(struct page *) * (sh->disks + 2); } static struct dma_async_tx_descriptor * ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) { int disks = sh->disks; struct page **xor_srcs = percpu->scribble; int target = sh->ops.target; struct r5dev *tgt = &sh->dev[target]; struct page *xor_dest = tgt->page; int count = 0; struct dma_async_tx_descriptor *tx; struct async_submit_ctl submit; int i; pr_debug("%s: stripe %llu block: %d\n", __func__, (unsigned long long)sh->sector, target); BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); for (i = disks; i--; ) if (i != target) xor_srcs[count++] = sh->dev[i].page; atomic_inc(&sh->count); init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, ops_complete_compute, sh, to_addr_conv(sh, percpu)); if (unlikely(count == 1)) tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); else tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); return tx; } /* set_syndrome_sources - populate source buffers for gen_syndrome * @srcs - (struct page *) array of size sh->disks * @sh - stripe_head to parse * * Populates srcs in proper layout order for the stripe and returns the * 'count' of sources to be used in a call to async_gen_syndrome. The P * destination buffer is recorded in srcs[count] and the Q destination * is recorded in srcs[count+1]]. */ static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh) { int disks = sh->disks; int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); int d0_idx = raid6_d0(sh); int count; int i; for (i = 0; i < disks; i++) srcs[i] = NULL; count = 0; i = d0_idx; do { int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); srcs[slot] = sh->dev[i].page; i = raid6_next_disk(i, disks); } while (i != d0_idx); return syndrome_disks; } static struct dma_async_tx_descriptor * ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) { int disks = sh->disks; struct page **blocks = percpu->scribble; int target; int qd_idx = sh->qd_idx; struct dma_async_tx_descriptor *tx; struct async_submit_ctl submit; struct r5dev *tgt; struct page *dest; int i; int count; if (sh->ops.target < 0) target = sh->ops.target2; else if (sh->ops.target2 < 0) target = sh->ops.target; else /* we should only have one valid target */ BUG(); BUG_ON(target < 0); pr_debug("%s: stripe %llu block: %d\n", __func__, (unsigned long long)sh->sector, target); tgt = &sh->dev[target]; BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); dest = tgt->page; atomic_inc(&sh->count); if (target == qd_idx) { count = set_syndrome_sources(blocks, sh); blocks[count] = NULL; /* regenerating p is not necessary */ BUG_ON(blocks[count+1] != dest); /* q should already be set */ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ops_complete_compute, sh, to_addr_conv(sh, percpu)); tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); } else { /* Compute any data- or p-drive using XOR */ count = 0; for (i = disks; i-- ; ) { if (i == target || i == qd_idx) continue; blocks[count++] = sh->dev[i].page; } init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, ops_complete_compute, sh, to_addr_conv(sh, percpu)); tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); } return tx; } static struct dma_async_tx_descriptor * ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) { int i, count, disks = sh->disks; int syndrome_disks = sh->ddf_layout ? disks : disks-2; int d0_idx = raid6_d0(sh); int faila = -1, failb = -1; int target = sh->ops.target; int target2 = sh->ops.target2; struct r5dev *tgt = &sh->dev[target]; struct r5dev *tgt2 = &sh->dev[target2]; struct dma_async_tx_descriptor *tx; struct page **blocks = percpu->scribble; struct async_submit_ctl submit; pr_debug("%s: stripe %llu block1: %d block2: %d\n", __func__, (unsigned long long)sh->sector, target, target2); BUG_ON(target < 0 || target2 < 0); BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); /* we need to open-code set_syndrome_sources to handle the * slot number conversion for 'faila' and 'failb' */ for (i = 0; i < disks ; i++) blocks[i] = NULL; count = 0; i = d0_idx; do { int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); blocks[slot] = sh->dev[i].page; if (i == target) faila = slot; if (i == target2) failb = slot; i = raid6_next_disk(i, disks); } while (i != d0_idx); BUG_ON(faila == failb); if (failb < faila) swap(faila, failb); pr_debug("%s: stripe: %llu faila: %d failb: %d\n", __func__, (unsigned long long)sh->sector, faila, failb); atomic_inc(&sh->count); if (failb == syndrome_disks+1) { /* Q disk is one of the missing disks */ if (faila == syndrome_disks) { /* Missing P+Q, just recompute */ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ops_complete_compute, sh, to_addr_conv(sh, percpu)); return async_gen_syndrome(blocks, 0, syndrome_disks+2, STRIPE_SIZE, &submit); } else { struct page *dest; int data_target; int qd_idx = sh->qd_idx; /* Missing D+Q: recompute D from P, then recompute Q */ if (target == qd_idx) data_target = target2; else data_target = target; count = 0; for (i = disks; i-- ; ) { if (i == data_target || i == qd_idx) continue; blocks[count++] = sh->dev[i].page; } dest = sh->dev[data_target].page; init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, NULL, NULL, to_addr_conv(sh, percpu)); tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); count = set_syndrome_sources(blocks, sh); init_async_submit(&submit, ASYNC_TX_FENCE, tx, ops_complete_compute, sh, to_addr_conv(sh, percpu)); return async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); } } else { init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ops_complete_compute, sh, to_addr_conv(sh, percpu)); if (failb == syndrome_disks) { /* We're missing D+P. */ return async_raid6_datap_recov(syndrome_disks+2, STRIPE_SIZE, faila, blocks, &submit); } else { /* We're missing D+D. */ return async_raid6_2data_recov(syndrome_disks+2, STRIPE_SIZE, faila, failb, blocks, &submit); } } } static void ops_complete_prexor(void *stripe_head_ref) { struct stripe_head *sh = stripe_head_ref; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); } static struct dma_async_tx_descriptor * ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu, struct dma_async_tx_descriptor *tx) { int disks = sh->disks; struct page **xor_srcs = percpu->scribble; int count = 0, pd_idx = sh->pd_idx, i; struct async_submit_ctl submit; /* existing parity data subtracted */ struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); for (i = disks; i--; ) { struct r5dev *dev = &sh->dev[i]; /* Only process blocks that are known to be uptodate */ if (test_bit(R5_Wantdrain, &dev->flags)) xor_srcs[count++] = dev->page; } init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, ops_complete_prexor, sh, to_addr_conv(sh, percpu)); tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); return tx; } static struct dma_async_tx_descriptor * ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) { int disks = sh->disks; int i; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); for (i = disks; i--; ) { struct r5dev *dev = &sh->dev[i]; struct bio *chosen; if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) { struct bio *wbi; spin_lock_irq(&sh->stripe_lock); chosen = dev->towrite; dev->towrite = NULL; BUG_ON(dev->written); wbi = dev->written = chosen; spin_unlock_irq(&sh->stripe_lock); while (wbi && wbi->bi_iter.bi_sector < dev->sector + STRIPE_SECTORS) { if (wbi->bi_rw & REQ_FUA) set_bit(R5_WantFUA, &dev->flags); if (wbi->bi_rw & REQ_SYNC) set_bit(R5_SyncIO, &dev->flags); if (wbi->bi_rw & REQ_DISCARD) set_bit(R5_Discard, &dev->flags); else tx = async_copy_data(1, wbi, dev->page, dev->sector, tx); wbi = r5_next_bio(wbi, dev->sector); } } } return tx; } static void ops_complete_reconstruct(void *stripe_head_ref) { struct stripe_head *sh = stripe_head_ref; int disks = sh->disks; int pd_idx = sh->pd_idx; int qd_idx = sh->qd_idx; int i; bool fua = false, sync = false, discard = false; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); for (i = disks; i--; ) { fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); discard |= test_bit(R5_Discard, &sh->dev[i].flags); } for (i = disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (dev->written || i == pd_idx || i == qd_idx) { if (!discard) set_bit(R5_UPTODATE, &dev->flags); if (fua) set_bit(R5_WantFUA, &dev->flags); if (sync) set_bit(R5_SyncIO, &dev->flags); } } if (sh->reconstruct_state == reconstruct_state_drain_run) sh->reconstruct_state = reconstruct_state_drain_result; else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) sh->reconstruct_state = reconstruct_state_prexor_drain_result; else { BUG_ON(sh->reconstruct_state != reconstruct_state_run); sh->reconstruct_state = reconstruct_state_result; } set_bit(STRIPE_HANDLE, &sh->state); release_stripe(sh); } static void ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, struct dma_async_tx_descriptor *tx) { int disks = sh->disks; struct page **xor_srcs = percpu->scribble; struct async_submit_ctl submit; int count = 0, pd_idx = sh->pd_idx, i; struct page *xor_dest; int prexor = 0; unsigned long flags; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); for (i = 0; i < sh->disks; i++) { if (pd_idx == i) continue; if (!test_bit(R5_Discard, &sh->dev[i].flags)) break; } if (i >= sh->disks) { atomic_inc(&sh->count); set_bit(R5_Discard, &sh->dev[pd_idx].flags); ops_complete_reconstruct(sh); return; } /* check if prexor is active which means only process blocks * that are part of a read-modify-write (written) */ if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { prexor = 1; xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; for (i = disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (dev->written) xor_srcs[count++] = dev->page; } } else { xor_dest = sh->dev[pd_idx].page; for (i = disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (i != pd_idx) xor_srcs[count++] = dev->page; } } /* 1/ if we prexor'd then the dest is reused as a source * 2/ if we did not prexor then we are redoing the parity * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST * for the synchronous xor case */ flags = ASYNC_TX_ACK | (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); atomic_inc(&sh->count); init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh, to_addr_conv(sh, percpu)); if (unlikely(count == 1)) tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); else tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); } static void ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, struct dma_async_tx_descriptor *tx) { struct async_submit_ctl submit; struct page **blocks = percpu->scribble; int count, i; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); for (i = 0; i < sh->disks; i++) { if (sh->pd_idx == i || sh->qd_idx == i) continue; if (!test_bit(R5_Discard, &sh->dev[i].flags)) break; } if (i >= sh->disks) { atomic_inc(&sh->count); set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); ops_complete_reconstruct(sh); return; } count = set_syndrome_sources(blocks, sh); atomic_inc(&sh->count); init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct, sh, to_addr_conv(sh, percpu)); async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); } static void ops_complete_check(void *stripe_head_ref) { struct stripe_head *sh = stripe_head_ref; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); sh->check_state = check_state_check_result; set_bit(STRIPE_HANDLE, &sh->state); release_stripe(sh); } static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) { int disks = sh->disks; int pd_idx = sh->pd_idx; int qd_idx = sh->qd_idx; struct page *xor_dest; struct page **xor_srcs = percpu->scribble; struct dma_async_tx_descriptor *tx; struct async_submit_ctl submit; int count; int i; pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); count = 0; xor_dest = sh->dev[pd_idx].page; xor_srcs[count++] = xor_dest; for (i = disks; i--; ) { if (i == pd_idx || i == qd_idx) continue; xor_srcs[count++] = sh->dev[i].page; } init_async_submit(&submit, 0, NULL, NULL, NULL, to_addr_conv(sh, percpu)); tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &sh->ops.zero_sum_result, &submit); atomic_inc(&sh->count); init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); tx = async_trigger_callback(&submit); } static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) { struct page **srcs = percpu->scribble; struct async_submit_ctl submit; int count; pr_debug("%s: stripe %llu checkp: %d\n", __func__, (unsigned long long)sh->sector, checkp); count = set_syndrome_sources(srcs, sh); if (!checkp) srcs[count] = NULL; atomic_inc(&sh->count); init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, sh, to_addr_conv(sh, percpu)); async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, &sh->ops.zero_sum_result, percpu->spare_page, &submit); } static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) { int overlap_clear = 0, i, disks = sh->disks; struct dma_async_tx_descriptor *tx = NULL; struct r5conf *conf = sh->raid_conf; int level = conf->level; struct raid5_percpu *percpu; unsigned long cpu; cpu = get_cpu(); percpu = per_cpu_ptr(conf->percpu, cpu); if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { ops_run_biofill(sh); overlap_clear++; } if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { if (level < 6) tx = ops_run_compute5(sh, percpu); else { if (sh->ops.target2 < 0 || sh->ops.target < 0) tx = ops_run_compute6_1(sh, percpu); else tx = ops_run_compute6_2(sh, percpu); } /* terminate the chain if reconstruct is not set to be run */ if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) async_tx_ack(tx); } if (test_bit(STRIPE_OP_PREXOR, &ops_request)) tx = ops_run_prexor(sh, percpu, tx); if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { tx = ops_run_biodrain(sh, tx); overlap_clear++; } if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { if (level < 6) ops_run_reconstruct5(sh, percpu, tx); else ops_run_reconstruct6(sh, percpu, tx); } if (test_bit(STRIPE_OP_CHECK, &ops_request)) { if (sh->check_state == check_state_run) ops_run_check_p(sh, percpu); else if (sh->check_state == check_state_run_q) ops_run_check_pq(sh, percpu, 0); else if (sh->check_state == check_state_run_pq) ops_run_check_pq(sh, percpu, 1); else BUG(); } if (overlap_clear) for (i = disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (test_and_clear_bit(R5_Overlap, &dev->flags)) wake_up(&sh->raid_conf->wait_for_overlap); } put_cpu(); } static int grow_one_stripe(struct r5conf *conf, int hash) { struct stripe_head *sh; sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL); if (!sh) return 0; sh->raid_conf = conf; spin_lock_init(&sh->stripe_lock); if (grow_buffers(sh)) { shrink_buffers(sh); kmem_cache_free(conf->slab_cache, sh); return 0; } sh->hash_lock_index = hash; /* we just created an active stripe so... */ atomic_set(&sh->count, 1); atomic_inc(&conf->active_stripes); INIT_LIST_HEAD(&sh->lru); release_stripe(sh); return 1; } static int grow_stripes(struct r5conf *conf, int num) { struct kmem_cache *sc; int devs = max(conf->raid_disks, conf->previous_raid_disks); int hash; if (conf->mddev->gendisk) sprintf(conf->cache_name[0], "raid%d-%s", conf->level, mdname(conf->mddev)); else sprintf(conf->cache_name[0], "raid%d-%p", conf->level, conf->mddev); sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); conf->active_name = 0; sc = kmem_cache_create(conf->cache_name[conf->active_name], sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), 0, 0, NULL); if (!sc) return 1; conf->slab_cache = sc; conf->pool_size = devs; hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; while (num--) { if (!grow_one_stripe(conf, hash)) return 1; conf->max_nr_stripes++; hash = (hash + 1) % NR_STRIPE_HASH_LOCKS; } return 0; } /** * scribble_len - return the required size of the scribble region * @num - total number of disks in the array * * The size must be enough to contain: * 1/ a struct page pointer for each device in the array +2 * 2/ room to convert each entry in (1) to its corresponding dma * (dma_map_page()) or page (page_address()) address. * * Note: the +2 is for the destination buffers of the ddf/raid6 case where we * calculate over all devices (not just the data blocks), using zeros in place * of the P and Q blocks. */ static size_t scribble_len(int num) { size_t len; len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); return len; } static int resize_stripes(struct r5conf *conf, int newsize) { /* Make all the stripes able to hold 'newsize' devices. * New slots in each stripe get 'page' set to a new page. * * This happens in stages: * 1/ create a new kmem_cache and allocate the required number of * stripe_heads. * 2/ gather all the old stripe_heads and transfer the pages across * to the new stripe_heads. This will have the side effect of * freezing the array as once all stripe_heads have been collected, * no IO will be possible. Old stripe heads are freed once their * pages have been transferred over, and the old kmem_cache is * freed when all stripes are done. * 3/ reallocate conf->disks to be suitable bigger. If this fails, * we simple return a failre status - no need to clean anything up. * 4/ allocate new pages for the new slots in the new stripe_heads. * If this fails, we don't bother trying the shrink the * stripe_heads down again, we just leave them as they are. * As each stripe_head is processed the new one is released into * active service. * * Once step2 is started, we cannot afford to wait for a write, * so we use GFP_NOIO allocations. */ struct stripe_head *osh, *nsh; LIST_HEAD(newstripes); struct disk_info *ndisks; unsigned long cpu; int err; struct kmem_cache *sc; int i; int hash, cnt; if (newsize <= conf->pool_size) return 0; /* never bother to shrink */ err = md_allow_write(conf->mddev); if (err) return err; /* Step 1 */ sc = kmem_cache_create(conf->cache_name[1-conf->active_name], sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), 0, 0, NULL); if (!sc) return -ENOMEM; for (i = conf->max_nr_stripes; i; i--) { nsh = kmem_cache_zalloc(sc, GFP_KERNEL); if (!nsh) break; nsh->raid_conf = conf; spin_lock_init(&nsh->stripe_lock); list_add(&nsh->lru, &newstripes); } if (i) { /* didn't get enough, give up */ while (!list_empty(&newstripes)) { nsh = list_entry(newstripes.next, struct stripe_head, lru); list_del(&nsh->lru); kmem_cache_free(sc, nsh); } kmem_cache_destroy(sc); return -ENOMEM; } /* Step 2 - Must use GFP_NOIO now. * OK, we have enough stripes, start collecting inactive * stripes and copying them over */ hash = 0; cnt = 0; list_for_each_entry(nsh, &newstripes, lru) { lock_device_hash_lock(conf, hash); wait_event_cmd(conf->wait_for_stripe, !list_empty(conf->inactive_list + hash), unlock_device_hash_lock(conf, hash), lock_device_hash_lock(conf, hash)); osh = get_free_stripe(conf, hash); unlock_device_hash_lock(conf, hash); atomic_set(&nsh->count, 1); for(i=0; ipool_size; i++) nsh->dev[i].page = osh->dev[i].page; for( ; idev[i].page = NULL; nsh->hash_lock_index = hash; kmem_cache_free(conf->slab_cache, osh); cnt++; if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { hash++; cnt = 0; } } kmem_cache_destroy(conf->slab_cache); /* Step 3. * At this point, we are holding all the stripes so the array * is completely stalled, so now is a good time to resize * conf->disks and the scribble region */ ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); if (ndisks) { for (i=0; iraid_disks; i++) ndisks[i] = conf->disks[i]; kfree(conf->disks); conf->disks = ndisks; } else err = -ENOMEM; get_online_cpus(); conf->scribble_len = scribble_len(newsize); for_each_present_cpu(cpu) { struct raid5_percpu *percpu; void *scribble; percpu = per_cpu_ptr(conf->percpu, cpu); scribble = kmalloc(conf->scribble_len, GFP_NOIO); if (scribble) { kfree(percpu->scribble); percpu->scribble = scribble; } else { err = -ENOMEM; break; } } put_online_cpus(); /* Step 4, return new stripes to service */ while(!list_empty(&newstripes)) { nsh = list_entry(newstripes.next, struct stripe_head, lru); list_del_init(&nsh->lru); for (i=conf->raid_disks; i < newsize; i++) if (nsh->dev[i].page == NULL) { struct page *p = alloc_page(GFP_NOIO); nsh->dev[i].page = p; if (!p) err = -ENOMEM; } release_stripe(nsh); } /* critical section pass, GFP_NOIO no longer needed */ conf->slab_cache = sc; conf->active_name = 1-conf->active_name; conf->pool_size = newsize; return err; } static int drop_one_stripe(struct r5conf *conf, int hash) { struct stripe_head *sh; spin_lock_irq(conf->hash_locks + hash); sh = get_free_stripe(conf, hash); spin_unlock_irq(conf->hash_locks + hash); if (!sh) return 0; BUG_ON(atomic_read(&sh->count)); shrink_buffers(sh); kmem_cache_free(conf->slab_cache, sh); atomic_dec(&conf->active_stripes); return 1; } static void shrink_stripes(struct r5conf *conf) { int hash; for (hash = 0; hash < NR_STRIPE_HASH_LOCKS; hash++) while (drop_one_stripe(conf, hash)) ; if (conf->slab_cache) kmem_cache_destroy(conf->slab_cache); conf->slab_cache = NULL; } static void raid5_end_read_request(struct bio * bi, int error) { struct stripe_head *sh = bi->bi_private; struct r5conf *conf = sh->raid_conf; int disks = sh->disks, i; int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); char b[BDEVNAME_SIZE]; struct md_rdev *rdev = NULL; sector_t s; for (i=0 ; idev[i].req) break; pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", (unsigned long long)sh->sector, i, atomic_read(&sh->count), uptodate); if (i == disks) { BUG(); return; } if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) /* If replacement finished while this request was outstanding, * 'replacement' might be NULL already. * In that case it moved down to 'rdev'. * rdev is not removed until all requests are finished. */ rdev = conf->disks[i].replacement; if (!rdev) rdev = conf->disks[i].rdev; if (use_new_offset(conf, sh)) s = sh->sector + rdev->new_data_offset; else s = sh->sector + rdev->data_offset; if (uptodate) { set_bit(R5_UPTODATE, &sh->dev[i].flags); if (test_bit(R5_ReadError, &sh->dev[i].flags)) { /* Note that this cannot happen on a * replacement device. We just fail those on * any error */ printk_ratelimited( KERN_INFO "md/raid:%s: read error corrected" " (%lu sectors at %llu on %s)\n", mdname(conf->mddev), STRIPE_SECTORS, (unsigned long long)s, bdevname(rdev->bdev, b)); atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); clear_bit(R5_ReadError, &sh->dev[i].flags); clear_bit(R5_ReWrite, &sh->dev[i].flags); } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); if (atomic_read(&rdev->read_errors)) atomic_set(&rdev->read_errors, 0); } else { const char *bdn = bdevname(rdev->bdev, b); int retry = 0; int set_bad = 0; clear_bit(R5_UPTODATE, &sh->dev[i].flags); atomic_inc(&rdev->read_errors); if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) printk_ratelimited( KERN_WARNING "md/raid:%s: read error on replacement device " "(sector %llu on %s).\n", mdname(conf->mddev), (unsigned long long)s, bdn); else if (conf->mddev->degraded >= conf->max_degraded) { set_bad = 1; printk_ratelimited( KERN_WARNING "md/raid:%s: read error not correctable " "(sector %llu on %s).\n", mdname(conf->mddev), (unsigned long long)s, bdn); } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { /* Oh, no!!! */ set_bad = 1; printk_ratelimited( KERN_WARNING "md/raid:%s: read error NOT corrected!! " "(sector %llu on %s).\n", mdname(conf->mddev), (unsigned long long)s, bdn); } else if (atomic_read(&rdev->read_errors) > conf->max_nr_stripes) printk(KERN_WARNING "md/raid:%s: Too many read errors, failing device %s.\n", mdname(conf->mddev), bdn); else retry = 1; if (set_bad && test_bit(In_sync, &rdev->flags) && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) retry = 1; if (retry) if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { set_bit(R5_ReadError, &sh->dev[i].flags); clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); } else set_bit(R5_ReadNoMerge, &sh->dev[i].flags); else { clear_bit(R5_ReadError, &sh->dev[i].flags); clear_bit(R5_ReWrite, &sh->dev[i].flags); if (!(set_bad && test_bit(In_sync, &rdev->flags) && rdev_set_badblocks( rdev, sh->sector, STRIPE_SECTORS, 0))) md_error(conf->mddev, rdev); } } rdev_dec_pending(rdev, conf->mddev); clear_bit(R5_LOCKED, &sh->dev[i].flags); set_bit(STRIPE_HANDLE, &sh->state); release_stripe(sh); } static void raid5_end_write_request(struct bio *bi, int error) { struct stripe_head *sh = bi->bi_private; struct r5conf *conf = sh->raid_conf; int disks = sh->disks, i; struct md_rdev *uninitialized_var(rdev); int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); sector_t first_bad; int bad_sectors; int replacement = 0; for (i = 0 ; i < disks; i++) { if (bi == &sh->dev[i].req) { rdev = conf->disks[i].rdev; break; } if (bi == &sh->dev[i].rreq) { rdev = conf->disks[i].replacement; if (rdev) replacement = 1; else /* rdev was removed and 'replacement' * replaced it. rdev is not removed * until all requests are finished. */ rdev = conf->disks[i].rdev; break; } } pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", (unsigned long long)sh->sector, i, atomic_read(&sh->count), uptodate); if (i == disks) { BUG(); return; } if (replacement) { if (!uptodate) md_error(conf->mddev, rdev); else if (is_badblock(rdev, sh->sector, STRIPE_SECTORS, &first_bad, &bad_sectors)) set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); } else { if (!uptodate) { set_bit(STRIPE_DEGRADED, &sh->state); set_bit(WriteErrorSeen, &rdev->flags); set_bit(R5_WriteError, &sh->dev[i].flags); if (!test_and_set_bit(WantReplacement, &rdev->flags)) set_bit(MD_RECOVERY_NEEDED, &rdev->mddev->recovery); } else if (is_badblock(rdev, sh->sector, STRIPE_SECTORS, &first_bad, &bad_sectors)) { set_bit(R5_MadeGood, &sh->dev[i].flags); if (test_bit(R5_ReadError, &sh->dev[i].flags)) /* That was a successful write so make * sure it looks like we already did * a re-write. */ set_bit(R5_ReWrite, &sh->dev[i].flags); } } rdev_dec_pending(rdev, conf->mddev); if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) clear_bit(R5_LOCKED, &sh->dev[i].flags); set_bit(STRIPE_HANDLE, &sh->state); release_stripe(sh); } static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); static void raid5_build_block(struct stripe_head *sh, int i, int previous) { struct r5dev *dev = &sh->dev[i]; bio_init(&dev->req); dev->req.bi_io_vec = &dev->vec; dev->req.bi_vcnt++; dev->req.bi_max_vecs++; dev->req.bi_private = sh; dev->vec.bv_page = dev->page; bio_init(&dev->rreq); dev->rreq.bi_io_vec = &dev->rvec; dev->rreq.bi_vcnt++; dev->rreq.bi_max_vecs++; dev->rreq.bi_private = sh; dev->rvec.bv_page = dev->page; dev->flags = 0; dev->sector = compute_blocknr(sh, i, previous); } static void error(struct mddev *mddev, struct md_rdev *rdev) { char b[BDEVNAME_SIZE]; struct r5conf *conf = mddev->private; unsigned long flags; pr_debug("raid456: error called\n"); spin_lock_irqsave(&conf->device_lock, flags); clear_bit(In_sync, &rdev->flags); mddev->degraded = calc_degraded(conf); spin_unlock_irqrestore(&conf->device_lock, flags); set_bit(MD_RECOVERY_INTR, &mddev->recovery); set_bit(Blocked, &rdev->flags); set_bit(Faulty, &rdev->flags); set_bit(MD_CHANGE_DEVS, &mddev->flags); printk(KERN_ALERT "md/raid:%s: Disk failure on %s, disabling device.\n" "md/raid:%s: Operation continuing on %d devices.\n", mdname(mddev), bdevname(rdev->bdev, b), mdname(mddev), conf->raid_disks - mddev->degraded); } /* * Input: a 'big' sector number, * Output: index of the data and parity disk, and the sector # in them. */ static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, int previous, int *dd_idx, struct stripe_head *sh) { sector_t stripe, stripe2; sector_t chunk_number; unsigned int chunk_offset; int pd_idx, qd_idx; int ddf_layout = 0; sector_t new_sector; int algorithm = previous ? conf->prev_algo : conf->algorithm; int sectors_per_chunk = previous ? conf->prev_chunk_sectors : conf->chunk_sectors; int raid_disks = previous ? conf->previous_raid_disks : conf->raid_disks; int data_disks = raid_disks - conf->max_degraded; /* First compute the information on this sector */ /* * Compute the chunk number and the sector offset inside the chunk */ chunk_offset = sector_div(r_sector, sectors_per_chunk); chunk_number = r_sector; /* * Compute the stripe number */ stripe = chunk_number; *dd_idx = sector_div(stripe, data_disks); stripe2 = stripe; /* * Select the parity disk based on the user selected algorithm. */ pd_idx = qd_idx = -1; switch(conf->level) { case 4: pd_idx = data_disks; break; case 5: switch (algorithm) { case ALGORITHM_LEFT_ASYMMETRIC: pd_idx = data_disks - sector_div(stripe2, raid_disks); if (*dd_idx >= pd_idx) (*dd_idx)++; break; case ALGORITHM_RIGHT_ASYMMETRIC: pd_idx = sector_div(stripe2, raid_disks); if (*dd_idx >= pd_idx) (*dd_idx)++; break; case ALGORITHM_LEFT_SYMMETRIC: pd_idx = data_disks - sector_div(stripe2, raid_disks); *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; break; case ALGORITHM_RIGHT_SYMMETRIC: pd_idx = sector_div(stripe2, raid_disks); *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; break; case ALGORITHM_PARITY_0: pd_idx = 0; (*dd_idx)++; break; case ALGORITHM_PARITY_N: pd_idx = data_disks; break; default: BUG(); } break; case 6: switch (algorithm) { case ALGORITHM_LEFT_ASYMMETRIC: pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); qd_idx = pd_idx + 1; if (pd_idx == raid_disks-1) { (*dd_idx)++; /* Q D D D P */ qd_idx = 0; } else if (*dd_idx >= pd_idx) (*dd_idx) += 2; /* D D P Q D */ break; case ALGORITHM_RIGHT_ASYMMETRIC: pd_idx = sector_div(stripe2, raid_disks); qd_idx = pd_idx + 1; if (pd_idx == raid_disks-1) { (*dd_idx)++; /* Q D D D P */ qd_idx = 0; } else if (*dd_idx >= pd_idx) (*dd_idx) += 2; /* D D P Q D */ break; case ALGORITHM_LEFT_SYMMETRIC: pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); qd_idx = (pd_idx + 1) % raid_disks; *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; break; case ALGORITHM_RIGHT_SYMMETRIC: pd_idx = sector_div(stripe2, raid_disks); qd_idx = (pd_idx + 1) % raid_disks; *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; break; case ALGORITHM_PARITY_0: pd_idx = 0; qd_idx = 1; (*dd_idx) += 2; break; case ALGORITHM_PARITY_N: pd_idx = data_disks; qd_idx = data_disks + 1; break; case ALGORITHM_ROTATING_ZERO_RESTART: /* Exactly the same as RIGHT_ASYMMETRIC, but or * of blocks for computing Q is different. */ pd_idx = sector_div(stripe2, raid_disks); qd_idx = pd_idx + 1; if (pd_idx == raid_disks-1) { (*dd_idx)++; /* Q D D D P */ qd_idx = 0; } else if (*dd_idx >= pd_idx) (*dd_idx) += 2; /* D D P Q D */ ddf_layout = 1; break; case ALGORITHM_ROTATING_N_RESTART: /* Same a left_asymmetric, by first stripe is * D D D P Q rather than * Q D D D P */ stripe2 += 1; pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); qd_idx = pd_idx + 1; if (pd_idx == raid_disks-1) { (*dd_idx)++; /* Q D D D P */ qd_idx = 0; } else if (*dd_idx >= pd_idx) (*dd_idx) += 2; /* D D P Q D */ ddf_layout = 1; break; case ALGORITHM_ROTATING_N_CONTINUE: /* Same as left_symmetric but Q is before P */ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); qd_idx = (pd_idx + raid_disks - 1) % raid_disks; *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; ddf_layout = 1; break; case ALGORITHM_LEFT_ASYMMETRIC_6: /* RAID5 left_asymmetric, with Q on last device */ pd_idx = data_disks - sector_div(stripe2, raid_disks-1); if (*dd_idx >= pd_idx) (*dd_idx)++; qd_idx = raid_disks - 1; break; case ALGORITHM_RIGHT_ASYMMETRIC_6: pd_idx = sector_div(stripe2, raid_disks-1); if (*dd_idx >= pd_idx) (*dd_idx)++; qd_idx = raid_disks - 1; break; case ALGORITHM_LEFT_SYMMETRIC_6: pd_idx = data_disks - sector_div(stripe2, raid_disks-1); *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); qd_idx = raid_disks - 1; break; case ALGORITHM_RIGHT_SYMMETRIC_6: pd_idx = sector_div(stripe2, raid_disks-1); *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); qd_idx = raid_disks - 1; break; case ALGORITHM_PARITY_0_6: pd_idx = 0; (*dd_idx)++; qd_idx = raid_disks - 1; break; default: BUG(); } break; } if (sh) { sh->pd_idx = pd_idx; sh->qd_idx = qd_idx; sh->ddf_layout = ddf_layout; } /* * Finally, compute the new sector number */ new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; return new_sector; } static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) { struct r5conf *conf = sh->raid_conf; int raid_disks = sh->disks; int data_disks = raid_disks - conf->max_degraded; sector_t new_sector = sh->sector, check; int sectors_per_chunk = previous ? conf->prev_chunk_sectors : conf->chunk_sectors; int algorithm = previous ? conf->prev_algo : conf->algorithm; sector_t stripe; int chunk_offset; sector_t chunk_number; int dummy1, dd_idx = i; sector_t r_sector; struct stripe_head sh2; chunk_offset = sector_div(new_sector, sectors_per_chunk); stripe = new_sector; if (i == sh->pd_idx) return 0; switch(conf->level) { case 4: break; case 5: switch (algorithm) { case ALGORITHM_LEFT_ASYMMETRIC: case ALGORITHM_RIGHT_ASYMMETRIC: if (i > sh->pd_idx) i--; break; case ALGORITHM_LEFT_SYMMETRIC: case ALGORITHM_RIGHT_SYMMETRIC: if (i < sh->pd_idx) i += raid_disks; i -= (sh->pd_idx + 1); break; case ALGORITHM_PARITY_0: i -= 1; break; case ALGORITHM_PARITY_N: break; default: BUG(); } break; case 6: if (i == sh->qd_idx) return 0; /* It is the Q disk */ switch (algorithm) { case ALGORITHM_LEFT_ASYMMETRIC: case ALGORITHM_RIGHT_ASYMMETRIC: case ALGORITHM_ROTATING_ZERO_RESTART: case ALGORITHM_ROTATING_N_RESTART: if (sh->pd_idx == raid_disks-1) i--; /* Q D D D P */ else if (i > sh->pd_idx) i -= 2; /* D D P Q D */ break; case ALGORITHM_LEFT_SYMMETRIC: case ALGORITHM_RIGHT_SYMMETRIC: if (sh->pd_idx == raid_disks-1) i--; /* Q D D D P */ else { /* D D P Q D */ if (i < sh->pd_idx) i += raid_disks; i -= (sh->pd_idx + 2); } break; case ALGORITHM_PARITY_0: i -= 2; break; case ALGORITHM_PARITY_N: break; case ALGORITHM_ROTATING_N_CONTINUE: /* Like left_symmetric, but P is before Q */ if (sh->pd_idx == 0) i--; /* P D D D Q */ else { /* D D Q P D */ if (i < sh->pd_idx) i += raid_disks; i -= (sh->pd_idx + 1); } break; case ALGORITHM_LEFT_ASYMMETRIC_6: case ALGORITHM_RIGHT_ASYMMETRIC_6: if (i > sh->pd_idx) i--; break; case ALGORITHM_LEFT_SYMMETRIC_6: case ALGORITHM_RIGHT_SYMMETRIC_6: if (i < sh->pd_idx) i += data_disks + 1; i -= (sh->pd_idx + 1); break; case ALGORITHM_PARITY_0_6: i -= 1; break; default: BUG(); } break; } chunk_number = stripe * data_disks + i; r_sector = chunk_number * sectors_per_chunk + chunk_offset; check = raid5_compute_sector(conf, r_sector, previous, &dummy1, &sh2); if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx || sh2.qd_idx != sh->qd_idx) { printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", mdname(conf->mddev)); return 0; } return r_sector; } static void schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, int rcw, int expand) { int i, pd_idx = sh->pd_idx, disks = sh->disks; struct r5conf *conf = sh->raid_conf; int level = conf->level; if (rcw) { for (i = disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (dev->towrite) { set_bit(R5_LOCKED, &dev->flags); set_bit(R5_Wantdrain, &dev->flags); if (!expand) clear_bit(R5_UPTODATE, &dev->flags); s->locked++; } } /* if we are not expanding this is a proper write request, and * there will be bios with new data to be drained into the * stripe cache */ if (!expand) { if (!s->locked) /* False alarm, nothing to do */ return; sh->reconstruct_state = reconstruct_state_drain_run; set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); } else sh->reconstruct_state = reconstruct_state_run; set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); if (s->locked + conf->max_degraded == disks) if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) atomic_inc(&conf->pending_full_writes); } else { BUG_ON(level == 6); BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); for (i = disks; i--; ) { struct r5dev *dev = &sh->dev[i]; if (i == pd_idx) continue; if (dev->towrite && (test_bit(R5_UPTODATE, &dev->flags) || test_bit(R5_Wantcompute, &dev->flags))) { set_bit(R5_Wantdrain, &dev->flags); set_bit(R5_LOCKED, &dev->flags); clear_bit(R5_UPTODATE, &dev->flags); s->locked++; } } if (!s->locked) /* False alarm - nothing to do */ return; sh->reconstruct_state = reconstruct_state_prexor_drain_run; set_bit(STRIPE_OP_PREXOR, &s->ops_request); set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); } /* keep the parity disk(s) locked while asynchronous operations * are in flight */ set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); s->locked++; if (level == 6) { int qd_idx = sh->qd_idx; struct r5dev *dev = &sh->dev[qd_idx]; set_bit(R5_LOCKED, &dev->flags); clear_bit(R5_UPTODATE, &dev->flags); s->locked++; } pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", __func__, (unsigned long long)sh->sector, s->locked, s->ops_request); } /* * Each stripe/dev can have one or more bion attached. * toread/towrite point to the first in a chain. * The bi_next chain must be in order. */ static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite) { struct bio **bip; struct r5conf *conf = sh->raid_conf; int firstwrite=0; pr_debug("adding bi b#%llu to stripe s#%llu\n", (unsigned long long)bi->bi_iter.bi_sector, (unsigned long long)sh->sector); /* * If several bio share a stripe. The bio bi_phys_segments acts as a * reference count to avoid race. The reference count should already be * increased before this func static unsigned long ev_a[EV],ev_b[EV]; static int ec = 0; static void EVENT(const char *s,unsigned long a,unsigned long b) { ev[ec] = s; ev_a[ec] = a; ev_b[ec] = b; ec = (ec+1) % EV; } static void event_dump(void) { int n,i; printk(KERN_NOTICE "----- event dump follows -----\n"); for (n = 0; n < EV; n++) { i = (ec+n) % EV; printk(KERN_NOTICE); printk(ev[i] ? ev[i] : "(null)",ev_a[i],ev_b[i]); } printk(KERN_NOTICE "----- event dump ends here -----\n"); } #endif /* CONFIG_ATM_ZATM_DEBUG */ #define RING_BUSY 1 /* indication from do_tx that PDU has to be backlogged */ static struct atm_dev *zatm_boards = NULL; static unsigned long dummy[2] = {0,0}; #define zin_n(r) inl(zatm_dev->base+r*4) #define zin(r) inl(zatm_dev->base+uPD98401_##r*4) #define zout(v,r) outl(v,zatm_dev->base+uPD98401_##r*4) #define zwait while (zin(CMR) & uPD98401_BUSY) /* RX0, RX1, TX0, TX1 */ static const int mbx_entries[NR_MBX] = { 1024,1024,1024,1024 }; static const int mbx_esize[NR_MBX] = { 16,16,4,4 }; /* entry size in bytes */ #define MBX_SIZE(i) (mbx_entries[i]*mbx_esize[i]) /*-------------------------------- utilities --------------------------------*/ static void zpokel(struct zatm_dev *zatm_dev,u32 value,u32 addr) { zwait; zout(value,CER); zout(uPD98401_IND_ACC | uPD98401_IA_BALL | (uPD98401_IA_TGT_CM << uPD98401_IA_TGT_SHIFT) | addr,CMR); } static u32 zpeekl(struct zatm_dev *zatm_dev,u32 addr) { zwait; zout(uPD98401_IND_ACC | uPD98401_IA_BALL | uPD98401_IA_RW | (uPD98401_IA_TGT_CM << uPD98401_IA_TGT_SHIFT) | addr,CMR); zwait; return zin(CER); } /*------------------------------- free lists --------------------------------*/ /* * Free buffer head structure: * [0] pointer to buffer (for SAR) * [1] buffer descr link pointer (for SAR) * [2] back pointer to skb (for poll_rx) * [3] data * ... */ struct rx_buffer_head { u32 buffer; /* pointer to buffer (for SAR) */ u32 link; /* buffer descriptor link pointer (for SAR) */ struct sk_buff *skb; /* back pointer to skb (for poll_rx) */ }; static void refill_pool(struct atm_dev *dev,int pool) { struct zatm_dev *zatm_dev; struct sk_buff *skb; struct rx_buffer_head *first; unsigned long flags; int align,offset,free,count,size; EVENT("refill_pool\n",0,0); zatm_dev = ZATM_DEV(dev); size = (64 << (pool <= ZATM_AAL5_POOL_BASE ? 0 : pool-ZATM_AAL5_POOL_BASE))+sizeof(struct rx_buffer_head); if (size < PAGE_SIZE) { align = 32; /* for 32 byte alignment */ offset = sizeof(struct rx_buffer_head); } else { align = 4096; offset = zatm_dev->pool_info[pool].offset+ sizeof(struct rx_buffer_head); } size += align; spin_lock_irqsave(&zatm_dev->lock, flags); free = zpeekl(zatm_dev,zatm_dev->pool_base+2*pool) & uPD98401_RXFP_REMAIN; spin_unlock_irqrestore(&zatm_dev->lock, flags); if (free >= zatm_dev->pool_info[pool].low_water) return; EVENT("starting ... POOL: 0x%x, 0x%x\n", zpeekl(zatm_dev,zatm_dev->pool_base+2*pool), zpeekl(zatm_dev,zatm_dev->pool_base+2*pool+1)); EVENT("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]); count = 0; first = NULL; while (free < zatm_dev->pool_info[pool].high_water) { struct rx_buffer_head *head; skb = alloc_skb(size,GFP_ATOMIC); if (!skb) { printk(KERN_WARNING DEV_LABEL "(Itf %d): got no new " "skb (%d) with %d free\n",dev->number,size,free); break; } skb_reserve(skb,(unsigned char *) ((((unsigned long) skb->data+ align+offset-1) & ~(unsigned long) (align-1))-offset)- skb->data); head = (struct rx_buffer_head *) skb->data; skb_reserve(skb,sizeof(struct rx_buffer_head)); if (!first) first = head; count++; head->buffer = virt_to_bus(skb->data); head->link = 0; head->skb = skb; EVENT("enq skb 0x%08lx/0x%08lx\n",(unsigned long) skb, (unsigned long) head); spin_lock_irqsave(&zatm_dev->lock, flags); if (zatm_dev->last_free[pool]) ((struct rx_buffer_head *) (zatm_dev->last_free[pool]-> data))[-1].link = virt_to_bus(head); zatm_dev->last_free[pool] = skb; skb_queue_tail(&zatm_dev->pool[pool],skb); spin_unlock_irqrestore(&zatm_dev->lock, flags); free++; } if (first) { spin_lock_irqsave(&zatm_dev->lock, flags); zwait; zout(virt_to_bus(first),CER); zout(uPD98401_ADD_BAT | (pool << uPD98401_POOL_SHIFT) | count, CMR); spin_unlock_irqrestore(&zatm_dev->lock, flags); EVENT ("POOL: 0x%x, 0x%x\n", zpeekl(zatm_dev,zatm_dev->pool_base+2*pool), zpeekl(zatm_dev,zatm_dev->pool_base+2*pool+1)); EVENT("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]); } } static void drain_free(struct atm_dev *dev,int pool) { skb_queue_purge(&ZATM_DEV(dev)->pool[pool]); } static int pool_index(int max_pdu) { int i; if (max_pdu % ATM_CELL_PAYLOAD) printk(KERN_ERR DEV_LABEL ": driver error in pool_index: " "max_pdu is %d\n",max_pdu); if (max_pdu > 65536) return -1; for (i = 0; (64 << i) < max_pdu; i++); return i+ZATM_AAL5_POOL_BASE; } /* use_pool isn't reentrant */ static void use_pool(struct atm_dev *dev,int pool) { struct zatm_dev *zatm_dev; unsigned long flags; int size; zatm_dev = ZATM_DEV(dev); if (!(zatm_dev->pool_info[pool].ref_count++)) { skb_queue_head_init(&zatm_dev->pool[pool]); size = pool-ZATM_AAL5_POOL_BASE; if (size < 0) size = 0; /* 64B... */ else if (size > 10) size = 10; /* ... 64kB */ spin_lock_irqsave(&zatm_dev->lock, flags); zpokel(zatm_dev,((zatm_dev->pool_info[pool].low_water/4) << uPD98401_RXFP_ALERT_SHIFT) | (1 << uPD98401_RXFP_BTSZ_SHIFT) | (size << uPD98401_RXFP_BFSZ_SHIFT), zatm_dev->pool_base+pool*2); zpokel(zatm_dev,(unsigned long) dummy,zatm_dev->pool_base+ pool*2+1); spin_unlock_irqrestore(&zatm_dev->lock, flags); zatm_dev->last_free[pool] = NULL; refill_pool(dev,pool); } DPRINTK("pool %d: %d\n",pool,zatm_dev->pool_info[pool].ref_count); } static void unuse_pool(struct atm_dev *dev,int pool) { if (!(--ZATM_DEV(dev)->pool_info[pool].ref_count)) drain_free(dev,pool); } /*----------------------------------- RX ------------------------------------*/ #if 0 static void exception(struct atm_vcc *vcc) { static int count = 0; struct zatm_dev *zatm_dev = ZATM_DEV(vcc->dev); struct zatm_vcc *zatm_vcc = ZATM_VCC(vcc); unsigned long *qrp; int i; if (count++ > 2) return; for (i = 0; i < 8; i++) printk("TX%d: 0x%08lx\n",i, zpeekl(zatm_dev,zatm_vcc->tx_chan*VC_SIZE/4+i)); for (i = 0; i < 5; i++) printk("SH%d: 0x%08lx\n",i, zpeekl(zatm_dev,uPD98401_IM(zatm_vcc->shaper)+16*i)); qrp = (unsigned long *) zpeekl(zatm_dev,zatm_vcc->tx_chan*VC_SIZE/4+ uPD98401_TXVC_QRP); printk("qrp=0x%08lx\n",(unsigned long) qrp); for (i = 0; i < 4; i++) printk("QRP[%d]: 0x%08lx",i,qrp[i]); } #endif static const char *err_txt[] = { "No error", "RX buf underflow", "RX FIFO overrun", "Maximum len violation", "CRC error", "User abort", "Length violation", "T1 error", "Deactivated", "???", "???", "???", "???", "???", "???", "???" }; static void poll_rx(struct atm_dev *dev,int mbx) { struct zatm_dev *zatm_dev; unsigned long pos; u32 x; int error; EVENT("poll_rx\n",0,0); zatm_dev = ZATM_DEV(dev); pos = (zatm_dev->mbx_start[mbx] & ~0xffffUL) | zin(MTA(mbx)); while (x = zin(MWA(mbx)), (pos & 0xffff) != x) { u32 *here; struct sk_buff *skb; struct atm_vcc *vcc; int cells,size,chan; EVENT("MBX: host 0x%lx, nic 0x%x\n",pos,x); here = (u32 *) pos; if (((pos += 16) & 0xffff) == zatm_dev->mbx_end[mbx]) pos = zatm_dev->mbx_start[mbx]; cells = here[0] & uPD98401_AAL5_SIZE; #if 0 printk("RX IND: 0x%x, 0x%x, 0x%x, 0x%x\n",here[0],here[1],here[2],here[3]); { unsigned long *x; printk("POOL: 0x%08x, 0x%08x\n",zpeekl(zatm_dev, zatm_dev->pool_base), zpeekl(zatm_dev,zatm_dev->pool_base+1)); x = (unsigned long *) here[2]; printk("[0..3] = 0x%08lx, 0x%08lx, 0x%08lx, 0x%08lx\n", x[0],x[1],x[2],x[3]); } #endif error = 0; if (here[3] & uPD98401_AAL5_ERR) { error = (here[3] & uPD98401_AAL5_ES) >> uPD98401_AAL5_ES_SHIFT; if (error == uPD98401_AAL5_ES_DEACT || error == uPD98401_AAL5_ES_FREE) continue; } EVENT("error code 0x%x/0x%x\n",(here[3] & uPD98401_AAL5_ES) >> uPD98401_AAL5_ES_SHIFT,error); skb = ((struct rx_buffer_head *) bus_to_virt(here[2]))->skb; do_gettimeofday(&skb->stamp); #if 0 printk("[-3..0] 0x%08lx 0x%08lx 0x%08lx 0x%08lx\n",((unsigned *) skb->data)[-3], ((unsigned *) skb->data)[-2],((unsigned *) skb->data)[-1], ((unsigned *) skb->data)[0]); #endif EVENT("skb 0x%lx, here 0x%lx\n",(unsigned long) skb, (unsigned long) here); #if 0 printk("dummy: 0x%08lx, 0x%08lx\n",dummy[0],dummy[1]); #endif size = error ? 0 : ntohs(((__be16 *) skb->data)[cells* ATM_CELL_PAYLOAD/sizeof(u16)-3]); EVENT("got skb 0x%lx, size %d\n",(unsigned long) skb,size); chan = (here[3] & uPD98401_AAL5_CHAN) >> uPD98401_AAL5_CHAN_SHIFT; if (chan < zatm_dev->chans && zatm_dev->rx_map[chan]) { vcc = zatm_dev->rx_map[chan]; if (skb == zatm_dev->last_free[ZATM_VCC(vcc)->pool]) zatm_dev->last_free[ZATM_VCC(vcc)->pool] = NULL; skb_unlink(skb); } else { printk(KERN_ERR DEV_LABEL "(itf %d): RX indication " "for non-existing channel\n",dev->number); size = 0; vcc = NULL; event_dump(); } if (error) { static unsigned long silence = 0; static int last_error = 0; if (error != last_error || time_after(jiffies, silence) || silence == 0){ printk(KERN_WARNING DEV_LABEL "(itf %d): " "chan %d error %s\n",dev->number,chan, err_txt[error]); last_error = error; silence = (jiffies+2*HZ)|1; } size = 0; } if (size && (size > cells*ATM_CELL_PAYLOAD-ATM_AAL5_TRAILER || size <= (cells-1)*ATM_CELL_PAYLOAD-ATM_AAL5_TRAILER)) { printk(KERN_ERR DEV_LABEL "(itf %d): size %d with %d " "cells\n",dev->number,size,cells); size = 0; event_dump(); } if (size > ATM_MAX_AAL5_PDU) { printk(KERN_ERR DEV_LABEL "(itf %d): size too big " "(%d)\n",dev->number,size); size = 0; event_dump(); } if (!size) { dev_kfree_skb_irq(skb); if (vcc) atomic_inc(&vcc->stats->rx_err); continue; } if (!atm_charge(vcc,skb->truesize)) { dev_kfree_skb_irq(skb); continue; } skb->len = size; ATM_SKB(skb)->vcc = vcc; vcc->push(vcc,skb); atomic_inc(&vcc->stats->rx); } zout(pos & 0xffff,MTA(mbx)); #if 0 /* probably a stupid idea */ refill_pool(dev,zatm_vcc->pool); /* maybe this saves us a few interrupts */ #endif } static int open_rx_first(struct atm_vcc *vcc) { struct zatm_dev *zatm_dev; struct zatm_vcc *zatm_vcc; unsigned long flags; unsigned short chan; int cells; DPRINTK("open_rx_first (0x%x)\n",inb_p(0xc053)); zatm_dev = ZATM_DEV(vcc->dev); zatm_vcc = ZATM_VCC(vcc); zatm_vcc->rx_chan = 0; if (vcc->qos.rxtp.traffic_class == ATM_NONE) return 0; if (vcc->qos.aal == ATM_AAL5) { if (vcc->qos.rxtp.max_sdu > 65464) vcc->qos.rxtp.max_sdu = 65464; /* fix this - we may want to receive 64kB SDUs later */ cells = (vcc->qos.rxtp.max_sdu+ATM_AAL5_TRAILER+ ATM_CELL_PAYLOAD-1)/ATM_CELL_PAYLOAD; zatm_vcc->pool = pool_index(cells*ATM_CELL_PAYLOAD); } else { cells = 1; zatm_vcc->pool = ZATM_AAL0_POOL; } if (zatm_vcc->pool < 0) return -EMSGSIZE; spin_lock_irqsave(&zatm_dev->lock, flags); zwait; zout(uPD98401_OPEN_CHAN,CMR); zwait; DPRINTK("0x%x 0x%x\n",zin(CMR),zin(CER)); chan = (zin(CMR) & uPD98401_CHAN_ADDR) >> uPD98401_CHAN_ADDR_SHIFT; spin_unlock_irqrestore(&zatm_dev->lock, flags); DPRINTK("chan is %d\n",chan); if (!chan) return -EAGAIN; use_pool(vcc->dev,zatm_vcc->pool); DPRINTK("pool %d\n",zatm_vcc->pool); /* set up VC descriptor */ spin_lock_irqsave(&zatm_dev->lock, flags); zpokel(zatm_dev,zatm_vcc->pool << uPD98401_RXVC_POOL_SHIFT, chan*VC_SIZE/4); zpokel(zatm_dev,uPD98401_RXVC_OD | (vcc->qos.aal == ATM_AAL5 ? uPD98401_RXVC_AR : 0) | cells,chan*VC_SIZE/4+1); zpokel(zatm_dev,0,chan*VC_SIZE/4+2); zatm_vcc->rx_chan = chan; zatm_dev->rx_map[chan] = vcc; spin_unlock_irqrestore(&zatm_dev->lock, flags); return 0; } static int open_rx_second(struct atm_vcc *vcc) { struct zatm_dev *zatm_dev; struct zatm_vcc *zatm_vcc; unsigned long flags; int pos,shift; DPRINTK("open_rx_second (0x%x)\n",inb_p(0xc053)); zatm_dev = ZATM_DEV(vcc->dev); zatm_vcc = ZATM_VCC(vcc); if (!zatm_vcc->rx_chan) return 0; spin_lock_irqsave(&zatm_dev->lock, flags); /* should also handle VPI @@@ */ pos = vcc->vci >> 1; shift = (1-(vcc->vci & 1)) << 4; zpokel(zatm_dev,(zpeekl(zatm_dev,pos) & ~(0xffff << shift)) | ((zatm_vcc->rx_chan | uPD98401_RXLT_ENBL) << shift),pos); spin_unlock_irqrestore(&zatm_dev->lock, flags); return 0; } static void close_rx(struct atm_vcc *vcc) { struct zatm_dev *zatm_dev; struct zatm_vcc *zatm_vcc; unsigned long flags; int pos,shift; zatm_vcc = ZATM_VCC(vcc); zatm_dev = ZATM_DEV(vcc->dev); if (!zatm_vcc->rx_chan) return; DPRINTK("close_rx\n"); /* disable receiver */ if (vcc->vpi != ATM_VPI_UNSPEC && vcc->vci != ATM_VCI_UNSPEC) { spin_lock_irqsave(&zatm_dev->lock, flags); pos = vcc->vci >> 1; shift = (1-(vcc->vci & 1)) << 4; zpokel(zatm_dev,zpeekl(zatm_dev,pos) & ~(0xffff << shift),pos); zwait; zout(uPD98401_NOP,CMR); zwait; zout(uPD98401_NOP,CMR); spin_unlock_irqrestore(&zatm_dev->lock, flags); } spin_lock_irqsave(&zatm_dev->lock, flags); zwait; zout(uPD98401_DEACT_CHAN | uPD98401_CHAN_RT | (zatm_vcc->rx_chan << uPD98401_CHAN_ADDR_SHIFT),CMR); zwait; udelay(10); /* why oh why ... ? */ zout(uPD98401_CLOSE_CHAN | uPD98401_CHAN_RT | (zatm_vcc->rx_chan << uPD98401_CHAN_ADDR_SHIFT),CMR); zwait; if (!(zin(CMR) & uPD98401_CHAN_ADDR)) printk(KERN_CRIT DEV_LABEL "(itf %d): can't close RX channel " "%d\n",vcc->dev->number,zatm_vcc->rx_chan); spin_unlock_irqrestore(&zatm_dev->lock, flags); zatm_dev->rx_map[zatm_vcc->rx_chan] = NULL; zatm_vcc->rx_chan = 0; unuse_pool(vcc->dev,zatm_vcc->pool); } static int start_rx(struct atm_dev *dev) { struct zatm_dev *zatm_dev; int size,i; DPRINTK("start_rx\n"); zatm_dev = ZATM_DEV(dev); size = sizeof(struct atm_vcc *)*zatm_dev->chans; zatm_dev->rx_map = (struct atm_vcc **) kmalloc(size,GFP_KERNEL); if (!zatm_dev->rx_map) return -ENOMEM; memset(zatm_dev->rx_map,0,size); /* set VPI/VCI split (use all VCIs and give what's left to VPIs) */ zpokel(zatm_dev,(1 << dev->ci_range.vci_bits)-1,uPD98401_VRR); /* prepare free buffer pools */ for (i = 0; i <= ZATM_LAST_POOL; i++) { zatm_dev->pool_info[i].ref_count = 0; zatm_dev->pool_info[i].rqa_count = 0; zatm_dev->pool_info[i].rqu_count = 0; zatm_dev->pool_info[i].low_water = LOW_MARK; zatm_dev->pool_info[i].high_water = HIGH_MARK; zatm_dev->pool_info[i].offset = 0; zatm_dev->pool_info[i].next_off = 0; zatm_dev->pool_info[i].next_cnt = 0; zatm_dev->pool_info[i].next_thres = OFF_CNG_THRES; } return 0; } /*----------------------------------- TX ------------------------------------*/ static int do_tx(struct sk_buff *skb) { struct atm_vcc *vcc; struct zatm_dev *zatm_dev; struct zatm_vcc *zatm_vcc; u32 *dsc; unsigned long flags; EVENT("do_tx\n",0,0); DPRINTK("sending skb %p\n",skb); vcc = ATM_SKB(skb)->vcc; zatm_dev = ZATM_DEV(vcc->dev); zatm_vcc = ZATM_VCC(vcc); EVENT("iovcnt=%d\n",skb_shinfo(skb)->nr_frags,0); spin_lock_irqsave(&zatm_dev->lock, flags); if (!skb_shinfo(skb)->nr_frags) { if (zatm_vcc->txing == RING_ENTRIES-1) { spin_unlock_irqrestore(&zatm_dev->lock, flags); return RING_BUSY; } zatm_vcc->txing++; dsc = zatm_vcc->ring+zatm_vcc->ring_curr; zatm_vcc->ring_curr = (zatm_vcc->ring_curr+RING_WORDS) & (RING_ENTRIES*RING_WORDS-1); dsc[1] = 0; dsc[2] = skb->len; dsc[3] = virt_to_bus(skb->data); mb(); dsc[0] = uPD98401_TXPD_V | uPD98401_TXPD_DP | uPD98401_TXPD_SM | (vcc->qos.aal == ATM_AAL5 ? uPD98401_TXPD_AAL5 : 0 | (ATM_SKB(skb)->atm_options & ATM_ATMOPT_CLP ? uPD98401_CLPM_1 : uPD98401_CLPM_0)); EVENT("dsc (0x%lx)\n",(unsigned long) dsc,0); } else { printk("NONONONOO!!!!\n"); dsc = NULL; #if 0 u32 *put; int i; dsc = (u32 *) kmalloc(uPD98401_TXPD_SIZE*2+ uPD98401_TXBD_SIZE*ATM_SKB(skb)->iovcnt,GFP_ATOMIC); if (!dsc) { if (vcc->pop) vcc->pop(vcc,skb); else dev_kfree_skb_irq(skb); return -EAGAIN; } /* @@@ should check alignment */ put = dsc+8; dsc[0] = uPD98401_TXPD_V | uPD98401_TXPD_DP | (vcc->aal == ATM_AAL5 ? uPD98401_TXPD_AAL5 : 0 | (ATM_SKB(skb)->atm_options & ATM_ATMOPT_CLP ? uPD98401_CLPM_1 : uPD98401_CLPM_0)); dsc[1] = 0; dsc[2] = ATM_SKB(skb)->iovcnt*uPD98401_TXBD_SIZE; dsc[3] = virt_to_bus(put); for (i = 0; i < ATM_SKB(skb)->iovcnt; i++) { *put++ = ((struct iovec *) skb->data)[i].iov_len; *put++ = virt_to_bus(((struct iovec *) skb->data)[i].iov_base); } put[-2] |= uPD98401_TXBD_LAST; #endif } ZATM_PRV_DSC(skb) = dsc; skb_queue_tail(&zatm_vcc->tx_queue,skb); DPRINTK("QRP=0x%08lx\n",zpeekl(zatm_dev,zatm_vcc->tx_chan*VC_SIZE/4+ uPD98401_TXVC_QRP)); zwait; zout(uPD98401_TX_READY | (zatm_vcc->tx_chan << uPD98401_CHAN_ADDR_SHIFT),CMR); spin_unlock_irqrestore(&zatm_dev->lock, flags); EVENT("done\n",0,0); return 0; } static inline void dequeue_tx(struct atm_vcc *vcc) { struct zatm_vcc *zatm_vcc; struct sk_buff *skb; EVENT("dequeue_tx\n",0,0); zatm_vcc = ZATM_VCC(vcc); skb = skb_dequeue(&zatm_vcc->tx_queue); if (!skb) { printk(KERN_CRIT DEV_LABEL "(itf %d): dequeue_tx but not " "txing\n",vcc->dev->number); return; } #if 0 /* @@@ would fail on CLP */ if (*ZATM_PRV_DSC(skb) != (uPD98401_TXPD_V | uPD98401_TXPD_DP | uPD98401_TXPD_SM | uPD98401_TXPD_AAL5)) printk("@#*$!!!! (%08x)\n", *ZATM_PRV_DSC(skb)); #endif *ZATM_PRV_DSC(skb) = 0; /* mark as invalid */ zatm_vcc->txing--; if (vcc->pop) vcc->pop(vcc,skb); else dev_kfree_skb_irq(skb); while ((skb = skb_dequeue(&zatm_vcc->backlog))) if (do_tx(skb) == RING_BUSY) { skb_queue_head(&zatm_vcc->backlog,skb); break; } atomic_inc(&vcc->stats->tx); wake_up(&zatm_vcc->tx_wait); } static void poll_tx(struct atm_dev *dev,int mbx) { struct zatm_dev *zatm_dev; unsigned long pos; u32 x; EVENT("poll_tx\n",0,0); zatm_dev = ZATM_DEV(dev); pos = (zatm_dev->mbx_start[mbx] & ~0xffffUL) | zin(MTA(mbx)); while (x = zin(MWA(mbx)), (pos & 0xffff) != x) { int chan; #if 1 u32 data,*addr; EVENT("MBX: host 0x%lx, nic 0x%x\n",pos,x); addr = (u32 *) pos; data = *addr; chan = (data & uPD98401_TXI_CONN) >> uPD98401_TXI_CONN_SHIFT; EVENT("addr = 0x%lx, data = 0x%08x,",(unsigned long) addr, data); EVENT("chan = %d\n",chan,0); #else NO ! chan = (zatm_dev->mbx_start[mbx][pos >> 2] & uPD98401_TXI_CONN) >> uPD98401_TXI_CONN_SHIFT; #endif if (chan < zatm_dev->chans && zatm_dev->tx_map[chan]) dequeue_tx(zatm_dev->tx_map[chan]); else { printk(KERN_CRIT DEV_LABEL "(itf %d): TX indication " "for non-existing channel %d\n",dev->number,chan); event_dump(); } if (((pos += 4) & 0xffff) == zatm_dev->mbx_end[mbx]) pos = zatm_dev->mbx_start[mbx]; } zout(pos & 0xffff,MTA(mbx)); } /* * BUG BUG BUG: Doesn't handle "new-style" rate specification yet. */ static int alloc_shaper(struct atm_dev *dev,int *pcr,int min,int max,int ubr) { struct zatm_dev *zatm_dev; unsigned long flags; unsigned long i,m,c; int shaper; DPRINTK("alloc_shaper (min = %d, max = %d)\n",min,max); zatm_dev = ZATM_DEV(dev); if (!zatm_dev->free_shapers) return -EAGAIN; for (shaper = 0; !((zatm_dev->free_shapers >> shaper) & 1); shaper++); zatm_dev->free_shapers &= ~1 << shaper; if (ubr) { c = 5; i = m = 1; zatm_dev->ubr_ref_cnt++; zatm_dev->ubr = shaper; } else { if (min) { if (min <= 255) { i = min; m = ATM_OC3_PCR; } else { i = 255; m = ATM_OC3_PCR*255/min; } } else { if (max > zatm_dev->tx_bw) max = zatm_dev->tx_bw; if (max <= 255) { i = max; m = ATM_OC3_PCR; } else { i = 255; m = (ATM_OC3_PCR*255+max-1)/max; } } if (i > m) { printk(KERN_CRIT DEV_LABEL "shaper algorithm botched " "[%d,%d] -> i=%ld,m=%ld\n",min,max,i,m); m = i; } *pcr = i*ATM_OC3_PCR/m; c = 20; /* @@@ should use max_cdv ! */ if ((min && *pcr < min) || (max && *pcr > max)) return -EINVAL; if (zatm_dev->tx_bw < *pcr) return -EAGAIN; zatm_dev->tx_bw -= *pcr; } spin_lock_irqsave(&zatm_dev->lock, flags); DPRINTK("i = %d, m = %d, PCR = %d\n",i,m,*pcr); zpokel(zatm_dev,(i << uPD98401_IM_I_SHIFT) | m,uPD98401_IM(shaper)); zpokel(zatm_dev,c << uPD98401_PC_C_SHIFT,uPD98401_PC(shaper)); zpokel(zatm_dev,0,uPD98401_X(shaper)); zpokel(zatm_dev,0,uPD98401_Y(shaper)); zpokel(zatm_dev,uPD98401_PS_E,uPD98401_PS(shaper)); spin_unlock_irqrestore(&zatm_dev->lock, flags); return shaper; } static void dealloc_shaper(struct atm_dev *dev,int shaper) { struct zatm_dev *zatm_dev; unsigned long flags; zatm_dev = ZATM_DEV(dev); if (shaper == zatm_dev->ubr) { if (--zatm_dev->ubr_ref_cnt) return; zatm_dev->ubr = -1; } spin_lock_irqsave(&zatm_dev->lock, flags); zpokel(zatm_dev,zpeekl(zatm_dev,uPD98401_PS(shaper)) & ~uPD98401_PS_E, uPD98401_PS(shaper)); spin_unlock_irqrestore(&zatm_dev->lock, flags); zatm_dev->free_shapers |= 1 << shaper; } static void close_tx(struct atm_vcc *vcc) { struct zatm_dev *zatm_dev; struct zatm_vcc *zatm_vcc; unsigned long flags; int chan; zatm_vcc = ZATM_VCC(vcc); zatm_dev = ZATM_DEV(vcc->dev); chan = zatm_vcc->tx_chan; if (!chan) return; DPRINTK("close_tx\n"); if (skb_peek(&zatm_vcc->backlog)) { printk("waiting for backlog to drain ...\n"); event_dump(); wait_event(zatm_vcc->tx_wait, !skb_peek(&zatm_vcc->backlog)); } if (skb_peek(&zatm_vcc->tx_queue)) { printk("waiting for TX queue to drain ...\n"); event_dump(); wait_event(zatm_vcc->tx_wait, !skb_peek(&zatm_vcc->tx_queue)); } spin_lock_irqsave(&zatm_dev->lock, flags); #if 0 zwait; zout(uPD98401_DEACT_CHAN | (chan << uPD98401_CHAN_ADDR_SHIFT),CMR); #endif zwait; zout(uPD98401_CLOSE_CHAN | (chan << uPD98401_CHAN_ADDR_SHIFT),CMR); zwait; if (!(zin(CMR) & uPD98401_CHAN_ADDR)) printk(KERN_CRIT DEV_LABEL "(itf %d): can't close TX channel " "%d\n",vcc->dev->number,chan); spin_unlock_irqrestore(&zatm_dev->lock, flags); zatm_vcc->tx_chan = 0; zatm_dev->tx_map[chan] = NULL; if (zatm_vcc->shaper != zatm_dev->ubr) { zatm_dev->tx_bw += vcc->qos.txtp.min_pcr; dealloc_shaper(vcc->dev,zatm_vcc->shaper); } if (zatm_vcc->ring) kfree(zatm_vcc->ring); } static int open_tx_first(struct atm_vcc *vcc) { struct zatm_dev *zatm_dev; struct zatm_vcc *zatm_vcc; unsigned long flags; u32 *loop; unsigned short chan; int pcr,unlimited; DPRINTK("open_tx_first\n"); zatm_dev = ZATM_DEV(vcc->dev); zatm_vcc = ZATM_VCC(vcc); zatm_vcc->tx_chan = 0; if (vcc->qos.txtp.traffic_class == ATM_NONE) return 0; spin_lock_irqsave(&zatm_dev->lock, flags); zwait; zout(uPD98401_OPEN_CHAN,CMR); zwait; DPRINTK("0x%x 0x%x\n",zin(CMR),zin(CER)); chan = (zin(CMR) & uPD98401_CHAN_ADDR) >> uPD98401_CHAN_ADDR_SHIFT; spin_unlock_irqrestore(&zatm_dev->lock, flags); DPRINTK("chan is %d\n",chan); if (!chan) return -EAGAIN; unlimited = vcc->qos.txtp.traffic_class == ATM_UBR && (!vcc->qos.txtp.max_pcr || vcc->qos.txtp.max_pcr == ATM_MAX_PCR || vcc->qos.txtp.max_pcr >= ATM_OC3_PCR); if (unlimited && zatm_dev->ubr != -1) zatm_vcc->shaper = zatm_dev->ubr; else { if (unlimited) vcc->qos.txtp.max_sdu = ATM_MAX_AAL5_PDU; if ((zatm_vcc->shaper = alloc_shaper(vcc->dev,&pcr, vcc->qos.txtp.min_pcr,vcc->qos.txtp.max_pcr,unlimited)) < 0) { close_tx(vcc); return zatm_vcc->shaper; } if (pcr > ATM_OC3_PCR) pcr = ATM_OC3_PCR; vcc->qos.txtp.min_pcr = vcc->qos.txtp.max_pcr = pcr; } zatm_vcc->tx_chan = chan; skb_queue_head_init(&zatm_vcc->tx_queue); init_waitqueue_head(&zatm_vcc->tx_wait); /* initialize ring */ zatm_vcc->ring = kmalloc(RING_SIZE,GFP_KERNEL); if (!zatm_vcc->ring) return -ENOMEM; memset(zatm_vcc->ring,0,RING_SIZE); loop = zatm_vcc->ring+RING_ENTRIES*RING_WORDS; loop[0] = uPD98401_TXPD_V; loop[1] = loop[2] = 0; loop[3] = virt_to_bus(zatm_vcc->ring); zatm_vcc->ring_curr = 0; zatm_vcc->txing = 0; skb_queue_head_init(&zatm_vcc->backlog); zpokel(zatm_dev,virt_to_bus(zatm_vcc->ring), chan*VC_SIZE/4+uPD98401_TXVC_QRP); return 0; } static int open_tx_second(struct atm_vcc *vcc) { struct zatm_dev *zatm_dev; struct zatm_vcc *zatm_vcc; unsigned long flags; DPRINTK("open_tx_second\n"); zatm_dev = ZATM_DEV(vcc->dev); zatm_vcc = ZATM_VCC(vcc); if (!zatm_vcc->tx_chan) return 0; /* set up VC descriptor */ spin_lock_irqsave(&zatm_dev->lock, flags); zpokel(zatm_dev,0,zatm_vcc->tx_chan*VC_SIZE/4); zpokel(zatm_dev,uPD98401_TXVC_L | (zatm_vcc->shaper << uPD98401_TXVC_SHP_SHIFT) | (vcc->vpi << uPD98401_TXVC_VPI_SHIFT) | vcc->vci,zatm_vcc->tx_chan*VC_SIZE/4+1); zpokel(zatm_dev,0,zatm_vcc->tx_chan*VC_SIZE/4+2); spin_unlock_irqrestore(&zatm_dev->lock, flags); zatm_dev->tx_map[zatm_vcc->tx_chan] = vcc; return 0; } static int start_tx(struct atm_dev *dev) { struct zatm_dev *zatm_dev; int i; DPRINTK("start_tx\n"); zatm_dev = ZATM_DEV(dev); zatm_dev->tx_map = (struct atm_vcc **) kmalloc(sizeof(struct atm_vcc *)* zatm_dev->chans,GFP_KERNEL); if (!zatm_dev->tx_map) return -ENOMEM; zatm_dev->tx_bw = ATM_OC3_PCR; zatm_dev->free_shapers = (1 << NR_SHAPERS)-1; zatm_dev->ubr = -1; zatm_dev->ubr_ref_cnt = 0; /* initialize shapers */ for (i = 0; i < NR_SHAPERS; i++) zpokel(zatm_dev,0,uPD98401_PS(i)); return 0; } /*------------------------------- interrupts --------------------------------*/ static irqreturn_t zatm_int(int irq,void *dev_id,struct pt_regs *regs) { struct atm_dev *dev;<