aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDavid Teigland <teigland@redhat.com>2007-05-18 10:01:26 -0400
committerSteven Whitehouse <swhiteho@redhat.com>2007-07-09 03:22:38 -0400
commitc85d65e91430db94ae9ce0cf38b56e496658b642 (patch)
tree5ab6e0399594df5fa53793ca8f922e0a55f0aca3
parentd7db923ea4990edb5583bf54af868ba687a1bc84 (diff)
[DLM] cancel in conversion deadlock [4/6]
When conversion deadlock is detected, cancel the conversion and return EDEADLK to the application. This is a new default behavior where before the dlm would allow the deadlock to exist indefinately. The DLM_LKF_NODLCKWT flag can now be used in a conversion to prevent the dlm from performing conversion deadlock detection/cancelation on it. The DLM_LKF_CONVDEADLK flag can continue to be used as before to tell the dlm to demote the granted mode of the lock being converted if it gets into a conversion deadlock. Signed-off-by: David Teigland <teigland@redhat.com> Signed-off-by: Steven Whitehouse <swhiteho@redhat.com>
-rw-r--r--fs/dlm/lock.c193
-rw-r--r--include/linux/dlm.h6
2 files changed, 137 insertions, 62 deletions
diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c
index ad3797a37942..3c4d570477b4 100644
--- a/fs/dlm/lock.c
+++ b/fs/dlm/lock.c
@@ -1408,10 +1408,8 @@ static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb)
1408 * queue for one resource. The granted mode of each lock blocks the requested 1408 * queue for one resource. The granted mode of each lock blocks the requested
1409 * mode of the other lock." 1409 * mode of the other lock."
1410 * 1410 *
1411 * Part 2: if the granted mode of lkb is preventing the first lkb in the 1411 * Part 2: if the granted mode of lkb is preventing an earlier lkb in the
1412 * convert queue from being granted, then demote lkb (set grmode to NL). 1412 * convert queue from being granted, then deadlk/demote lkb.
1413 * This second form requires that we check for conv-deadlk even when
1414 * now == 0 in _can_be_granted().
1415 * 1413 *
1416 * Example: 1414 * Example:
1417 * Granted Queue: empty 1415 * Granted Queue: empty
@@ -1420,41 +1418,52 @@ static int queue_conflict(struct list_head *head, struct dlm_lkb *lkb)
1420 * 1418 *
1421 * The first lock can't be granted because of the granted mode of the second 1419 * The first lock can't be granted because of the granted mode of the second
1422 * lock and the second lock can't be granted because it's not first in the 1420 * lock and the second lock can't be granted because it's not first in the
1423 * list. We demote the granted mode of the second lock (the lkb passed to this 1421 * list. We either cancel lkb's conversion (PR->EX) and return EDEADLK, or we
1424 * function). 1422 * demote the granted mode of lkb (from PR to NL) if it has the CONVDEADLK
1423 * flag set and return DEMOTED in the lksb flags.
1425 * 1424 *
1426 * After the resolution, the "grant pending" function needs to go back and try 1425 * Originally, this function detected conv-deadlk in a more limited scope:
1427 * to grant locks on the convert queue again since the first lock can now be 1426 * - if !modes_compat(lkb1, lkb2) && !modes_compat(lkb2, lkb1), or
1428 * granted. 1427 * - if lkb1 was the first entry in the queue (not just earlier), and was
1428 * blocked by the granted mode of lkb2, and there was nothing on the
1429 * granted queue preventing lkb1 from being granted immediately, i.e.
1430 * lkb2 was the only thing preventing lkb1 from being granted.
1431 *
1432 * That second condition meant we'd only say there was conv-deadlk if
1433 * resolving it (by demotion) would lead to the first lock on the convert
1434 * queue being granted right away. It allowed conversion deadlocks to exist
1435 * between locks on the convert queue while they couldn't be granted anyway.
1436 *
1437 * Now, we detect and take action on conversion deadlocks immediately when
1438 * they're created, even if they may not be immediately consequential. If
1439 * lkb1 exists anywhere in the convert queue and lkb2 comes in with a granted
1440 * mode that would prevent lkb1's conversion from being granted, we do a
1441 * deadlk/demote on lkb2 right away and don't let it onto the convert queue.
1442 * I think this means that the lkb_is_ahead condition below should always
1443 * be zero, i.e. there will never be conv-deadlk between two locks that are
1444 * both already on the convert queue.
1429 */ 1445 */
1430 1446
1431static int conversion_deadlock_detect(struct dlm_rsb *rsb, struct dlm_lkb *lkb) 1447static int conversion_deadlock_detect(struct dlm_rsb *r, struct dlm_lkb *lkb2)
1432{ 1448{
1433 struct dlm_lkb *this, *first = NULL, *self = NULL; 1449 struct dlm_lkb *lkb1;
1450 int lkb_is_ahead = 0;
1434 1451
1435 list_for_each_entry(this, &rsb->res_convertqueue, lkb_statequeue) { 1452 list_for_each_entry(lkb1, &r->res_convertqueue, lkb_statequeue) {
1436 if (!first) 1453 if (lkb1 == lkb2) {
1437 first = this; 1454 lkb_is_ahead = 1;
1438 if (this == lkb) {
1439 self = lkb;
1440 continue; 1455 continue;
1441 } 1456 }
1442 1457
1443 if (!modes_compat(this, lkb) && !modes_compat(lkb, this)) 1458 if (!lkb_is_ahead) {
1444 return 1; 1459 if (!modes_compat(lkb2, lkb1))
1445 } 1460 return 1;
1446 1461 } else {
1447 /* if lkb is on the convert queue and is preventing the first 1462 if (!modes_compat(lkb2, lkb1) &&
1448 from being granted, then there's deadlock and we demote lkb. 1463 !modes_compat(lkb1, lkb2))
1449 multiple converting locks may need to do this before the first 1464 return 1;
1450 converting lock can be granted. */ 1465 }
1451
1452 if (self && self != first) {
1453 if (!modes_compat(lkb, first) &&
1454 !queue_conflict(&rsb->res_grantqueue, first))
1455 return 1;
1456 } 1466 }
1457
1458 return 0; 1467 return 0;
1459} 1468}
1460 1469
@@ -1583,42 +1592,57 @@ static int _can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now)
1583 if (!now && !conv && list_empty(&r->res_convertqueue) && 1592 if (!now && !conv && list_empty(&r->res_convertqueue) &&
1584 first_in_list(lkb, &r->res_waitqueue)) 1593 first_in_list(lkb, &r->res_waitqueue))
1585 return 1; 1594 return 1;
1586
1587 out: 1595 out:
1588 /*
1589 * The following, enabled by CONVDEADLK, departs from VMS.
1590 */
1591
1592 if (conv && (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) &&
1593 conversion_deadlock_detect(r, lkb)) {
1594 lkb->lkb_grmode = DLM_LOCK_NL;
1595 lkb->lkb_sbflags |= DLM_SBF_DEMOTED;
1596 }
1597
1598 return 0; 1596 return 0;
1599} 1597}
1600 1598
1601/* 1599static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now,
1602 * The ALTPR and ALTCW flags aren't traditional lock manager flags, but are a 1600 int *err)
1603 * simple way to provide a big optimization to applications that can use them.
1604 */
1605
1606static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now)
1607{ 1601{
1608 uint32_t flags = lkb->lkb_exflags;
1609 int rv; 1602 int rv;
1610 int8_t alt = 0, rqmode = lkb->lkb_rqmode; 1603 int8_t alt = 0, rqmode = lkb->lkb_rqmode;
1604 int8_t is_convert = (lkb->lkb_grmode != DLM_LOCK_IV);
1605
1606 if (err)
1607 *err = 0;
1611 1608
1612 rv = _can_be_granted(r, lkb, now); 1609 rv = _can_be_granted(r, lkb, now);
1613 if (rv) 1610 if (rv)
1614 goto out; 1611 goto out;
1615 1612
1616 if (lkb->lkb_sbflags & DLM_SBF_DEMOTED) 1613 /*
1614 * The CONVDEADLK flag is non-standard and tells the dlm to resolve
1615 * conversion deadlocks by demoting grmode to NL, otherwise the dlm
1616 * cancels one of the locks.
1617 */
1618
1619 if (is_convert && can_be_queued(lkb) &&
1620 conversion_deadlock_detect(r, lkb)) {
1621 if (lkb->lkb_exflags & DLM_LKF_CONVDEADLK) {
1622 lkb->lkb_grmode = DLM_LOCK_NL;
1623 lkb->lkb_sbflags |= DLM_SBF_DEMOTED;
1624 } else if (!(lkb->lkb_exflags & DLM_LKF_NODLCKWT)) {
1625 if (err)
1626 *err = -EDEADLK;
1627 else {
1628 log_print("can_be_granted deadlock %x now %d",
1629 lkb->lkb_id, now);
1630 dlm_dump_rsb(r);
1631 }
1632 }
1617 goto out; 1633 goto out;
1634 }
1618 1635
1619 if (rqmode != DLM_LOCK_PR && flags & DLM_LKF_ALTPR) 1636 /*
1637 * The ALTPR and ALTCW flags are non-standard and tell the dlm to try
1638 * to grant a request in a mode other than the normal rqmode. It's a
1639 * simple way to provide a big optimization to applications that can
1640 * use them.
1641 */
1642
1643 if (rqmode != DLM_LOCK_PR && (lkb->lkb_exflags & DLM_LKF_ALTPR))
1620 alt = DLM_LOCK_PR; 1644 alt = DLM_LOCK_PR;
1621 else if (rqmode != DLM_LOCK_CW && flags & DLM_LKF_ALTCW) 1645 else if (rqmode != DLM_LOCK_CW && (lkb->lkb_exflags & DLM_LKF_ALTCW))
1622 alt = DLM_LOCK_CW; 1646 alt = DLM_LOCK_CW;
1623 1647
1624 if (alt) { 1648 if (alt) {
@@ -1633,10 +1657,20 @@ static int can_be_granted(struct dlm_rsb *r, struct dlm_lkb *lkb, int now)
1633 return rv; 1657 return rv;
1634} 1658}
1635 1659
1660/* FIXME: I don't think that can_be_granted() can/will demote or find deadlock
1661 for locks pending on the convert list. Once verified (watch for these
1662 log_prints), we should be able to just call _can_be_granted() and not
1663 bother with the demote/deadlk cases here (and there's no easy way to deal
1664 with a deadlk here, we'd have to generate something like grant_lock with
1665 the deadlk error.) */
1666
1667/* returns the highest requested mode of all blocked conversions */
1668
1636static int grant_pending_convert(struct dlm_rsb *r, int high) 1669static int grant_pending_convert(struct dlm_rsb *r, int high)
1637{ 1670{
1638 struct dlm_lkb *lkb, *s; 1671 struct dlm_lkb *lkb, *s;
1639 int hi, demoted, quit, grant_restart, demote_restart; 1672 int hi, demoted, quit, grant_restart, demote_restart;
1673 int deadlk;
1640 1674
1641 quit = 0; 1675 quit = 0;
1642 restart: 1676 restart:
@@ -1646,14 +1680,29 @@ static int grant_pending_convert(struct dlm_rsb *r, int high)
1646 1680
1647 list_for_each_entry_safe(lkb, s, &r->res_convertqueue, lkb_statequeue) { 1681 list_for_each_entry_safe(lkb, s, &r->res_convertqueue, lkb_statequeue) {
1648 demoted = is_demoted(lkb); 1682 demoted = is_demoted(lkb);
1649 if (can_be_granted(r, lkb, 0)) { 1683 deadlk = 0;
1684
1685 if (can_be_granted(r, lkb, 0, &deadlk)) {
1650 grant_lock_pending(r, lkb); 1686 grant_lock_pending(r, lkb);
1651 grant_restart = 1; 1687 grant_restart = 1;
1652 } else { 1688 continue;
1653 hi = max_t(int, lkb->lkb_rqmode, hi);
1654 if (!demoted && is_demoted(lkb))
1655 demote_restart = 1;
1656 } 1689 }
1690
1691 if (!demoted && is_demoted(lkb)) {
1692 log_print("WARN: pending demoted %x node %d %s",
1693 lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
1694 demote_restart = 1;
1695 continue;
1696 }
1697
1698 if (deadlk) {
1699 log_print("WARN: pending deadlock %x node %d %s",
1700 lkb->lkb_id, lkb->lkb_nodeid, r->res_name);
1701 dlm_dump_rsb(r);
1702 continue;
1703 }
1704
1705 hi = max_t(int, lkb->lkb_rqmode, hi);
1657 } 1706 }
1658 1707
1659 if (grant_restart) 1708 if (grant_restart)
@@ -1671,7 +1720,7 @@ static int grant_pending_wait(struct dlm_rsb *r, int high)
1671 struct dlm_lkb *lkb, *s; 1720 struct dlm_lkb *lkb, *s;
1672 1721
1673 list_for_each_entry_safe(lkb, s, &r->res_waitqueue, lkb_statequeue) { 1722 list_for_each_entry_safe(lkb, s, &r->res_waitqueue, lkb_statequeue) {
1674 if (can_be_granted(r, lkb, 0)) 1723 if (can_be_granted(r, lkb, 0, NULL))
1675 grant_lock_pending(r, lkb); 1724 grant_lock_pending(r, lkb);
1676 else 1725 else
1677 high = max_t(int, lkb->lkb_rqmode, high); 1726 high = max_t(int, lkb->lkb_rqmode, high);
@@ -2121,7 +2170,7 @@ static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb)
2121{ 2170{
2122 int error = 0; 2171 int error = 0;
2123 2172
2124 if (can_be_granted(r, lkb, 1)) { 2173 if (can_be_granted(r, lkb, 1, NULL)) {
2125 grant_lock(r, lkb); 2174 grant_lock(r, lkb);
2126 queue_cast(r, lkb, 0); 2175 queue_cast(r, lkb, 0);
2127 goto out; 2176 goto out;
@@ -2147,16 +2196,32 @@ static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb)
2147static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) 2196static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb)
2148{ 2197{
2149 int error = 0; 2198 int error = 0;
2199 int deadlk = 0;
2150 2200
2151 /* changing an existing lock may allow others to be granted */ 2201 /* changing an existing lock may allow others to be granted */
2152 2202
2153 if (can_be_granted(r, lkb, 1)) { 2203 if (can_be_granted(r, lkb, 1, &deadlk)) {
2154 grant_lock(r, lkb); 2204 grant_lock(r, lkb);
2155 queue_cast(r, lkb, 0); 2205 queue_cast(r, lkb, 0);
2156 grant_pending_locks(r); 2206 grant_pending_locks(r);
2157 goto out; 2207 goto out;
2158 } 2208 }
2159 2209
2210 /* can_be_granted() detected that this lock would block in a conversion
2211 deadlock, so we leave it on the granted queue and return EDEADLK in
2212 the ast for the convert. */
2213
2214 if (deadlk) {
2215 /* it's left on the granted queue */
2216 log_debug(r->res_ls, "deadlock %x node %d sts%d g%d r%d %s",
2217 lkb->lkb_id, lkb->lkb_nodeid, lkb->lkb_status,
2218 lkb->lkb_grmode, lkb->lkb_rqmode, r->res_name);
2219 revert_lock(r, lkb);
2220 queue_cast(r, lkb, -EDEADLK);
2221 error = -EDEADLK;
2222 goto out;
2223 }
2224
2160 /* is_demoted() means the can_be_granted() above set the grmode 2225 /* is_demoted() means the can_be_granted() above set the grmode
2161 to NL, and left us on the granted queue. This auto-demotion 2226 to NL, and left us on the granted queue. This auto-demotion
2162 (due to CONVDEADLK) might mean other locks, and/or this lock, are 2227 (due to CONVDEADLK) might mean other locks, and/or this lock, are
@@ -2438,7 +2503,7 @@ int dlm_lock(dlm_lockspace_t *lockspace,
2438 out_put: 2503 out_put:
2439 if (convert || error) 2504 if (convert || error)
2440 __put_lkb(ls, lkb); 2505 __put_lkb(ls, lkb);
2441 if (error == -EAGAIN) 2506 if (error == -EAGAIN || error == -EDEADLK)
2442 error = 0; 2507 error = 0;
2443 out: 2508 out:
2444 dlm_unlock_recovery(ls); 2509 dlm_unlock_recovery(ls);
@@ -3312,6 +3377,12 @@ static void __receive_convert_reply(struct dlm_rsb *r, struct dlm_lkb *lkb,
3312 queue_cast(r, lkb, -EAGAIN); 3377 queue_cast(r, lkb, -EAGAIN);
3313 break; 3378 break;
3314 3379
3380 case -EDEADLK:
3381 receive_flags_reply(lkb, ms);
3382 revert_lock_pc(r, lkb);
3383 queue_cast(r, lkb, -EDEADLK);
3384 break;
3385
3315 case -EINPROGRESS: 3386 case -EINPROGRESS:
3316 /* convert was queued on remote master */ 3387 /* convert was queued on remote master */
3317 receive_flags_reply(lkb, ms); 3388 receive_flags_reply(lkb, ms);
@@ -4284,7 +4355,7 @@ int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
4284 4355
4285 error = convert_lock(ls, lkb, &args); 4356 error = convert_lock(ls, lkb, &args);
4286 4357
4287 if (error == -EINPROGRESS || error == -EAGAIN) 4358 if (error == -EINPROGRESS || error == -EAGAIN || error == -EDEADLK)
4288 error = 0; 4359 error = 0;
4289 out_put: 4360 out_put:
4290 dlm_put_lkb(lkb); 4361 dlm_put_lkb(lkb);
diff --git a/include/linux/dlm.h b/include/linux/dlm.h
index 975f17d8aa53..5227a9594f9d 100644
--- a/include/linux/dlm.h
+++ b/include/linux/dlm.h
@@ -85,7 +85,11 @@
85 * Only relevant to locks originating in userspace. A persistent lock will not 85 * Only relevant to locks originating in userspace. A persistent lock will not
86 * be removed if the process holding the lock exits. 86 * be removed if the process holding the lock exits.
87 * 87 *
88 * DLM_LKF_NODLKWT 88 * DLM_LKF_NODLCKWT
89 *
90 * Do not cancel the lock if it gets into conversion deadlock.
91 * Exclude this lock from being monitored due to DLM_LSFL_TIMEWARN.
92 *
89 * DLM_LKF_NODLCKBLK 93 * DLM_LKF_NODLCKBLK
90 * 94 *
91 * net yet implemented 95 * net yet implemented