From 7fe2b3190b8b299409f13cf3a6f85c2bd371f8bb Mon Sep 17 00:00:00 2001 From: David Teigland Date: Wed, 24 Feb 2010 11:08:18 -0600 Subject: dlm: fix ordering of bast and cast When both blocking and completion callbacks are queued for lock, the dlm would always deliver the completion callback (cast) first. In some cases the blocking callback (bast) is queued before the cast, though, and should be delivered first. This patch keeps track of the order in which they were queued and delivers them in that order. This patch also keeps track of the granted mode in the last cast and eliminates the following bast if the bast mode is compatible with the preceding cast mode. This happens when a remotely mastered lock is demoted, e.g. EX->NL, in which case the local node queues a cast immediately after sending the demote message. In this way a cast can be queued for a mode, e.g. NL, that makes an in-transit bast extraneous. Signed-off-by: David Teigland --- fs/dlm/ast.c | 74 ++++++++++++++++++++++++++++++++++++++++----------- fs/dlm/ast.h | 4 +-- fs/dlm/dlm_internal.h | 10 +++++-- fs/dlm/lock.c | 4 +-- fs/dlm/user.c | 10 ++++--- fs/dlm/user.h | 4 +-- 6 files changed, 78 insertions(+), 28 deletions(-) (limited to 'fs') diff --git a/fs/dlm/ast.c b/fs/dlm/ast.c index dc2ad6008b2d..4314f0d48d85 100644 --- a/fs/dlm/ast.c +++ b/fs/dlm/ast.c @@ -2,7 +2,7 @@ ******************************************************************************* ** ** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. +** Copyright (C) 2004-2010 Red Hat, Inc. All rights reserved. ** ** This copyrighted material is made available to anyone wishing to use, ** modify, copy, or redistribute it subject to the terms and conditions @@ -33,10 +33,10 @@ void dlm_del_ast(struct dlm_lkb *lkb) spin_unlock(&ast_queue_lock); } -void dlm_add_ast(struct dlm_lkb *lkb, int type, int bastmode) +void dlm_add_ast(struct dlm_lkb *lkb, int type, int mode) { if (lkb->lkb_flags & DLM_IFL_USER) { - dlm_user_add_ast(lkb, type, bastmode); + dlm_user_add_ast(lkb, type, mode); return; } @@ -44,10 +44,21 @@ void dlm_add_ast(struct dlm_lkb *lkb, int type, int bastmode) if (!(lkb->lkb_ast_type & (AST_COMP | AST_BAST))) { kref_get(&lkb->lkb_ref); list_add_tail(&lkb->lkb_astqueue, &ast_queue); + lkb->lkb_ast_first = type; } + + /* sanity check, this should not happen */ + + if ((type == AST_COMP) && (lkb->lkb_ast_type & AST_COMP)) + log_print("repeat cast %d castmode %d lock %x %s", + mode, lkb->lkb_castmode, + lkb->lkb_id, lkb->lkb_resource->res_name); + lkb->lkb_ast_type |= type; - if (bastmode) - lkb->lkb_bastmode = bastmode; + if (type == AST_BAST) + lkb->lkb_bastmode = mode; + else + lkb->lkb_castmode = mode; spin_unlock(&ast_queue_lock); set_bit(WAKE_ASTS, &astd_wakeflags); @@ -59,9 +70,9 @@ static void process_asts(void) struct dlm_ls *ls = NULL; struct dlm_rsb *r = NULL; struct dlm_lkb *lkb; - void (*cast) (void *astparam); - void (*bast) (void *astparam, int mode); - int type = 0, bastmode; + void (*castfn) (void *astparam); + void (*bastfn) (void *astparam, int mode); + int type, first, bastmode, castmode, do_bast, do_cast, last_castmode; repeat: spin_lock(&ast_queue_lock); @@ -75,17 +86,48 @@ repeat: list_del(&lkb->lkb_astqueue); type = lkb->lkb_ast_type; lkb->lkb_ast_type = 0; + first = lkb->lkb_ast_first; + lkb->lkb_ast_first = 0; bastmode = lkb->lkb_bastmode; - + castmode = lkb->lkb_castmode; + castfn = lkb->lkb_astfn; + bastfn = lkb->lkb_bastfn; spin_unlock(&ast_queue_lock); - cast = lkb->lkb_astfn; - bast = lkb->lkb_bastfn; - - if ((type & AST_COMP) && cast) - cast(lkb->lkb_astparam); - if ((type & AST_BAST) && bast) - bast(lkb->lkb_astparam, bastmode); + do_cast = (type & AST_COMP) && castfn; + do_bast = (type & AST_BAST) && bastfn; + + /* Skip a bast if its blocking mode is compatible with the + granted mode of the preceding cast. */ + + if (do_bast) { + if (first == AST_COMP) + last_castmode = castmode; + else + last_castmode = lkb->lkb_castmode_done; + if (dlm_modes_compat(bastmode, last_castmode)) + do_bast = 0; + } + + if (first == AST_COMP) { + if (do_cast) + castfn(lkb->lkb_astparam); + if (do_bast) + bastfn(lkb->lkb_astparam, bastmode); + } else if (first == AST_BAST) { + if (do_bast) + bastfn(lkb->lkb_astparam, bastmode); + if (do_cast) + castfn(lkb->lkb_astparam); + } else { + log_error(ls, "bad ast_first %d ast_type %d", + first, type); + } + + if (do_cast) + lkb->lkb_castmode_done = castmode; + if (do_bast) + lkb->lkb_bastmode_done = bastmode; /* this removes the reference added by dlm_add_ast and may result in the lkb being freed */ diff --git a/fs/dlm/ast.h b/fs/dlm/ast.h index 1b5fc5f428fd..bcb1aaba519d 100644 --- a/fs/dlm/ast.h +++ b/fs/dlm/ast.h @@ -1,7 +1,7 @@ /****************************************************************************** ******************************************************************************* ** -** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. +** Copyright (C) 2005-2010 Red Hat, Inc. All rights reserved. ** ** This copyrighted material is made available to anyone wishing to use, ** modify, copy, or redistribute it subject to the terms and conditions @@ -13,7 +13,7 @@ #ifndef __ASTD_DOT_H__ #define __ASTD_DOT_H__ -void dlm_add_ast(struct dlm_lkb *lkb, int type, int bastmode); +void dlm_add_ast(struct dlm_lkb *lkb, int type, int mode); void dlm_del_ast(struct dlm_lkb *lkb); void dlm_astd_wake(void); diff --git a/fs/dlm/dlm_internal.h b/fs/dlm/dlm_internal.h index 826d3dc6e0ab..f632b58cd222 100644 --- a/fs/dlm/dlm_internal.h +++ b/fs/dlm/dlm_internal.h @@ -2,7 +2,7 @@ ******************************************************************************* ** ** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. -** Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. +** Copyright (C) 2004-2010 Red Hat, Inc. All rights reserved. ** ** This copyrighted material is made available to anyone wishing to use, ** modify, copy, or redistribute it subject to the terms and conditions @@ -232,11 +232,17 @@ struct dlm_lkb { int8_t lkb_status; /* granted, waiting, convert */ int8_t lkb_rqmode; /* requested lock mode */ int8_t lkb_grmode; /* granted lock mode */ - int8_t lkb_bastmode; /* requested mode */ int8_t lkb_highbast; /* highest mode bast sent for */ + int8_t lkb_wait_type; /* type of reply waiting for */ int8_t lkb_wait_count; int8_t lkb_ast_type; /* type of ast queued for */ + int8_t lkb_ast_first; /* type of first ast queued */ + + int8_t lkb_bastmode; /* req mode of queued bast */ + int8_t lkb_castmode; /* gr mode of queued cast */ + int8_t lkb_bastmode_done; /* last delivered bastmode */ + int8_t lkb_castmode_done; /* last delivered castmode */ struct list_head lkb_idtbl_list; /* lockspace lkbtbl */ struct list_head lkb_statequeue; /* rsb g/c/w list */ diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index 9c0c1db1e105..e08ea93432bc 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -1,7 +1,7 @@ /****************************************************************************** ******************************************************************************* ** -** Copyright (C) 2005-2008 Red Hat, Inc. All rights reserved. +** Copyright (C) 2005-2010 Red Hat, Inc. All rights reserved. ** ** This copyrighted material is made available to anyone wishing to use, ** modify, copy, or redistribute it subject to the terms and conditions @@ -307,7 +307,7 @@ static void queue_cast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rv) lkb->lkb_lksb->sb_status = rv; lkb->lkb_lksb->sb_flags = lkb->lkb_sbflags; - dlm_add_ast(lkb, AST_COMP, 0); + dlm_add_ast(lkb, AST_COMP, lkb->lkb_grmode); } static inline void queue_cast_overlap(struct dlm_rsb *r, struct dlm_lkb *lkb) diff --git a/fs/dlm/user.c b/fs/dlm/user.c index e73a4bb572aa..a4bfd31ac45b 100644 --- a/fs/dlm/user.c +++ b/fs/dlm/user.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2006-2009 Red Hat, Inc. All rights reserved. + * Copyright (C) 2006-2010 Red Hat, Inc. All rights reserved. * * This copyrighted material is made available to anyone wishing to use, * modify, copy, or redistribute it subject to the terms and conditions @@ -173,7 +173,7 @@ static int lkb_is_endoflife(struct dlm_lkb *lkb, int sb_status, int type) /* we could possibly check if the cancel of an orphan has resulted in the lkb being removed and then remove that lkb from the orphans list and free it */ -void dlm_user_add_ast(struct dlm_lkb *lkb, int type, int bastmode) +void dlm_user_add_ast(struct dlm_lkb *lkb, int type, int mode) { struct dlm_ls *ls; struct dlm_user_args *ua; @@ -206,8 +206,10 @@ void dlm_user_add_ast(struct dlm_lkb *lkb, int type, int bastmode) ast_type = lkb->lkb_ast_type; lkb->lkb_ast_type |= type; - if (bastmode) - lkb->lkb_bastmode = bastmode; + if (type == AST_BAST) + lkb->lkb_bastmode = mode; + else + lkb->lkb_castmode = mode; if (!ast_type) { kref_get(&lkb->lkb_ref); diff --git a/fs/dlm/user.h b/fs/dlm/user.h index 1c9686492286..f196091dd7ff 100644 --- a/fs/dlm/user.h +++ b/fs/dlm/user.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved. + * Copyright (C) 2006-2010 Red Hat, Inc. All rights reserved. * * This copyrighted material is made available to anyone wishing to use, * modify, copy, or redistribute it subject to the terms and conditions @@ -9,7 +9,7 @@ #ifndef __USER_DOT_H__ #define __USER_DOT_H__ -void dlm_user_add_ast(struct dlm_lkb *lkb, int type, int bastmode); +void dlm_user_add_ast(struct dlm_lkb *lkb, int type, int mode); int dlm_user_init(void); void dlm_user_exit(void); int dlm_device_deregister(struct dlm_ls *ls); -- cgit v1.2.3 From cf6620acc0f6fac57968aafef79ab372bdcf6157 Mon Sep 17 00:00:00 2001 From: David Teigland Date: Wed, 24 Feb 2010 11:59:23 -0600 Subject: dlm: send reply before bast When the lock master processes a successful operation (request, convert, cancel, or unlock), it will process the effects of the change before sending the reply for the operation. The "effects" of the operation are: - blocking callbacks (basts) for any newly granted locks - waiting or converting locks that can now be granted The cast is queued on the local node when the reply from the lock master is received. This means that a lock holder can receive a bast for a lock mode that is doesn't yet know has been granted. Signed-off-by: David Teigland --- fs/dlm/lock.c | 110 ++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 26 deletions(-) (limited to 'fs') diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index e08ea93432bc..d0e43a3da887 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -2280,20 +2280,30 @@ static int do_request(struct dlm_rsb *r, struct dlm_lkb *lkb) if (can_be_queued(lkb)) { error = -EINPROGRESS; add_lkb(r, lkb, DLM_LKSTS_WAITING); - send_blocking_asts(r, lkb); add_timeout(lkb); goto out; } error = -EAGAIN; - if (force_blocking_asts(lkb)) - send_blocking_asts_all(r, lkb); queue_cast(r, lkb, -EAGAIN); - out: return error; } +static void do_request_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + switch (error) { + case -EAGAIN: + if (force_blocking_asts(lkb)) + send_blocking_asts_all(r, lkb); + break; + case -EINPROGRESS: + send_blocking_asts(r, lkb); + break; + } +} + static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) { int error = 0; @@ -2304,7 +2314,6 @@ static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) if (can_be_granted(r, lkb, 1, &deadlk)) { grant_lock(r, lkb); queue_cast(r, lkb, 0); - grant_pending_locks(r); goto out; } @@ -2334,7 +2343,6 @@ static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) if (_can_be_granted(r, lkb, 1)) { grant_lock(r, lkb); queue_cast(r, lkb, 0); - grant_pending_locks(r); goto out; } /* else fall through and move to convert queue */ @@ -2344,28 +2352,47 @@ static int do_convert(struct dlm_rsb *r, struct dlm_lkb *lkb) error = -EINPROGRESS; del_lkb(r, lkb); add_lkb(r, lkb, DLM_LKSTS_CONVERT); - send_blocking_asts(r, lkb); add_timeout(lkb); goto out; } error = -EAGAIN; - if (force_blocking_asts(lkb)) - send_blocking_asts_all(r, lkb); queue_cast(r, lkb, -EAGAIN); - out: return error; } +static void do_convert_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + switch (error) { + case 0: + grant_pending_locks(r); + /* grant_pending_locks also sends basts */ + break; + case -EAGAIN: + if (force_blocking_asts(lkb)) + send_blocking_asts_all(r, lkb); + break; + case -EINPROGRESS: + send_blocking_asts(r, lkb); + break; + } +} + static int do_unlock(struct dlm_rsb *r, struct dlm_lkb *lkb) { remove_lock(r, lkb); queue_cast(r, lkb, -DLM_EUNLOCK); - grant_pending_locks(r); return -DLM_EUNLOCK; } +static void do_unlock_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + grant_pending_locks(r); +} + /* returns: 0 did nothing, -DLM_ECANCEL canceled lock */ static int do_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb) @@ -2375,12 +2402,18 @@ static int do_cancel(struct dlm_rsb *r, struct dlm_lkb *lkb) error = revert_lock(r, lkb); if (error) { queue_cast(r, lkb, -DLM_ECANCEL); - grant_pending_locks(r); return -DLM_ECANCEL; } return 0; } +static void do_cancel_effects(struct dlm_rsb *r, struct dlm_lkb *lkb, + int error) +{ + if (error) + grant_pending_locks(r); +} + /* * Four stage 3 varieties: * _request_lock(), _convert_lock(), _unlock_lock(), _cancel_lock() @@ -2402,11 +2435,15 @@ static int _request_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) goto out; } - if (is_remote(r)) + if (is_remote(r)) { /* receive_request() calls do_request() on remote node */ error = send_request(r, lkb); - else + } else { error = do_request(r, lkb); + /* for remote locks the request_reply is sent + between do_request and do_request_effects */ + do_request_effects(r, lkb, error); + } out: return error; } @@ -2417,11 +2454,15 @@ static int _convert_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) { int error; - if (is_remote(r)) + if (is_remote(r)) { /* receive_convert() calls do_convert() on remote node */ error = send_convert(r, lkb); - else + } else { error = do_convert(r, lkb); + /* for remote locks the convert_reply is sent + between do_convert and do_convert_effects */ + do_convert_effects(r, lkb, error); + } return error; } @@ -2432,11 +2473,15 @@ static int _unlock_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) { int error; - if (is_remote(r)) + if (is_remote(r)) { /* receive_unlock() calls do_unlock() on remote node */ error = send_unlock(r, lkb); - else + } else { error = do_unlock(r, lkb); + /* for remote locks the unlock_reply is sent + between do_unlock and do_unlock_effects */ + do_unlock_effects(r, lkb, error); + } return error; } @@ -2447,11 +2492,15 @@ static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb) { int error; - if (is_remote(r)) + if (is_remote(r)) { /* receive_cancel() calls do_cancel() on remote node */ error = send_cancel(r, lkb); - else + } else { error = do_cancel(r, lkb); + /* for remote locks the cancel_reply is sent + between do_cancel and do_cancel_effects */ + do_cancel_effects(r, lkb, error); + } return error; } @@ -3191,6 +3240,7 @@ static void receive_request(struct dlm_ls *ls, struct dlm_message *ms) attach_lkb(r, lkb); error = do_request(r, lkb); send_request_reply(r, lkb, error); + do_request_effects(r, lkb, error); unlock_rsb(r); put_rsb(r); @@ -3226,15 +3276,19 @@ static void receive_convert(struct dlm_ls *ls, struct dlm_message *ms) goto out; receive_flags(lkb, ms); + error = receive_convert_args(ls, lkb, ms); - if (error) - goto out_reply; + if (error) { + send_convert_reply(r, lkb, error); + goto out; + } + reply = !down_conversion(lkb); error = do_convert(r, lkb); - out_reply: if (reply) send_convert_reply(r, lkb, error); + do_convert_effects(r, lkb, error); out: unlock_rsb(r); put_rsb(r); @@ -3266,13 +3320,16 @@ static void receive_unlock(struct dlm_ls *ls, struct dlm_message *ms) goto out; receive_flags(lkb, ms); + error = receive_unlock_args(ls, lkb, ms); - if (error) - goto out_reply; + if (error) { + send_unlock_reply(r, lkb, error); + goto out; + } error = do_unlock(r, lkb); - out_reply: send_unlock_reply(r, lkb, error); + do_unlock_effects(r, lkb, error); out: unlock_rsb(r); put_rsb(r); @@ -3307,6 +3364,7 @@ static void receive_cancel(struct dlm_ls *ls, struct dlm_message *ms) error = do_cancel(r, lkb); send_cancel_reply(r, lkb, error); + do_cancel_effects(r, lkb, error); out: unlock_rsb(r); put_rsb(r); -- cgit v1.2.3 From b4a5d4bc377e49239374f266f0a0e2772c29749c Mon Sep 17 00:00:00 2001 From: Steven Whitehouse Date: Wed, 17 Feb 2010 09:41:34 +0000 Subject: dlm: Send lockspace name with uevents Although it is possible to get this information from the path, its much easier to provide the lockspace as a seperate env variable. Signed-off-by: Steven Whitehouse Signed-off-by: David Teigland --- fs/dlm/lockspace.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'fs') diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c index c010ecfc0d29..26a8bd40400a 100644 --- a/fs/dlm/lockspace.c +++ b/fs/dlm/lockspace.c @@ -191,6 +191,18 @@ static int do_uevent(struct dlm_ls *ls, int in) return error; } +static int dlm_uevent(struct kset *kset, struct kobject *kobj, + struct kobj_uevent_env *env) +{ + struct dlm_ls *ls = container_of(kobj, struct dlm_ls, ls_kobj); + + add_uevent_var(env, "LOCKSPACE=%s", ls->ls_name); + return 0; +} + +static struct kset_uevent_ops dlm_uevent_ops = { + .uevent = dlm_uevent, +}; int __init dlm_lockspace_init(void) { @@ -199,7 +211,7 @@ int __init dlm_lockspace_init(void) INIT_LIST_HEAD(&lslist); spin_lock_init(&lslist_lock); - dlm_kset = kset_create_and_add("dlm", NULL, kernel_kobj); + dlm_kset = kset_create_and_add("dlm", &dlm_uevent_ops, kernel_kobj); if (!dlm_kset) { printk(KERN_WARNING "%s: can not create kset\n", __func__); return -ENOMEM; -- cgit v1.2.3 From b6fa8796b2da0390e9f4115e8789a01004fc1c9b Mon Sep 17 00:00:00 2001 From: David Teigland Date: Thu, 25 Feb 2010 12:20:57 -0600 Subject: dlm: use bastmode in debugfs output The bast mode that appears in the debugfs output should be useful on both master and process nodes. lkb_highbast is currently printed, and is only useful on the master node. lkb_bastmode is only useful on the process node. This patch sets lkb_bastmode on the master node as well, and uses that value in the debugfs print. Signed-off-by: David Teigland --- fs/dlm/debug_fs.c | 2 +- fs/dlm/lock.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'fs') diff --git a/fs/dlm/debug_fs.c b/fs/dlm/debug_fs.c index 375a2359b3bf..29d6139c35fc 100644 --- a/fs/dlm/debug_fs.c +++ b/fs/dlm/debug_fs.c @@ -256,7 +256,7 @@ static int print_format3_lock(struct seq_file *s, struct dlm_lkb *lkb, lkb->lkb_status, lkb->lkb_grmode, lkb->lkb_rqmode, - lkb->lkb_highbast, + lkb->lkb_bastmode, rsb_lookup, lkb->lkb_wait_type, lkb->lkb_lvbseq, diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index d0e43a3da887..46ffd3eeaaf7 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -320,10 +320,12 @@ static void queue_bast(struct dlm_rsb *r, struct dlm_lkb *lkb, int rqmode) { lkb->lkb_time_bast = ktime_get(); - if (is_master_copy(lkb)) + if (is_master_copy(lkb)) { + lkb->lkb_bastmode = rqmode; /* printed by debugfs */ send_bast(r, lkb, rqmode); - else + } else { dlm_add_ast(lkb, AST_BAST, rqmode); + } } /* -- cgit v1.2.3