diff --git a/include/linux/wait-simple.h b/include/linux/wait-simple.h index a9f24692cbe1..4efba4d28c48 100644 --- a/include/linux/wait-simple.h +++ b/include/linux/wait-simple.h @@ -22,12 +22,14 @@ struct swait_head { struct list_head list; }; -#define DEFINE_SWAIT_HEAD(name) \ - struct swait_head name = { \ +#define SWAIT_HEAD_INITIALIZER(name) { \ .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ .list = LIST_HEAD_INIT((name).list), \ } +#define DEFINE_SWAIT_HEAD(name) \ + struct swait_head name = SWAIT_HEAD_INITIALIZER(name) + extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key); #define init_swait_head(swh) \ @@ -40,63 +42,25 @@ extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key); /* * Waiter functions */ -static inline bool swaiter_enqueued(struct swaiter *w) -{ - return w->task != NULL; -} - +extern void swait_prepare_locked(struct swait_head *head, struct swaiter *w); extern void swait_prepare(struct swait_head *head, struct swaiter *w, int state); +extern void swait_finish_locked(struct swait_head *head, struct swaiter *w); extern void swait_finish(struct swait_head *head, struct swaiter *w); -/* - * Adds w to head->list. Must be called with head->lock locked. - */ -static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w) -{ - list_add(&w->node, &head->list); - /* We can't let the condition leak before the setting of head */ - smp_mb(); -} - -/* - * Removes w from head->list. Must be called with head->lock locked. - */ -static inline void __swait_dequeue(struct swaiter *w) -{ - list_del_init(&w->node); -} - -/* - * Check whether a head has waiters enqueued - */ -static inline bool swait_head_has_waiters(struct swait_head *h) -{ - /* Make sure the condition is visible before checking list_empty() */ - smp_mb(); - return !list_empty(&h->list); -} - /* * Wakeup functions */ -extern int __swait_wake(struct swait_head *head, unsigned int state); +extern unsigned int __swait_wake(struct swait_head *head, unsigned int state, unsigned int num); +extern unsigned int __swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num); -static inline int swait_wake(struct swait_head *head) -{ - return swait_head_has_waiters(head) ? - __swait_wake(head, TASK_NORMAL) : 0; -} - -static inline int swait_wake_interruptible(struct swait_head *head) -{ - return swait_head_has_waiters(head) ? - __swait_wake(head, TASK_INTERRUPTIBLE) : 0; -} +#define swait_wake(head) __swait_wake(head, TASK_NORMAL, 1) +#define swait_wake_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 1) +#define swait_wake_all(head) __swait_wake(head, TASK_NORMAL, 0) +#define swait_wake_all_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 0) /* * Event API */ - #define __swait_event(wq, condition) \ do { \ DEFINE_SWAITER(__wait); \ diff --git a/kernel/sched/wait-simple.c b/kernel/sched/wait-simple.c index 040d7146e4df..2c856267445b 100644 --- a/kernel/sched/wait-simple.c +++ b/kernel/sched/wait-simple.c @@ -12,6 +12,28 @@ #include #include +/* Adds w to head->list. Must be called with head->lock locked. */ +static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w) +{ + list_add(&w->node, &head->list); + /* We can't let the condition leak before the setting of head */ + smp_mb(); +} + +/* Removes w from head->list. Must be called with head->lock locked. */ +static inline void __swait_dequeue(struct swaiter *w) +{ + list_del_init(&w->node); +} + +/* Check whether a head has waiters enqueued */ +static inline bool swait_head_has_waiters(struct swait_head *h) +{ + /* Make sure the condition is visible before checking list_empty() */ + smp_mb(); + return !list_empty(&h->list); +} + void __init_swait_head(struct swait_head *head, struct lock_class_key *key) { raw_spin_lock_init(&head->lock); @@ -20,19 +42,31 @@ void __init_swait_head(struct swait_head *head, struct lock_class_key *key) } EXPORT_SYMBOL(__init_swait_head); +void swait_prepare_locked(struct swait_head *head, struct swaiter *w) +{ + w->task = current; + if (list_empty(&w->node)) + __swait_enqueue(head, w); +} + void swait_prepare(struct swait_head *head, struct swaiter *w, int state) { unsigned long flags; raw_spin_lock_irqsave(&head->lock, flags); - w->task = current; - if (list_empty(&w->node)) - __swait_enqueue(head, w); - set_current_state(state); + swait_prepare_locked(head, w); + __set_current_state(state); raw_spin_unlock_irqrestore(&head->lock, flags); } EXPORT_SYMBOL(swait_prepare); +void swait_finish_locked(struct swait_head *head, struct swaiter *w) +{ + __set_current_state(TASK_RUNNING); + if (w->task) + __swait_dequeue(w); +} + void swait_finish(struct swait_head *head, struct swaiter *w) { unsigned long flags; @@ -46,22 +80,43 @@ void swait_finish(struct swait_head *head, struct swaiter *w) } EXPORT_SYMBOL(swait_finish); -int __swait_wake(struct swait_head *head, unsigned int state) +unsigned int +__swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num) { struct swaiter *curr, *next; - unsigned long flags; int woken = 0; - raw_spin_lock_irqsave(&head->lock, flags); - list_for_each_entry_safe(curr, next, &head->list, node) { if (wake_up_state(curr->task, state)) { __swait_dequeue(curr); + /* + * The waiting task can free the waiter as + * soon as curr->task = NULL is written, + * without taking any locks. A memory barrier + * is required here to prevent the following + * store to curr->task from getting ahead of + * the dequeue operation. + */ + smp_wmb(); curr->task = NULL; - woken++; + if (++woken == num) + break; } } + return woken; +} +unsigned int +__swait_wake(struct swait_head *head, unsigned int state, unsigned int num) +{ + unsigned long flags; + int woken; + + if (!swait_head_has_waiters(head)) + return 0; + + raw_spin_lock_irqsave(&head->lock, flags); + woken = __swait_wake_locked(head, state, num); raw_spin_unlock_irqrestore(&head->lock, flags); return woken; }