2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* linux/kernel/workqueue.c
|
|
|
|
*
|
|
|
|
* Generic mechanism for defining kernel helper threads for running
|
|
|
|
* arbitrary tasks in process context.
|
|
|
|
*
|
|
|
|
* Started by Ingo Molnar, Copyright (C) 2002
|
|
|
|
*
|
|
|
|
* Derived from the taskqueue/keventd code by:
|
|
|
|
*
|
|
|
|
* David Woodhouse <dwmw2@infradead.org>
|
2008-10-16 07:01:59 +02:00
|
|
|
* Andrew Morton
|
2005-04-17 00:20:36 +02:00
|
|
|
* Kai Petzke <wpp@marie.physik.tu-berlin.de>
|
|
|
|
* Theodore Ts'o <tytso@mit.edu>
|
2005-10-31 00:01:59 +01:00
|
|
|
*
|
2008-07-04 18:59:22 +02:00
|
|
|
* Made to use alloc_percpu by Christoph Lameter.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/sched.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/signal.h>
|
|
|
|
#include <linux/completion.h>
|
|
|
|
#include <linux/workqueue.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/kthread.h>
|
2006-02-23 19:43:43 +01:00
|
|
|
#include <linux/hardirq.h>
|
2006-10-11 10:21:26 +02:00
|
|
|
#include <linux/mempolicy.h>
|
2006-12-07 05:34:49 +01:00
|
|
|
#include <linux/freezer.h>
|
2006-12-07 05:37:26 +01:00
|
|
|
#include <linux/kallsyms.h>
|
|
|
|
#include <linux/debug_locks.h>
|
2007-10-19 08:39:55 +02:00
|
|
|
#include <linux/lockdep.h>
|
2010-06-29 10:07:11 +02:00
|
|
|
#include <linux/idr.h>
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
/*
|
|
|
|
* Structure fields follow one of the following exclusion rules.
|
|
|
|
*
|
|
|
|
* I: Set during initialization and read-only afterwards.
|
|
|
|
*
|
|
|
|
* L: cwq->lock protected. Access with cwq->lock held.
|
|
|
|
*
|
2010-06-29 10:07:11 +02:00
|
|
|
* F: wq->flush_mutex protected.
|
|
|
|
*
|
2010-06-29 10:07:10 +02:00
|
|
|
* W: workqueue_lock protected.
|
|
|
|
*/
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
struct cpu_workqueue_struct;
|
|
|
|
|
|
|
|
struct worker {
|
|
|
|
struct work_struct *current_work; /* L: work being processed */
|
2010-06-29 10:07:12 +02:00
|
|
|
struct list_head scheduled; /* L: scheduled works */
|
2010-06-29 10:07:11 +02:00
|
|
|
struct task_struct *task; /* I: worker task */
|
|
|
|
struct cpu_workqueue_struct *cwq; /* I: the associated cwq */
|
|
|
|
int id; /* I: worker id */
|
|
|
|
};
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
2006-01-08 10:05:12 +01:00
|
|
|
* The per-CPU workqueue (if single thread, we always use the first
|
2010-06-29 10:07:11 +02:00
|
|
|
* possible cpu). The lower WORK_STRUCT_FLAG_BITS of
|
|
|
|
* work_struct->data are used for flags and thus cwqs need to be
|
|
|
|
* aligned at two's power of the number of flag bits.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
|
|
|
struct cpu_workqueue_struct {
|
|
|
|
|
|
|
|
spinlock_t lock;
|
|
|
|
|
|
|
|
struct list_head worklist;
|
|
|
|
wait_queue_head_t more_work;
|
2010-06-29 10:07:11 +02:00
|
|
|
unsigned int cpu;
|
2010-06-29 10:07:11 +02:00
|
|
|
struct worker *worker;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
struct workqueue_struct *wq; /* I: the owning workqueue */
|
2010-06-29 10:07:11 +02:00
|
|
|
int work_color; /* L: current color */
|
|
|
|
int flush_color; /* L: flushing color */
|
|
|
|
int nr_in_flight[WORK_NR_COLORS];
|
|
|
|
/* L: nr of in_flight works */
|
2010-06-29 10:07:12 +02:00
|
|
|
int nr_active; /* L: nr of active works */
|
2010-06-29 10:07:12 +02:00
|
|
|
int max_active; /* L: max active works */
|
2010-06-29 10:07:12 +02:00
|
|
|
struct list_head delayed_works; /* L: delayed works */
|
2010-06-29 10:07:11 +02:00
|
|
|
};
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
/*
|
|
|
|
* Structure used to wait for workqueue flush.
|
|
|
|
*/
|
|
|
|
struct wq_flusher {
|
|
|
|
struct list_head list; /* F: list of flushers */
|
|
|
|
int flush_color; /* F: flush color waiting for */
|
|
|
|
struct completion done; /* flush completion */
|
|
|
|
};
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
/*
|
|
|
|
* The externally visible workqueue abstraction is an array of
|
|
|
|
* per-CPU workqueues:
|
|
|
|
*/
|
|
|
|
struct workqueue_struct {
|
2010-06-29 10:07:10 +02:00
|
|
|
unsigned int flags; /* I: WQ_* flags */
|
2010-06-29 10:07:10 +02:00
|
|
|
struct cpu_workqueue_struct *cpu_wq; /* I: cwq's */
|
|
|
|
struct list_head list; /* W: list of all workqueues */
|
2010-06-29 10:07:11 +02:00
|
|
|
|
|
|
|
struct mutex flush_mutex; /* protects wq flushing */
|
|
|
|
int work_color; /* F: current work color */
|
|
|
|
int flush_color; /* F: current flush color */
|
|
|
|
atomic_t nr_cwqs_to_flush; /* flush in progress */
|
|
|
|
struct wq_flusher *first_flusher; /* F: first flusher */
|
|
|
|
struct list_head flusher_queue; /* F: flush waiters */
|
|
|
|
struct list_head flusher_overflow; /* F: flush overflow list */
|
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
int saved_max_active; /* I: saved cwq max_active */
|
2010-06-29 10:07:10 +02:00
|
|
|
const char *name; /* I: workqueue name */
|
2007-10-19 08:39:55 +02:00
|
|
|
#ifdef CONFIG_LOCKDEP
|
2010-06-29 10:07:10 +02:00
|
|
|
struct lockdep_map lockdep_map;
|
2007-10-19 08:39:55 +02:00
|
|
|
#endif
|
2005-04-17 00:20:36 +02:00
|
|
|
};
|
|
|
|
|
2009-11-15 17:09:48 +01:00
|
|
|
#ifdef CONFIG_DEBUG_OBJECTS_WORK
|
|
|
|
|
|
|
|
static struct debug_obj_descr work_debug_descr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_init is called when:
|
|
|
|
* - an active object is initialized
|
|
|
|
*/
|
|
|
|
static int work_fixup_init(void *addr, enum debug_obj_state state)
|
|
|
|
{
|
|
|
|
struct work_struct *work = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
cancel_work_sync(work);
|
|
|
|
debug_object_init(work, &work_debug_descr);
|
|
|
|
return 1;
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_activate is called when:
|
|
|
|
* - an active object is activated
|
|
|
|
* - an unknown object is activated (might be a statically initialized object)
|
|
|
|
*/
|
|
|
|
static int work_fixup_activate(void *addr, enum debug_obj_state state)
|
|
|
|
{
|
|
|
|
struct work_struct *work = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
|
|
|
|
case ODEBUG_STATE_NOTAVAILABLE:
|
|
|
|
/*
|
|
|
|
* This is not really a fixup. The work struct was
|
|
|
|
* statically initialized. We just make sure that it
|
|
|
|
* is tracked in the object tracker.
|
|
|
|
*/
|
2010-06-29 10:07:10 +02:00
|
|
|
if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
|
2009-11-15 17:09:48 +01:00
|
|
|
debug_object_init(work, &work_debug_descr);
|
|
|
|
debug_object_activate(work, &work_debug_descr);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
WARN_ON_ONCE(1);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
WARN_ON(1);
|
|
|
|
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* fixup_free is called when:
|
|
|
|
* - an active object is freed
|
|
|
|
*/
|
|
|
|
static int work_fixup_free(void *addr, enum debug_obj_state state)
|
|
|
|
{
|
|
|
|
struct work_struct *work = addr;
|
|
|
|
|
|
|
|
switch (state) {
|
|
|
|
case ODEBUG_STATE_ACTIVE:
|
|
|
|
cancel_work_sync(work);
|
|
|
|
debug_object_free(work, &work_debug_descr);
|
|
|
|
return 1;
|
|
|
|
default:
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct debug_obj_descr work_debug_descr = {
|
|
|
|
.name = "work_struct",
|
|
|
|
.fixup_init = work_fixup_init,
|
|
|
|
.fixup_activate = work_fixup_activate,
|
|
|
|
.fixup_free = work_fixup_free,
|
|
|
|
};
|
|
|
|
|
|
|
|
static inline void debug_work_activate(struct work_struct *work)
|
|
|
|
{
|
|
|
|
debug_object_activate(work, &work_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void debug_work_deactivate(struct work_struct *work)
|
|
|
|
{
|
|
|
|
debug_object_deactivate(work, &work_debug_descr);
|
|
|
|
}
|
|
|
|
|
|
|
|
void __init_work(struct work_struct *work, int onstack)
|
|
|
|
{
|
|
|
|
if (onstack)
|
|
|
|
debug_object_init_on_stack(work, &work_debug_descr);
|
|
|
|
else
|
|
|
|
debug_object_init(work, &work_debug_descr);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__init_work);
|
|
|
|
|
|
|
|
void destroy_work_on_stack(struct work_struct *work)
|
|
|
|
{
|
|
|
|
debug_object_free(work, &work_debug_descr);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(destroy_work_on_stack);
|
|
|
|
|
|
|
|
#else
|
|
|
|
static inline void debug_work_activate(struct work_struct *work) { }
|
|
|
|
static inline void debug_work_deactivate(struct work_struct *work) { }
|
|
|
|
#endif
|
|
|
|
|
2008-01-25 21:08:02 +01:00
|
|
|
/* Serializes the accesses to the list of workqueues. */
|
|
|
|
static DEFINE_SPINLOCK(workqueue_lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
static LIST_HEAD(workqueues);
|
2010-06-29 10:07:11 +02:00
|
|
|
static DEFINE_PER_CPU(struct ida, worker_ida);
|
2010-06-29 10:07:12 +02:00
|
|
|
static bool workqueue_freezing; /* W: have wqs started freezing? */
|
2010-06-29 10:07:11 +02:00
|
|
|
|
|
|
|
static int worker_thread(void *__worker);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-05-09 11:34:09 +02:00
|
|
|
static int singlethread_cpu __read_mostly;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
static struct cpu_workqueue_struct *get_cwq(unsigned int cpu,
|
|
|
|
struct workqueue_struct *wq)
|
2007-05-09 11:34:12 +02:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
return per_cpu_ptr(wq->cpu_wq, cpu);
|
2007-05-09 11:34:12 +02:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
static struct cpu_workqueue_struct *target_cwq(unsigned int cpu,
|
|
|
|
struct workqueue_struct *wq)
|
2007-05-09 11:34:17 +02:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
if (unlikely(wq->flags & WQ_SINGLE_THREAD))
|
2007-05-09 11:34:17 +02:00
|
|
|
cpu = singlethread_cpu;
|
2010-06-29 10:07:11 +02:00
|
|
|
return get_cwq(cpu, wq);
|
2007-05-09 11:34:17 +02:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
static unsigned int work_color_to_flags(int color)
|
|
|
|
{
|
|
|
|
return color << WORK_STRUCT_COLOR_SHIFT;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int get_work_color(struct work_struct *work)
|
|
|
|
{
|
|
|
|
return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
|
|
|
|
((1 << WORK_STRUCT_COLOR_BITS) - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int work_next_color(int color)
|
|
|
|
{
|
|
|
|
return (color + 1) % WORK_NR_COLORS;
|
|
|
|
}
|
|
|
|
|
[PATCH] WorkStruct: Use direct assignment rather than cmpxchg()
Use direct assignment rather than cmpxchg() as the latter is unavailable
and unimplementable on some platforms and is actually unnecessary.
The use of cmpxchg() was to guard against two possibilities, neither of
which can actually occur:
(1) The pending flag may have been unset or may be cleared. However, given
where it's called, the pending flag is _always_ set. I don't think it
can be unset whilst we're in set_wq_data().
Once the work is enqueued to be actually run, the only way off the queue
is for it to be actually run.
If it's a delayed work item, then the bit can't be cleared by the timer
because we haven't started the timer yet. Also, the pending bit can't be
cleared by cancelling the delayed work _until_ the work item has had its
timer started.
(2) The workqueue pointer might change. This can only happen in two cases:
(a) The work item has just been queued to actually run, and so we're
protected by the appropriate workqueue spinlock.
(b) A delayed work item is being queued, and so the timer hasn't been
started yet, and so no one else knows about the work item or can
access it (the pending bit protects us).
Besides, set_wq_data() _sets_ the workqueue pointer unconditionally, so
it can be assigned instead.
So, replacing the set_wq_data() with a straight assignment would be okay
in most cases.
The problem is where we end up tangling with test_and_set_bit() emulated
using spinlocks, and even then it's not a problem _provided_
test_and_set_bit() doesn't attempt to modify the word if the bit was
set.
If that's a problem, then a bitops-proofed assignment will be required -
equivalent to atomic_set() vs other atomic_xxx() ops.
Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-07 12:33:26 +01:00
|
|
|
/*
|
|
|
|
* Set the workqueue on which a work item is to be run
|
|
|
|
* - Must *only* be called if the pending flag is set
|
|
|
|
*/
|
2007-05-09 11:34:16 +02:00
|
|
|
static inline void set_wq_data(struct work_struct *work,
|
2010-06-29 10:07:10 +02:00
|
|
|
struct cpu_workqueue_struct *cwq,
|
|
|
|
unsigned long extra_flags)
|
2006-11-22 15:54:49 +01:00
|
|
|
{
|
[PATCH] WorkStruct: Use direct assignment rather than cmpxchg()
Use direct assignment rather than cmpxchg() as the latter is unavailable
and unimplementable on some platforms and is actually unnecessary.
The use of cmpxchg() was to guard against two possibilities, neither of
which can actually occur:
(1) The pending flag may have been unset or may be cleared. However, given
where it's called, the pending flag is _always_ set. I don't think it
can be unset whilst we're in set_wq_data().
Once the work is enqueued to be actually run, the only way off the queue
is for it to be actually run.
If it's a delayed work item, then the bit can't be cleared by the timer
because we haven't started the timer yet. Also, the pending bit can't be
cleared by cancelling the delayed work _until_ the work item has had its
timer started.
(2) The workqueue pointer might change. This can only happen in two cases:
(a) The work item has just been queued to actually run, and so we're
protected by the appropriate workqueue spinlock.
(b) A delayed work item is being queued, and so the timer hasn't been
started yet, and so no one else knows about the work item or can
access it (the pending bit protects us).
Besides, set_wq_data() _sets_ the workqueue pointer unconditionally, so
it can be assigned instead.
So, replacing the set_wq_data() with a straight assignment would be okay
in most cases.
The problem is where we end up tangling with test_and_set_bit() emulated
using spinlocks, and even then it's not a problem _provided_
test_and_set_bit() doesn't attempt to modify the word if the bit was
set.
If that's a problem, then a bitops-proofed assignment will be required -
equivalent to atomic_set() vs other atomic_xxx() ops.
Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
2006-12-07 12:33:26 +01:00
|
|
|
BUG_ON(!work_pending(work));
|
2006-11-22 15:54:49 +01:00
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
atomic_long_set(&work->data, (unsigned long)cwq | work_static(work) |
|
2010-06-29 10:07:10 +02:00
|
|
|
WORK_STRUCT_PENDING | extra_flags);
|
2006-11-22 15:54:49 +01:00
|
|
|
}
|
|
|
|
|
2010-04-23 17:40:40 +02:00
|
|
|
/*
|
|
|
|
* Clear WORK_STRUCT_PENDING and the workqueue on which it was queued.
|
|
|
|
*/
|
|
|
|
static inline void clear_wq_data(struct work_struct *work)
|
|
|
|
{
|
2010-06-29 10:07:10 +02:00
|
|
|
atomic_long_set(&work->data, work_static(work));
|
2010-04-23 17:40:40 +02:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
static inline struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
|
2006-11-22 15:54:49 +01:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
return (void *)(atomic_long_read(&work->data) &
|
|
|
|
WORK_STRUCT_WQ_DATA_MASK);
|
2006-11-22 15:54:49 +01:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
/**
|
|
|
|
* insert_work - insert a work into cwq
|
|
|
|
* @cwq: cwq @work belongs to
|
|
|
|
* @work: work to insert
|
|
|
|
* @head: insertion point
|
|
|
|
* @extra_flags: extra WORK_STRUCT_* flags to set
|
|
|
|
*
|
|
|
|
* Insert @work into @cwq after @head.
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* spin_lock_irq(cwq->lock).
|
|
|
|
*/
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
static void insert_work(struct cpu_workqueue_struct *cwq,
|
2010-06-29 10:07:10 +02:00
|
|
|
struct work_struct *work, struct list_head *head,
|
|
|
|
unsigned int extra_flags)
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
{
|
2010-06-29 10:07:10 +02:00
|
|
|
/* we own @work, set data and link */
|
|
|
|
set_wq_data(work, cwq, extra_flags);
|
|
|
|
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
/*
|
|
|
|
* Ensure that we get the right work->data if we see the
|
|
|
|
* result of list_add() below, see try_to_grab_pending().
|
|
|
|
*/
|
|
|
|
smp_wmb();
|
2010-06-29 10:07:10 +02:00
|
|
|
|
2008-07-25 10:47:47 +02:00
|
|
|
list_add_tail(&work->entry, head);
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
wake_up(&cwq->more_work);
|
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
static void __queue_work(unsigned int cpu, struct workqueue_struct *wq,
|
2005-04-17 00:20:36 +02:00
|
|
|
struct work_struct *work)
|
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
struct cpu_workqueue_struct *cwq = target_cwq(cpu, wq);
|
2010-06-29 10:07:12 +02:00
|
|
|
struct list_head *worklist;
|
2005-04-17 00:20:36 +02:00
|
|
|
unsigned long flags;
|
|
|
|
|
2009-11-15 17:09:48 +01:00
|
|
|
debug_work_activate(work);
|
2010-06-29 10:07:12 +02:00
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_lock_irqsave(&cwq->lock, flags);
|
2010-06-29 10:07:10 +02:00
|
|
|
BUG_ON(!list_empty(&work->entry));
|
2010-06-29 10:07:12 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq->nr_in_flight[cwq->work_color]++;
|
2010-06-29 10:07:12 +02:00
|
|
|
|
|
|
|
if (likely(cwq->nr_active < cwq->max_active)) {
|
|
|
|
cwq->nr_active++;
|
|
|
|
worklist = &cwq->worklist;
|
|
|
|
} else
|
|
|
|
worklist = &cwq->delayed_works;
|
|
|
|
|
|
|
|
insert_work(cwq, work, worklist, work_color_to_flags(cwq->work_color));
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
spin_unlock_irqrestore(&cwq->lock, flags);
|
|
|
|
}
|
|
|
|
|
2006-07-30 12:03:42 +02:00
|
|
|
/**
|
|
|
|
* queue_work - queue work on a workqueue
|
|
|
|
* @wq: workqueue to use
|
|
|
|
* @work: work to queue
|
|
|
|
*
|
2006-10-28 19:38:58 +02:00
|
|
|
* Returns 0 if @work was already on a queue, non-zero otherwise.
|
2005-04-17 00:20:36 +02:00
|
|
|
*
|
2008-04-29 10:00:27 +02:00
|
|
|
* We queue the work to the CPU on which it was submitted, but if the CPU dies
|
|
|
|
* it can be processed by another CPU.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
2008-02-08 13:19:53 +01:00
|
|
|
int queue_work(struct workqueue_struct *wq, struct work_struct *work)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2008-07-25 10:47:53 +02:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = queue_work_on(get_cpu(), wq, work);
|
|
|
|
put_cpu();
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
return ret;
|
|
|
|
}
|
2006-06-30 07:40:45 +02:00
|
|
|
EXPORT_SYMBOL_GPL(queue_work);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2008-07-24 06:28:39 +02:00
|
|
|
/**
|
|
|
|
* queue_work_on - queue work on specific cpu
|
|
|
|
* @cpu: CPU number to execute work on
|
|
|
|
* @wq: workqueue to use
|
|
|
|
* @work: work to queue
|
|
|
|
*
|
|
|
|
* Returns 0 if @work was already on a queue, non-zero otherwise.
|
|
|
|
*
|
|
|
|
* We queue the work to a specific CPU, the caller must ensure it
|
|
|
|
* can't go away.
|
|
|
|
*/
|
|
|
|
int
|
|
|
|
queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work)
|
|
|
|
{
|
|
|
|
int ret = 0;
|
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
|
2010-06-29 10:07:10 +02:00
|
|
|
__queue_work(cpu, wq, work);
|
2008-07-24 06:28:39 +02:00
|
|
|
ret = 1;
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(queue_work_on);
|
|
|
|
|
2008-02-08 13:21:09 +01:00
|
|
|
static void delayed_work_timer_fn(unsigned long __data)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-11-22 15:54:01 +01:00
|
|
|
struct delayed_work *dwork = (struct delayed_work *)__data;
|
2007-05-09 11:34:16 +02:00
|
|
|
struct cpu_workqueue_struct *cwq = get_wq_data(&dwork->work);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
__queue_work(smp_processor_id(), cwq->wq, &dwork->work);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2006-07-30 12:03:42 +02:00
|
|
|
/**
|
|
|
|
* queue_delayed_work - queue work on a workqueue after delay
|
|
|
|
* @wq: workqueue to use
|
2006-12-22 10:06:52 +01:00
|
|
|
* @dwork: delayable work to queue
|
2006-07-30 12:03:42 +02:00
|
|
|
* @delay: number of jiffies to wait before queueing
|
|
|
|
*
|
2006-10-28 19:38:58 +02:00
|
|
|
* Returns 0 if @work was already on a queue, non-zero otherwise.
|
2006-07-30 12:03:42 +02:00
|
|
|
*/
|
2008-02-08 13:19:53 +01:00
|
|
|
int queue_delayed_work(struct workqueue_struct *wq,
|
2006-11-22 15:54:01 +01:00
|
|
|
struct delayed_work *dwork, unsigned long delay)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-11-22 15:54:01 +01:00
|
|
|
if (delay == 0)
|
2007-05-09 11:34:16 +02:00
|
|
|
return queue_work(wq, &dwork->work);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-05-09 11:34:16 +02:00
|
|
|
return queue_delayed_work_on(-1, wq, dwork, delay);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-06-30 07:40:45 +02:00
|
|
|
EXPORT_SYMBOL_GPL(queue_delayed_work);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2006-07-30 12:03:42 +02:00
|
|
|
/**
|
|
|
|
* queue_delayed_work_on - queue work on specific CPU after delay
|
|
|
|
* @cpu: CPU number to execute work on
|
|
|
|
* @wq: workqueue to use
|
2006-12-22 10:06:52 +01:00
|
|
|
* @dwork: work to queue
|
2006-07-30 12:03:42 +02:00
|
|
|
* @delay: number of jiffies to wait before queueing
|
|
|
|
*
|
2006-10-28 19:38:58 +02:00
|
|
|
* Returns 0 if @work was already on a queue, non-zero otherwise.
|
2006-07-30 12:03:42 +02:00
|
|
|
*/
|
2006-06-28 22:50:33 +02:00
|
|
|
int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
|
2006-11-22 15:54:01 +01:00
|
|
|
struct delayed_work *dwork, unsigned long delay)
|
2006-06-28 22:50:33 +02:00
|
|
|
{
|
|
|
|
int ret = 0;
|
2006-11-22 15:54:01 +01:00
|
|
|
struct timer_list *timer = &dwork->timer;
|
|
|
|
struct work_struct *work = &dwork->work;
|
2006-06-28 22:50:33 +02:00
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
|
2006-06-28 22:50:33 +02:00
|
|
|
BUG_ON(timer_pending(timer));
|
|
|
|
BUG_ON(!list_empty(&work->entry));
|
|
|
|
|
2008-05-01 13:35:14 +02:00
|
|
|
timer_stats_timer_set_start_info(&dwork->timer);
|
|
|
|
|
2007-05-09 11:34:16 +02:00
|
|
|
/* This stores cwq for the moment, for the timer_fn */
|
2010-06-29 10:07:11 +02:00
|
|
|
set_wq_data(work, target_cwq(raw_smp_processor_id(), wq), 0);
|
2006-06-28 22:50:33 +02:00
|
|
|
timer->expires = jiffies + delay;
|
2006-11-22 15:54:01 +01:00
|
|
|
timer->data = (unsigned long)dwork;
|
2006-06-28 22:50:33 +02:00
|
|
|
timer->function = delayed_work_timer_fn;
|
2007-05-09 11:34:16 +02:00
|
|
|
|
|
|
|
if (unlikely(cpu >= 0))
|
|
|
|
add_timer_on(timer, cpu);
|
|
|
|
else
|
|
|
|
add_timer(timer);
|
2006-06-28 22:50:33 +02:00
|
|
|
ret = 1;
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
2006-06-30 07:40:45 +02:00
|
|
|
EXPORT_SYMBOL_GPL(queue_delayed_work_on);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
static struct worker *alloc_worker(void)
|
|
|
|
{
|
|
|
|
struct worker *worker;
|
|
|
|
|
|
|
|
worker = kzalloc(sizeof(*worker), GFP_KERNEL);
|
2010-06-29 10:07:12 +02:00
|
|
|
if (worker)
|
|
|
|
INIT_LIST_HEAD(&worker->scheduled);
|
2010-06-29 10:07:11 +02:00
|
|
|
return worker;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* create_worker - create a new workqueue worker
|
|
|
|
* @cwq: cwq the new worker will belong to
|
|
|
|
* @bind: whether to set affinity to @cpu or not
|
|
|
|
*
|
|
|
|
* Create a new worker which is bound to @cwq. The returned worker
|
|
|
|
* can be started by calling start_worker() or destroyed using
|
|
|
|
* destroy_worker().
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* Might sleep. Does GFP_KERNEL allocations.
|
|
|
|
*
|
|
|
|
* RETURNS:
|
|
|
|
* Pointer to the newly created worker.
|
|
|
|
*/
|
|
|
|
static struct worker *create_worker(struct cpu_workqueue_struct *cwq, bool bind)
|
|
|
|
{
|
|
|
|
int id = -1;
|
|
|
|
struct worker *worker = NULL;
|
|
|
|
|
|
|
|
spin_lock(&workqueue_lock);
|
|
|
|
while (ida_get_new(&per_cpu(worker_ida, cwq->cpu), &id)) {
|
|
|
|
spin_unlock(&workqueue_lock);
|
|
|
|
if (!ida_pre_get(&per_cpu(worker_ida, cwq->cpu), GFP_KERNEL))
|
|
|
|
goto fail;
|
|
|
|
spin_lock(&workqueue_lock);
|
|
|
|
}
|
|
|
|
spin_unlock(&workqueue_lock);
|
|
|
|
|
|
|
|
worker = alloc_worker();
|
|
|
|
if (!worker)
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
worker->cwq = cwq;
|
|
|
|
worker->id = id;
|
|
|
|
|
|
|
|
worker->task = kthread_create(worker_thread, worker, "kworker/%u:%d",
|
|
|
|
cwq->cpu, id);
|
|
|
|
if (IS_ERR(worker->task))
|
|
|
|
goto fail;
|
|
|
|
|
|
|
|
if (bind)
|
|
|
|
kthread_bind(worker->task, cwq->cpu);
|
|
|
|
|
|
|
|
return worker;
|
|
|
|
fail:
|
|
|
|
if (id >= 0) {
|
|
|
|
spin_lock(&workqueue_lock);
|
|
|
|
ida_remove(&per_cpu(worker_ida, cwq->cpu), id);
|
|
|
|
spin_unlock(&workqueue_lock);
|
|
|
|
}
|
|
|
|
kfree(worker);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* start_worker - start a newly created worker
|
|
|
|
* @worker: worker to start
|
|
|
|
*
|
|
|
|
* Start @worker.
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* spin_lock_irq(cwq->lock).
|
|
|
|
*/
|
|
|
|
static void start_worker(struct worker *worker)
|
|
|
|
{
|
|
|
|
wake_up_process(worker->task);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* destroy_worker - destroy a workqueue worker
|
|
|
|
* @worker: worker to be destroyed
|
|
|
|
*
|
|
|
|
* Destroy @worker.
|
|
|
|
*/
|
|
|
|
static void destroy_worker(struct worker *worker)
|
|
|
|
{
|
|
|
|
int cpu = worker->cwq->cpu;
|
|
|
|
int id = worker->id;
|
|
|
|
|
|
|
|
/* sanity check frenzy */
|
|
|
|
BUG_ON(worker->current_work);
|
2010-06-29 10:07:12 +02:00
|
|
|
BUG_ON(!list_empty(&worker->scheduled));
|
2010-06-29 10:07:11 +02:00
|
|
|
|
|
|
|
kthread_stop(worker->task);
|
|
|
|
kfree(worker);
|
|
|
|
|
|
|
|
spin_lock(&workqueue_lock);
|
|
|
|
ida_remove(&per_cpu(worker_ida, cpu), id);
|
|
|
|
spin_unlock(&workqueue_lock);
|
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
/**
|
|
|
|
* move_linked_works - move linked works to a list
|
|
|
|
* @work: start of series of works to be scheduled
|
|
|
|
* @head: target list to append @work to
|
|
|
|
* @nextp: out paramter for nested worklist walking
|
|
|
|
*
|
|
|
|
* Schedule linked works starting from @work to @head. Work series to
|
|
|
|
* be scheduled starts at @work and includes any consecutive work with
|
|
|
|
* WORK_STRUCT_LINKED set in its predecessor.
|
|
|
|
*
|
|
|
|
* If @nextp is not NULL, it's updated to point to the next work of
|
|
|
|
* the last scheduled work. This allows move_linked_works() to be
|
|
|
|
* nested inside outer list_for_each_entry_safe().
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* spin_lock_irq(cwq->lock).
|
|
|
|
*/
|
|
|
|
static void move_linked_works(struct work_struct *work, struct list_head *head,
|
|
|
|
struct work_struct **nextp)
|
|
|
|
{
|
|
|
|
struct work_struct *n;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Linked worklist will always end before the end of the list,
|
|
|
|
* use NULL for list head.
|
|
|
|
*/
|
|
|
|
list_for_each_entry_safe_from(work, n, NULL, entry) {
|
|
|
|
list_move_tail(&work->entry, head);
|
|
|
|
if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we're already inside safe list traversal and have moved
|
|
|
|
* multiple works to the scheduled queue, the next position
|
|
|
|
* needs to be updated.
|
|
|
|
*/
|
|
|
|
if (nextp)
|
|
|
|
*nextp = n;
|
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
static void cwq_activate_first_delayed(struct cpu_workqueue_struct *cwq)
|
|
|
|
{
|
|
|
|
struct work_struct *work = list_first_entry(&cwq->delayed_works,
|
|
|
|
struct work_struct, entry);
|
|
|
|
|
|
|
|
move_linked_works(work, &cwq->worklist, NULL);
|
|
|
|
cwq->nr_active++;
|
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
/**
|
|
|
|
* cwq_dec_nr_in_flight - decrement cwq's nr_in_flight
|
|
|
|
* @cwq: cwq of interest
|
|
|
|
* @color: color of work which left the queue
|
|
|
|
*
|
|
|
|
* A work either has completed or is removed from pending queue,
|
|
|
|
* decrement nr_in_flight of its cwq and handle workqueue flushing.
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* spin_lock_irq(cwq->lock).
|
|
|
|
*/
|
|
|
|
static void cwq_dec_nr_in_flight(struct cpu_workqueue_struct *cwq, int color)
|
|
|
|
{
|
|
|
|
/* ignore uncolored works */
|
|
|
|
if (color == WORK_NO_COLOR)
|
|
|
|
return;
|
|
|
|
|
|
|
|
cwq->nr_in_flight[color]--;
|
2010-06-29 10:07:12 +02:00
|
|
|
cwq->nr_active--;
|
|
|
|
|
|
|
|
/* one down, submit a delayed one */
|
|
|
|
if (!list_empty(&cwq->delayed_works) &&
|
|
|
|
cwq->nr_active < cwq->max_active)
|
|
|
|
cwq_activate_first_delayed(cwq);
|
2010-06-29 10:07:11 +02:00
|
|
|
|
|
|
|
/* is flush in progress and are we at the flushing tip? */
|
|
|
|
if (likely(cwq->flush_color != color))
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* are there still in-flight works? */
|
|
|
|
if (cwq->nr_in_flight[color])
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* this cwq is done, clear flush_color */
|
|
|
|
cwq->flush_color = -1;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If this was the last cwq, wake up the first flusher. It
|
|
|
|
* will handle the rest.
|
|
|
|
*/
|
|
|
|
if (atomic_dec_and_test(&cwq->wq->nr_cwqs_to_flush))
|
|
|
|
complete(&cwq->wq->first_flusher->done);
|
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
/**
|
|
|
|
* process_one_work - process single work
|
2010-06-29 10:07:11 +02:00
|
|
|
* @worker: self
|
2010-06-29 10:07:10 +02:00
|
|
|
* @work: work to process
|
|
|
|
*
|
|
|
|
* Process @work. This function contains all the logics necessary to
|
|
|
|
* process a single work including synchronization against and
|
|
|
|
* interaction with other workers on the same cpu, queueing and
|
|
|
|
* flushing. As long as context requirement is met, any worker can
|
|
|
|
* call this function to process a work.
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* spin_lock_irq(cwq->lock) which is released and regrabbed.
|
|
|
|
*/
|
2010-06-29 10:07:11 +02:00
|
|
|
static void process_one_work(struct worker *worker, struct work_struct *work)
|
2010-06-29 10:07:10 +02:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
struct cpu_workqueue_struct *cwq = worker->cwq;
|
2010-06-29 10:07:10 +02:00
|
|
|
work_func_t f = work->func;
|
2010-06-29 10:07:11 +02:00
|
|
|
int work_color;
|
2010-06-29 10:07:10 +02:00
|
|
|
#ifdef CONFIG_LOCKDEP
|
|
|
|
/*
|
|
|
|
* It is permissible to free the struct work_struct from
|
|
|
|
* inside the function that is called from it, this we need to
|
|
|
|
* take into account for lockdep too. To avoid bogus "held
|
|
|
|
* lock freed" warnings as well as problems when looking into
|
|
|
|
* work->lockdep_map, make a copy and use that here.
|
|
|
|
*/
|
|
|
|
struct lockdep_map lockdep_map = work->lockdep_map;
|
|
|
|
#endif
|
|
|
|
/* claim and process */
|
|
|
|
debug_work_deactivate(work);
|
2010-06-29 10:07:11 +02:00
|
|
|
worker->current_work = work;
|
2010-06-29 10:07:11 +02:00
|
|
|
work_color = get_work_color(work);
|
2010-06-29 10:07:10 +02:00
|
|
|
list_del_init(&work->entry);
|
|
|
|
|
|
|
|
spin_unlock_irq(&cwq->lock);
|
|
|
|
|
|
|
|
BUG_ON(get_wq_data(work) != cwq);
|
|
|
|
work_clear_pending(work);
|
|
|
|
lock_map_acquire(&cwq->wq->lockdep_map);
|
|
|
|
lock_map_acquire(&lockdep_map);
|
|
|
|
f(work);
|
|
|
|
lock_map_release(&lockdep_map);
|
|
|
|
lock_map_release(&cwq->wq->lockdep_map);
|
|
|
|
|
|
|
|
if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
|
|
|
|
printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
|
|
|
|
"%s/0x%08x/%d\n",
|
|
|
|
current->comm, preempt_count(), task_pid_nr(current));
|
|
|
|
printk(KERN_ERR " last function: ");
|
|
|
|
print_symbol("%s\n", (unsigned long)f);
|
|
|
|
debug_show_held_locks(current);
|
|
|
|
dump_stack();
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_lock_irq(&cwq->lock);
|
|
|
|
|
|
|
|
/* we're done with it, release */
|
2010-06-29 10:07:11 +02:00
|
|
|
worker->current_work = NULL;
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq_dec_nr_in_flight(cwq, work_color);
|
2010-06-29 10:07:10 +02:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
/**
|
|
|
|
* process_scheduled_works - process scheduled works
|
|
|
|
* @worker: self
|
|
|
|
*
|
|
|
|
* Process all scheduled works. Please note that the scheduled list
|
|
|
|
* may change while processing a work, so this function repeatedly
|
|
|
|
* fetches a work from the top and executes it.
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* spin_lock_irq(cwq->lock) which may be released and regrabbed
|
|
|
|
* multiple times.
|
|
|
|
*/
|
|
|
|
static void process_scheduled_works(struct worker *worker)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2010-06-29 10:07:12 +02:00
|
|
|
while (!list_empty(&worker->scheduled)) {
|
|
|
|
struct work_struct *work = list_first_entry(&worker->scheduled,
|
2005-04-17 00:20:36 +02:00
|
|
|
struct work_struct, entry);
|
2010-06-29 10:07:11 +02:00
|
|
|
process_one_work(worker, work);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
/**
|
|
|
|
* worker_thread - the worker thread function
|
2010-06-29 10:07:11 +02:00
|
|
|
* @__worker: self
|
2010-06-29 10:07:10 +02:00
|
|
|
*
|
|
|
|
* The cwq worker thread function.
|
|
|
|
*/
|
2010-06-29 10:07:11 +02:00
|
|
|
static int worker_thread(void *__worker)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
struct worker *worker = __worker;
|
|
|
|
struct cpu_workqueue_struct *cwq = worker->cwq;
|
2007-05-09 11:34:09 +02:00
|
|
|
DEFINE_WAIT(wait);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-05-09 11:34:09 +02:00
|
|
|
for (;;) {
|
|
|
|
prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
|
2010-06-29 10:07:12 +02:00
|
|
|
if (!kthread_should_stop() &&
|
2007-05-23 22:57:57 +02:00
|
|
|
list_empty(&cwq->worklist))
|
2005-04-17 00:20:36 +02:00
|
|
|
schedule();
|
2007-05-09 11:34:09 +02:00
|
|
|
finish_wait(&cwq->more_work, &wait);
|
|
|
|
|
2007-05-23 22:57:57 +02:00
|
|
|
if (kthread_should_stop())
|
2007-05-09 11:34:09 +02:00
|
|
|
break;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
if (unlikely(!cpumask_equal(&worker->task->cpus_allowed,
|
2010-06-29 10:07:11 +02:00
|
|
|
get_cpu_mask(cwq->cpu))))
|
2010-06-29 10:07:11 +02:00
|
|
|
set_cpus_allowed_ptr(worker->task,
|
2010-06-29 10:07:11 +02:00
|
|
|
get_cpu_mask(cwq->cpu));
|
2010-06-29 10:07:12 +02:00
|
|
|
|
|
|
|
spin_lock_irq(&cwq->lock);
|
|
|
|
|
|
|
|
while (!list_empty(&cwq->worklist)) {
|
|
|
|
struct work_struct *work =
|
|
|
|
list_first_entry(&cwq->worklist,
|
|
|
|
struct work_struct, entry);
|
|
|
|
|
|
|
|
if (likely(!(*work_data_bits(work) &
|
|
|
|
WORK_STRUCT_LINKED))) {
|
|
|
|
/* optimization path, not strictly necessary */
|
|
|
|
process_one_work(worker, work);
|
|
|
|
if (unlikely(!list_empty(&worker->scheduled)))
|
|
|
|
process_scheduled_works(worker);
|
|
|
|
} else {
|
|
|
|
move_linked_works(work, &worker->scheduled,
|
|
|
|
NULL);
|
|
|
|
process_scheduled_works(worker);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock_irq(&cwq->lock);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2007-05-09 11:34:09 +02:00
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-05-09 11:33:51 +02:00
|
|
|
struct wq_barrier {
|
|
|
|
struct work_struct work;
|
|
|
|
struct completion done;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void wq_barrier_func(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
|
|
|
|
complete(&barr->done);
|
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
/**
|
|
|
|
* insert_wq_barrier - insert a barrier work
|
|
|
|
* @cwq: cwq to insert barrier into
|
|
|
|
* @barr: wq_barrier to insert
|
2010-06-29 10:07:12 +02:00
|
|
|
* @target: target work to attach @barr to
|
|
|
|
* @worker: worker currently executing @target, NULL if @target is not executing
|
2010-06-29 10:07:10 +02:00
|
|
|
*
|
2010-06-29 10:07:12 +02:00
|
|
|
* @barr is linked to @target such that @barr is completed only after
|
|
|
|
* @target finishes execution. Please note that the ordering
|
|
|
|
* guarantee is observed only with respect to @target and on the local
|
|
|
|
* cpu.
|
|
|
|
*
|
|
|
|
* Currently, a queued barrier can't be canceled. This is because
|
|
|
|
* try_to_grab_pending() can't determine whether the work to be
|
|
|
|
* grabbed is at the head of the queue and thus can't clear LINKED
|
|
|
|
* flag of the previous work while there must be a valid next work
|
|
|
|
* after a work with LINKED flag set.
|
|
|
|
*
|
|
|
|
* Note that when @worker is non-NULL, @target may be modified
|
|
|
|
* underneath us, so we can't reliably determine cwq from @target.
|
2010-06-29 10:07:10 +02:00
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* spin_lock_irq(cwq->lock).
|
|
|
|
*/
|
2007-05-09 11:33:54 +02:00
|
|
|
static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
|
2010-06-29 10:07:12 +02:00
|
|
|
struct wq_barrier *barr,
|
|
|
|
struct work_struct *target, struct worker *worker)
|
2007-05-09 11:33:51 +02:00
|
|
|
{
|
2010-06-29 10:07:12 +02:00
|
|
|
struct list_head *head;
|
|
|
|
unsigned int linked = 0;
|
|
|
|
|
2009-11-15 17:09:48 +01:00
|
|
|
/*
|
|
|
|
* debugobject calls are safe here even with cwq->lock locked
|
|
|
|
* as we know for sure that this will not trigger any of the
|
|
|
|
* checks and call back into the fixup functions where we
|
|
|
|
* might deadlock.
|
|
|
|
*/
|
|
|
|
INIT_WORK_ON_STACK(&barr->work, wq_barrier_func);
|
2010-06-29 10:07:10 +02:00
|
|
|
__set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
|
2007-05-09 11:33:51 +02:00
|
|
|
init_completion(&barr->done);
|
2007-05-09 11:33:54 +02:00
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
/*
|
|
|
|
* If @target is currently being executed, schedule the
|
|
|
|
* barrier to the worker; otherwise, put it after @target.
|
|
|
|
*/
|
|
|
|
if (worker)
|
|
|
|
head = worker->scheduled.next;
|
|
|
|
else {
|
|
|
|
unsigned long *bits = work_data_bits(target);
|
|
|
|
|
|
|
|
head = target->entry.next;
|
|
|
|
/* there can already be other linked works, inherit and set */
|
|
|
|
linked = *bits & WORK_STRUCT_LINKED;
|
|
|
|
__set_bit(WORK_STRUCT_LINKED_BIT, bits);
|
|
|
|
}
|
|
|
|
|
2009-11-15 17:09:48 +01:00
|
|
|
debug_work_activate(&barr->work);
|
2010-06-29 10:07:12 +02:00
|
|
|
insert_work(cwq, &barr->work, head,
|
|
|
|
work_color_to_flags(WORK_NO_COLOR) | linked);
|
2007-05-09 11:33:51 +02:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
/**
|
|
|
|
* flush_workqueue_prep_cwqs - prepare cwqs for workqueue flushing
|
|
|
|
* @wq: workqueue being flushed
|
|
|
|
* @flush_color: new flush color, < 0 for no-op
|
|
|
|
* @work_color: new work color, < 0 for no-op
|
|
|
|
*
|
|
|
|
* Prepare cwqs for workqueue flushing.
|
|
|
|
*
|
|
|
|
* If @flush_color is non-negative, flush_color on all cwqs should be
|
|
|
|
* -1. If no cwq has in-flight commands at the specified color, all
|
|
|
|
* cwq->flush_color's stay at -1 and %false is returned. If any cwq
|
|
|
|
* has in flight commands, its cwq->flush_color is set to
|
|
|
|
* @flush_color, @wq->nr_cwqs_to_flush is updated accordingly, cwq
|
|
|
|
* wakeup logic is armed and %true is returned.
|
|
|
|
*
|
|
|
|
* The caller should have initialized @wq->first_flusher prior to
|
|
|
|
* calling this function with non-negative @flush_color. If
|
|
|
|
* @flush_color is negative, no flush color update is done and %false
|
|
|
|
* is returned.
|
|
|
|
*
|
|
|
|
* If @work_color is non-negative, all cwqs should have the same
|
|
|
|
* work_color which is previous to @work_color and all will be
|
|
|
|
* advanced to @work_color.
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* mutex_lock(wq->flush_mutex).
|
|
|
|
*
|
|
|
|
* RETURNS:
|
|
|
|
* %true if @flush_color >= 0 and there's something to flush. %false
|
|
|
|
* otherwise.
|
|
|
|
*/
|
|
|
|
static bool flush_workqueue_prep_cwqs(struct workqueue_struct *wq,
|
|
|
|
int flush_color, int work_color)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
bool wait = false;
|
|
|
|
unsigned int cpu;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
if (flush_color >= 0) {
|
|
|
|
BUG_ON(atomic_read(&wq->nr_cwqs_to_flush));
|
|
|
|
atomic_set(&wq->nr_cwqs_to_flush, 1);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2009-04-03 01:58:24 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
|
|
|
|
|
|
|
|
spin_lock_irq(&cwq->lock);
|
|
|
|
|
|
|
|
if (flush_color >= 0) {
|
|
|
|
BUG_ON(cwq->flush_color != -1);
|
|
|
|
|
|
|
|
if (cwq->nr_in_flight[flush_color]) {
|
|
|
|
cwq->flush_color = flush_color;
|
|
|
|
atomic_inc(&wq->nr_cwqs_to_flush);
|
|
|
|
wait = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (work_color >= 0) {
|
|
|
|
BUG_ON(work_color != work_next_color(cwq->work_color));
|
|
|
|
cwq->work_color = work_color;
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock_irq(&cwq->lock);
|
2009-11-15 17:09:48 +01:00
|
|
|
}
|
2007-05-23 22:57:57 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_cwqs_to_flush))
|
|
|
|
complete(&wq->first_flusher->done);
|
|
|
|
|
|
|
|
return wait;
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2006-07-30 12:03:42 +02:00
|
|
|
/**
|
2005-04-17 00:20:36 +02:00
|
|
|
* flush_workqueue - ensure that any scheduled work has run to completion.
|
2006-07-30 12:03:42 +02:00
|
|
|
* @wq: workqueue to flush
|
2005-04-17 00:20:36 +02:00
|
|
|
*
|
|
|
|
* Forces execution of the workqueue and blocks until its completion.
|
|
|
|
* This is typically used in driver shutdown handlers.
|
|
|
|
*
|
2007-05-09 11:33:51 +02:00
|
|
|
* We sleep until all works which were queued on entry have been handled,
|
|
|
|
* but we are not livelocked by new incoming ones.
|
2005-04-17 00:20:36 +02:00
|
|
|
*/
|
2008-02-08 13:19:53 +01:00
|
|
|
void flush_workqueue(struct workqueue_struct *wq)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
struct wq_flusher this_flusher = {
|
|
|
|
.list = LIST_HEAD_INIT(this_flusher.list),
|
|
|
|
.flush_color = -1,
|
|
|
|
.done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
|
|
|
|
};
|
|
|
|
int next_color;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2008-08-11 10:30:30 +02:00
|
|
|
lock_map_acquire(&wq->lockdep_map);
|
|
|
|
lock_map_release(&wq->lockdep_map);
|
2010-06-29 10:07:11 +02:00
|
|
|
|
|
|
|
mutex_lock(&wq->flush_mutex);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Start-to-wait phase
|
|
|
|
*/
|
|
|
|
next_color = work_next_color(wq->work_color);
|
|
|
|
|
|
|
|
if (next_color != wq->flush_color) {
|
|
|
|
/*
|
|
|
|
* Color space is not full. The current work_color
|
|
|
|
* becomes our flush_color and work_color is advanced
|
|
|
|
* by one.
|
|
|
|
*/
|
|
|
|
BUG_ON(!list_empty(&wq->flusher_overflow));
|
|
|
|
this_flusher.flush_color = wq->work_color;
|
|
|
|
wq->work_color = next_color;
|
|
|
|
|
|
|
|
if (!wq->first_flusher) {
|
|
|
|
/* no flush in progress, become the first flusher */
|
|
|
|
BUG_ON(wq->flush_color != this_flusher.flush_color);
|
|
|
|
|
|
|
|
wq->first_flusher = &this_flusher;
|
|
|
|
|
|
|
|
if (!flush_workqueue_prep_cwqs(wq, wq->flush_color,
|
|
|
|
wq->work_color)) {
|
|
|
|
/* nothing to flush, done */
|
|
|
|
wq->flush_color = next_color;
|
|
|
|
wq->first_flusher = NULL;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/* wait in queue */
|
|
|
|
BUG_ON(wq->flush_color == this_flusher.flush_color);
|
|
|
|
list_add_tail(&this_flusher.list, &wq->flusher_queue);
|
|
|
|
flush_workqueue_prep_cwqs(wq, -1, wq->work_color);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* Oops, color space is full, wait on overflow queue.
|
|
|
|
* The next flush completion will assign us
|
|
|
|
* flush_color and transfer to flusher_queue.
|
|
|
|
*/
|
|
|
|
list_add_tail(&this_flusher.list, &wq->flusher_overflow);
|
|
|
|
}
|
|
|
|
|
|
|
|
mutex_unlock(&wq->flush_mutex);
|
|
|
|
|
|
|
|
wait_for_completion(&this_flusher.done);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Wake-up-and-cascade phase
|
|
|
|
*
|
|
|
|
* First flushers are responsible for cascading flushes and
|
|
|
|
* handling overflow. Non-first flushers can simply return.
|
|
|
|
*/
|
|
|
|
if (wq->first_flusher != &this_flusher)
|
|
|
|
return;
|
|
|
|
|
|
|
|
mutex_lock(&wq->flush_mutex);
|
|
|
|
|
|
|
|
wq->first_flusher = NULL;
|
|
|
|
|
|
|
|
BUG_ON(!list_empty(&this_flusher.list));
|
|
|
|
BUG_ON(wq->flush_color != this_flusher.flush_color);
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
struct wq_flusher *next, *tmp;
|
|
|
|
|
|
|
|
/* complete all the flushers sharing the current flush color */
|
|
|
|
list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
|
|
|
|
if (next->flush_color != wq->flush_color)
|
|
|
|
break;
|
|
|
|
list_del_init(&next->list);
|
|
|
|
complete(&next->done);
|
|
|
|
}
|
|
|
|
|
|
|
|
BUG_ON(!list_empty(&wq->flusher_overflow) &&
|
|
|
|
wq->flush_color != work_next_color(wq->work_color));
|
|
|
|
|
|
|
|
/* this flush_color is finished, advance by one */
|
|
|
|
wq->flush_color = work_next_color(wq->flush_color);
|
|
|
|
|
|
|
|
/* one color has been freed, handle overflow queue */
|
|
|
|
if (!list_empty(&wq->flusher_overflow)) {
|
|
|
|
/*
|
|
|
|
* Assign the same color to all overflowed
|
|
|
|
* flushers, advance work_color and append to
|
|
|
|
* flusher_queue. This is the start-to-wait
|
|
|
|
* phase for these overflowed flushers.
|
|
|
|
*/
|
|
|
|
list_for_each_entry(tmp, &wq->flusher_overflow, list)
|
|
|
|
tmp->flush_color = wq->work_color;
|
|
|
|
|
|
|
|
wq->work_color = work_next_color(wq->work_color);
|
|
|
|
|
|
|
|
list_splice_tail_init(&wq->flusher_overflow,
|
|
|
|
&wq->flusher_queue);
|
|
|
|
flush_workqueue_prep_cwqs(wq, -1, wq->work_color);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (list_empty(&wq->flusher_queue)) {
|
|
|
|
BUG_ON(wq->flush_color != wq->work_color);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Need to flush more colors. Make the next flusher
|
|
|
|
* the new first flusher and arm cwqs.
|
|
|
|
*/
|
|
|
|
BUG_ON(wq->flush_color == wq->work_color);
|
|
|
|
BUG_ON(wq->flush_color != next->flush_color);
|
|
|
|
|
|
|
|
list_del_init(&next->list);
|
|
|
|
wq->first_flusher = next;
|
|
|
|
|
|
|
|
if (flush_workqueue_prep_cwqs(wq, wq->flush_color, -1))
|
|
|
|
break;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Meh... this color is already done, clear first
|
|
|
|
* flusher and repeat cascading.
|
|
|
|
*/
|
|
|
|
wq->first_flusher = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
out_unlock:
|
|
|
|
mutex_unlock(&wq->flush_mutex);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-06-30 07:40:45 +02:00
|
|
|
EXPORT_SYMBOL_GPL(flush_workqueue);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
/**
|
|
|
|
* flush_work - block until a work_struct's callback has terminated
|
|
|
|
* @work: the work which is to be flushed
|
|
|
|
*
|
2008-07-25 10:47:52 +02:00
|
|
|
* Returns false if @work has already terminated.
|
|
|
|
*
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
* It is expected that, prior to calling flush_work(), the caller has
|
|
|
|
* arranged for the work to not be requeued, otherwise it doesn't make
|
|
|
|
* sense to use this function.
|
|
|
|
*/
|
|
|
|
int flush_work(struct work_struct *work)
|
|
|
|
{
|
2010-06-29 10:07:12 +02:00
|
|
|
struct worker *worker = NULL;
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
struct cpu_workqueue_struct *cwq;
|
|
|
|
struct wq_barrier barr;
|
|
|
|
|
|
|
|
might_sleep();
|
|
|
|
cwq = get_wq_data(work);
|
|
|
|
if (!cwq)
|
|
|
|
return 0;
|
|
|
|
|
2008-08-11 10:30:30 +02:00
|
|
|
lock_map_acquire(&cwq->wq->lockdep_map);
|
|
|
|
lock_map_release(&cwq->wq->lockdep_map);
|
2008-07-25 10:47:52 +02:00
|
|
|
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
spin_lock_irq(&cwq->lock);
|
|
|
|
if (!list_empty(&work->entry)) {
|
|
|
|
/*
|
|
|
|
* See the comment near try_to_grab_pending()->smp_rmb().
|
|
|
|
* If it was re-queued under us we are not going to wait.
|
|
|
|
*/
|
|
|
|
smp_rmb();
|
|
|
|
if (unlikely(cwq != get_wq_data(work)))
|
2010-06-29 10:07:10 +02:00
|
|
|
goto already_gone;
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
} else {
|
2010-06-29 10:07:12 +02:00
|
|
|
if (cwq->worker && cwq->worker->current_work == work)
|
|
|
|
worker = cwq->worker;
|
|
|
|
if (!worker)
|
2010-06-29 10:07:10 +02:00
|
|
|
goto already_gone;
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
insert_wq_barrier(cwq, &barr, work, worker);
|
2010-06-29 10:07:10 +02:00
|
|
|
spin_unlock_irq(&cwq->lock);
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
wait_for_completion(&barr.done);
|
2009-11-15 17:09:48 +01:00
|
|
|
destroy_work_on_stack(&barr.work);
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
return 1;
|
2010-06-29 10:07:10 +02:00
|
|
|
already_gone:
|
|
|
|
spin_unlock_irq(&cwq->lock);
|
|
|
|
return 0;
|
workqueues: implement flush_work()
Most of users of flush_workqueue() can be changed to use cancel_work_sync(),
but sometimes we really need to wait for the completion and cancelling is not
an option. schedule_on_each_cpu() is good example.
Add the new helper, flush_work(work), which waits for the completion of the
specific work_struct. More precisely, it "flushes" the result of of the last
queue_work() which is visible to the caller.
For example, this code
queue_work(wq, work);
/* WINDOW */
queue_work(wq, work);
flush_work(work);
doesn't necessary work "as expected". What can happen in the WINDOW above is
- wq starts the execution of work->func()
- the caller migrates to another CPU
now, after the 2nd queue_work() this work is active on the previous CPU, and
at the same time it is queued on another. In this case flush_work(work) may
return before the first work->func() completes.
It is trivial to add another helper
int flush_work_sync(struct work_struct *work)
{
return flush_work(work) || wait_on_work(work);
}
which works "more correctly", but it has to iterate over all CPUs and thus
it much slower than flush_work().
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Acked-by: Max Krasnyansky <maxk@qualcomm.com>
Acked-by: Jarek Poplawski <jarkao2@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2008-07-25 10:47:49 +02:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(flush_work);
|
|
|
|
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
/*
|
2007-07-16 08:41:44 +02:00
|
|
|
* Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
* so this work can't be re-armed in any way.
|
|
|
|
*/
|
|
|
|
static int try_to_grab_pending(struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct cpu_workqueue_struct *cwq;
|
2007-07-16 08:41:44 +02:00
|
|
|
int ret = -1;
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
|
2007-07-16 08:41:44 +02:00
|
|
|
return 0;
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The queueing is in progress, or it is already queued. Try to
|
|
|
|
* steal it from ->worklist without clearing WORK_STRUCT_PENDING.
|
|
|
|
*/
|
|
|
|
|
|
|
|
cwq = get_wq_data(work);
|
|
|
|
if (!cwq)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
spin_lock_irq(&cwq->lock);
|
|
|
|
if (!list_empty(&work->entry)) {
|
|
|
|
/*
|
|
|
|
* This work is queued, but perhaps we locked the wrong cwq.
|
|
|
|
* In that case we must see the new value after rmb(), see
|
|
|
|
* insert_work()->wmb().
|
|
|
|
*/
|
|
|
|
smp_rmb();
|
|
|
|
if (cwq == get_wq_data(work)) {
|
2009-11-15 17:09:48 +01:00
|
|
|
debug_work_deactivate(work);
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
list_del_init(&work->entry);
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq_dec_nr_in_flight(cwq, get_work_color(work));
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
ret = 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
spin_unlock_irq(&cwq->lock);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq,
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
struct work_struct *work)
|
|
|
|
{
|
|
|
|
struct wq_barrier barr;
|
2010-06-29 10:07:12 +02:00
|
|
|
struct worker *worker;
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
|
|
|
|
spin_lock_irq(&cwq->lock);
|
2010-06-29 10:07:12 +02:00
|
|
|
|
|
|
|
worker = NULL;
|
2010-06-29 10:07:11 +02:00
|
|
|
if (unlikely(cwq->worker && cwq->worker->current_work == work)) {
|
2010-06-29 10:07:12 +02:00
|
|
|
worker = cwq->worker;
|
|
|
|
insert_wq_barrier(cwq, &barr, work, worker);
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
}
|
2010-06-29 10:07:12 +02:00
|
|
|
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
spin_unlock_irq(&cwq->lock);
|
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
if (unlikely(worker)) {
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
wait_for_completion(&barr.done);
|
2009-11-15 17:09:48 +01:00
|
|
|
destroy_work_on_stack(&barr.work);
|
|
|
|
}
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
}
|
|
|
|
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
static void wait_on_work(struct work_struct *work)
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
{
|
|
|
|
struct cpu_workqueue_struct *cwq;
|
2007-05-09 11:34:22 +02:00
|
|
|
struct workqueue_struct *wq;
|
2007-05-09 11:34:12 +02:00
|
|
|
int cpu;
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
|
2007-05-09 11:34:10 +02:00
|
|
|
might_sleep();
|
|
|
|
|
2008-08-11 10:30:30 +02:00
|
|
|
lock_map_acquire(&work->lockdep_map);
|
|
|
|
lock_map_release(&work->lockdep_map);
|
2007-10-19 08:39:55 +02:00
|
|
|
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
cwq = get_wq_data(work);
|
|
|
|
if (!cwq)
|
2007-05-09 11:34:09 +02:00
|
|
|
return;
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
|
2007-05-09 11:34:22 +02:00
|
|
|
wq = cwq->wq;
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
for_each_possible_cpu(cpu)
|
2010-06-29 10:07:10 +02:00
|
|
|
wait_on_cpu_work(get_cwq(cpu, wq), work);
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
}
|
|
|
|
|
2007-07-16 08:41:44 +02:00
|
|
|
static int __cancel_work_timer(struct work_struct *work,
|
|
|
|
struct timer_list* timer)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
do {
|
|
|
|
ret = (timer && likely(del_timer(timer)));
|
|
|
|
if (!ret)
|
|
|
|
ret = try_to_grab_pending(work);
|
|
|
|
wait_on_work(work);
|
|
|
|
} while (unlikely(ret < 0));
|
|
|
|
|
2010-04-23 17:40:40 +02:00
|
|
|
clear_wq_data(work);
|
2007-07-16 08:41:44 +02:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
/**
|
|
|
|
* cancel_work_sync - block until a work_struct's callback has terminated
|
|
|
|
* @work: the work which is to be flushed
|
|
|
|
*
|
2007-07-16 08:41:44 +02:00
|
|
|
* Returns true if @work was pending.
|
|
|
|
*
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
* cancel_work_sync() will cancel the work if it is queued. If the work's
|
|
|
|
* callback appears to be running, cancel_work_sync() will block until it
|
|
|
|
* has completed.
|
|
|
|
*
|
|
|
|
* It is possible to use this function if the work re-queues itself. It can
|
|
|
|
* cancel the work even if it migrates to another workqueue, however in that
|
|
|
|
* case it only guarantees that work->func() has completed on the last queued
|
|
|
|
* workqueue.
|
|
|
|
*
|
|
|
|
* cancel_work_sync(&delayed_work->work) should be used only if ->timer is not
|
|
|
|
* pending, otherwise it goes into a busy-wait loop until the timer expires.
|
|
|
|
*
|
|
|
|
* The caller must ensure that workqueue_struct on which this work was last
|
|
|
|
* queued can't be destroyed before this function returns.
|
|
|
|
*/
|
2007-07-16 08:41:44 +02:00
|
|
|
int cancel_work_sync(struct work_struct *work)
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
{
|
2007-07-16 08:41:44 +02:00
|
|
|
return __cancel_work_timer(work, NULL);
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
}
|
2007-05-09 11:34:22 +02:00
|
|
|
EXPORT_SYMBOL_GPL(cancel_work_sync);
|
implement flush_work()
A basic problem with flush_scheduled_work() is that it blocks behind _all_
presently-queued works, rather than just the work whcih the caller wants to
flush. If the caller holds some lock, and if one of the queued work happens
to want that lock as well then accidental deadlocks can occur.
One example of this is the phy layer: it wants to flush work while holding
rtnl_lock(). But if a linkwatch event happens to be queued, the phy code will
deadlock because the linkwatch callback function takes rtnl_lock.
So we implement a new function which will flush a *single* work - just the one
which the caller wants to free up. Thus we avoid the accidental deadlocks
which can arise from unrelated subsystems' callbacks taking shared locks.
flush_work() non-blockingly dequeues the work_struct which we want to kill,
then it waits for its handler to complete on all CPUs.
Add ->current_work to the "struct cpu_workqueue_struct", it points to
currently running "struct work_struct". When flush_work(work) detects
->current_work == work, it inserts a barrier at the _head_ of ->worklist
(and thus right _after_ that work) and waits for completition. This means
that the next work fired on that CPU will be this barrier, or another
barrier queued by concurrent flush_work(), so the caller of flush_work()
will be woken before any "regular" work has a chance to run.
When wait_on_work() unlocks workqueue_mutex (or whatever we choose to protect
against CPU hotplug), CPU may go away. But in that case take_over_work() will
move a barrier we queued to another CPU, it will be fired sometime, and
wait_on_work() will be woken.
Actually, we are doing cleanup_workqueue_thread()->kthread_stop() before
take_over_work(), so cwq->thread should complete its ->worklist (and thus
the barrier), because currently we don't check kthread_should_stop() in
run_workqueue(). But even if we did, everything should be ok.
[akpm@osdl.org: cleanup]
[akpm@osdl.org: add flush_work_keventd() wrapper]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:33:52 +02:00
|
|
|
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
/**
|
2007-07-16 08:41:44 +02:00
|
|
|
* cancel_delayed_work_sync - reliably kill off a delayed work.
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
* @dwork: the delayed work struct
|
|
|
|
*
|
2007-07-16 08:41:44 +02:00
|
|
|
* Returns true if @dwork was pending.
|
|
|
|
*
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
* It is possible to use this function if @dwork rearms itself via queue_work()
|
|
|
|
* or queue_delayed_work(). See also the comment for cancel_work_sync().
|
|
|
|
*/
|
2007-07-16 08:41:44 +02:00
|
|
|
int cancel_delayed_work_sync(struct delayed_work *dwork)
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
{
|
2007-07-16 08:41:44 +02:00
|
|
|
return __cancel_work_timer(&dwork->work, &dwork->timer);
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
}
|
2007-07-16 08:41:44 +02:00
|
|
|
EXPORT_SYMBOL(cancel_delayed_work_sync);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
make cancel_rearming_delayed_work() reliable
Thanks to Jarek Poplawski for the ideas and for spotting the bug in the
initial draft patch.
cancel_rearming_delayed_work() currently has many limitations, because it
requires that dwork always re-arms itself via queue_delayed_work(). So it
hangs forever if dwork doesn't do this, or cancel_rearming_delayed_work/
cancel_delayed_work was already called. It uses flush_workqueue() in a
loop, so it can't be used if workqueue was freezed, and it is potentially
live- lockable on busy system if delay is small.
With this patch cancel_rearming_delayed_work() doesn't make any assumptions
about dwork, it can re-arm itself via queue_delayed_work(), or
queue_work(), or do nothing.
As a "side effect", cancel_work_sync() was changed to handle re-arming works
as well.
Disadvantages:
- this patch adds wmb() to insert_work().
- slowdowns the fast path (when del_timer() succeeds on entry) of
cancel_rearming_delayed_work(), because wait_on_work() is called
unconditionally. In that case, compared to the old version, we are
doing "unneeded" lock/unlock for each online CPU.
On the other hand, this means we don't need to use cancel_work_sync()
after cancel_rearming_delayed_work().
- complicates the code (.text grows by 130 bytes).
[akpm@linux-foundation.org: fix speling]
Signed-off-by: Oleg Nesterov <oleg@tv-sign.ru>
Cc: David Chinner <dgc@sgi.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Gautham Shenoy <ego@in.ibm.com>
Acked-by: Jarek Poplawski <jarkao2@o2.pl>
Cc: Srivatsa Vaddagiri <vatsa@in.ibm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-05-09 11:34:46 +02:00
|
|
|
static struct workqueue_struct *keventd_wq __read_mostly;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2006-07-30 12:03:42 +02:00
|
|
|
/**
|
|
|
|
* schedule_work - put work task in global workqueue
|
|
|
|
* @work: job to be done
|
|
|
|
*
|
2009-07-30 19:00:53 +02:00
|
|
|
* Returns zero if @work was already on the kernel-global workqueue and
|
|
|
|
* non-zero otherwise.
|
|
|
|
*
|
|
|
|
* This puts a job in the kernel-global workqueue if it was not already
|
|
|
|
* queued and leaves it in the same position on the kernel-global
|
|
|
|
* workqueue otherwise.
|
2006-07-30 12:03:42 +02:00
|
|
|
*/
|
2008-02-08 13:19:53 +01:00
|
|
|
int schedule_work(struct work_struct *work)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
|
|
|
return queue_work(keventd_wq, work);
|
|
|
|
}
|
2006-06-30 07:40:45 +02:00
|
|
|
EXPORT_SYMBOL(schedule_work);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2008-07-24 06:28:39 +02:00
|
|
|
/*
|
|
|
|
* schedule_work_on - put work task on a specific cpu
|
|
|
|
* @cpu: cpu to put the work task on
|
|
|
|
* @work: job to be done
|
|
|
|
*
|
|
|
|
* This puts a job on a specific cpu
|
|
|
|
*/
|
|
|
|
int schedule_work_on(int cpu, struct work_struct *work)
|
|
|
|
{
|
|
|
|
return queue_work_on(cpu, keventd_wq, work);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(schedule_work_on);
|
|
|
|
|
2006-07-30 12:03:42 +02:00
|
|
|
/**
|
|
|
|
* schedule_delayed_work - put work task in global workqueue after delay
|
2006-11-22 15:54:01 +01:00
|
|
|
* @dwork: job to be done
|
|
|
|
* @delay: number of jiffies to wait or 0 for immediate execution
|
2006-07-30 12:03:42 +02:00
|
|
|
*
|
|
|
|
* After waiting for a given time this puts a job in the kernel-global
|
|
|
|
* workqueue.
|
|
|
|
*/
|
2008-02-08 13:19:53 +01:00
|
|
|
int schedule_delayed_work(struct delayed_work *dwork,
|
[PATCH] Add debugging feature /proc/timer_stat
Add /proc/timer_stats support: debugging feature to profile timer expiration.
Both the starting site, process/PID and the expiration function is captured.
This allows the quick identification of timer event sources in a system.
Sample output:
# echo 1 > /proc/timer_stats
# cat /proc/timer_stats
Timer Stats Version: v0.1
Sample period: 4.010 s
24, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
11, 0 swapper sk_reset_timer (tcp_delack_timer)
6, 0 swapper hrtimer_stop_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
17, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
2, 1 swapper queue_delayed_work_on (delayed_work_timer_fn)
4, 2050 pcscd do_nanosleep (hrtimer_wakeup)
5, 4179 sshd sk_reset_timer (tcp_write_timer)
4, 2248 yum-updatesd schedule_timeout (process_timeout)
18, 0 swapper hrtimer_restart_sched_tick (hrtimer_sched_tick)
3, 0 swapper sk_reset_timer (tcp_delack_timer)
1, 1 swapper neigh_table_init_no_netlink (neigh_periodic_timer)
2, 1 swapper e1000_up (e1000_watchdog)
1, 1 init schedule_timeout (process_timeout)
100 total events, 25.24 events/sec
[ cleanups and hrtimers support from Thomas Gleixner <tglx@linutronix.de> ]
[bunk@stusta.de: nr_entries can become static]
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: john stultz <johnstul@us.ibm.com>
Cc: Roman Zippel <zippel@linux-m68k.org>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: Adrian Bunk <bunk@stusta.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-02-16 10:28:13 +01:00
|
|
|
unsigned long delay)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-11-22 15:54:01 +01:00
|
|
|
return queue_delayed_work(keventd_wq, dwork, delay);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-06-30 07:40:45 +02:00
|
|
|
EXPORT_SYMBOL(schedule_delayed_work);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2009-10-14 18:16:42 +02:00
|
|
|
/**
|
|
|
|
* flush_delayed_work - block until a dwork_struct's callback has terminated
|
|
|
|
* @dwork: the delayed work which is to be flushed
|
|
|
|
*
|
|
|
|
* Any timeout is cancelled, and any pending work is run immediately.
|
|
|
|
*/
|
|
|
|
void flush_delayed_work(struct delayed_work *dwork)
|
|
|
|
{
|
|
|
|
if (del_timer_sync(&dwork->timer)) {
|
2010-06-29 10:07:10 +02:00
|
|
|
__queue_work(get_cpu(), get_wq_data(&dwork->work)->wq,
|
|
|
|
&dwork->work);
|
2009-10-14 18:16:42 +02:00
|
|
|
put_cpu();
|
|
|
|
}
|
|
|
|
flush_work(&dwork->work);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(flush_delayed_work);
|
|
|
|
|
2006-07-30 12:03:42 +02:00
|
|
|
/**
|
|
|
|
* schedule_delayed_work_on - queue work in global workqueue on CPU after delay
|
|
|
|
* @cpu: cpu to use
|
2006-11-22 15:54:01 +01:00
|
|
|
* @dwork: job to be done
|
2006-07-30 12:03:42 +02:00
|
|
|
* @delay: number of jiffies to wait
|
|
|
|
*
|
|
|
|
* After waiting for a given time this puts a job in the kernel-global
|
|
|
|
* workqueue on the specified CPU.
|
|
|
|
*/
|
2005-04-17 00:20:36 +02:00
|
|
|
int schedule_delayed_work_on(int cpu,
|
2006-11-22 15:54:01 +01:00
|
|
|
struct delayed_work *dwork, unsigned long delay)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2006-11-22 15:54:01 +01:00
|
|
|
return queue_delayed_work_on(cpu, keventd_wq, dwork, delay);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
2006-06-30 07:40:45 +02:00
|
|
|
EXPORT_SYMBOL(schedule_delayed_work_on);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2006-06-25 14:47:49 +02:00
|
|
|
/**
|
|
|
|
* schedule_on_each_cpu - call a function on each online CPU from keventd
|
|
|
|
* @func: the function to call
|
|
|
|
*
|
|
|
|
* Returns zero on success.
|
|
|
|
* Returns -ve errno on failure.
|
|
|
|
*
|
|
|
|
* schedule_on_each_cpu() is very slow.
|
|
|
|
*/
|
2006-11-22 15:55:48 +01:00
|
|
|
int schedule_on_each_cpu(work_func_t func)
|
2006-01-08 10:00:43 +01:00
|
|
|
{
|
|
|
|
int cpu;
|
2009-10-14 06:22:47 +02:00
|
|
|
int orig = -1;
|
2006-06-25 14:47:49 +02:00
|
|
|
struct work_struct *works;
|
2006-01-08 10:00:43 +01:00
|
|
|
|
2006-06-25 14:47:49 +02:00
|
|
|
works = alloc_percpu(struct work_struct);
|
|
|
|
if (!works)
|
2006-01-08 10:00:43 +01:00
|
|
|
return -ENOMEM;
|
2006-06-25 14:47:49 +02:00
|
|
|
|
2009-11-17 23:06:20 +01:00
|
|
|
get_online_cpus();
|
|
|
|
|
2009-10-14 06:22:47 +02:00
|
|
|
/*
|
2009-11-17 23:06:20 +01:00
|
|
|
* When running in keventd don't schedule a work item on
|
|
|
|
* itself. Can just call directly because the work queue is
|
|
|
|
* already bound. This also is faster.
|
2009-10-14 06:22:47 +02:00
|
|
|
*/
|
2009-11-17 23:06:20 +01:00
|
|
|
if (current_is_keventd())
|
2009-10-14 06:22:47 +02:00
|
|
|
orig = raw_smp_processor_id();
|
|
|
|
|
2006-01-08 10:00:43 +01:00
|
|
|
for_each_online_cpu(cpu) {
|
2006-12-18 20:05:09 +01:00
|
|
|
struct work_struct *work = per_cpu_ptr(works, cpu);
|
|
|
|
|
|
|
|
INIT_WORK(work, func);
|
2009-10-14 06:22:47 +02:00
|
|
|
if (cpu != orig)
|
2009-11-17 23:06:20 +01:00
|
|
|
schedule_work_on(cpu, work);
|
2009-10-14 06:22:47 +02:00
|
|
|
}
|
2009-11-17 23:06:20 +01:00
|
|
|
if (orig >= 0)
|
|
|
|
func(per_cpu_ptr(works, orig));
|
|
|
|
|
|
|
|
for_each_online_cpu(cpu)
|
|
|
|
flush_work(per_cpu_ptr(works, cpu));
|
|
|
|
|
2008-01-25 21:08:02 +01:00
|
|
|
put_online_cpus();
|
2006-06-25 14:47:49 +02:00
|
|
|
free_percpu(works);
|
2006-01-08 10:00:43 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-02-12 09:39:21 +01:00
|
|
|
/**
|
|
|
|
* flush_scheduled_work - ensure that any scheduled work has run to completion.
|
|
|
|
*
|
|
|
|
* Forces execution of the kernel-global workqueue and blocks until its
|
|
|
|
* completion.
|
|
|
|
*
|
|
|
|
* Think twice before calling this function! It's very easy to get into
|
|
|
|
* trouble if you don't take great care. Either of the following situations
|
|
|
|
* will lead to deadlock:
|
|
|
|
*
|
|
|
|
* One of the work items currently on the workqueue needs to acquire
|
|
|
|
* a lock held by your code or its caller.
|
|
|
|
*
|
|
|
|
* Your code is running in the context of a work routine.
|
|
|
|
*
|
|
|
|
* They will be detected by lockdep when they occur, but the first might not
|
|
|
|
* occur very often. It depends on what work items are on the workqueue and
|
|
|
|
* what locks they need, which you have no control over.
|
|
|
|
*
|
|
|
|
* In most situations flushing the entire workqueue is overkill; you merely
|
|
|
|
* need to know that a particular work item isn't queued and isn't running.
|
|
|
|
* In such cases you should use cancel_delayed_work_sync() or
|
|
|
|
* cancel_work_sync() instead.
|
|
|
|
*/
|
2005-04-17 00:20:36 +02:00
|
|
|
void flush_scheduled_work(void)
|
|
|
|
{
|
|
|
|
flush_workqueue(keventd_wq);
|
|
|
|
}
|
2006-06-30 07:40:45 +02:00
|
|
|
EXPORT_SYMBOL(flush_scheduled_work);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2006-02-23 19:43:43 +01:00
|
|
|
/**
|
|
|
|
* execute_in_process_context - reliably execute the routine with user context
|
|
|
|
* @fn: the function to execute
|
|
|
|
* @ew: guaranteed storage for the execute work structure (must
|
|
|
|
* be available when the work executes)
|
|
|
|
*
|
|
|
|
* Executes the function immediately if process context is available,
|
|
|
|
* otherwise schedules the function for delayed execution.
|
|
|
|
*
|
|
|
|
* Returns: 0 - function was executed
|
|
|
|
* 1 - function was scheduled for execution
|
|
|
|
*/
|
2006-11-22 15:55:48 +01:00
|
|
|
int execute_in_process_context(work_func_t fn, struct execute_work *ew)
|
2006-02-23 19:43:43 +01:00
|
|
|
{
|
|
|
|
if (!in_interrupt()) {
|
2006-11-22 15:55:48 +01:00
|
|
|
fn(&ew->work);
|
2006-02-23 19:43:43 +01:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2006-11-22 15:55:48 +01:00
|
|
|
INIT_WORK(&ew->work, fn);
|
2006-02-23 19:43:43 +01:00
|
|
|
schedule_work(&ew->work);
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(execute_in_process_context);
|
|
|
|
|
2005-04-17 00:20:36 +02:00
|
|
|
int keventd_up(void)
|
|
|
|
{
|
|
|
|
return keventd_wq != NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
int current_is_keventd(void)
|
|
|
|
{
|
|
|
|
struct cpu_workqueue_struct *cwq;
|
2007-08-27 17:06:19 +02:00
|
|
|
int cpu = raw_smp_processor_id(); /* preempt-safe: keventd is per-cpu */
|
2005-04-17 00:20:36 +02:00
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
BUG_ON(!keventd_wq);
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq = get_cwq(cpu, keventd_wq);
|
2010-06-29 10:07:11 +02:00
|
|
|
if (current == cwq->worker->task)
|
2005-04-17 00:20:36 +02:00
|
|
|
ret = 1;
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
static struct cpu_workqueue_struct *alloc_cwqs(void)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* cwqs are forced aligned according to WORK_STRUCT_FLAG_BITS.
|
|
|
|
* Make sure that the alignment isn't lower than that of
|
|
|
|
* unsigned long long.
|
|
|
|
*/
|
|
|
|
const size_t size = sizeof(struct cpu_workqueue_struct);
|
|
|
|
const size_t align = max_t(size_t, 1 << WORK_STRUCT_FLAG_BITS,
|
|
|
|
__alignof__(unsigned long long));
|
|
|
|
struct cpu_workqueue_struct *cwqs;
|
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
void *ptr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* On UP, percpu allocator doesn't honor alignment parameter
|
|
|
|
* and simply uses arch-dependent default. Allocate enough
|
|
|
|
* room to align cwq and put an extra pointer at the end
|
|
|
|
* pointing back to the originally allocated pointer which
|
|
|
|
* will be used for free.
|
|
|
|
*
|
|
|
|
* FIXME: This really belongs to UP percpu code. Update UP
|
|
|
|
* percpu code to honor alignment and remove this ugliness.
|
|
|
|
*/
|
|
|
|
ptr = __alloc_percpu(size + align + sizeof(void *), 1);
|
|
|
|
cwqs = PTR_ALIGN(ptr, align);
|
|
|
|
*(void **)per_cpu_ptr(cwqs + 1, 0) = ptr;
|
|
|
|
#else
|
|
|
|
/* On SMP, percpu allocator can do it itself */
|
|
|
|
cwqs = __alloc_percpu(size, align);
|
|
|
|
#endif
|
|
|
|
/* just in case, make sure it's actually aligned */
|
|
|
|
BUG_ON(!IS_ALIGNED((unsigned long)cwqs, align));
|
|
|
|
return cwqs;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void free_cwqs(struct cpu_workqueue_struct *cwqs)
|
|
|
|
{
|
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
/* on UP, the pointer to free is stored right after the cwq */
|
|
|
|
if (cwqs)
|
|
|
|
free_percpu(*(void **)per_cpu_ptr(cwqs + 1, 0));
|
|
|
|
#else
|
|
|
|
free_percpu(cwqs);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2007-10-19 08:39:55 +02:00
|
|
|
struct workqueue_struct *__create_workqueue_key(const char *name,
|
2010-06-29 10:07:10 +02:00
|
|
|
unsigned int flags,
|
2010-06-29 10:07:12 +02:00
|
|
|
int max_active,
|
2008-01-16 09:51:58 +01:00
|
|
|
struct lock_class_key *key,
|
|
|
|
const char *lock_name)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
bool singlethread = flags & WQ_SINGLE_THREAD;
|
2005-04-17 00:20:36 +02:00
|
|
|
struct workqueue_struct *wq;
|
2010-06-29 10:07:11 +02:00
|
|
|
bool failed = false;
|
|
|
|
unsigned int cpu;
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
max_active = clamp_val(max_active, 1, INT_MAX);
|
|
|
|
|
2007-05-09 11:34:09 +02:00
|
|
|
wq = kzalloc(sizeof(*wq), GFP_KERNEL);
|
|
|
|
if (!wq)
|
2010-06-29 10:07:10 +02:00
|
|
|
goto err;
|
2007-05-09 11:34:09 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
wq->cpu_wq = alloc_cwqs();
|
2010-06-29 10:07:10 +02:00
|
|
|
if (!wq->cpu_wq)
|
|
|
|
goto err;
|
2007-05-09 11:34:09 +02:00
|
|
|
|
2010-06-29 10:07:10 +02:00
|
|
|
wq->flags = flags;
|
2010-06-29 10:07:12 +02:00
|
|
|
wq->saved_max_active = max_active;
|
2010-06-29 10:07:11 +02:00
|
|
|
mutex_init(&wq->flush_mutex);
|
|
|
|
atomic_set(&wq->nr_cwqs_to_flush, 0);
|
|
|
|
INIT_LIST_HEAD(&wq->flusher_queue);
|
|
|
|
INIT_LIST_HEAD(&wq->flusher_overflow);
|
2007-05-09 11:34:09 +02:00
|
|
|
wq->name = name;
|
2008-01-16 09:51:58 +01:00
|
|
|
lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
|
2007-05-09 11:34:13 +02:00
|
|
|
INIT_LIST_HEAD(&wq->list);
|
2007-05-09 11:34:09 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
cpu_maps_update_begin();
|
|
|
|
/*
|
|
|
|
* We must initialize cwqs for each possible cpu even if we
|
|
|
|
* are going to call destroy_workqueue() finally. Otherwise
|
|
|
|
* cpu_up() can hit the uninitialized cwq once we drop the
|
|
|
|
* lock.
|
|
|
|
*/
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK);
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq->cpu = cpu;
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq->wq = wq;
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq->flush_color = -1;
|
2010-06-29 10:07:12 +02:00
|
|
|
cwq->max_active = max_active;
|
2010-06-29 10:07:11 +02:00
|
|
|
spin_lock_init(&cwq->lock);
|
|
|
|
INIT_LIST_HEAD(&cwq->worklist);
|
2010-06-29 10:07:12 +02:00
|
|
|
INIT_LIST_HEAD(&cwq->delayed_works);
|
2010-06-29 10:07:11 +02:00
|
|
|
init_waitqueue_head(&cwq->more_work);
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
if (failed)
|
2010-06-29 10:07:11 +02:00
|
|
|
continue;
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq->worker = create_worker(cwq,
|
|
|
|
cpu_online(cpu) && !singlethread);
|
|
|
|
if (cwq->worker)
|
|
|
|
start_worker(cwq->worker);
|
2010-06-29 10:07:11 +02:00
|
|
|
else
|
2010-06-29 10:07:11 +02:00
|
|
|
failed = true;
|
2007-05-09 11:34:09 +02:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
/*
|
|
|
|
* workqueue_lock protects global freeze state and workqueues
|
|
|
|
* list. Grab it, set max_active accordingly and add the new
|
|
|
|
* workqueue to workqueues list.
|
|
|
|
*/
|
2010-06-29 10:07:11 +02:00
|
|
|
spin_lock(&workqueue_lock);
|
2010-06-29 10:07:12 +02:00
|
|
|
|
|
|
|
if (workqueue_freezing && wq->flags & WQ_FREEZEABLE)
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
get_cwq(cpu, wq)->max_active = 0;
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
list_add(&wq->list, &workqueues);
|
2010-06-29 10:07:12 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
spin_unlock(&workqueue_lock);
|
|
|
|
|
|
|
|
cpu_maps_update_done();
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
if (failed) {
|
2007-05-09 11:34:09 +02:00
|
|
|
destroy_workqueue(wq);
|
|
|
|
wq = NULL;
|
|
|
|
}
|
|
|
|
return wq;
|
2010-06-29 10:07:10 +02:00
|
|
|
err:
|
|
|
|
if (wq) {
|
2010-06-29 10:07:11 +02:00
|
|
|
free_cwqs(wq->cpu_wq);
|
2010-06-29 10:07:10 +02:00
|
|
|
kfree(wq);
|
|
|
|
}
|
|
|
|
return NULL;
|
2007-05-09 11:34:09 +02:00
|
|
|
}
|
2007-10-19 08:39:55 +02:00
|
|
|
EXPORT_SYMBOL_GPL(__create_workqueue_key);
|
2005-04-17 00:20:36 +02:00
|
|
|
|
2007-05-09 11:34:09 +02:00
|
|
|
/**
|
|
|
|
* destroy_workqueue - safely terminate a workqueue
|
|
|
|
* @wq: target workqueue
|
|
|
|
*
|
|
|
|
* Safely destroy a workqueue. All work currently pending will be done first.
|
|
|
|
*/
|
|
|
|
void destroy_workqueue(struct workqueue_struct *wq)
|
|
|
|
{
|
2007-05-09 11:34:12 +02:00
|
|
|
int cpu;
|
2007-05-09 11:34:09 +02:00
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
flush_workqueue(wq);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* wq list is used to freeze wq, remove from list after
|
|
|
|
* flushing is complete in case freeze races us.
|
|
|
|
*/
|
2008-07-25 10:47:50 +02:00
|
|
|
cpu_maps_update_begin();
|
2008-01-25 21:08:02 +01:00
|
|
|
spin_lock(&workqueue_lock);
|
2007-05-09 11:34:12 +02:00
|
|
|
list_del(&wq->list);
|
2008-01-25 21:08:02 +01:00
|
|
|
spin_unlock(&workqueue_lock);
|
2010-06-29 10:07:11 +02:00
|
|
|
cpu_maps_update_done();
|
2007-05-09 11:34:09 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
|
|
|
|
int i;
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
if (cwq->worker) {
|
|
|
|
destroy_worker(cwq->worker);
|
|
|
|
cwq->worker = NULL;
|
2010-06-29 10:07:11 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < WORK_NR_COLORS; i++)
|
|
|
|
BUG_ON(cwq->nr_in_flight[i]);
|
2010-06-29 10:07:12 +02:00
|
|
|
BUG_ON(cwq->nr_active);
|
|
|
|
BUG_ON(!list_empty(&cwq->delayed_works));
|
2010-06-29 10:07:11 +02:00
|
|
|
}
|
2006-08-14 08:24:26 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
free_cwqs(wq->cpu_wq);
|
2007-05-09 11:34:09 +02:00
|
|
|
kfree(wq);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(destroy_workqueue);
|
|
|
|
|
|
|
|
static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
|
|
|
|
unsigned long action,
|
|
|
|
void *hcpu)
|
|
|
|
{
|
|
|
|
unsigned int cpu = (unsigned long)hcpu;
|
|
|
|
struct cpu_workqueue_struct *cwq;
|
|
|
|
struct workqueue_struct *wq;
|
|
|
|
|
2007-05-09 11:35:10 +02:00
|
|
|
action &= ~CPU_TASKS_FROZEN;
|
|
|
|
|
2007-05-09 11:34:09 +02:00
|
|
|
list_for_each_entry(wq, &workqueues, list) {
|
2010-06-29 10:07:11 +02:00
|
|
|
if (wq->flags & WQ_SINGLE_THREAD)
|
|
|
|
continue;
|
2007-05-09 11:34:09 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
cwq = get_cwq(cpu, wq);
|
2007-05-09 11:34:09 +02:00
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
switch (action) {
|
2008-07-25 10:47:50 +02:00
|
|
|
case CPU_POST_DEAD:
|
2010-06-29 10:07:11 +02:00
|
|
|
flush_workqueue(wq);
|
2007-05-09 11:34:09 +02:00
|
|
|
break;
|
|
|
|
}
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2010-06-29 10:07:11 +02:00
|
|
|
return notifier_from_errno(0);
|
2005-04-17 00:20:36 +02:00
|
|
|
}
|
|
|
|
|
2008-11-05 03:39:10 +01:00
|
|
|
#ifdef CONFIG_SMP
|
2009-01-17 00:31:15 +01:00
|
|
|
|
2008-11-05 03:39:10 +01:00
|
|
|
struct work_for_cpu {
|
2009-04-09 17:50:37 +02:00
|
|
|
struct completion completion;
|
2008-11-05 03:39:10 +01:00
|
|
|
long (*fn)(void *);
|
|
|
|
void *arg;
|
|
|
|
long ret;
|
|
|
|
};
|
|
|
|
|
2009-04-09 17:50:37 +02:00
|
|
|
static int do_work_for_cpu(void *_wfc)
|
2008-11-05 03:39:10 +01:00
|
|
|
{
|
2009-04-09 17:50:37 +02:00
|
|
|
struct work_for_cpu *wfc = _wfc;
|
2008-11-05 03:39:10 +01:00
|
|
|
wfc->ret = wfc->fn(wfc->arg);
|
2009-04-09 17:50:37 +02:00
|
|
|
complete(&wfc->completion);
|
|
|
|
return 0;
|
2008-11-05 03:39:10 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* work_on_cpu - run a function in user context on a particular cpu
|
|
|
|
* @cpu: the cpu to run on
|
|
|
|
* @fn: the function to run
|
|
|
|
* @arg: the function arg
|
|
|
|
*
|
2009-01-17 00:31:15 +01:00
|
|
|
* This will return the value @fn returns.
|
|
|
|
* It is up to the caller to ensure that the cpu doesn't go offline.
|
2009-04-09 17:50:37 +02:00
|
|
|
* The caller must not hold any locks which would prevent @fn from completing.
|
2008-11-05 03:39:10 +01:00
|
|
|
*/
|
|
|
|
long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg)
|
|
|
|
{
|
2009-04-09 17:50:37 +02:00
|
|
|
struct task_struct *sub_thread;
|
|
|
|
struct work_for_cpu wfc = {
|
|
|
|
.completion = COMPLETION_INITIALIZER_ONSTACK(wfc.completion),
|
|
|
|
.fn = fn,
|
|
|
|
.arg = arg,
|
|
|
|
};
|
|
|
|
|
|
|
|
sub_thread = kthread_create(do_work_for_cpu, &wfc, "work_for_cpu");
|
|
|
|
if (IS_ERR(sub_thread))
|
|
|
|
return PTR_ERR(sub_thread);
|
|
|
|
kthread_bind(sub_thread, cpu);
|
|
|
|
wake_up_process(sub_thread);
|
|
|
|
wait_for_completion(&wfc.completion);
|
2008-11-05 03:39:10 +01:00
|
|
|
return wfc.ret;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(work_on_cpu);
|
|
|
|
#endif /* CONFIG_SMP */
|
|
|
|
|
2010-06-29 10:07:12 +02:00
|
|
|
#ifdef CONFIG_FREEZER
|
|
|
|
|
|
|
|
/**
|
|
|
|
* freeze_workqueues_begin - begin freezing workqueues
|
|
|
|
*
|
|
|
|
* Start freezing workqueues. After this function returns, all
|
|
|
|
* freezeable workqueues will queue new works to their frozen_works
|
|
|
|
* list instead of the cwq ones.
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* Grabs and releases workqueue_lock and cwq->lock's.
|
|
|
|
*/
|
|
|
|
void freeze_workqueues_begin(void)
|
|
|
|
{
|
|
|
|
struct workqueue_struct *wq;
|
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
spin_lock(&workqueue_lock);
|
|
|
|
|
|
|
|
BUG_ON(workqueue_freezing);
|
|
|
|
workqueue_freezing = true;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
list_for_each_entry(wq, &workqueues, list) {
|
|
|
|
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
|
|
|
|
|
|
|
|
spin_lock_irq(&cwq->lock);
|
|
|
|
|
|
|
|
if (wq->flags & WQ_FREEZEABLE)
|
|
|
|
cwq->max_active = 0;
|
|
|
|
|
|
|
|
spin_unlock_irq(&cwq->lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock(&workqueue_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* freeze_workqueues_busy - are freezeable workqueues still busy?
|
|
|
|
*
|
|
|
|
* Check whether freezing is complete. This function must be called
|
|
|
|
* between freeze_workqueues_begin() and thaw_workqueues().
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* Grabs and releases workqueue_lock.
|
|
|
|
*
|
|
|
|
* RETURNS:
|
|
|
|
* %true if some freezeable workqueues are still busy. %false if
|
|
|
|
* freezing is complete.
|
|
|
|
*/
|
|
|
|
bool freeze_workqueues_busy(void)
|
|
|
|
{
|
|
|
|
struct workqueue_struct *wq;
|
|
|
|
unsigned int cpu;
|
|
|
|
bool busy = false;
|
|
|
|
|
|
|
|
spin_lock(&workqueue_lock);
|
|
|
|
|
|
|
|
BUG_ON(!workqueue_freezing);
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
/*
|
|
|
|
* nr_active is monotonically decreasing. It's safe
|
|
|
|
* to peek without lock.
|
|
|
|
*/
|
|
|
|
list_for_each_entry(wq, &workqueues, list) {
|
|
|
|
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
|
|
|
|
|
|
|
|
if (!(wq->flags & WQ_FREEZEABLE))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
BUG_ON(cwq->nr_active < 0);
|
|
|
|
if (cwq->nr_active) {
|
|
|
|
busy = true;
|
|
|
|
goto out_unlock;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
out_unlock:
|
|
|
|
spin_unlock(&workqueue_lock);
|
|
|
|
return busy;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* thaw_workqueues - thaw workqueues
|
|
|
|
*
|
|
|
|
* Thaw workqueues. Normal queueing is restored and all collected
|
|
|
|
* frozen works are transferred to their respective cwq worklists.
|
|
|
|
*
|
|
|
|
* CONTEXT:
|
|
|
|
* Grabs and releases workqueue_lock and cwq->lock's.
|
|
|
|
*/
|
|
|
|
void thaw_workqueues(void)
|
|
|
|
{
|
|
|
|
struct workqueue_struct *wq;
|
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
spin_lock(&workqueue_lock);
|
|
|
|
|
|
|
|
if (!workqueue_freezing)
|
|
|
|
goto out_unlock;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
list_for_each_entry(wq, &workqueues, list) {
|
|
|
|
struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);
|
|
|
|
|
|
|
|
if (!(wq->flags & WQ_FREEZEABLE))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
spin_lock_irq(&cwq->lock);
|
|
|
|
|
|
|
|
/* restore max_active and repopulate worklist */
|
|
|
|
cwq->max_active = wq->saved_max_active;
|
|
|
|
|
|
|
|
while (!list_empty(&cwq->delayed_works) &&
|
|
|
|
cwq->nr_active < cwq->max_active)
|
|
|
|
cwq_activate_first_delayed(cwq);
|
|
|
|
|
|
|
|
wake_up(&cwq->more_work);
|
|
|
|
|
|
|
|
spin_unlock_irq(&cwq->lock);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
workqueue_freezing = false;
|
|
|
|
out_unlock:
|
|
|
|
spin_unlock(&workqueue_lock);
|
|
|
|
}
|
|
|
|
#endif /* CONFIG_FREEZER */
|
|
|
|
|
2007-05-09 11:34:14 +02:00
|
|
|
void __init init_workqueues(void)
|
2005-04-17 00:20:36 +02:00
|
|
|
{
|
2010-06-29 10:07:11 +02:00
|
|
|
unsigned int cpu;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
ida_init(&per_cpu(worker_ida, cpu));
|
|
|
|
|
2009-01-01 00:42:25 +01:00
|
|
|
singlethread_cpu = cpumask_first(cpu_possible_mask);
|
2005-04-17 00:20:36 +02:00
|
|
|
hotcpu_notifier(workqueue_cpu_callback, 0);
|
|
|
|
keventd_wq = create_workqueue("events");
|
|
|
|
BUG_ON(!keventd_wq);
|
|
|
|
}
|