workqueue.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833
  1. /*
  2. * linux/kernel/workqueue.c
  3. *
  4. * Generic mechanism for defining kernel helper threads for running
  5. * arbitrary tasks in process context.
  6. *
  7. * Started by Ingo Molnar, Copyright (C) 2002
  8. *
  9. * Derived from the taskqueue/keventd code by:
  10. *
  11. * David Woodhouse <dwmw2@infradead.org>
  12. * Andrew Morton <andrewm@uow.edu.au>
  13. * Kai Petzke <wpp@marie.physik.tu-berlin.de>
  14. * Theodore Ts'o <tytso@mit.edu>
  15. *
  16. * Made to use alloc_percpu by Christoph Lameter <clameter@sgi.com>.
  17. */
  18. #include <linux/module.h>
  19. #include <linux/kernel.h>
  20. #include <linux/sched.h>
  21. #include <linux/init.h>
  22. #include <linux/signal.h>
  23. #include <linux/completion.h>
  24. #include <linux/workqueue.h>
  25. #include <linux/slab.h>
  26. #include <linux/cpu.h>
  27. #include <linux/notifier.h>
  28. #include <linux/kthread.h>
  29. #include <linux/hardirq.h>
  30. #include <linux/mempolicy.h>
  31. #include <linux/freezer.h>
  32. #include <linux/kallsyms.h>
  33. #include <linux/debug_locks.h>
  34. /*
  35. * The per-CPU workqueue (if single thread, we always use the first
  36. * possible cpu).
  37. */
  38. struct cpu_workqueue_struct {
  39. spinlock_t lock;
  40. struct list_head worklist;
  41. wait_queue_head_t more_work;
  42. struct work_struct *current_work;
  43. struct workqueue_struct *wq;
  44. struct task_struct *thread;
  45. int should_stop;
  46. int run_depth; /* Detect run_workqueue() recursion depth */
  47. } ____cacheline_aligned;
  48. /*
  49. * The externally visible workqueue abstraction is an array of
  50. * per-CPU workqueues:
  51. */
  52. struct workqueue_struct {
  53. struct cpu_workqueue_struct *cpu_wq;
  54. const char *name;
  55. struct list_head list; /* Empty if single thread */
  56. int freezeable; /* Freeze threads during suspend */
  57. };
  58. /* All the per-cpu workqueues on the system, for hotplug cpu to add/remove
  59. threads to each one as cpus come/go. */
  60. static DEFINE_MUTEX(workqueue_mutex);
  61. static LIST_HEAD(workqueues);
  62. static int singlethread_cpu __read_mostly;
  63. /* optimization, we could use cpu_possible_map */
  64. static cpumask_t cpu_populated_map __read_mostly;
  65. /* If it's single threaded, it isn't in the list of workqueues. */
  66. static inline int is_single_threaded(struct workqueue_struct *wq)
  67. {
  68. return list_empty(&wq->list);
  69. }
  70. /*
  71. * Set the workqueue on which a work item is to be run
  72. * - Must *only* be called if the pending flag is set
  73. */
  74. static inline void set_wq_data(struct work_struct *work, void *wq)
  75. {
  76. unsigned long new;
  77. BUG_ON(!work_pending(work));
  78. new = (unsigned long) wq | (1UL << WORK_STRUCT_PENDING);
  79. new |= WORK_STRUCT_FLAG_MASK & *work_data_bits(work);
  80. atomic_long_set(&work->data, new);
  81. }
  82. static inline void *get_wq_data(struct work_struct *work)
  83. {
  84. return (void *) (atomic_long_read(&work->data) & WORK_STRUCT_WQ_DATA_MASK);
  85. }
  86. static void insert_work(struct cpu_workqueue_struct *cwq,
  87. struct work_struct *work, int tail)
  88. {
  89. set_wq_data(work, cwq);
  90. if (tail)
  91. list_add_tail(&work->entry, &cwq->worklist);
  92. else
  93. list_add(&work->entry, &cwq->worklist);
  94. wake_up(&cwq->more_work);
  95. }
  96. /* Preempt must be disabled. */
  97. static void __queue_work(struct cpu_workqueue_struct *cwq,
  98. struct work_struct *work)
  99. {
  100. unsigned long flags;
  101. spin_lock_irqsave(&cwq->lock, flags);
  102. insert_work(cwq, work, 1);
  103. spin_unlock_irqrestore(&cwq->lock, flags);
  104. }
  105. /**
  106. * queue_work - queue work on a workqueue
  107. * @wq: workqueue to use
  108. * @work: work to queue
  109. *
  110. * Returns 0 if @work was already on a queue, non-zero otherwise.
  111. *
  112. * We queue the work to the CPU it was submitted, but there is no
  113. * guarantee that it will be processed by that CPU.
  114. */
  115. int fastcall queue_work(struct workqueue_struct *wq, struct work_struct *work)
  116. {
  117. int ret = 0, cpu = get_cpu();
  118. if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
  119. if (unlikely(is_single_threaded(wq)))
  120. cpu = singlethread_cpu;
  121. BUG_ON(!list_empty(&work->entry));
  122. __queue_work(per_cpu_ptr(wq->cpu_wq, cpu), work);
  123. ret = 1;
  124. }
  125. put_cpu();
  126. return ret;
  127. }
  128. EXPORT_SYMBOL_GPL(queue_work);
  129. void delayed_work_timer_fn(unsigned long __data)
  130. {
  131. struct delayed_work *dwork = (struct delayed_work *)__data;
  132. struct workqueue_struct *wq = get_wq_data(&dwork->work);
  133. int cpu = smp_processor_id();
  134. if (unlikely(is_single_threaded(wq)))
  135. cpu = singlethread_cpu;
  136. __queue_work(per_cpu_ptr(wq->cpu_wq, cpu), &dwork->work);
  137. }
  138. /**
  139. * queue_delayed_work - queue work on a workqueue after delay
  140. * @wq: workqueue to use
  141. * @dwork: delayable work to queue
  142. * @delay: number of jiffies to wait before queueing
  143. *
  144. * Returns 0 if @work was already on a queue, non-zero otherwise.
  145. */
  146. int fastcall queue_delayed_work(struct workqueue_struct *wq,
  147. struct delayed_work *dwork, unsigned long delay)
  148. {
  149. int ret = 0;
  150. struct timer_list *timer = &dwork->timer;
  151. struct work_struct *work = &dwork->work;
  152. timer_stats_timer_set_start_info(timer);
  153. if (delay == 0)
  154. return queue_work(wq, work);
  155. if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
  156. BUG_ON(timer_pending(timer));
  157. BUG_ON(!list_empty(&work->entry));
  158. /* This stores wq for the moment, for the timer_fn */
  159. set_wq_data(work, wq);
  160. timer->expires = jiffies + delay;
  161. timer->data = (unsigned long)dwork;
  162. timer->function = delayed_work_timer_fn;
  163. add_timer(timer);
  164. ret = 1;
  165. }
  166. return ret;
  167. }
  168. EXPORT_SYMBOL_GPL(queue_delayed_work);
  169. /**
  170. * queue_delayed_work_on - queue work on specific CPU after delay
  171. * @cpu: CPU number to execute work on
  172. * @wq: workqueue to use
  173. * @dwork: work to queue
  174. * @delay: number of jiffies to wait before queueing
  175. *
  176. * Returns 0 if @work was already on a queue, non-zero otherwise.
  177. */
  178. int queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
  179. struct delayed_work *dwork, unsigned long delay)
  180. {
  181. int ret = 0;
  182. struct timer_list *timer = &dwork->timer;
  183. struct work_struct *work = &dwork->work;
  184. if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
  185. BUG_ON(timer_pending(timer));
  186. BUG_ON(!list_empty(&work->entry));
  187. /* This stores wq for the moment, for the timer_fn */
  188. set_wq_data(work, wq);
  189. timer->expires = jiffies + delay;
  190. timer->data = (unsigned long)dwork;
  191. timer->function = delayed_work_timer_fn;
  192. add_timer_on(timer, cpu);
  193. ret = 1;
  194. }
  195. return ret;
  196. }
  197. EXPORT_SYMBOL_GPL(queue_delayed_work_on);
  198. static void run_workqueue(struct cpu_workqueue_struct *cwq)
  199. {
  200. spin_lock_irq(&cwq->lock);
  201. cwq->run_depth++;
  202. if (cwq->run_depth > 3) {
  203. /* morton gets to eat his hat */
  204. printk("%s: recursion depth exceeded: %d\n",
  205. __FUNCTION__, cwq->run_depth);
  206. dump_stack();
  207. }
  208. while (!list_empty(&cwq->worklist)) {
  209. struct work_struct *work = list_entry(cwq->worklist.next,
  210. struct work_struct, entry);
  211. work_func_t f = work->func;
  212. cwq->current_work = work;
  213. list_del_init(cwq->worklist.next);
  214. spin_unlock_irq(&cwq->lock);
  215. BUG_ON(get_wq_data(work) != cwq);
  216. if (!test_bit(WORK_STRUCT_NOAUTOREL, work_data_bits(work)))
  217. work_release(work);
  218. f(work);
  219. if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
  220. printk(KERN_ERR "BUG: workqueue leaked lock or atomic: "
  221. "%s/0x%08x/%d\n",
  222. current->comm, preempt_count(),
  223. current->pid);
  224. printk(KERN_ERR " last function: ");
  225. print_symbol("%s\n", (unsigned long)f);
  226. debug_show_held_locks(current);
  227. dump_stack();
  228. }
  229. spin_lock_irq(&cwq->lock);
  230. cwq->current_work = NULL;
  231. }
  232. cwq->run_depth--;
  233. spin_unlock_irq(&cwq->lock);
  234. }
  235. /*
  236. * NOTE: the caller must not touch *cwq if this func returns true
  237. */
  238. static int cwq_should_stop(struct cpu_workqueue_struct *cwq)
  239. {
  240. int should_stop = cwq->should_stop;
  241. if (unlikely(should_stop)) {
  242. spin_lock_irq(&cwq->lock);
  243. should_stop = cwq->should_stop && list_empty(&cwq->worklist);
  244. if (should_stop)
  245. cwq->thread = NULL;
  246. spin_unlock_irq(&cwq->lock);
  247. }
  248. return should_stop;
  249. }
  250. static int worker_thread(void *__cwq)
  251. {
  252. struct cpu_workqueue_struct *cwq = __cwq;
  253. DEFINE_WAIT(wait);
  254. struct k_sigaction sa;
  255. sigset_t blocked;
  256. if (!cwq->wq->freezeable)
  257. current->flags |= PF_NOFREEZE;
  258. set_user_nice(current, -5);
  259. /* Block and flush all signals */
  260. sigfillset(&blocked);
  261. sigprocmask(SIG_BLOCK, &blocked, NULL);
  262. flush_signals(current);
  263. /*
  264. * We inherited MPOL_INTERLEAVE from the booting kernel.
  265. * Set MPOL_DEFAULT to insure node local allocations.
  266. */
  267. numa_default_policy();
  268. /* SIG_IGN makes children autoreap: see do_notify_parent(). */
  269. sa.sa.sa_handler = SIG_IGN;
  270. sa.sa.sa_flags = 0;
  271. siginitset(&sa.sa.sa_mask, sigmask(SIGCHLD));
  272. do_sigaction(SIGCHLD, &sa, (struct k_sigaction *)0);
  273. for (;;) {
  274. if (cwq->wq->freezeable)
  275. try_to_freeze();
  276. prepare_to_wait(&cwq->more_work, &wait, TASK_INTERRUPTIBLE);
  277. if (!cwq->should_stop && list_empty(&cwq->worklist))
  278. schedule();
  279. finish_wait(&cwq->more_work, &wait);
  280. if (cwq_should_stop(cwq))
  281. break;
  282. run_workqueue(cwq);
  283. }
  284. return 0;
  285. }
  286. struct wq_barrier {
  287. struct work_struct work;
  288. struct completion done;
  289. };
  290. static void wq_barrier_func(struct work_struct *work)
  291. {
  292. struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
  293. complete(&barr->done);
  294. }
  295. static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
  296. struct wq_barrier *barr, int tail)
  297. {
  298. INIT_WORK(&barr->work, wq_barrier_func);
  299. __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work));
  300. init_completion(&barr->done);
  301. insert_work(cwq, &barr->work, tail);
  302. }
  303. static void flush_cpu_workqueue(struct cpu_workqueue_struct *cwq)
  304. {
  305. if (cwq->thread == current) {
  306. /*
  307. * Probably keventd trying to flush its own queue. So simply run
  308. * it by hand rather than deadlocking.
  309. */
  310. run_workqueue(cwq);
  311. } else {
  312. struct wq_barrier barr;
  313. int active = 0;
  314. spin_lock_irq(&cwq->lock);
  315. if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) {
  316. insert_wq_barrier(cwq, &barr, 1);
  317. active = 1;
  318. }
  319. spin_unlock_irq(&cwq->lock);
  320. if (active)
  321. wait_for_completion(&barr.done);
  322. }
  323. }
  324. /**
  325. * flush_workqueue - ensure that any scheduled work has run to completion.
  326. * @wq: workqueue to flush
  327. *
  328. * Forces execution of the workqueue and blocks until its completion.
  329. * This is typically used in driver shutdown handlers.
  330. *
  331. * We sleep until all works which were queued on entry have been handled,
  332. * but we are not livelocked by new incoming ones.
  333. *
  334. * This function used to run the workqueues itself. Now we just wait for the
  335. * helper threads to do it.
  336. */
  337. void fastcall flush_workqueue(struct workqueue_struct *wq)
  338. {
  339. might_sleep();
  340. if (is_single_threaded(wq))
  341. flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, singlethread_cpu));
  342. else {
  343. int cpu;
  344. for_each_cpu_mask(cpu, cpu_populated_map)
  345. flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu));
  346. }
  347. }
  348. EXPORT_SYMBOL_GPL(flush_workqueue);
  349. static void wait_on_work(struct cpu_workqueue_struct *cwq,
  350. struct work_struct *work)
  351. {
  352. struct wq_barrier barr;
  353. int running = 0;
  354. spin_lock_irq(&cwq->lock);
  355. if (unlikely(cwq->current_work == work)) {
  356. insert_wq_barrier(cwq, &barr, 0);
  357. running = 1;
  358. }
  359. spin_unlock_irq(&cwq->lock);
  360. if (unlikely(running))
  361. wait_for_completion(&barr.done);
  362. }
  363. /**
  364. * flush_work - block until a work_struct's callback has terminated
  365. * @wq: the workqueue on which the work is queued
  366. * @work: the work which is to be flushed
  367. *
  368. * flush_work() will attempt to cancel the work if it is queued. If the work's
  369. * callback appears to be running, flush_work() will block until it has
  370. * completed.
  371. *
  372. * flush_work() is designed to be used when the caller is tearing down data
  373. * structures which the callback function operates upon. It is expected that,
  374. * prior to calling flush_work(), the caller has arranged for the work to not
  375. * be requeued.
  376. */
  377. void flush_work(struct workqueue_struct *wq, struct work_struct *work)
  378. {
  379. struct cpu_workqueue_struct *cwq;
  380. might_sleep();
  381. cwq = get_wq_data(work);
  382. /* Was it ever queued ? */
  383. if (!cwq)
  384. return;
  385. /*
  386. * This work can't be re-queued, no need to re-check that
  387. * get_wq_data() is still the same when we take cwq->lock.
  388. */
  389. spin_lock_irq(&cwq->lock);
  390. list_del_init(&work->entry);
  391. work_release(work);
  392. spin_unlock_irq(&cwq->lock);
  393. if (is_single_threaded(wq))
  394. wait_on_work(per_cpu_ptr(wq->cpu_wq, singlethread_cpu), work);
  395. else {
  396. int cpu;
  397. for_each_cpu_mask(cpu, cpu_populated_map)
  398. wait_on_work(per_cpu_ptr(wq->cpu_wq, cpu), work);
  399. }
  400. }
  401. EXPORT_SYMBOL_GPL(flush_work);
  402. static struct workqueue_struct *keventd_wq;
  403. /**
  404. * schedule_work - put work task in global workqueue
  405. * @work: job to be done
  406. *
  407. * This puts a job in the kernel-global workqueue.
  408. */
  409. int fastcall schedule_work(struct work_struct *work)
  410. {
  411. return queue_work(keventd_wq, work);
  412. }
  413. EXPORT_SYMBOL(schedule_work);
  414. /**
  415. * schedule_delayed_work - put work task in global workqueue after delay
  416. * @dwork: job to be done
  417. * @delay: number of jiffies to wait or 0 for immediate execution
  418. *
  419. * After waiting for a given time this puts a job in the kernel-global
  420. * workqueue.
  421. */
  422. int fastcall schedule_delayed_work(struct delayed_work *dwork,
  423. unsigned long delay)
  424. {
  425. timer_stats_timer_set_start_info(&dwork->timer);
  426. return queue_delayed_work(keventd_wq, dwork, delay);
  427. }
  428. EXPORT_SYMBOL(schedule_delayed_work);
  429. /**
  430. * schedule_delayed_work_on - queue work in global workqueue on CPU after delay
  431. * @cpu: cpu to use
  432. * @dwork: job to be done
  433. * @delay: number of jiffies to wait
  434. *
  435. * After waiting for a given time this puts a job in the kernel-global
  436. * workqueue on the specified CPU.
  437. */
  438. int schedule_delayed_work_on(int cpu,
  439. struct delayed_work *dwork, unsigned long delay)
  440. {
  441. return queue_delayed_work_on(cpu, keventd_wq, dwork, delay);
  442. }
  443. EXPORT_SYMBOL(schedule_delayed_work_on);
  444. /**
  445. * schedule_on_each_cpu - call a function on each online CPU from keventd
  446. * @func: the function to call
  447. *
  448. * Returns zero on success.
  449. * Returns -ve errno on failure.
  450. *
  451. * Appears to be racy against CPU hotplug.
  452. *
  453. * schedule_on_each_cpu() is very slow.
  454. */
  455. int schedule_on_each_cpu(work_func_t func)
  456. {
  457. int cpu;
  458. struct work_struct *works;
  459. works = alloc_percpu(struct work_struct);
  460. if (!works)
  461. return -ENOMEM;
  462. preempt_disable(); /* CPU hotplug */
  463. for_each_online_cpu(cpu) {
  464. struct work_struct *work = per_cpu_ptr(works, cpu);
  465. INIT_WORK(work, func);
  466. set_bit(WORK_STRUCT_PENDING, work_data_bits(work));
  467. __queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), work);
  468. }
  469. preempt_enable();
  470. flush_workqueue(keventd_wq);
  471. free_percpu(works);
  472. return 0;
  473. }
  474. void flush_scheduled_work(void)
  475. {
  476. flush_workqueue(keventd_wq);
  477. }
  478. EXPORT_SYMBOL(flush_scheduled_work);
  479. void flush_work_keventd(struct work_struct *work)
  480. {
  481. flush_work(keventd_wq, work);
  482. }
  483. EXPORT_SYMBOL(flush_work_keventd);
  484. /**
  485. * cancel_rearming_delayed_workqueue - reliably kill off a delayed work whose handler rearms the delayed work.
  486. * @wq: the controlling workqueue structure
  487. * @dwork: the delayed work struct
  488. */
  489. void cancel_rearming_delayed_workqueue(struct workqueue_struct *wq,
  490. struct delayed_work *dwork)
  491. {
  492. while (!cancel_delayed_work(dwork))
  493. flush_workqueue(wq);
  494. }
  495. EXPORT_SYMBOL(cancel_rearming_delayed_workqueue);
  496. /**
  497. * cancel_rearming_delayed_work - reliably kill off a delayed keventd work whose handler rearms the delayed work.
  498. * @dwork: the delayed work struct
  499. */
  500. void cancel_rearming_delayed_work(struct delayed_work *dwork)
  501. {
  502. cancel_rearming_delayed_workqueue(keventd_wq, dwork);
  503. }
  504. EXPORT_SYMBOL(cancel_rearming_delayed_work);
  505. /**
  506. * execute_in_process_context - reliably execute the routine with user context
  507. * @fn: the function to execute
  508. * @ew: guaranteed storage for the execute work structure (must
  509. * be available when the work executes)
  510. *
  511. * Executes the function immediately if process context is available,
  512. * otherwise schedules the function for delayed execution.
  513. *
  514. * Returns: 0 - function was executed
  515. * 1 - function was scheduled for execution
  516. */
  517. int execute_in_process_context(work_func_t fn, struct execute_work *ew)
  518. {
  519. if (!in_interrupt()) {
  520. fn(&ew->work);
  521. return 0;
  522. }
  523. INIT_WORK(&ew->work, fn);
  524. schedule_work(&ew->work);
  525. return 1;
  526. }
  527. EXPORT_SYMBOL_GPL(execute_in_process_context);
  528. int keventd_up(void)
  529. {
  530. return keventd_wq != NULL;
  531. }
  532. int current_is_keventd(void)
  533. {
  534. struct cpu_workqueue_struct *cwq;
  535. int cpu = smp_processor_id(); /* preempt-safe: keventd is per-cpu */
  536. int ret = 0;
  537. BUG_ON(!keventd_wq);
  538. cwq = per_cpu_ptr(keventd_wq->cpu_wq, cpu);
  539. if (current == cwq->thread)
  540. ret = 1;
  541. return ret;
  542. }
  543. static struct cpu_workqueue_struct *
  544. init_cpu_workqueue(struct workqueue_struct *wq, int cpu)
  545. {
  546. struct cpu_workqueue_struct *cwq = per_cpu_ptr(wq->cpu_wq, cpu);
  547. cwq->wq = wq;
  548. spin_lock_init(&cwq->lock);
  549. INIT_LIST_HEAD(&cwq->worklist);
  550. init_waitqueue_head(&cwq->more_work);
  551. return cwq;
  552. }
  553. static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
  554. {
  555. struct workqueue_struct *wq = cwq->wq;
  556. const char *fmt = is_single_threaded(wq) ? "%s" : "%s/%d";
  557. struct task_struct *p;
  558. p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);
  559. /*
  560. * Nobody can add the work_struct to this cwq,
  561. * if (caller is __create_workqueue)
  562. * nobody should see this wq
  563. * else // caller is CPU_UP_PREPARE
  564. * cpu is not on cpu_online_map
  565. * so we can abort safely.
  566. */
  567. if (IS_ERR(p))
  568. return PTR_ERR(p);
  569. cwq->thread = p;
  570. cwq->should_stop = 0;
  571. if (!is_single_threaded(wq))
  572. kthread_bind(p, cpu);
  573. if (is_single_threaded(wq) || cpu_online(cpu))
  574. wake_up_process(p);
  575. return 0;
  576. }
  577. struct workqueue_struct *__create_workqueue(const char *name,
  578. int singlethread, int freezeable)
  579. {
  580. struct workqueue_struct *wq;
  581. struct cpu_workqueue_struct *cwq;
  582. int err = 0, cpu;
  583. wq = kzalloc(sizeof(*wq), GFP_KERNEL);
  584. if (!wq)
  585. return NULL;
  586. wq->cpu_wq = alloc_percpu(struct cpu_workqueue_struct);
  587. if (!wq->cpu_wq) {
  588. kfree(wq);
  589. return NULL;
  590. }
  591. wq->name = name;
  592. wq->freezeable = freezeable;
  593. if (singlethread) {
  594. INIT_LIST_HEAD(&wq->list);
  595. cwq = init_cpu_workqueue(wq, singlethread_cpu);
  596. err = create_workqueue_thread(cwq, singlethread_cpu);
  597. } else {
  598. mutex_lock(&workqueue_mutex);
  599. list_add(&wq->list, &workqueues);
  600. for_each_possible_cpu(cpu) {
  601. cwq = init_cpu_workqueue(wq, cpu);
  602. if (err || !cpu_online(cpu))
  603. continue;
  604. err = create_workqueue_thread(cwq, cpu);
  605. }
  606. mutex_unlock(&workqueue_mutex);
  607. }
  608. if (err) {
  609. destroy_workqueue(wq);
  610. wq = NULL;
  611. }
  612. return wq;
  613. }
  614. EXPORT_SYMBOL_GPL(__create_workqueue);
  615. static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
  616. {
  617. struct wq_barrier barr;
  618. int alive = 0;
  619. spin_lock_irq(&cwq->lock);
  620. if (cwq->thread != NULL) {
  621. insert_wq_barrier(cwq, &barr, 1);
  622. cwq->should_stop = 1;
  623. alive = 1;
  624. }
  625. spin_unlock_irq(&cwq->lock);
  626. if (alive) {
  627. wait_for_completion(&barr.done);
  628. while (unlikely(cwq->thread != NULL))
  629. cpu_relax();
  630. /*
  631. * Wait until cwq->thread unlocks cwq->lock,
  632. * it won't touch *cwq after that.
  633. */
  634. smp_rmb();
  635. spin_unlock_wait(&cwq->lock);
  636. }
  637. }
  638. /**
  639. * destroy_workqueue - safely terminate a workqueue
  640. * @wq: target workqueue
  641. *
  642. * Safely destroy a workqueue. All work currently pending will be done first.
  643. */
  644. void destroy_workqueue(struct workqueue_struct *wq)
  645. {
  646. struct cpu_workqueue_struct *cwq;
  647. if (is_single_threaded(wq)) {
  648. cwq = per_cpu_ptr(wq->cpu_wq, singlethread_cpu);
  649. cleanup_workqueue_thread(cwq, singlethread_cpu);
  650. } else {
  651. int cpu;
  652. mutex_lock(&workqueue_mutex);
  653. list_del(&wq->list);
  654. mutex_unlock(&workqueue_mutex);
  655. for_each_cpu_mask(cpu, cpu_populated_map) {
  656. cwq = per_cpu_ptr(wq->cpu_wq, cpu);
  657. cleanup_workqueue_thread(cwq, cpu);
  658. }
  659. }
  660. free_percpu(wq->cpu_wq);
  661. kfree(wq);
  662. }
  663. EXPORT_SYMBOL_GPL(destroy_workqueue);
  664. static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
  665. unsigned long action,
  666. void *hcpu)
  667. {
  668. unsigned int cpu = (unsigned long)hcpu;
  669. struct cpu_workqueue_struct *cwq;
  670. struct workqueue_struct *wq;
  671. switch (action) {
  672. case CPU_LOCK_ACQUIRE:
  673. mutex_lock(&workqueue_mutex);
  674. return NOTIFY_OK;
  675. case CPU_LOCK_RELEASE:
  676. mutex_unlock(&workqueue_mutex);
  677. return NOTIFY_OK;
  678. case CPU_UP_PREPARE:
  679. cpu_set(cpu, cpu_populated_map);
  680. }
  681. list_for_each_entry(wq, &workqueues, list) {
  682. cwq = per_cpu_ptr(wq->cpu_wq, cpu);
  683. switch (action) {
  684. case CPU_UP_PREPARE:
  685. if (!create_workqueue_thread(cwq, cpu))
  686. break;
  687. printk(KERN_ERR "workqueue for %i failed\n", cpu);
  688. return NOTIFY_BAD;
  689. case CPU_ONLINE:
  690. wake_up_process(cwq->thread);
  691. break;
  692. case CPU_UP_CANCELED:
  693. if (cwq->thread)
  694. wake_up_process(cwq->thread);
  695. case CPU_DEAD:
  696. cleanup_workqueue_thread(cwq, cpu);
  697. break;
  698. }
  699. }
  700. return NOTIFY_OK;
  701. }
  702. void init_workqueues(void)
  703. {
  704. cpu_populated_map = cpu_online_map;
  705. singlethread_cpu = first_cpu(cpu_possible_map);
  706. hotcpu_notifier(workqueue_cpu_callback, 0);
  707. keventd_wq = create_workqueue("events");
  708. BUG_ON(!keventd_wq);
  709. }