slow-work.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761
  1. /* Worker thread pool for slow items, such as filesystem lookups or mkdirs
  2. *
  3. * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
  4. * Written by David Howells (dhowells@redhat.com)
  5. *
  6. * This program is free software; you can redistribute it and/or
  7. * modify it under the terms of the GNU General Public Licence
  8. * as published by the Free Software Foundation; either version
  9. * 2 of the Licence, or (at your option) any later version.
  10. *
  11. * See Documentation/slow-work.txt
  12. */
  13. #include <linux/module.h>
  14. #include <linux/slow-work.h>
  15. #include <linux/kthread.h>
  16. #include <linux/freezer.h>
  17. #include <linux/wait.h>
  18. #define SLOW_WORK_CULL_TIMEOUT (5 * HZ) /* cull threads 5s after running out of
  19. * things to do */
  20. #define SLOW_WORK_OOM_TIMEOUT (5 * HZ) /* can't start new threads for 5s after
  21. * OOM */
  22. #define SLOW_WORK_THREAD_LIMIT 255 /* abs maximum number of slow-work threads */
  23. static void slow_work_cull_timeout(unsigned long);
  24. static void slow_work_oom_timeout(unsigned long);
  25. #ifdef CONFIG_SYSCTL
  26. static int slow_work_min_threads_sysctl(struct ctl_table *, int,
  27. void __user *, size_t *, loff_t *);
  28. static int slow_work_max_threads_sysctl(struct ctl_table *, int ,
  29. void __user *, size_t *, loff_t *);
  30. #endif
  31. /*
  32. * The pool of threads has at least min threads in it as long as someone is
  33. * using the facility, and may have as many as max.
  34. *
  35. * A portion of the pool may be processing very slow operations.
  36. */
  37. static unsigned slow_work_min_threads = 2;
  38. static unsigned slow_work_max_threads = 4;
  39. static unsigned vslow_work_proportion = 50; /* % of threads that may process
  40. * very slow work */
  41. #ifdef CONFIG_SYSCTL
  42. static const int slow_work_min_min_threads = 2;
  43. static int slow_work_max_max_threads = SLOW_WORK_THREAD_LIMIT;
  44. static const int slow_work_min_vslow = 1;
  45. static const int slow_work_max_vslow = 99;
  46. ctl_table slow_work_sysctls[] = {
  47. {
  48. .ctl_name = CTL_UNNUMBERED,
  49. .procname = "min-threads",
  50. .data = &slow_work_min_threads,
  51. .maxlen = sizeof(unsigned),
  52. .mode = 0644,
  53. .proc_handler = slow_work_min_threads_sysctl,
  54. .extra1 = (void *) &slow_work_min_min_threads,
  55. .extra2 = &slow_work_max_threads,
  56. },
  57. {
  58. .ctl_name = CTL_UNNUMBERED,
  59. .procname = "max-threads",
  60. .data = &slow_work_max_threads,
  61. .maxlen = sizeof(unsigned),
  62. .mode = 0644,
  63. .proc_handler = slow_work_max_threads_sysctl,
  64. .extra1 = &slow_work_min_threads,
  65. .extra2 = (void *) &slow_work_max_max_threads,
  66. },
  67. {
  68. .ctl_name = CTL_UNNUMBERED,
  69. .procname = "vslow-percentage",
  70. .data = &vslow_work_proportion,
  71. .maxlen = sizeof(unsigned),
  72. .mode = 0644,
  73. .proc_handler = &proc_dointvec_minmax,
  74. .extra1 = (void *) &slow_work_min_vslow,
  75. .extra2 = (void *) &slow_work_max_vslow,
  76. },
  77. { .ctl_name = 0 }
  78. };
  79. #endif
  80. /*
  81. * The active state of the thread pool
  82. */
  83. static atomic_t slow_work_thread_count;
  84. static atomic_t vslow_work_executing_count;
  85. static bool slow_work_may_not_start_new_thread;
  86. static bool slow_work_cull; /* cull a thread due to lack of activity */
  87. static DEFINE_TIMER(slow_work_cull_timer, slow_work_cull_timeout, 0, 0);
  88. static DEFINE_TIMER(slow_work_oom_timer, slow_work_oom_timeout, 0, 0);
  89. static struct slow_work slow_work_new_thread; /* new thread starter */
  90. /*
  91. * slow work ID allocation (use slow_work_queue_lock)
  92. */
  93. static DECLARE_BITMAP(slow_work_ids, SLOW_WORK_THREAD_LIMIT);
  94. /*
  95. * Unregistration tracking to prevent put_ref() from disappearing during module
  96. * unload
  97. */
  98. #ifdef CONFIG_MODULES
  99. static struct module *slow_work_thread_processing[SLOW_WORK_THREAD_LIMIT];
  100. static struct module *slow_work_unreg_module;
  101. static struct slow_work *slow_work_unreg_work_item;
  102. static DECLARE_WAIT_QUEUE_HEAD(slow_work_unreg_wq);
  103. static DEFINE_MUTEX(slow_work_unreg_sync_lock);
  104. #endif
  105. /*
  106. * The queues of work items and the lock governing access to them. These are
  107. * shared between all the CPUs. It doesn't make sense to have per-CPU queues
  108. * as the number of threads bears no relation to the number of CPUs.
  109. *
  110. * There are two queues of work items: one for slow work items, and one for
  111. * very slow work items.
  112. */
  113. static LIST_HEAD(slow_work_queue);
  114. static LIST_HEAD(vslow_work_queue);
  115. static DEFINE_SPINLOCK(slow_work_queue_lock);
  116. /*
  117. * The thread controls. A variable used to signal to the threads that they
  118. * should exit when the queue is empty, a waitqueue used by the threads to wait
  119. * for signals, and a completion set by the last thread to exit.
  120. */
  121. static bool slow_work_threads_should_exit;
  122. static DECLARE_WAIT_QUEUE_HEAD(slow_work_thread_wq);
  123. static DECLARE_COMPLETION(slow_work_last_thread_exited);
  124. /*
  125. * The number of users of the thread pool and its lock. Whilst this is zero we
  126. * have no threads hanging around, and when this reaches zero, we wait for all
  127. * active or queued work items to complete and kill all the threads we do have.
  128. */
  129. static int slow_work_user_count;
  130. static DEFINE_MUTEX(slow_work_user_lock);
  131. static inline int slow_work_get_ref(struct slow_work *work)
  132. {
  133. if (work->ops->get_ref)
  134. return work->ops->get_ref(work);
  135. return 0;
  136. }
  137. static inline void slow_work_put_ref(struct slow_work *work)
  138. {
  139. if (work->ops->put_ref)
  140. work->ops->put_ref(work);
  141. }
  142. /*
  143. * Calculate the maximum number of active threads in the pool that are
  144. * permitted to process very slow work items.
  145. *
  146. * The answer is rounded up to at least 1, but may not equal or exceed the
  147. * maximum number of the threads in the pool. This means we always have at
  148. * least one thread that can process slow work items, and we always have at
  149. * least one thread that won't get tied up doing so.
  150. */
  151. static unsigned slow_work_calc_vsmax(void)
  152. {
  153. unsigned vsmax;
  154. vsmax = atomic_read(&slow_work_thread_count) * vslow_work_proportion;
  155. vsmax /= 100;
  156. vsmax = max(vsmax, 1U);
  157. return min(vsmax, slow_work_max_threads - 1);
  158. }
  159. /*
  160. * Attempt to execute stuff queued on a slow thread. Return true if we managed
  161. * it, false if there was nothing to do.
  162. */
  163. static bool slow_work_execute(int id)
  164. {
  165. #ifdef CONFIG_MODULES
  166. struct module *module;
  167. #endif
  168. struct slow_work *work = NULL;
  169. unsigned vsmax;
  170. bool very_slow;
  171. vsmax = slow_work_calc_vsmax();
  172. /* see if we can schedule a new thread to be started if we're not
  173. * keeping up with the work */
  174. if (!waitqueue_active(&slow_work_thread_wq) &&
  175. (!list_empty(&slow_work_queue) || !list_empty(&vslow_work_queue)) &&
  176. atomic_read(&slow_work_thread_count) < slow_work_max_threads &&
  177. !slow_work_may_not_start_new_thread)
  178. slow_work_enqueue(&slow_work_new_thread);
  179. /* find something to execute */
  180. spin_lock_irq(&slow_work_queue_lock);
  181. if (!list_empty(&vslow_work_queue) &&
  182. atomic_read(&vslow_work_executing_count) < vsmax) {
  183. work = list_entry(vslow_work_queue.next,
  184. struct slow_work, link);
  185. if (test_and_set_bit_lock(SLOW_WORK_EXECUTING, &work->flags))
  186. BUG();
  187. list_del_init(&work->link);
  188. atomic_inc(&vslow_work_executing_count);
  189. very_slow = true;
  190. } else if (!list_empty(&slow_work_queue)) {
  191. work = list_entry(slow_work_queue.next,
  192. struct slow_work, link);
  193. if (test_and_set_bit_lock(SLOW_WORK_EXECUTING, &work->flags))
  194. BUG();
  195. list_del_init(&work->link);
  196. very_slow = false;
  197. } else {
  198. very_slow = false; /* avoid the compiler warning */
  199. }
  200. #ifdef CONFIG_MODULES
  201. if (work)
  202. slow_work_thread_processing[id] = work->owner;
  203. #endif
  204. spin_unlock_irq(&slow_work_queue_lock);
  205. if (!work)
  206. return false;
  207. if (!test_and_clear_bit(SLOW_WORK_PENDING, &work->flags))
  208. BUG();
  209. work->ops->execute(work);
  210. if (very_slow)
  211. atomic_dec(&vslow_work_executing_count);
  212. clear_bit_unlock(SLOW_WORK_EXECUTING, &work->flags);
  213. /* if someone tried to enqueue the item whilst we were executing it,
  214. * then it'll be left unenqueued to avoid multiple threads trying to
  215. * execute it simultaneously
  216. *
  217. * there is, however, a race between us testing the pending flag and
  218. * getting the spinlock, and between the enqueuer setting the pending
  219. * flag and getting the spinlock, so we use a deferral bit to tell us
  220. * if the enqueuer got there first
  221. */
  222. if (test_bit(SLOW_WORK_PENDING, &work->flags)) {
  223. spin_lock_irq(&slow_work_queue_lock);
  224. if (!test_bit(SLOW_WORK_EXECUTING, &work->flags) &&
  225. test_and_clear_bit(SLOW_WORK_ENQ_DEFERRED, &work->flags))
  226. goto auto_requeue;
  227. spin_unlock_irq(&slow_work_queue_lock);
  228. }
  229. /* sort out the race between module unloading and put_ref() */
  230. slow_work_put_ref(work);
  231. #ifdef CONFIG_MODULES
  232. module = slow_work_thread_processing[id];
  233. slow_work_thread_processing[id] = NULL;
  234. smp_mb();
  235. if (slow_work_unreg_work_item == work ||
  236. slow_work_unreg_module == module)
  237. wake_up_all(&slow_work_unreg_wq);
  238. #endif
  239. return true;
  240. auto_requeue:
  241. /* we must complete the enqueue operation
  242. * - we transfer our ref on the item back to the appropriate queue
  243. * - don't wake another thread up as we're awake already
  244. */
  245. if (test_bit(SLOW_WORK_VERY_SLOW, &work->flags))
  246. list_add_tail(&work->link, &vslow_work_queue);
  247. else
  248. list_add_tail(&work->link, &slow_work_queue);
  249. spin_unlock_irq(&slow_work_queue_lock);
  250. slow_work_thread_processing[id] = NULL;
  251. return true;
  252. }
  253. /**
  254. * slow_work_enqueue - Schedule a slow work item for processing
  255. * @work: The work item to queue
  256. *
  257. * Schedule a slow work item for processing. If the item is already undergoing
  258. * execution, this guarantees not to re-enter the execution routine until the
  259. * first execution finishes.
  260. *
  261. * The item is pinned by this function as it retains a reference to it, managed
  262. * through the item operations. The item is unpinned once it has been
  263. * executed.
  264. *
  265. * An item may hog the thread that is running it for a relatively large amount
  266. * of time, sufficient, for example, to perform several lookup, mkdir, create
  267. * and setxattr operations. It may sleep on I/O and may sleep to obtain locks.
  268. *
  269. * Conversely, if a number of items are awaiting processing, it may take some
  270. * time before any given item is given attention. The number of threads in the
  271. * pool may be increased to deal with demand, but only up to a limit.
  272. *
  273. * If SLOW_WORK_VERY_SLOW is set on the work item, then it will be placed in
  274. * the very slow queue, from which only a portion of the threads will be
  275. * allowed to pick items to execute. This ensures that very slow items won't
  276. * overly block ones that are just ordinarily slow.
  277. *
  278. * Returns 0 if successful, -EAGAIN if not.
  279. */
  280. int slow_work_enqueue(struct slow_work *work)
  281. {
  282. unsigned long flags;
  283. BUG_ON(slow_work_user_count <= 0);
  284. BUG_ON(!work);
  285. BUG_ON(!work->ops);
  286. /* when honouring an enqueue request, we only promise that we will run
  287. * the work function in the future; we do not promise to run it once
  288. * per enqueue request
  289. *
  290. * we use the PENDING bit to merge together repeat requests without
  291. * having to disable IRQs and take the spinlock, whilst still
  292. * maintaining our promise
  293. */
  294. if (!test_and_set_bit_lock(SLOW_WORK_PENDING, &work->flags)) {
  295. spin_lock_irqsave(&slow_work_queue_lock, flags);
  296. /* we promise that we will not attempt to execute the work
  297. * function in more than one thread simultaneously
  298. *
  299. * this, however, leaves us with a problem if we're asked to
  300. * enqueue the work whilst someone is executing the work
  301. * function as simply queueing the work immediately means that
  302. * another thread may try executing it whilst it is already
  303. * under execution
  304. *
  305. * to deal with this, we set the ENQ_DEFERRED bit instead of
  306. * enqueueing, and the thread currently executing the work
  307. * function will enqueue the work item when the work function
  308. * returns and it has cleared the EXECUTING bit
  309. */
  310. if (test_bit(SLOW_WORK_EXECUTING, &work->flags)) {
  311. set_bit(SLOW_WORK_ENQ_DEFERRED, &work->flags);
  312. } else {
  313. if (slow_work_get_ref(work) < 0)
  314. goto cant_get_ref;
  315. if (test_bit(SLOW_WORK_VERY_SLOW, &work->flags))
  316. list_add_tail(&work->link, &vslow_work_queue);
  317. else
  318. list_add_tail(&work->link, &slow_work_queue);
  319. wake_up(&slow_work_thread_wq);
  320. }
  321. spin_unlock_irqrestore(&slow_work_queue_lock, flags);
  322. }
  323. return 0;
  324. cant_get_ref:
  325. spin_unlock_irqrestore(&slow_work_queue_lock, flags);
  326. return -EAGAIN;
  327. }
  328. EXPORT_SYMBOL(slow_work_enqueue);
  329. /*
  330. * Schedule a cull of the thread pool at some time in the near future
  331. */
  332. static void slow_work_schedule_cull(void)
  333. {
  334. mod_timer(&slow_work_cull_timer,
  335. round_jiffies(jiffies + SLOW_WORK_CULL_TIMEOUT));
  336. }
  337. /*
  338. * Worker thread culling algorithm
  339. */
  340. static bool slow_work_cull_thread(void)
  341. {
  342. unsigned long flags;
  343. bool do_cull = false;
  344. spin_lock_irqsave(&slow_work_queue_lock, flags);
  345. if (slow_work_cull) {
  346. slow_work_cull = false;
  347. if (list_empty(&slow_work_queue) &&
  348. list_empty(&vslow_work_queue) &&
  349. atomic_read(&slow_work_thread_count) >
  350. slow_work_min_threads) {
  351. slow_work_schedule_cull();
  352. do_cull = true;
  353. }
  354. }
  355. spin_unlock_irqrestore(&slow_work_queue_lock, flags);
  356. return do_cull;
  357. }
  358. /*
  359. * Determine if there is slow work available for dispatch
  360. */
  361. static inline bool slow_work_available(int vsmax)
  362. {
  363. return !list_empty(&slow_work_queue) ||
  364. (!list_empty(&vslow_work_queue) &&
  365. atomic_read(&vslow_work_executing_count) < vsmax);
  366. }
  367. /*
  368. * Worker thread dispatcher
  369. */
  370. static int slow_work_thread(void *_data)
  371. {
  372. int vsmax, id;
  373. DEFINE_WAIT(wait);
  374. set_freezable();
  375. set_user_nice(current, -5);
  376. /* allocate ourselves an ID */
  377. spin_lock_irq(&slow_work_queue_lock);
  378. id = find_first_zero_bit(slow_work_ids, SLOW_WORK_THREAD_LIMIT);
  379. BUG_ON(id < 0 || id >= SLOW_WORK_THREAD_LIMIT);
  380. __set_bit(id, slow_work_ids);
  381. spin_unlock_irq(&slow_work_queue_lock);
  382. sprintf(current->comm, "kslowd%03u", id);
  383. for (;;) {
  384. vsmax = vslow_work_proportion;
  385. vsmax *= atomic_read(&slow_work_thread_count);
  386. vsmax /= 100;
  387. prepare_to_wait_exclusive(&slow_work_thread_wq, &wait,
  388. TASK_INTERRUPTIBLE);
  389. if (!freezing(current) &&
  390. !slow_work_threads_should_exit &&
  391. !slow_work_available(vsmax) &&
  392. !slow_work_cull)
  393. schedule();
  394. finish_wait(&slow_work_thread_wq, &wait);
  395. try_to_freeze();
  396. vsmax = vslow_work_proportion;
  397. vsmax *= atomic_read(&slow_work_thread_count);
  398. vsmax /= 100;
  399. if (slow_work_available(vsmax) && slow_work_execute(id)) {
  400. cond_resched();
  401. if (list_empty(&slow_work_queue) &&
  402. list_empty(&vslow_work_queue) &&
  403. atomic_read(&slow_work_thread_count) >
  404. slow_work_min_threads)
  405. slow_work_schedule_cull();
  406. continue;
  407. }
  408. if (slow_work_threads_should_exit)
  409. break;
  410. if (slow_work_cull && slow_work_cull_thread())
  411. break;
  412. }
  413. spin_lock_irq(&slow_work_queue_lock);
  414. __clear_bit(id, slow_work_ids);
  415. spin_unlock_irq(&slow_work_queue_lock);
  416. if (atomic_dec_and_test(&slow_work_thread_count))
  417. complete_and_exit(&slow_work_last_thread_exited, 0);
  418. return 0;
  419. }
  420. /*
  421. * Handle thread cull timer expiration
  422. */
  423. static void slow_work_cull_timeout(unsigned long data)
  424. {
  425. slow_work_cull = true;
  426. wake_up(&slow_work_thread_wq);
  427. }
  428. /*
  429. * Start a new slow work thread
  430. */
  431. static void slow_work_new_thread_execute(struct slow_work *work)
  432. {
  433. struct task_struct *p;
  434. if (slow_work_threads_should_exit)
  435. return;
  436. if (atomic_read(&slow_work_thread_count) >= slow_work_max_threads)
  437. return;
  438. if (!mutex_trylock(&slow_work_user_lock))
  439. return;
  440. slow_work_may_not_start_new_thread = true;
  441. atomic_inc(&slow_work_thread_count);
  442. p = kthread_run(slow_work_thread, NULL, "kslowd");
  443. if (IS_ERR(p)) {
  444. printk(KERN_DEBUG "Slow work thread pool: OOM\n");
  445. if (atomic_dec_and_test(&slow_work_thread_count))
  446. BUG(); /* we're running on a slow work thread... */
  447. mod_timer(&slow_work_oom_timer,
  448. round_jiffies(jiffies + SLOW_WORK_OOM_TIMEOUT));
  449. } else {
  450. /* ratelimit the starting of new threads */
  451. mod_timer(&slow_work_oom_timer, jiffies + 1);
  452. }
  453. mutex_unlock(&slow_work_user_lock);
  454. }
  455. static const struct slow_work_ops slow_work_new_thread_ops = {
  456. .owner = THIS_MODULE,
  457. .execute = slow_work_new_thread_execute,
  458. };
  459. /*
  460. * post-OOM new thread start suppression expiration
  461. */
  462. static void slow_work_oom_timeout(unsigned long data)
  463. {
  464. slow_work_may_not_start_new_thread = false;
  465. }
  466. #ifdef CONFIG_SYSCTL
  467. /*
  468. * Handle adjustment of the minimum number of threads
  469. */
  470. static int slow_work_min_threads_sysctl(struct ctl_table *table, int write,
  471. void __user *buffer,
  472. size_t *lenp, loff_t *ppos)
  473. {
  474. int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  475. int n;
  476. if (ret == 0) {
  477. mutex_lock(&slow_work_user_lock);
  478. if (slow_work_user_count > 0) {
  479. /* see if we need to start or stop threads */
  480. n = atomic_read(&slow_work_thread_count) -
  481. slow_work_min_threads;
  482. if (n < 0 && !slow_work_may_not_start_new_thread)
  483. slow_work_enqueue(&slow_work_new_thread);
  484. else if (n > 0)
  485. slow_work_schedule_cull();
  486. }
  487. mutex_unlock(&slow_work_user_lock);
  488. }
  489. return ret;
  490. }
  491. /*
  492. * Handle adjustment of the maximum number of threads
  493. */
  494. static int slow_work_max_threads_sysctl(struct ctl_table *table, int write,
  495. void __user *buffer,
  496. size_t *lenp, loff_t *ppos)
  497. {
  498. int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  499. int n;
  500. if (ret == 0) {
  501. mutex_lock(&slow_work_user_lock);
  502. if (slow_work_user_count > 0) {
  503. /* see if we need to stop threads */
  504. n = slow_work_max_threads -
  505. atomic_read(&slow_work_thread_count);
  506. if (n < 0)
  507. slow_work_schedule_cull();
  508. }
  509. mutex_unlock(&slow_work_user_lock);
  510. }
  511. return ret;
  512. }
  513. #endif /* CONFIG_SYSCTL */
  514. /**
  515. * slow_work_register_user - Register a user of the facility
  516. * @module: The module about to make use of the facility
  517. *
  518. * Register a user of the facility, starting up the initial threads if there
  519. * aren't any other users at this point. This will return 0 if successful, or
  520. * an error if not.
  521. */
  522. int slow_work_register_user(struct module *module)
  523. {
  524. struct task_struct *p;
  525. int loop;
  526. mutex_lock(&slow_work_user_lock);
  527. if (slow_work_user_count == 0) {
  528. printk(KERN_NOTICE "Slow work thread pool: Starting up\n");
  529. init_completion(&slow_work_last_thread_exited);
  530. slow_work_threads_should_exit = false;
  531. slow_work_init(&slow_work_new_thread,
  532. &slow_work_new_thread_ops);
  533. slow_work_may_not_start_new_thread = false;
  534. slow_work_cull = false;
  535. /* start the minimum number of threads */
  536. for (loop = 0; loop < slow_work_min_threads; loop++) {
  537. atomic_inc(&slow_work_thread_count);
  538. p = kthread_run(slow_work_thread, NULL, "kslowd");
  539. if (IS_ERR(p))
  540. goto error;
  541. }
  542. printk(KERN_NOTICE "Slow work thread pool: Ready\n");
  543. }
  544. slow_work_user_count++;
  545. mutex_unlock(&slow_work_user_lock);
  546. return 0;
  547. error:
  548. if (atomic_dec_and_test(&slow_work_thread_count))
  549. complete(&slow_work_last_thread_exited);
  550. if (loop > 0) {
  551. printk(KERN_ERR "Slow work thread pool:"
  552. " Aborting startup on ENOMEM\n");
  553. slow_work_threads_should_exit = true;
  554. wake_up_all(&slow_work_thread_wq);
  555. wait_for_completion(&slow_work_last_thread_exited);
  556. printk(KERN_ERR "Slow work thread pool: Aborted\n");
  557. }
  558. mutex_unlock(&slow_work_user_lock);
  559. return PTR_ERR(p);
  560. }
  561. EXPORT_SYMBOL(slow_work_register_user);
  562. /*
  563. * wait for all outstanding items from the calling module to complete
  564. * - note that more items may be queued whilst we're waiting
  565. */
  566. static void slow_work_wait_for_items(struct module *module)
  567. {
  568. DECLARE_WAITQUEUE(myself, current);
  569. struct slow_work *work;
  570. int loop;
  571. mutex_lock(&slow_work_unreg_sync_lock);
  572. add_wait_queue(&slow_work_unreg_wq, &myself);
  573. for (;;) {
  574. spin_lock_irq(&slow_work_queue_lock);
  575. /* first of all, we wait for the last queued item in each list
  576. * to be processed */
  577. list_for_each_entry_reverse(work, &vslow_work_queue, link) {
  578. if (work->owner == module) {
  579. set_current_state(TASK_UNINTERRUPTIBLE);
  580. slow_work_unreg_work_item = work;
  581. goto do_wait;
  582. }
  583. }
  584. list_for_each_entry_reverse(work, &slow_work_queue, link) {
  585. if (work->owner == module) {
  586. set_current_state(TASK_UNINTERRUPTIBLE);
  587. slow_work_unreg_work_item = work;
  588. goto do_wait;
  589. }
  590. }
  591. /* then we wait for the items being processed to finish */
  592. slow_work_unreg_module = module;
  593. smp_mb();
  594. for (loop = 0; loop < SLOW_WORK_THREAD_LIMIT; loop++) {
  595. if (slow_work_thread_processing[loop] == module)
  596. goto do_wait;
  597. }
  598. spin_unlock_irq(&slow_work_queue_lock);
  599. break; /* okay, we're done */
  600. do_wait:
  601. spin_unlock_irq(&slow_work_queue_lock);
  602. schedule();
  603. slow_work_unreg_work_item = NULL;
  604. slow_work_unreg_module = NULL;
  605. }
  606. remove_wait_queue(&slow_work_unreg_wq, &myself);
  607. mutex_unlock(&slow_work_unreg_sync_lock);
  608. }
  609. /**
  610. * slow_work_unregister_user - Unregister a user of the facility
  611. * @module: The module whose items should be cleared
  612. *
  613. * Unregister a user of the facility, killing all the threads if this was the
  614. * last one.
  615. *
  616. * This waits for all the work items belonging to the nominated module to go
  617. * away before proceeding.
  618. */
  619. void slow_work_unregister_user(struct module *module)
  620. {
  621. /* first of all, wait for all outstanding items from the calling module
  622. * to complete */
  623. if (module)
  624. slow_work_wait_for_items(module);
  625. /* then we can actually go about shutting down the facility if need
  626. * be */
  627. mutex_lock(&slow_work_user_lock);
  628. BUG_ON(slow_work_user_count <= 0);
  629. slow_work_user_count--;
  630. if (slow_work_user_count == 0) {
  631. printk(KERN_NOTICE "Slow work thread pool: Shutting down\n");
  632. slow_work_threads_should_exit = true;
  633. del_timer_sync(&slow_work_cull_timer);
  634. del_timer_sync(&slow_work_oom_timer);
  635. wake_up_all(&slow_work_thread_wq);
  636. wait_for_completion(&slow_work_last_thread_exited);
  637. printk(KERN_NOTICE "Slow work thread pool:"
  638. " Shut down complete\n");
  639. }
  640. mutex_unlock(&slow_work_user_lock);
  641. }
  642. EXPORT_SYMBOL(slow_work_unregister_user);
  643. /*
  644. * Initialise the slow work facility
  645. */
  646. static int __init init_slow_work(void)
  647. {
  648. unsigned nr_cpus = num_possible_cpus();
  649. if (slow_work_max_threads < nr_cpus)
  650. slow_work_max_threads = nr_cpus;
  651. #ifdef CONFIG_SYSCTL
  652. if (slow_work_max_max_threads < nr_cpus * 2)
  653. slow_work_max_max_threads = nr_cpus * 2;
  654. #endif
  655. return 0;
  656. }
  657. subsys_initcall(init_slow_work);