mutex.c 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962
  1. /*
  2. * kernel/mutex.c
  3. *
  4. * Mutexes: blocking mutual exclusion locks
  5. *
  6. * Started by Ingo Molnar:
  7. *
  8. * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
  9. *
  10. * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
  11. * David Howells for suggestions and improvements.
  12. *
  13. * - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
  14. * from the -rt tree, where it was originally implemented for rtmutexes
  15. * by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
  16. * and Sven Dietrich.
  17. *
  18. * Also see Documentation/mutex-design.txt.
  19. */
  20. #include <linux/mutex.h>
  21. #include <linux/sched.h>
  22. #include <linux/sched/rt.h>
  23. #include <linux/export.h>
  24. #include <linux/spinlock.h>
  25. #include <linux/interrupt.h>
  26. #include <linux/debug_locks.h>
  27. /*
  28. * In the DEBUG case we are using the "NULL fastpath" for mutexes,
  29. * which forces all calls into the slowpath:
  30. */
  31. #ifdef CONFIG_DEBUG_MUTEXES
  32. # include "mutex-debug.h"
  33. # include <asm-generic/mutex-null.h>
  34. #else
  35. # include "mutex.h"
  36. # include <asm/mutex.h>
  37. #endif
  38. /*
  39. * A negative mutex count indicates that waiters are sleeping waiting for the
  40. * mutex.
  41. */
  42. #define MUTEX_SHOW_NO_WAITER(mutex) (atomic_read(&(mutex)->count) >= 0)
  43. void
  44. __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
  45. {
  46. atomic_set(&lock->count, 1);
  47. spin_lock_init(&lock->wait_lock);
  48. INIT_LIST_HEAD(&lock->wait_list);
  49. mutex_clear_owner(lock);
  50. #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
  51. lock->spin_mlock = NULL;
  52. #endif
  53. debug_mutex_init(lock, name, key);
  54. }
  55. EXPORT_SYMBOL(__mutex_init);
  56. #ifndef CONFIG_DEBUG_LOCK_ALLOC
  57. /*
  58. * We split the mutex lock/unlock logic into separate fastpath and
  59. * slowpath functions, to reduce the register pressure on the fastpath.
  60. * We also put the fastpath first in the kernel image, to make sure the
  61. * branch is predicted by the CPU as default-untaken.
  62. */
  63. static __used noinline void __sched
  64. __mutex_lock_slowpath(atomic_t *lock_count);
  65. /**
  66. * mutex_lock - acquire the mutex
  67. * @lock: the mutex to be acquired
  68. *
  69. * Lock the mutex exclusively for this task. If the mutex is not
  70. * available right now, it will sleep until it can get it.
  71. *
  72. * The mutex must later on be released by the same task that
  73. * acquired it. Recursive locking is not allowed. The task
  74. * may not exit without first unlocking the mutex. Also, kernel
  75. * memory where the mutex resides mutex must not be freed with
  76. * the mutex still locked. The mutex must first be initialized
  77. * (or statically defined) before it can be locked. memset()-ing
  78. * the mutex to 0 is not allowed.
  79. *
  80. * ( The CONFIG_DEBUG_MUTEXES .config option turns on debugging
  81. * checks that will enforce the restrictions and will also do
  82. * deadlock debugging. )
  83. *
  84. * This function is similar to (but not equivalent to) down().
  85. */
  86. void __sched mutex_lock(struct mutex *lock)
  87. {
  88. might_sleep();
  89. /*
  90. * The locking fastpath is the 1->0 transition from
  91. * 'unlocked' into 'locked' state.
  92. */
  93. __mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
  94. mutex_set_owner(lock);
  95. }
  96. EXPORT_SYMBOL(mutex_lock);
  97. #endif
  98. #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
  99. /*
  100. * In order to avoid a stampede of mutex spinners from acquiring the mutex
  101. * more or less simultaneously, the spinners need to acquire a MCS lock
  102. * first before spinning on the owner field.
  103. *
  104. * We don't inline mspin_lock() so that perf can correctly account for the
  105. * time spent in this lock function.
  106. */
  107. struct mspin_node {
  108. struct mspin_node *next ;
  109. int locked; /* 1 if lock acquired */
  110. };
  111. #define MLOCK(mutex) ((struct mspin_node **)&((mutex)->spin_mlock))
  112. static noinline
  113. void mspin_lock(struct mspin_node **lock, struct mspin_node *node)
  114. {
  115. struct mspin_node *prev;
  116. /* Init node */
  117. node->locked = 0;
  118. node->next = NULL;
  119. prev = xchg(lock, node);
  120. if (likely(prev == NULL)) {
  121. /* Lock acquired */
  122. node->locked = 1;
  123. return;
  124. }
  125. ACCESS_ONCE(prev->next) = node;
  126. smp_wmb();
  127. /* Wait until the lock holder passes the lock down */
  128. while (!ACCESS_ONCE(node->locked))
  129. arch_mutex_cpu_relax();
  130. }
  131. static void mspin_unlock(struct mspin_node **lock, struct mspin_node *node)
  132. {
  133. struct mspin_node *next = ACCESS_ONCE(node->next);
  134. if (likely(!next)) {
  135. /*
  136. * Release the lock by setting it to NULL
  137. */
  138. if (cmpxchg(lock, node, NULL) == node)
  139. return;
  140. /* Wait until the next pointer is set */
  141. while (!(next = ACCESS_ONCE(node->next)))
  142. arch_mutex_cpu_relax();
  143. }
  144. ACCESS_ONCE(next->locked) = 1;
  145. smp_wmb();
  146. }
  147. /*
  148. * Mutex spinning code migrated from kernel/sched/core.c
  149. */
  150. static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
  151. {
  152. if (lock->owner != owner)
  153. return false;
  154. /*
  155. * Ensure we emit the owner->on_cpu, dereference _after_ checking
  156. * lock->owner still matches owner, if that fails, owner might
  157. * point to free()d memory, if it still matches, the rcu_read_lock()
  158. * ensures the memory stays valid.
  159. */
  160. barrier();
  161. return owner->on_cpu;
  162. }
  163. /*
  164. * Look out! "owner" is an entirely speculative pointer
  165. * access and not reliable.
  166. */
  167. static noinline
  168. int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
  169. {
  170. rcu_read_lock();
  171. while (owner_running(lock, owner)) {
  172. if (need_resched())
  173. break;
  174. arch_mutex_cpu_relax();
  175. }
  176. rcu_read_unlock();
  177. /*
  178. * We break out the loop above on need_resched() and when the
  179. * owner changed, which is a sign for heavy contention. Return
  180. * success only when lock->owner is NULL.
  181. */
  182. return lock->owner == NULL;
  183. }
  184. /*
  185. * Initial check for entering the mutex spinning loop
  186. */
  187. static inline int mutex_can_spin_on_owner(struct mutex *lock)
  188. {
  189. int retval = 1;
  190. rcu_read_lock();
  191. if (lock->owner)
  192. retval = lock->owner->on_cpu;
  193. rcu_read_unlock();
  194. /*
  195. * if lock->owner is not set, the mutex owner may have just acquired
  196. * it and not set the owner yet or the mutex has been released.
  197. */
  198. return retval;
  199. }
  200. #endif
  201. static __used noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count);
  202. /**
  203. * mutex_unlock - release the mutex
  204. * @lock: the mutex to be released
  205. *
  206. * Unlock a mutex that has been locked by this task previously.
  207. *
  208. * This function must not be used in interrupt context. Unlocking
  209. * of a not locked mutex is not allowed.
  210. *
  211. * This function is similar to (but not equivalent to) up().
  212. */
  213. void __sched mutex_unlock(struct mutex *lock)
  214. {
  215. /*
  216. * The unlocking fastpath is the 0->1 transition from 'locked'
  217. * into 'unlocked' state:
  218. */
  219. #ifndef CONFIG_DEBUG_MUTEXES
  220. /*
  221. * When debugging is enabled we must not clear the owner before time,
  222. * the slow path will always be taken, and that clears the owner field
  223. * after verifying that it was indeed current.
  224. */
  225. mutex_clear_owner(lock);
  226. #endif
  227. __mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
  228. }
  229. EXPORT_SYMBOL(mutex_unlock);
  230. /**
  231. * ww_mutex_unlock - release the w/w mutex
  232. * @lock: the mutex to be released
  233. *
  234. * Unlock a mutex that has been locked by this task previously with any of the
  235. * ww_mutex_lock* functions (with or without an acquire context). It is
  236. * forbidden to release the locks after releasing the acquire context.
  237. *
  238. * This function must not be used in interrupt context. Unlocking
  239. * of a unlocked mutex is not allowed.
  240. */
  241. void __sched ww_mutex_unlock(struct ww_mutex *lock)
  242. {
  243. /*
  244. * The unlocking fastpath is the 0->1 transition from 'locked'
  245. * into 'unlocked' state:
  246. */
  247. if (lock->ctx) {
  248. #ifdef CONFIG_DEBUG_MUTEXES
  249. DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
  250. #endif
  251. if (lock->ctx->acquired > 0)
  252. lock->ctx->acquired--;
  253. lock->ctx = NULL;
  254. }
  255. #ifndef CONFIG_DEBUG_MUTEXES
  256. /*
  257. * When debugging is enabled we must not clear the owner before time,
  258. * the slow path will always be taken, and that clears the owner field
  259. * after verifying that it was indeed current.
  260. */
  261. mutex_clear_owner(&lock->base);
  262. #endif
  263. __mutex_fastpath_unlock(&lock->base.count, __mutex_unlock_slowpath);
  264. }
  265. EXPORT_SYMBOL(ww_mutex_unlock);
  266. static inline int __sched
  267. __mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx)
  268. {
  269. struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
  270. struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
  271. if (!hold_ctx)
  272. return 0;
  273. if (unlikely(ctx == hold_ctx))
  274. return -EALREADY;
  275. if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
  276. (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
  277. #ifdef CONFIG_DEBUG_MUTEXES
  278. DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
  279. ctx->contending_lock = ww;
  280. #endif
  281. return -EDEADLK;
  282. }
  283. return 0;
  284. }
  285. static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
  286. struct ww_acquire_ctx *ww_ctx)
  287. {
  288. #ifdef CONFIG_DEBUG_MUTEXES
  289. /*
  290. * If this WARN_ON triggers, you used ww_mutex_lock to acquire,
  291. * but released with a normal mutex_unlock in this call.
  292. *
  293. * This should never happen, always use ww_mutex_unlock.
  294. */
  295. DEBUG_LOCKS_WARN_ON(ww->ctx);
  296. /*
  297. * Not quite done after calling ww_acquire_done() ?
  298. */
  299. DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);
  300. if (ww_ctx->contending_lock) {
  301. /*
  302. * After -EDEADLK you tried to
  303. * acquire a different ww_mutex? Bad!
  304. */
  305. DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);
  306. /*
  307. * You called ww_mutex_lock after receiving -EDEADLK,
  308. * but 'forgot' to unlock everything else first?
  309. */
  310. DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
  311. ww_ctx->contending_lock = NULL;
  312. }
  313. /*
  314. * Naughty, using a different class will lead to undefined behavior!
  315. */
  316. DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
  317. #endif
  318. ww_ctx->acquired++;
  319. }
  320. /*
  321. * after acquiring lock with fastpath or when we lost out in contested
  322. * slowpath, set ctx and wake up any waiters so they can recheck.
  323. *
  324. * This function is never called when CONFIG_DEBUG_LOCK_ALLOC is set,
  325. * as the fastpath and opportunistic spinning are disabled in that case.
  326. */
  327. static __always_inline void
  328. ww_mutex_set_context_fastpath(struct ww_mutex *lock,
  329. struct ww_acquire_ctx *ctx)
  330. {
  331. unsigned long flags;
  332. struct mutex_waiter *cur;
  333. ww_mutex_lock_acquired(lock, ctx);
  334. lock->ctx = ctx;
  335. /*
  336. * The lock->ctx update should be visible on all cores before
  337. * the atomic read is done, otherwise contended waiters might be
  338. * missed. The contended waiters will either see ww_ctx == NULL
  339. * and keep spinning, or it will acquire wait_lock, add itself
  340. * to waiter list and sleep.
  341. */
  342. smp_mb(); /* ^^^ */
  343. /*
  344. * Check if lock is contended, if not there is nobody to wake up
  345. */
  346. if (likely(atomic_read(&lock->base.count) == 0))
  347. return;
  348. /*
  349. * Uh oh, we raced in fastpath, wake up everyone in this case,
  350. * so they can see the new lock->ctx.
  351. */
  352. spin_lock_mutex(&lock->base.wait_lock, flags);
  353. list_for_each_entry(cur, &lock->base.wait_list, list) {
  354. debug_mutex_wake_waiter(&lock->base, cur);
  355. wake_up_process(cur->task);
  356. }
  357. spin_unlock_mutex(&lock->base.wait_lock, flags);
  358. }
  359. /*
  360. * Lock a mutex (possibly interruptible), slowpath:
  361. */
  362. static __always_inline int __sched
  363. __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
  364. struct lockdep_map *nest_lock, unsigned long ip,
  365. struct ww_acquire_ctx *ww_ctx)
  366. {
  367. struct task_struct *task = current;
  368. struct mutex_waiter waiter;
  369. unsigned long flags;
  370. int ret;
  371. preempt_disable();
  372. mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
  373. #ifdef CONFIG_MUTEX_SPIN_ON_OWNER
  374. /*
  375. * Optimistic spinning.
  376. *
  377. * We try to spin for acquisition when we find that there are no
  378. * pending waiters and the lock owner is currently running on a
  379. * (different) CPU.
  380. *
  381. * The rationale is that if the lock owner is running, it is likely to
  382. * release the lock soon.
  383. *
  384. * Since this needs the lock owner, and this mutex implementation
  385. * doesn't track the owner atomically in the lock field, we need to
  386. * track it non-atomically.
  387. *
  388. * We can't do this for DEBUG_MUTEXES because that relies on wait_lock
  389. * to serialize everything.
  390. *
  391. * The mutex spinners are queued up using MCS lock so that only one
  392. * spinner can compete for the mutex. However, if mutex spinning isn't
  393. * going to happen, there is no point in going through the lock/unlock
  394. * overhead.
  395. */
  396. if (!mutex_can_spin_on_owner(lock))
  397. goto slowpath;
  398. for (;;) {
  399. struct task_struct *owner;
  400. struct mspin_node node;
  401. if (!__builtin_constant_p(ww_ctx == NULL) && ww_ctx->acquired > 0) {
  402. struct ww_mutex *ww;
  403. ww = container_of(lock, struct ww_mutex, base);
  404. /*
  405. * If ww->ctx is set the contents are undefined, only
  406. * by acquiring wait_lock there is a guarantee that
  407. * they are not invalid when reading.
  408. *
  409. * As such, when deadlock detection needs to be
  410. * performed the optimistic spinning cannot be done.
  411. */
  412. if (ACCESS_ONCE(ww->ctx))
  413. break;
  414. }
  415. /*
  416. * If there's an owner, wait for it to either
  417. * release the lock or go to sleep.
  418. */
  419. mspin_lock(MLOCK(lock), &node);
  420. owner = ACCESS_ONCE(lock->owner);
  421. if (owner && !mutex_spin_on_owner(lock, owner)) {
  422. mspin_unlock(MLOCK(lock), &node);
  423. break;
  424. }
  425. if ((atomic_read(&lock->count) == 1) &&
  426. (atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
  427. lock_acquired(&lock->dep_map, ip);
  428. if (!__builtin_constant_p(ww_ctx == NULL)) {
  429. struct ww_mutex *ww;
  430. ww = container_of(lock, struct ww_mutex, base);
  431. ww_mutex_set_context_fastpath(ww, ww_ctx);
  432. }
  433. mutex_set_owner(lock);
  434. mspin_unlock(MLOCK(lock), &node);
  435. preempt_enable();
  436. return 0;
  437. }
  438. mspin_unlock(MLOCK(lock), &node);
  439. /*
  440. * When there's no owner, we might have preempted between the
  441. * owner acquiring the lock and setting the owner field. If
  442. * we're an RT task that will live-lock because we won't let
  443. * the owner complete.
  444. */
  445. if (!owner && (need_resched() || rt_task(task)))
  446. break;
  447. /*
  448. * The cpu_relax() call is a compiler barrier which forces
  449. * everything in this loop to be re-loaded. We don't need
  450. * memory barriers as we'll eventually observe the right
  451. * values at the cost of a few extra spins.
  452. */
  453. arch_mutex_cpu_relax();
  454. }
  455. slowpath:
  456. #endif
  457. spin_lock_mutex(&lock->wait_lock, flags);
  458. debug_mutex_lock_common(lock, &waiter);
  459. debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));
  460. /* add waiting tasks to the end of the waitqueue (FIFO): */
  461. list_add_tail(&waiter.list, &lock->wait_list);
  462. waiter.task = task;
  463. if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, -1) == 1))
  464. goto done;
  465. lock_contended(&lock->dep_map, ip);
  466. for (;;) {
  467. /*
  468. * Lets try to take the lock again - this is needed even if
  469. * we get here for the first time (shortly after failing to
  470. * acquire the lock), to make sure that we get a wakeup once
  471. * it's unlocked. Later on, if we sleep, this is the
  472. * operation that gives us the lock. We xchg it to -1, so
  473. * that when we release the lock, we properly wake up the
  474. * other waiters:
  475. */
  476. if (MUTEX_SHOW_NO_WAITER(lock) &&
  477. (atomic_xchg(&lock->count, -1) == 1))
  478. break;
  479. /*
  480. * got a signal? (This code gets eliminated in the
  481. * TASK_UNINTERRUPTIBLE case.)
  482. */
  483. if (unlikely(signal_pending_state(state, task))) {
  484. ret = -EINTR;
  485. goto err;
  486. }
  487. if (!__builtin_constant_p(ww_ctx == NULL) && ww_ctx->acquired > 0) {
  488. ret = __mutex_lock_check_stamp(lock, ww_ctx);
  489. if (ret)
  490. goto err;
  491. }
  492. __set_task_state(task, state);
  493. /* didn't get the lock, go to sleep: */
  494. spin_unlock_mutex(&lock->wait_lock, flags);
  495. schedule_preempt_disabled();
  496. spin_lock_mutex(&lock->wait_lock, flags);
  497. }
  498. done:
  499. lock_acquired(&lock->dep_map, ip);
  500. /* got the lock - rejoice! */
  501. mutex_remove_waiter(lock, &waiter, current_thread_info());
  502. mutex_set_owner(lock);
  503. if (!__builtin_constant_p(ww_ctx == NULL)) {
  504. struct ww_mutex *ww = container_of(lock,
  505. struct ww_mutex,
  506. base);
  507. struct mutex_waiter *cur;
  508. /*
  509. * This branch gets optimized out for the common case,
  510. * and is only important for ww_mutex_lock.
  511. */
  512. ww_mutex_lock_acquired(ww, ww_ctx);
  513. ww->ctx = ww_ctx;
  514. /*
  515. * Give any possible sleeping processes the chance to wake up,
  516. * so they can recheck if they have to back off.
  517. */
  518. list_for_each_entry(cur, &lock->wait_list, list) {
  519. debug_mutex_wake_waiter(lock, cur);
  520. wake_up_process(cur->task);
  521. }
  522. }
  523. /* set it to 0 if there are no waiters left: */
  524. if (likely(list_empty(&lock->wait_list)))
  525. atomic_set(&lock->count, 0);
  526. spin_unlock_mutex(&lock->wait_lock, flags);
  527. debug_mutex_free_waiter(&waiter);
  528. preempt_enable();
  529. return 0;
  530. err:
  531. mutex_remove_waiter(lock, &waiter, task_thread_info(task));
  532. spin_unlock_mutex(&lock->wait_lock, flags);
  533. debug_mutex_free_waiter(&waiter);
  534. mutex_release(&lock->dep_map, 1, ip);
  535. preempt_enable();
  536. return ret;
  537. }
  538. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  539. void __sched
  540. mutex_lock_nested(struct mutex *lock, unsigned int subclass)
  541. {
  542. might_sleep();
  543. __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
  544. subclass, NULL, _RET_IP_, NULL);
  545. }
  546. EXPORT_SYMBOL_GPL(mutex_lock_nested);
  547. void __sched
  548. _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
  549. {
  550. might_sleep();
  551. __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE,
  552. 0, nest, _RET_IP_, NULL);
  553. }
  554. EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
  555. int __sched
  556. mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
  557. {
  558. might_sleep();
  559. return __mutex_lock_common(lock, TASK_KILLABLE,
  560. subclass, NULL, _RET_IP_, NULL);
  561. }
  562. EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
  563. int __sched
  564. mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
  565. {
  566. might_sleep();
  567. return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
  568. subclass, NULL, _RET_IP_, NULL);
  569. }
  570. EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
  571. static inline int
  572. ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  573. {
  574. #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
  575. unsigned tmp;
  576. if (ctx->deadlock_inject_countdown-- == 0) {
  577. tmp = ctx->deadlock_inject_interval;
  578. if (tmp > UINT_MAX/4)
  579. tmp = UINT_MAX;
  580. else
  581. tmp = tmp*2 + tmp + tmp/2;
  582. ctx->deadlock_inject_interval = tmp;
  583. ctx->deadlock_inject_countdown = tmp;
  584. ctx->contending_lock = lock;
  585. ww_mutex_unlock(lock);
  586. return -EDEADLK;
  587. }
  588. #endif
  589. return 0;
  590. }
  591. int __sched
  592. __ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  593. {
  594. int ret;
  595. might_sleep();
  596. ret = __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE,
  597. 0, &ctx->dep_map, _RET_IP_, ctx);
  598. if (!ret && ctx->acquired > 0)
  599. return ww_mutex_deadlock_injection(lock, ctx);
  600. return ret;
  601. }
  602. EXPORT_SYMBOL_GPL(__ww_mutex_lock);
  603. int __sched
  604. __ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  605. {
  606. int ret;
  607. might_sleep();
  608. ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE,
  609. 0, &ctx->dep_map, _RET_IP_, ctx);
  610. if (!ret && ctx->acquired > 0)
  611. return ww_mutex_deadlock_injection(lock, ctx);
  612. return ret;
  613. }
  614. EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);
  615. #endif
  616. /*
  617. * Release the lock, slowpath:
  618. */
  619. static inline void
  620. __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
  621. {
  622. struct mutex *lock = container_of(lock_count, struct mutex, count);
  623. unsigned long flags;
  624. spin_lock_mutex(&lock->wait_lock, flags);
  625. mutex_release(&lock->dep_map, nested, _RET_IP_);
  626. debug_mutex_unlock(lock);
  627. /*
  628. * some architectures leave the lock unlocked in the fastpath failure
  629. * case, others need to leave it locked. In the later case we have to
  630. * unlock it here
  631. */
  632. if (__mutex_slowpath_needs_to_unlock())
  633. atomic_set(&lock->count, 1);
  634. if (!list_empty(&lock->wait_list)) {
  635. /* get the first entry from the wait-list: */
  636. struct mutex_waiter *waiter =
  637. list_entry(lock->wait_list.next,
  638. struct mutex_waiter, list);
  639. debug_mutex_wake_waiter(lock, waiter);
  640. wake_up_process(waiter->task);
  641. }
  642. spin_unlock_mutex(&lock->wait_lock, flags);
  643. }
  644. /*
  645. * Release the lock, slowpath:
  646. */
  647. static __used noinline void
  648. __mutex_unlock_slowpath(atomic_t *lock_count)
  649. {
  650. __mutex_unlock_common_slowpath(lock_count, 1);
  651. }
  652. #ifndef CONFIG_DEBUG_LOCK_ALLOC
  653. /*
  654. * Here come the less common (and hence less performance-critical) APIs:
  655. * mutex_lock_interruptible() and mutex_trylock().
  656. */
  657. static noinline int __sched
  658. __mutex_lock_killable_slowpath(struct mutex *lock);
  659. static noinline int __sched
  660. __mutex_lock_interruptible_slowpath(struct mutex *lock);
  661. /**
  662. * mutex_lock_interruptible - acquire the mutex, interruptible
  663. * @lock: the mutex to be acquired
  664. *
  665. * Lock the mutex like mutex_lock(), and return 0 if the mutex has
  666. * been acquired or sleep until the mutex becomes available. If a
  667. * signal arrives while waiting for the lock then this function
  668. * returns -EINTR.
  669. *
  670. * This function is similar to (but not equivalent to) down_interruptible().
  671. */
  672. int __sched mutex_lock_interruptible(struct mutex *lock)
  673. {
  674. int ret;
  675. might_sleep();
  676. ret = __mutex_fastpath_lock_retval(&lock->count);
  677. if (likely(!ret)) {
  678. mutex_set_owner(lock);
  679. return 0;
  680. } else
  681. return __mutex_lock_interruptible_slowpath(lock);
  682. }
  683. EXPORT_SYMBOL(mutex_lock_interruptible);
  684. int __sched mutex_lock_killable(struct mutex *lock)
  685. {
  686. int ret;
  687. might_sleep();
  688. ret = __mutex_fastpath_lock_retval(&lock->count);
  689. if (likely(!ret)) {
  690. mutex_set_owner(lock);
  691. return 0;
  692. } else
  693. return __mutex_lock_killable_slowpath(lock);
  694. }
  695. EXPORT_SYMBOL(mutex_lock_killable);
  696. static __used noinline void __sched
  697. __mutex_lock_slowpath(atomic_t *lock_count)
  698. {
  699. struct mutex *lock = container_of(lock_count, struct mutex, count);
  700. __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0,
  701. NULL, _RET_IP_, NULL);
  702. }
  703. static noinline int __sched
  704. __mutex_lock_killable_slowpath(struct mutex *lock)
  705. {
  706. return __mutex_lock_common(lock, TASK_KILLABLE, 0,
  707. NULL, _RET_IP_, NULL);
  708. }
  709. static noinline int __sched
  710. __mutex_lock_interruptible_slowpath(struct mutex *lock)
  711. {
  712. return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0,
  713. NULL, _RET_IP_, NULL);
  714. }
  715. static noinline int __sched
  716. __ww_mutex_lock_slowpath(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  717. {
  718. return __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE, 0,
  719. NULL, _RET_IP_, ctx);
  720. }
  721. static noinline int __sched
  722. __ww_mutex_lock_interruptible_slowpath(struct ww_mutex *lock,
  723. struct ww_acquire_ctx *ctx)
  724. {
  725. return __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE, 0,
  726. NULL, _RET_IP_, ctx);
  727. }
  728. #endif
  729. /*
  730. * Spinlock based trylock, we take the spinlock and check whether we
  731. * can get the lock:
  732. */
  733. static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
  734. {
  735. struct mutex *lock = container_of(lock_count, struct mutex, count);
  736. unsigned long flags;
  737. int prev;
  738. spin_lock_mutex(&lock->wait_lock, flags);
  739. prev = atomic_xchg(&lock->count, -1);
  740. if (likely(prev == 1)) {
  741. mutex_set_owner(lock);
  742. mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
  743. }
  744. /* Set it back to 0 if there are no waiters: */
  745. if (likely(list_empty(&lock->wait_list)))
  746. atomic_set(&lock->count, 0);
  747. spin_unlock_mutex(&lock->wait_lock, flags);
  748. return prev == 1;
  749. }
  750. /**
  751. * mutex_trylock - try to acquire the mutex, without waiting
  752. * @lock: the mutex to be acquired
  753. *
  754. * Try to acquire the mutex atomically. Returns 1 if the mutex
  755. * has been acquired successfully, and 0 on contention.
  756. *
  757. * NOTE: this function follows the spin_trylock() convention, so
  758. * it is negated from the down_trylock() return values! Be careful
  759. * about this when converting semaphore users to mutexes.
  760. *
  761. * This function must not be used in interrupt context. The
  762. * mutex must be released by the same task that acquired it.
  763. */
  764. int __sched mutex_trylock(struct mutex *lock)
  765. {
  766. int ret;
  767. ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath);
  768. if (ret)
  769. mutex_set_owner(lock);
  770. return ret;
  771. }
  772. EXPORT_SYMBOL(mutex_trylock);
  773. #ifndef CONFIG_DEBUG_LOCK_ALLOC
  774. int __sched
  775. __ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  776. {
  777. int ret;
  778. might_sleep();
  779. ret = __mutex_fastpath_lock_retval(&lock->base.count);
  780. if (likely(!ret)) {
  781. ww_mutex_set_context_fastpath(lock, ctx);
  782. mutex_set_owner(&lock->base);
  783. } else
  784. ret = __ww_mutex_lock_slowpath(lock, ctx);
  785. return ret;
  786. }
  787. EXPORT_SYMBOL(__ww_mutex_lock);
  788. int __sched
  789. __ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
  790. {
  791. int ret;
  792. might_sleep();
  793. ret = __mutex_fastpath_lock_retval(&lock->base.count);
  794. if (likely(!ret)) {
  795. ww_mutex_set_context_fastpath(lock, ctx);
  796. mutex_set_owner(&lock->base);
  797. } else
  798. ret = __ww_mutex_lock_interruptible_slowpath(lock, ctx);
  799. return ret;
  800. }
  801. EXPORT_SYMBOL(__ww_mutex_lock_interruptible);
  802. #endif
  803. /**
  804. * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
  805. * @cnt: the atomic which we are to dec
  806. * @lock: the mutex to return holding if we dec to 0
  807. *
  808. * return true and hold lock if we dec to 0, return false otherwise
  809. */
  810. int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
  811. {
  812. /* dec if we can't possibly hit 0 */
  813. if (atomic_add_unless(cnt, -1, 1))
  814. return 0;
  815. /* we might hit 0, so take the lock */
  816. mutex_lock(lock);
  817. if (!atomic_dec_and_test(cnt)) {
  818. /* when we actually did the dec, we didn't hit 0 */
  819. mutex_unlock(lock);
  820. return 0;
  821. }
  822. /* we hit 0, and we hold the lock */
  823. return 1;
  824. }
  825. EXPORT_SYMBOL(atomic_dec_and_mutex_lock);