rcutree_plugin.h 67 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252
  1. /*
  2. * Read-Copy Update mechanism for mutual exclusion (tree-based version)
  3. * Internal non-public definitions that provide either classic
  4. * or preemptible semantics.
  5. *
  6. * This program is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU General Public License as published by
  8. * the Free Software Foundation; either version 2 of the License, or
  9. * (at your option) any later version.
  10. *
  11. * This program is distributed in the hope that it will be useful,
  12. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. * GNU General Public License for more details.
  15. *
  16. * You should have received a copy of the GNU General Public License
  17. * along with this program; if not, write to the Free Software
  18. * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  19. *
  20. * Copyright Red Hat, 2009
  21. * Copyright IBM Corporation, 2009
  22. *
  23. * Author: Ingo Molnar <mingo@elte.hu>
  24. * Paul E. McKenney <paulmck@linux.vnet.ibm.com>
  25. */
  26. #include <linux/delay.h>
  27. #define RCU_KTHREAD_PRIO 1
  28. #ifdef CONFIG_RCU_BOOST
  29. #define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO
  30. #else
  31. #define RCU_BOOST_PRIO RCU_KTHREAD_PRIO
  32. #endif
  33. /*
  34. * Check the RCU kernel configuration parameters and print informative
  35. * messages about anything out of the ordinary. If you like #ifdef, you
  36. * will love this function.
  37. */
  38. static void __init rcu_bootup_announce_oddness(void)
  39. {
  40. #ifdef CONFIG_RCU_TRACE
  41. printk(KERN_INFO "\tRCU debugfs-based tracing is enabled.\n");
  42. #endif
  43. #if (defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) || (!defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32)
  44. printk(KERN_INFO "\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
  45. CONFIG_RCU_FANOUT);
  46. #endif
  47. #ifdef CONFIG_RCU_FANOUT_EXACT
  48. printk(KERN_INFO "\tHierarchical RCU autobalancing is disabled.\n");
  49. #endif
  50. #ifdef CONFIG_RCU_FAST_NO_HZ
  51. printk(KERN_INFO
  52. "\tRCU dyntick-idle grace-period acceleration is enabled.\n");
  53. #endif
  54. #ifdef CONFIG_PROVE_RCU
  55. printk(KERN_INFO "\tRCU lockdep checking is enabled.\n");
  56. #endif
  57. #ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
  58. printk(KERN_INFO "\tRCU torture testing starts during boot.\n");
  59. #endif
  60. #if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
  61. printk(KERN_INFO "\tDump stacks of tasks blocking RCU-preempt GP.\n");
  62. #endif
  63. #if defined(CONFIG_RCU_CPU_STALL_INFO)
  64. printk(KERN_INFO "\tAdditional per-CPU info printed with stalls.\n");
  65. #endif
  66. #if NUM_RCU_LVL_4 != 0
  67. printk(KERN_INFO "\tFour-level hierarchy is enabled.\n");
  68. #endif
  69. if (rcu_fanout_leaf != CONFIG_RCU_FANOUT_LEAF)
  70. printk(KERN_INFO "\tExperimental boot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf);
  71. if (nr_cpu_ids != NR_CPUS)
  72. printk(KERN_INFO "\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids);
  73. }
  74. #ifdef CONFIG_TREE_PREEMPT_RCU
  75. struct rcu_state rcu_preempt_state =
  76. RCU_STATE_INITIALIZER(rcu_preempt, call_rcu);
  77. DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
  78. static struct rcu_state *rcu_state = &rcu_preempt_state;
  79. static void rcu_read_unlock_special(struct task_struct *t);
  80. static int rcu_preempted_readers_exp(struct rcu_node *rnp);
  81. /*
  82. * Tell them what RCU they are running.
  83. */
  84. static void __init rcu_bootup_announce(void)
  85. {
  86. printk(KERN_INFO "Preemptible hierarchical RCU implementation.\n");
  87. rcu_bootup_announce_oddness();
  88. }
  89. /*
  90. * Return the number of RCU-preempt batches processed thus far
  91. * for debug and statistics.
  92. */
  93. long rcu_batches_completed_preempt(void)
  94. {
  95. return rcu_preempt_state.completed;
  96. }
  97. EXPORT_SYMBOL_GPL(rcu_batches_completed_preempt);
  98. /*
  99. * Return the number of RCU batches processed thus far for debug & stats.
  100. */
  101. long rcu_batches_completed(void)
  102. {
  103. return rcu_batches_completed_preempt();
  104. }
  105. EXPORT_SYMBOL_GPL(rcu_batches_completed);
  106. /*
  107. * Force a quiescent state for preemptible RCU.
  108. */
  109. void rcu_force_quiescent_state(void)
  110. {
  111. force_quiescent_state(&rcu_preempt_state, 0);
  112. }
  113. EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
  114. /*
  115. * Record a preemptible-RCU quiescent state for the specified CPU. Note
  116. * that this just means that the task currently running on the CPU is
  117. * not in a quiescent state. There might be any number of tasks blocked
  118. * while in an RCU read-side critical section.
  119. *
  120. * Unlike the other rcu_*_qs() functions, callers to this function
  121. * must disable irqs in order to protect the assignment to
  122. * ->rcu_read_unlock_special.
  123. */
  124. static void rcu_preempt_qs(int cpu)
  125. {
  126. struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
  127. rdp->passed_quiesce_gpnum = rdp->gpnum;
  128. barrier();
  129. if (rdp->passed_quiesce == 0)
  130. trace_rcu_grace_period("rcu_preempt", rdp->gpnum, "cpuqs");
  131. rdp->passed_quiesce = 1;
  132. current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
  133. }
  134. /*
  135. * We have entered the scheduler, and the current task might soon be
  136. * context-switched away from. If this task is in an RCU read-side
  137. * critical section, we will no longer be able to rely on the CPU to
  138. * record that fact, so we enqueue the task on the blkd_tasks list.
  139. * The task will dequeue itself when it exits the outermost enclosing
  140. * RCU read-side critical section. Therefore, the current grace period
  141. * cannot be permitted to complete until the blkd_tasks list entries
  142. * predating the current grace period drain, in other words, until
  143. * rnp->gp_tasks becomes NULL.
  144. *
  145. * Caller must disable preemption.
  146. */
  147. static void rcu_preempt_note_context_switch(int cpu)
  148. {
  149. struct task_struct *t = current;
  150. unsigned long flags;
  151. struct rcu_data *rdp;
  152. struct rcu_node *rnp;
  153. if (t->rcu_read_lock_nesting > 0 &&
  154. (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
  155. /* Possibly blocking in an RCU read-side critical section. */
  156. rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
  157. rnp = rdp->mynode;
  158. raw_spin_lock_irqsave(&rnp->lock, flags);
  159. t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
  160. t->rcu_blocked_node = rnp;
  161. /*
  162. * If this CPU has already checked in, then this task
  163. * will hold up the next grace period rather than the
  164. * current grace period. Queue the task accordingly.
  165. * If the task is queued for the current grace period
  166. * (i.e., this CPU has not yet passed through a quiescent
  167. * state for the current grace period), then as long
  168. * as that task remains queued, the current grace period
  169. * cannot end. Note that there is some uncertainty as
  170. * to exactly when the current grace period started.
  171. * We take a conservative approach, which can result
  172. * in unnecessarily waiting on tasks that started very
  173. * slightly after the current grace period began. C'est
  174. * la vie!!!
  175. *
  176. * But first, note that the current CPU must still be
  177. * on line!
  178. */
  179. WARN_ON_ONCE((rdp->grpmask & rnp->qsmaskinit) == 0);
  180. WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
  181. if ((rnp->qsmask & rdp->grpmask) && rnp->gp_tasks != NULL) {
  182. list_add(&t->rcu_node_entry, rnp->gp_tasks->prev);
  183. rnp->gp_tasks = &t->rcu_node_entry;
  184. #ifdef CONFIG_RCU_BOOST
  185. if (rnp->boost_tasks != NULL)
  186. rnp->boost_tasks = rnp->gp_tasks;
  187. #endif /* #ifdef CONFIG_RCU_BOOST */
  188. } else {
  189. list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
  190. if (rnp->qsmask & rdp->grpmask)
  191. rnp->gp_tasks = &t->rcu_node_entry;
  192. }
  193. trace_rcu_preempt_task(rdp->rsp->name,
  194. t->pid,
  195. (rnp->qsmask & rdp->grpmask)
  196. ? rnp->gpnum
  197. : rnp->gpnum + 1);
  198. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  199. } else if (t->rcu_read_lock_nesting < 0 &&
  200. t->rcu_read_unlock_special) {
  201. /*
  202. * Complete exit from RCU read-side critical section on
  203. * behalf of preempted instance of __rcu_read_unlock().
  204. */
  205. rcu_read_unlock_special(t);
  206. }
  207. /*
  208. * Either we were not in an RCU read-side critical section to
  209. * begin with, or we have now recorded that critical section
  210. * globally. Either way, we can now note a quiescent state
  211. * for this CPU. Again, if we were in an RCU read-side critical
  212. * section, and if that critical section was blocking the current
  213. * grace period, then the fact that the task has been enqueued
  214. * means that we continue to block the current grace period.
  215. */
  216. local_irq_save(flags);
  217. rcu_preempt_qs(cpu);
  218. local_irq_restore(flags);
  219. }
  220. /*
  221. * Tree-preemptible RCU implementation for rcu_read_lock().
  222. * Just increment ->rcu_read_lock_nesting, shared state will be updated
  223. * if we block.
  224. */
  225. void __rcu_read_lock(void)
  226. {
  227. current->rcu_read_lock_nesting++;
  228. barrier(); /* needed if we ever invoke rcu_read_lock in rcutree.c */
  229. }
  230. EXPORT_SYMBOL_GPL(__rcu_read_lock);
  231. /*
  232. * Check for preempted RCU readers blocking the current grace period
  233. * for the specified rcu_node structure. If the caller needs a reliable
  234. * answer, it must hold the rcu_node's ->lock.
  235. */
  236. static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
  237. {
  238. return rnp->gp_tasks != NULL;
  239. }
  240. /*
  241. * Record a quiescent state for all tasks that were previously queued
  242. * on the specified rcu_node structure and that were blocking the current
  243. * RCU grace period. The caller must hold the specified rnp->lock with
  244. * irqs disabled, and this lock is released upon return, but irqs remain
  245. * disabled.
  246. */
  247. static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
  248. __releases(rnp->lock)
  249. {
  250. unsigned long mask;
  251. struct rcu_node *rnp_p;
  252. if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
  253. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  254. return; /* Still need more quiescent states! */
  255. }
  256. rnp_p = rnp->parent;
  257. if (rnp_p == NULL) {
  258. /*
  259. * Either there is only one rcu_node in the tree,
  260. * or tasks were kicked up to root rcu_node due to
  261. * CPUs going offline.
  262. */
  263. rcu_report_qs_rsp(&rcu_preempt_state, flags);
  264. return;
  265. }
  266. /* Report up the rest of the hierarchy. */
  267. mask = rnp->grpmask;
  268. raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
  269. raw_spin_lock(&rnp_p->lock); /* irqs already disabled. */
  270. rcu_report_qs_rnp(mask, &rcu_preempt_state, rnp_p, flags);
  271. }
  272. /*
  273. * Advance a ->blkd_tasks-list pointer to the next entry, instead
  274. * returning NULL if at the end of the list.
  275. */
  276. static struct list_head *rcu_next_node_entry(struct task_struct *t,
  277. struct rcu_node *rnp)
  278. {
  279. struct list_head *np;
  280. np = t->rcu_node_entry.next;
  281. if (np == &rnp->blkd_tasks)
  282. np = NULL;
  283. return np;
  284. }
  285. /*
  286. * Handle special cases during rcu_read_unlock(), such as needing to
  287. * notify RCU core processing or task having blocked during the RCU
  288. * read-side critical section.
  289. */
  290. static noinline void rcu_read_unlock_special(struct task_struct *t)
  291. {
  292. int empty;
  293. int empty_exp;
  294. int empty_exp_now;
  295. unsigned long flags;
  296. struct list_head *np;
  297. #ifdef CONFIG_RCU_BOOST
  298. struct rt_mutex *rbmp = NULL;
  299. #endif /* #ifdef CONFIG_RCU_BOOST */
  300. struct rcu_node *rnp;
  301. int special;
  302. /* NMI handlers cannot block and cannot safely manipulate state. */
  303. if (in_nmi())
  304. return;
  305. local_irq_save(flags);
  306. /*
  307. * If RCU core is waiting for this CPU to exit critical section,
  308. * let it know that we have done so.
  309. */
  310. special = t->rcu_read_unlock_special;
  311. if (special & RCU_READ_UNLOCK_NEED_QS) {
  312. rcu_preempt_qs(smp_processor_id());
  313. }
  314. /* Hardware IRQ handlers cannot block. */
  315. if (in_irq() || in_serving_softirq()) {
  316. local_irq_restore(flags);
  317. return;
  318. }
  319. /* Clean up if blocked during RCU read-side critical section. */
  320. if (special & RCU_READ_UNLOCK_BLOCKED) {
  321. t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BLOCKED;
  322. /*
  323. * Remove this task from the list it blocked on. The
  324. * task can migrate while we acquire the lock, but at
  325. * most one time. So at most two passes through loop.
  326. */
  327. for (;;) {
  328. rnp = t->rcu_blocked_node;
  329. raw_spin_lock(&rnp->lock); /* irqs already disabled. */
  330. if (rnp == t->rcu_blocked_node)
  331. break;
  332. raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
  333. }
  334. empty = !rcu_preempt_blocked_readers_cgp(rnp);
  335. empty_exp = !rcu_preempted_readers_exp(rnp);
  336. smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
  337. np = rcu_next_node_entry(t, rnp);
  338. list_del_init(&t->rcu_node_entry);
  339. t->rcu_blocked_node = NULL;
  340. trace_rcu_unlock_preempted_task("rcu_preempt",
  341. rnp->gpnum, t->pid);
  342. if (&t->rcu_node_entry == rnp->gp_tasks)
  343. rnp->gp_tasks = np;
  344. if (&t->rcu_node_entry == rnp->exp_tasks)
  345. rnp->exp_tasks = np;
  346. #ifdef CONFIG_RCU_BOOST
  347. if (&t->rcu_node_entry == rnp->boost_tasks)
  348. rnp->boost_tasks = np;
  349. /* Snapshot/clear ->rcu_boost_mutex with rcu_node lock held. */
  350. if (t->rcu_boost_mutex) {
  351. rbmp = t->rcu_boost_mutex;
  352. t->rcu_boost_mutex = NULL;
  353. }
  354. #endif /* #ifdef CONFIG_RCU_BOOST */
  355. /*
  356. * If this was the last task on the current list, and if
  357. * we aren't waiting on any CPUs, report the quiescent state.
  358. * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
  359. * so we must take a snapshot of the expedited state.
  360. */
  361. empty_exp_now = !rcu_preempted_readers_exp(rnp);
  362. if (!empty && !rcu_preempt_blocked_readers_cgp(rnp)) {
  363. trace_rcu_quiescent_state_report("preempt_rcu",
  364. rnp->gpnum,
  365. 0, rnp->qsmask,
  366. rnp->level,
  367. rnp->grplo,
  368. rnp->grphi,
  369. !!rnp->gp_tasks);
  370. rcu_report_unblock_qs_rnp(rnp, flags);
  371. } else
  372. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  373. #ifdef CONFIG_RCU_BOOST
  374. /* Unboost if we were boosted. */
  375. if (rbmp)
  376. rt_mutex_unlock(rbmp);
  377. #endif /* #ifdef CONFIG_RCU_BOOST */
  378. /*
  379. * If this was the last task on the expedited lists,
  380. * then we need to report up the rcu_node hierarchy.
  381. */
  382. if (!empty_exp && empty_exp_now)
  383. rcu_report_exp_rnp(&rcu_preempt_state, rnp, true);
  384. } else {
  385. local_irq_restore(flags);
  386. }
  387. }
  388. /*
  389. * Tree-preemptible RCU implementation for rcu_read_unlock().
  390. * Decrement ->rcu_read_lock_nesting. If the result is zero (outermost
  391. * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
  392. * invoke rcu_read_unlock_special() to clean up after a context switch
  393. * in an RCU read-side critical section and other special cases.
  394. */
  395. void __rcu_read_unlock(void)
  396. {
  397. struct task_struct *t = current;
  398. if (t->rcu_read_lock_nesting != 1)
  399. --t->rcu_read_lock_nesting;
  400. else {
  401. barrier(); /* critical section before exit code. */
  402. t->rcu_read_lock_nesting = INT_MIN;
  403. barrier(); /* assign before ->rcu_read_unlock_special load */
  404. if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
  405. rcu_read_unlock_special(t);
  406. barrier(); /* ->rcu_read_unlock_special load before assign */
  407. t->rcu_read_lock_nesting = 0;
  408. }
  409. #ifdef CONFIG_PROVE_LOCKING
  410. {
  411. int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
  412. WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
  413. }
  414. #endif /* #ifdef CONFIG_PROVE_LOCKING */
  415. }
  416. EXPORT_SYMBOL_GPL(__rcu_read_unlock);
  417. #ifdef CONFIG_RCU_CPU_STALL_VERBOSE
  418. /*
  419. * Dump detailed information for all tasks blocking the current RCU
  420. * grace period on the specified rcu_node structure.
  421. */
  422. static void rcu_print_detail_task_stall_rnp(struct rcu_node *rnp)
  423. {
  424. unsigned long flags;
  425. struct task_struct *t;
  426. if (!rcu_preempt_blocked_readers_cgp(rnp))
  427. return;
  428. raw_spin_lock_irqsave(&rnp->lock, flags);
  429. t = list_entry(rnp->gp_tasks,
  430. struct task_struct, rcu_node_entry);
  431. list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry)
  432. sched_show_task(t);
  433. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  434. }
  435. /*
  436. * Dump detailed information for all tasks blocking the current RCU
  437. * grace period.
  438. */
  439. static void rcu_print_detail_task_stall(struct rcu_state *rsp)
  440. {
  441. struct rcu_node *rnp = rcu_get_root(rsp);
  442. rcu_print_detail_task_stall_rnp(rnp);
  443. rcu_for_each_leaf_node(rsp, rnp)
  444. rcu_print_detail_task_stall_rnp(rnp);
  445. }
  446. #else /* #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */
  447. static void rcu_print_detail_task_stall(struct rcu_state *rsp)
  448. {
  449. }
  450. #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */
  451. #ifdef CONFIG_RCU_CPU_STALL_INFO
  452. static void rcu_print_task_stall_begin(struct rcu_node *rnp)
  453. {
  454. printk(KERN_ERR "\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
  455. rnp->level, rnp->grplo, rnp->grphi);
  456. }
  457. static void rcu_print_task_stall_end(void)
  458. {
  459. printk(KERN_CONT "\n");
  460. }
  461. #else /* #ifdef CONFIG_RCU_CPU_STALL_INFO */
  462. static void rcu_print_task_stall_begin(struct rcu_node *rnp)
  463. {
  464. }
  465. static void rcu_print_task_stall_end(void)
  466. {
  467. }
  468. #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_INFO */
  469. /*
  470. * Scan the current list of tasks blocked within RCU read-side critical
  471. * sections, printing out the tid of each.
  472. */
  473. static int rcu_print_task_stall(struct rcu_node *rnp)
  474. {
  475. struct task_struct *t;
  476. int ndetected = 0;
  477. if (!rcu_preempt_blocked_readers_cgp(rnp))
  478. return 0;
  479. rcu_print_task_stall_begin(rnp);
  480. t = list_entry(rnp->gp_tasks,
  481. struct task_struct, rcu_node_entry);
  482. list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
  483. printk(KERN_CONT " P%d", t->pid);
  484. ndetected++;
  485. }
  486. rcu_print_task_stall_end();
  487. return ndetected;
  488. }
  489. /*
  490. * Check that the list of blocked tasks for the newly completed grace
  491. * period is in fact empty. It is a serious bug to complete a grace
  492. * period that still has RCU readers blocked! This function must be
  493. * invoked -before- updating this rnp's ->gpnum, and the rnp's ->lock
  494. * must be held by the caller.
  495. *
  496. * Also, if there are blocked tasks on the list, they automatically
  497. * block the newly created grace period, so set up ->gp_tasks accordingly.
  498. */
  499. static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
  500. {
  501. WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp));
  502. if (!list_empty(&rnp->blkd_tasks))
  503. rnp->gp_tasks = rnp->blkd_tasks.next;
  504. WARN_ON_ONCE(rnp->qsmask);
  505. }
  506. #ifdef CONFIG_HOTPLUG_CPU
  507. /*
  508. * Handle tasklist migration for case in which all CPUs covered by the
  509. * specified rcu_node have gone offline. Move them up to the root
  510. * rcu_node. The reason for not just moving them to the immediate
  511. * parent is to remove the need for rcu_read_unlock_special() to
  512. * make more than two attempts to acquire the target rcu_node's lock.
  513. * Returns true if there were tasks blocking the current RCU grace
  514. * period.
  515. *
  516. * Returns 1 if there was previously a task blocking the current grace
  517. * period on the specified rcu_node structure.
  518. *
  519. * The caller must hold rnp->lock with irqs disabled.
  520. */
  521. static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
  522. struct rcu_node *rnp,
  523. struct rcu_data *rdp)
  524. {
  525. struct list_head *lp;
  526. struct list_head *lp_root;
  527. int retval = 0;
  528. struct rcu_node *rnp_root = rcu_get_root(rsp);
  529. struct task_struct *t;
  530. if (rnp == rnp_root) {
  531. WARN_ONCE(1, "Last CPU thought to be offlined?");
  532. return 0; /* Shouldn't happen: at least one CPU online. */
  533. }
  534. /* If we are on an internal node, complain bitterly. */
  535. WARN_ON_ONCE(rnp != rdp->mynode);
  536. /*
  537. * Move tasks up to root rcu_node. Don't try to get fancy for
  538. * this corner-case operation -- just put this node's tasks
  539. * at the head of the root node's list, and update the root node's
  540. * ->gp_tasks and ->exp_tasks pointers to those of this node's,
  541. * if non-NULL. This might result in waiting for more tasks than
  542. * absolutely necessary, but this is a good performance/complexity
  543. * tradeoff.
  544. */
  545. if (rcu_preempt_blocked_readers_cgp(rnp) && rnp->qsmask == 0)
  546. retval |= RCU_OFL_TASKS_NORM_GP;
  547. if (rcu_preempted_readers_exp(rnp))
  548. retval |= RCU_OFL_TASKS_EXP_GP;
  549. lp = &rnp->blkd_tasks;
  550. lp_root = &rnp_root->blkd_tasks;
  551. while (!list_empty(lp)) {
  552. t = list_entry(lp->next, typeof(*t), rcu_node_entry);
  553. raw_spin_lock(&rnp_root->lock); /* irqs already disabled */
  554. list_del(&t->rcu_node_entry);
  555. t->rcu_blocked_node = rnp_root;
  556. list_add(&t->rcu_node_entry, lp_root);
  557. if (&t->rcu_node_entry == rnp->gp_tasks)
  558. rnp_root->gp_tasks = rnp->gp_tasks;
  559. if (&t->rcu_node_entry == rnp->exp_tasks)
  560. rnp_root->exp_tasks = rnp->exp_tasks;
  561. #ifdef CONFIG_RCU_BOOST
  562. if (&t->rcu_node_entry == rnp->boost_tasks)
  563. rnp_root->boost_tasks = rnp->boost_tasks;
  564. #endif /* #ifdef CONFIG_RCU_BOOST */
  565. raw_spin_unlock(&rnp_root->lock); /* irqs still disabled */
  566. }
  567. #ifdef CONFIG_RCU_BOOST
  568. /* In case root is being boosted and leaf is not. */
  569. raw_spin_lock(&rnp_root->lock); /* irqs already disabled */
  570. if (rnp_root->boost_tasks != NULL &&
  571. rnp_root->boost_tasks != rnp_root->gp_tasks)
  572. rnp_root->boost_tasks = rnp_root->gp_tasks;
  573. raw_spin_unlock(&rnp_root->lock); /* irqs still disabled */
  574. #endif /* #ifdef CONFIG_RCU_BOOST */
  575. rnp->gp_tasks = NULL;
  576. rnp->exp_tasks = NULL;
  577. return retval;
  578. }
  579. #endif /* #ifdef CONFIG_HOTPLUG_CPU */
  580. /*
  581. * Check for a quiescent state from the current CPU. When a task blocks,
  582. * the task is recorded in the corresponding CPU's rcu_node structure,
  583. * which is checked elsewhere.
  584. *
  585. * Caller must disable hard irqs.
  586. */
  587. static void rcu_preempt_check_callbacks(int cpu)
  588. {
  589. struct task_struct *t = current;
  590. if (t->rcu_read_lock_nesting == 0) {
  591. rcu_preempt_qs(cpu);
  592. return;
  593. }
  594. if (t->rcu_read_lock_nesting > 0 &&
  595. per_cpu(rcu_preempt_data, cpu).qs_pending)
  596. t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
  597. }
  598. #ifdef CONFIG_RCU_BOOST
  599. static void rcu_preempt_do_callbacks(void)
  600. {
  601. rcu_do_batch(&rcu_preempt_state, &__get_cpu_var(rcu_preempt_data));
  602. }
  603. #endif /* #ifdef CONFIG_RCU_BOOST */
  604. /*
  605. * Queue a preemptible-RCU callback for invocation after a grace period.
  606. */
  607. void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
  608. {
  609. __call_rcu(head, func, &rcu_preempt_state, 0);
  610. }
  611. EXPORT_SYMBOL_GPL(call_rcu);
  612. /*
  613. * Queue an RCU callback for lazy invocation after a grace period.
  614. * This will likely be later named something like "call_rcu_lazy()",
  615. * but this change will require some way of tagging the lazy RCU
  616. * callbacks in the list of pending callbacks. Until then, this
  617. * function may only be called from __kfree_rcu().
  618. */
  619. void kfree_call_rcu(struct rcu_head *head,
  620. void (*func)(struct rcu_head *rcu))
  621. {
  622. __call_rcu(head, func, &rcu_preempt_state, 1);
  623. }
  624. EXPORT_SYMBOL_GPL(kfree_call_rcu);
  625. /**
  626. * synchronize_rcu - wait until a grace period has elapsed.
  627. *
  628. * Control will return to the caller some time after a full grace
  629. * period has elapsed, in other words after all currently executing RCU
  630. * read-side critical sections have completed. Note, however, that
  631. * upon return from synchronize_rcu(), the caller might well be executing
  632. * concurrently with new RCU read-side critical sections that began while
  633. * synchronize_rcu() was waiting. RCU read-side critical sections are
  634. * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
  635. */
  636. void synchronize_rcu(void)
  637. {
  638. rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
  639. !lock_is_held(&rcu_lock_map) &&
  640. !lock_is_held(&rcu_sched_lock_map),
  641. "Illegal synchronize_rcu() in RCU read-side critical section");
  642. if (!rcu_scheduler_active)
  643. return;
  644. wait_rcu_gp(call_rcu);
  645. }
  646. EXPORT_SYMBOL_GPL(synchronize_rcu);
  647. static DECLARE_WAIT_QUEUE_HEAD(sync_rcu_preempt_exp_wq);
  648. static long sync_rcu_preempt_exp_count;
  649. static DEFINE_MUTEX(sync_rcu_preempt_exp_mutex);
  650. /*
  651. * Return non-zero if there are any tasks in RCU read-side critical
  652. * sections blocking the current preemptible-RCU expedited grace period.
  653. * If there is no preemptible-RCU expedited grace period currently in
  654. * progress, returns zero unconditionally.
  655. */
  656. static int rcu_preempted_readers_exp(struct rcu_node *rnp)
  657. {
  658. return rnp->exp_tasks != NULL;
  659. }
  660. /*
  661. * return non-zero if there is no RCU expedited grace period in progress
  662. * for the specified rcu_node structure, in other words, if all CPUs and
  663. * tasks covered by the specified rcu_node structure have done their bit
  664. * for the current expedited grace period. Works only for preemptible
  665. * RCU -- other RCU implementation use other means.
  666. *
  667. * Caller must hold sync_rcu_preempt_exp_mutex.
  668. */
  669. static int sync_rcu_preempt_exp_done(struct rcu_node *rnp)
  670. {
  671. return !rcu_preempted_readers_exp(rnp) &&
  672. ACCESS_ONCE(rnp->expmask) == 0;
  673. }
  674. /*
  675. * Report the exit from RCU read-side critical section for the last task
  676. * that queued itself during or before the current expedited preemptible-RCU
  677. * grace period. This event is reported either to the rcu_node structure on
  678. * which the task was queued or to one of that rcu_node structure's ancestors,
  679. * recursively up the tree. (Calm down, calm down, we do the recursion
  680. * iteratively!)
  681. *
  682. * Most callers will set the "wake" flag, but the task initiating the
  683. * expedited grace period need not wake itself.
  684. *
  685. * Caller must hold sync_rcu_preempt_exp_mutex.
  686. */
  687. static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
  688. bool wake)
  689. {
  690. unsigned long flags;
  691. unsigned long mask;
  692. raw_spin_lock_irqsave(&rnp->lock, flags);
  693. for (;;) {
  694. if (!sync_rcu_preempt_exp_done(rnp)) {
  695. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  696. break;
  697. }
  698. if (rnp->parent == NULL) {
  699. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  700. if (wake)
  701. wake_up(&sync_rcu_preempt_exp_wq);
  702. break;
  703. }
  704. mask = rnp->grpmask;
  705. raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
  706. rnp = rnp->parent;
  707. raw_spin_lock(&rnp->lock); /* irqs already disabled */
  708. rnp->expmask &= ~mask;
  709. }
  710. }
  711. /*
  712. * Snapshot the tasks blocking the newly started preemptible-RCU expedited
  713. * grace period for the specified rcu_node structure. If there are no such
  714. * tasks, report it up the rcu_node hierarchy.
  715. *
  716. * Caller must hold sync_rcu_preempt_exp_mutex and rsp->onofflock.
  717. */
  718. static void
  719. sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp)
  720. {
  721. unsigned long flags;
  722. int must_wait = 0;
  723. raw_spin_lock_irqsave(&rnp->lock, flags);
  724. if (list_empty(&rnp->blkd_tasks))
  725. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  726. else {
  727. rnp->exp_tasks = rnp->blkd_tasks.next;
  728. rcu_initiate_boost(rnp, flags); /* releases rnp->lock */
  729. must_wait = 1;
  730. }
  731. if (!must_wait)
  732. rcu_report_exp_rnp(rsp, rnp, false); /* Don't wake self. */
  733. }
  734. /**
  735. * synchronize_rcu_expedited - Brute-force RCU grace period
  736. *
  737. * Wait for an RCU-preempt grace period, but expedite it. The basic
  738. * idea is to invoke synchronize_sched_expedited() to push all the tasks to
  739. * the ->blkd_tasks lists and wait for this list to drain. This consumes
  740. * significant time on all CPUs and is unfriendly to real-time workloads,
  741. * so is thus not recommended for any sort of common-case code.
  742. * In fact, if you are using synchronize_rcu_expedited() in a loop,
  743. * please restructure your code to batch your updates, and then Use a
  744. * single synchronize_rcu() instead.
  745. *
  746. * Note that it is illegal to call this function while holding any lock
  747. * that is acquired by a CPU-hotplug notifier. And yes, it is also illegal
  748. * to call this function from a CPU-hotplug notifier. Failing to observe
  749. * these restriction will result in deadlock.
  750. */
  751. void synchronize_rcu_expedited(void)
  752. {
  753. unsigned long flags;
  754. struct rcu_node *rnp;
  755. struct rcu_state *rsp = &rcu_preempt_state;
  756. long snap;
  757. int trycount = 0;
  758. smp_mb(); /* Caller's modifications seen first by other CPUs. */
  759. snap = ACCESS_ONCE(sync_rcu_preempt_exp_count) + 1;
  760. smp_mb(); /* Above access cannot bleed into critical section. */
  761. /*
  762. * Acquire lock, falling back to synchronize_rcu() if too many
  763. * lock-acquisition failures. Of course, if someone does the
  764. * expedited grace period for us, just leave.
  765. */
  766. while (!mutex_trylock(&sync_rcu_preempt_exp_mutex)) {
  767. if (trycount++ < 10)
  768. udelay(trycount * num_online_cpus());
  769. else {
  770. synchronize_rcu();
  771. return;
  772. }
  773. if ((ACCESS_ONCE(sync_rcu_preempt_exp_count) - snap) > 0)
  774. goto mb_ret; /* Others did our work for us. */
  775. }
  776. if ((ACCESS_ONCE(sync_rcu_preempt_exp_count) - snap) > 0)
  777. goto unlock_mb_ret; /* Others did our work for us. */
  778. /* force all RCU readers onto ->blkd_tasks lists. */
  779. synchronize_sched_expedited();
  780. raw_spin_lock_irqsave(&rsp->onofflock, flags);
  781. /* Initialize ->expmask for all non-leaf rcu_node structures. */
  782. rcu_for_each_nonleaf_node_breadth_first(rsp, rnp) {
  783. raw_spin_lock(&rnp->lock); /* irqs already disabled. */
  784. rnp->expmask = rnp->qsmaskinit;
  785. raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
  786. }
  787. /* Snapshot current state of ->blkd_tasks lists. */
  788. rcu_for_each_leaf_node(rsp, rnp)
  789. sync_rcu_preempt_exp_init(rsp, rnp);
  790. if (NUM_RCU_NODES > 1)
  791. sync_rcu_preempt_exp_init(rsp, rcu_get_root(rsp));
  792. raw_spin_unlock_irqrestore(&rsp->onofflock, flags);
  793. /* Wait for snapshotted ->blkd_tasks lists to drain. */
  794. rnp = rcu_get_root(rsp);
  795. wait_event(sync_rcu_preempt_exp_wq,
  796. sync_rcu_preempt_exp_done(rnp));
  797. /* Clean up and exit. */
  798. smp_mb(); /* ensure expedited GP seen before counter increment. */
  799. ACCESS_ONCE(sync_rcu_preempt_exp_count)++;
  800. unlock_mb_ret:
  801. mutex_unlock(&sync_rcu_preempt_exp_mutex);
  802. mb_ret:
  803. smp_mb(); /* ensure subsequent action seen after grace period. */
  804. }
  805. EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
  806. /**
  807. * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
  808. */
  809. void rcu_barrier(void)
  810. {
  811. _rcu_barrier(&rcu_preempt_state);
  812. }
  813. EXPORT_SYMBOL_GPL(rcu_barrier);
  814. /*
  815. * Initialize preemptible RCU's state structures.
  816. */
  817. static void __init __rcu_init_preempt(void)
  818. {
  819. rcu_init_one(&rcu_preempt_state, &rcu_preempt_data);
  820. }
  821. #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
  822. static struct rcu_state *rcu_state = &rcu_sched_state;
  823. /*
  824. * Tell them what RCU they are running.
  825. */
  826. static void __init rcu_bootup_announce(void)
  827. {
  828. printk(KERN_INFO "Hierarchical RCU implementation.\n");
  829. rcu_bootup_announce_oddness();
  830. }
  831. /*
  832. * Return the number of RCU batches processed thus far for debug & stats.
  833. */
  834. long rcu_batches_completed(void)
  835. {
  836. return rcu_batches_completed_sched();
  837. }
  838. EXPORT_SYMBOL_GPL(rcu_batches_completed);
  839. /*
  840. * Force a quiescent state for RCU, which, because there is no preemptible
  841. * RCU, becomes the same as rcu-sched.
  842. */
  843. void rcu_force_quiescent_state(void)
  844. {
  845. rcu_sched_force_quiescent_state();
  846. }
  847. EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
  848. /*
  849. * Because preemptible RCU does not exist, we never have to check for
  850. * CPUs being in quiescent states.
  851. */
  852. static void rcu_preempt_note_context_switch(int cpu)
  853. {
  854. }
  855. /*
  856. * Because preemptible RCU does not exist, there are never any preempted
  857. * RCU readers.
  858. */
  859. static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
  860. {
  861. return 0;
  862. }
  863. #ifdef CONFIG_HOTPLUG_CPU
  864. /* Because preemptible RCU does not exist, no quieting of tasks. */
  865. static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
  866. {
  867. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  868. }
  869. #endif /* #ifdef CONFIG_HOTPLUG_CPU */
  870. /*
  871. * Because preemptible RCU does not exist, we never have to check for
  872. * tasks blocked within RCU read-side critical sections.
  873. */
  874. static void rcu_print_detail_task_stall(struct rcu_state *rsp)
  875. {
  876. }
  877. /*
  878. * Because preemptible RCU does not exist, we never have to check for
  879. * tasks blocked within RCU read-side critical sections.
  880. */
  881. static int rcu_print_task_stall(struct rcu_node *rnp)
  882. {
  883. return 0;
  884. }
  885. /*
  886. * Because there is no preemptible RCU, there can be no readers blocked,
  887. * so there is no need to check for blocked tasks. So check only for
  888. * bogus qsmask values.
  889. */
  890. static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
  891. {
  892. WARN_ON_ONCE(rnp->qsmask);
  893. }
  894. #ifdef CONFIG_HOTPLUG_CPU
  895. /*
  896. * Because preemptible RCU does not exist, it never needs to migrate
  897. * tasks that were blocked within RCU read-side critical sections, and
  898. * such non-existent tasks cannot possibly have been blocking the current
  899. * grace period.
  900. */
  901. static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
  902. struct rcu_node *rnp,
  903. struct rcu_data *rdp)
  904. {
  905. return 0;
  906. }
  907. #endif /* #ifdef CONFIG_HOTPLUG_CPU */
  908. /*
  909. * Because preemptible RCU does not exist, it never has any callbacks
  910. * to check.
  911. */
  912. static void rcu_preempt_check_callbacks(int cpu)
  913. {
  914. }
  915. /*
  916. * Queue an RCU callback for lazy invocation after a grace period.
  917. * This will likely be later named something like "call_rcu_lazy()",
  918. * but this change will require some way of tagging the lazy RCU
  919. * callbacks in the list of pending callbacks. Until then, this
  920. * function may only be called from __kfree_rcu().
  921. *
  922. * Because there is no preemptible RCU, we use RCU-sched instead.
  923. */
  924. void kfree_call_rcu(struct rcu_head *head,
  925. void (*func)(struct rcu_head *rcu))
  926. {
  927. __call_rcu(head, func, &rcu_sched_state, 1);
  928. }
  929. EXPORT_SYMBOL_GPL(kfree_call_rcu);
  930. /*
  931. * Wait for an rcu-preempt grace period, but make it happen quickly.
  932. * But because preemptible RCU does not exist, map to rcu-sched.
  933. */
  934. void synchronize_rcu_expedited(void)
  935. {
  936. synchronize_sched_expedited();
  937. }
  938. EXPORT_SYMBOL_GPL(synchronize_rcu_expedited);
  939. #ifdef CONFIG_HOTPLUG_CPU
  940. /*
  941. * Because preemptible RCU does not exist, there is never any need to
  942. * report on tasks preempted in RCU read-side critical sections during
  943. * expedited RCU grace periods.
  944. */
  945. static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp,
  946. bool wake)
  947. {
  948. }
  949. #endif /* #ifdef CONFIG_HOTPLUG_CPU */
  950. /*
  951. * Because preemptible RCU does not exist, rcu_barrier() is just
  952. * another name for rcu_barrier_sched().
  953. */
  954. void rcu_barrier(void)
  955. {
  956. rcu_barrier_sched();
  957. }
  958. EXPORT_SYMBOL_GPL(rcu_barrier);
  959. /*
  960. * Because preemptible RCU does not exist, it need not be initialized.
  961. */
  962. static void __init __rcu_init_preempt(void)
  963. {
  964. }
  965. #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
  966. #ifdef CONFIG_RCU_BOOST
  967. #include "rtmutex_common.h"
  968. #ifdef CONFIG_RCU_TRACE
  969. static void rcu_initiate_boost_trace(struct rcu_node *rnp)
  970. {
  971. if (list_empty(&rnp->blkd_tasks))
  972. rnp->n_balk_blkd_tasks++;
  973. else if (rnp->exp_tasks == NULL && rnp->gp_tasks == NULL)
  974. rnp->n_balk_exp_gp_tasks++;
  975. else if (rnp->gp_tasks != NULL && rnp->boost_tasks != NULL)
  976. rnp->n_balk_boost_tasks++;
  977. else if (rnp->gp_tasks != NULL && rnp->qsmask != 0)
  978. rnp->n_balk_notblocked++;
  979. else if (rnp->gp_tasks != NULL &&
  980. ULONG_CMP_LT(jiffies, rnp->boost_time))
  981. rnp->n_balk_notyet++;
  982. else
  983. rnp->n_balk_nos++;
  984. }
  985. #else /* #ifdef CONFIG_RCU_TRACE */
  986. static void rcu_initiate_boost_trace(struct rcu_node *rnp)
  987. {
  988. }
  989. #endif /* #else #ifdef CONFIG_RCU_TRACE */
  990. /*
  991. * Carry out RCU priority boosting on the task indicated by ->exp_tasks
  992. * or ->boost_tasks, advancing the pointer to the next task in the
  993. * ->blkd_tasks list.
  994. *
  995. * Note that irqs must be enabled: boosting the task can block.
  996. * Returns 1 if there are more tasks needing to be boosted.
  997. */
  998. static int rcu_boost(struct rcu_node *rnp)
  999. {
  1000. unsigned long flags;
  1001. struct rt_mutex mtx;
  1002. struct task_struct *t;
  1003. struct list_head *tb;
  1004. if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL)
  1005. return 0; /* Nothing left to boost. */
  1006. raw_spin_lock_irqsave(&rnp->lock, flags);
  1007. /*
  1008. * Recheck under the lock: all tasks in need of boosting
  1009. * might exit their RCU read-side critical sections on their own.
  1010. */
  1011. if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
  1012. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  1013. return 0;
  1014. }
  1015. /*
  1016. * Preferentially boost tasks blocking expedited grace periods.
  1017. * This cannot starve the normal grace periods because a second
  1018. * expedited grace period must boost all blocked tasks, including
  1019. * those blocking the pre-existing normal grace period.
  1020. */
  1021. if (rnp->exp_tasks != NULL) {
  1022. tb = rnp->exp_tasks;
  1023. rnp->n_exp_boosts++;
  1024. } else {
  1025. tb = rnp->boost_tasks;
  1026. rnp->n_normal_boosts++;
  1027. }
  1028. rnp->n_tasks_boosted++;
  1029. /*
  1030. * We boost task t by manufacturing an rt_mutex that appears to
  1031. * be held by task t. We leave a pointer to that rt_mutex where
  1032. * task t can find it, and task t will release the mutex when it
  1033. * exits its outermost RCU read-side critical section. Then
  1034. * simply acquiring this artificial rt_mutex will boost task
  1035. * t's priority. (Thanks to tglx for suggesting this approach!)
  1036. *
  1037. * Note that task t must acquire rnp->lock to remove itself from
  1038. * the ->blkd_tasks list, which it will do from exit() if from
  1039. * nowhere else. We therefore are guaranteed that task t will
  1040. * stay around at least until we drop rnp->lock. Note that
  1041. * rnp->lock also resolves races between our priority boosting
  1042. * and task t's exiting its outermost RCU read-side critical
  1043. * section.
  1044. */
  1045. t = container_of(tb, struct task_struct, rcu_node_entry);
  1046. rt_mutex_init_proxy_locked(&mtx, t);
  1047. t->rcu_boost_mutex = &mtx;
  1048. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  1049. rt_mutex_lock(&mtx); /* Side effect: boosts task t's priority. */
  1050. rt_mutex_unlock(&mtx); /* Keep lockdep happy. */
  1051. return ACCESS_ONCE(rnp->exp_tasks) != NULL ||
  1052. ACCESS_ONCE(rnp->boost_tasks) != NULL;
  1053. }
  1054. /*
  1055. * Timer handler to initiate waking up of boost kthreads that
  1056. * have yielded the CPU due to excessive numbers of tasks to
  1057. * boost. We wake up the per-rcu_node kthread, which in turn
  1058. * will wake up the booster kthread.
  1059. */
  1060. static void rcu_boost_kthread_timer(unsigned long arg)
  1061. {
  1062. invoke_rcu_node_kthread((struct rcu_node *)arg);
  1063. }
  1064. /*
  1065. * Priority-boosting kthread. One per leaf rcu_node and one for the
  1066. * root rcu_node.
  1067. */
  1068. static int rcu_boost_kthread(void *arg)
  1069. {
  1070. struct rcu_node *rnp = (struct rcu_node *)arg;
  1071. int spincnt = 0;
  1072. int more2boost;
  1073. trace_rcu_utilization("Start boost kthread@init");
  1074. for (;;) {
  1075. rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
  1076. trace_rcu_utilization("End boost kthread@rcu_wait");
  1077. rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
  1078. trace_rcu_utilization("Start boost kthread@rcu_wait");
  1079. rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
  1080. more2boost = rcu_boost(rnp);
  1081. if (more2boost)
  1082. spincnt++;
  1083. else
  1084. spincnt = 0;
  1085. if (spincnt > 10) {
  1086. trace_rcu_utilization("End boost kthread@rcu_yield");
  1087. rcu_yield(rcu_boost_kthread_timer, (unsigned long)rnp);
  1088. trace_rcu_utilization("Start boost kthread@rcu_yield");
  1089. spincnt = 0;
  1090. }
  1091. }
  1092. /* NOTREACHED */
  1093. trace_rcu_utilization("End boost kthread@notreached");
  1094. return 0;
  1095. }
  1096. /*
  1097. * Check to see if it is time to start boosting RCU readers that are
  1098. * blocking the current grace period, and, if so, tell the per-rcu_node
  1099. * kthread to start boosting them. If there is an expedited grace
  1100. * period in progress, it is always time to boost.
  1101. *
  1102. * The caller must hold rnp->lock, which this function releases,
  1103. * but irqs remain disabled. The ->boost_kthread_task is immortal,
  1104. * so we don't need to worry about it going away.
  1105. */
  1106. static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
  1107. {
  1108. struct task_struct *t;
  1109. if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
  1110. rnp->n_balk_exp_gp_tasks++;
  1111. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  1112. return;
  1113. }
  1114. if (rnp->exp_tasks != NULL ||
  1115. (rnp->gp_tasks != NULL &&
  1116. rnp->boost_tasks == NULL &&
  1117. rnp->qsmask == 0 &&
  1118. ULONG_CMP_GE(jiffies, rnp->boost_time))) {
  1119. if (rnp->exp_tasks == NULL)
  1120. rnp->boost_tasks = rnp->gp_tasks;
  1121. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  1122. t = rnp->boost_kthread_task;
  1123. if (t != NULL)
  1124. wake_up_process(t);
  1125. } else {
  1126. rcu_initiate_boost_trace(rnp);
  1127. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  1128. }
  1129. }
  1130. /*
  1131. * Wake up the per-CPU kthread to invoke RCU callbacks.
  1132. */
  1133. static void invoke_rcu_callbacks_kthread(void)
  1134. {
  1135. unsigned long flags;
  1136. local_irq_save(flags);
  1137. __this_cpu_write(rcu_cpu_has_work, 1);
  1138. if (__this_cpu_read(rcu_cpu_kthread_task) != NULL &&
  1139. current != __this_cpu_read(rcu_cpu_kthread_task))
  1140. wake_up_process(__this_cpu_read(rcu_cpu_kthread_task));
  1141. local_irq_restore(flags);
  1142. }
  1143. /*
  1144. * Is the current CPU running the RCU-callbacks kthread?
  1145. * Caller must have preemption disabled.
  1146. */
  1147. static bool rcu_is_callbacks_kthread(void)
  1148. {
  1149. return __get_cpu_var(rcu_cpu_kthread_task) == current;
  1150. }
  1151. /*
  1152. * Set the affinity of the boost kthread. The CPU-hotplug locks are
  1153. * held, so no one should be messing with the existence of the boost
  1154. * kthread.
  1155. */
  1156. static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,
  1157. cpumask_var_t cm)
  1158. {
  1159. struct task_struct *t;
  1160. t = rnp->boost_kthread_task;
  1161. if (t != NULL)
  1162. set_cpus_allowed_ptr(rnp->boost_kthread_task, cm);
  1163. }
  1164. #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
  1165. /*
  1166. * Do priority-boost accounting for the start of a new grace period.
  1167. */
  1168. static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
  1169. {
  1170. rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
  1171. }
  1172. /*
  1173. * Create an RCU-boost kthread for the specified node if one does not
  1174. * already exist. We only create this kthread for preemptible RCU.
  1175. * Returns zero if all is well, a negated errno otherwise.
  1176. */
  1177. static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
  1178. struct rcu_node *rnp,
  1179. int rnp_index)
  1180. {
  1181. unsigned long flags;
  1182. struct sched_param sp;
  1183. struct task_struct *t;
  1184. if (&rcu_preempt_state != rsp)
  1185. return 0;
  1186. rsp->boost = 1;
  1187. if (rnp->boost_kthread_task != NULL)
  1188. return 0;
  1189. t = kthread_create(rcu_boost_kthread, (void *)rnp,
  1190. "rcub/%d", rnp_index);
  1191. if (IS_ERR(t))
  1192. return PTR_ERR(t);
  1193. raw_spin_lock_irqsave(&rnp->lock, flags);
  1194. rnp->boost_kthread_task = t;
  1195. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  1196. sp.sched_priority = RCU_BOOST_PRIO;
  1197. sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
  1198. wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
  1199. return 0;
  1200. }
  1201. #ifdef CONFIG_HOTPLUG_CPU
  1202. /*
  1203. * Stop the RCU's per-CPU kthread when its CPU goes offline,.
  1204. */
  1205. static void rcu_stop_cpu_kthread(int cpu)
  1206. {
  1207. struct task_struct *t;
  1208. /* Stop the CPU's kthread. */
  1209. t = per_cpu(rcu_cpu_kthread_task, cpu);
  1210. if (t != NULL) {
  1211. per_cpu(rcu_cpu_kthread_task, cpu) = NULL;
  1212. kthread_stop(t);
  1213. }
  1214. }
  1215. #endif /* #ifdef CONFIG_HOTPLUG_CPU */
  1216. static void rcu_kthread_do_work(void)
  1217. {
  1218. rcu_do_batch(&rcu_sched_state, &__get_cpu_var(rcu_sched_data));
  1219. rcu_do_batch(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
  1220. rcu_preempt_do_callbacks();
  1221. }
  1222. /*
  1223. * Wake up the specified per-rcu_node-structure kthread.
  1224. * Because the per-rcu_node kthreads are immortal, we don't need
  1225. * to do anything to keep them alive.
  1226. */
  1227. static void invoke_rcu_node_kthread(struct rcu_node *rnp)
  1228. {
  1229. struct task_struct *t;
  1230. t = rnp->node_kthread_task;
  1231. if (t != NULL)
  1232. wake_up_process(t);
  1233. }
  1234. /*
  1235. * Set the specified CPU's kthread to run RT or not, as specified by
  1236. * the to_rt argument. The CPU-hotplug locks are held, so the task
  1237. * is not going away.
  1238. */
  1239. static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
  1240. {
  1241. int policy;
  1242. struct sched_param sp;
  1243. struct task_struct *t;
  1244. t = per_cpu(rcu_cpu_kthread_task, cpu);
  1245. if (t == NULL)
  1246. return;
  1247. if (to_rt) {
  1248. policy = SCHED_FIFO;
  1249. sp.sched_priority = RCU_KTHREAD_PRIO;
  1250. } else {
  1251. policy = SCHED_NORMAL;
  1252. sp.sched_priority = 0;
  1253. }
  1254. sched_setscheduler_nocheck(t, policy, &sp);
  1255. }
  1256. /*
  1257. * Timer handler to initiate the waking up of per-CPU kthreads that
  1258. * have yielded the CPU due to excess numbers of RCU callbacks.
  1259. * We wake up the per-rcu_node kthread, which in turn will wake up
  1260. * the booster kthread.
  1261. */
  1262. static void rcu_cpu_kthread_timer(unsigned long arg)
  1263. {
  1264. struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, arg);
  1265. struct rcu_node *rnp = rdp->mynode;
  1266. atomic_or(rdp->grpmask, &rnp->wakemask);
  1267. invoke_rcu_node_kthread(rnp);
  1268. }
  1269. /*
  1270. * Drop to non-real-time priority and yield, but only after posting a
  1271. * timer that will cause us to regain our real-time priority if we
  1272. * remain preempted. Either way, we restore our real-time priority
  1273. * before returning.
  1274. */
  1275. static void rcu_yield(void (*f)(unsigned long), unsigned long arg)
  1276. {
  1277. struct sched_param sp;
  1278. struct timer_list yield_timer;
  1279. int prio = current->rt_priority;
  1280. setup_timer_on_stack(&yield_timer, f, arg);
  1281. mod_timer(&yield_timer, jiffies + 2);
  1282. sp.sched_priority = 0;
  1283. sched_setscheduler_nocheck(current, SCHED_NORMAL, &sp);
  1284. set_user_nice(current, 19);
  1285. schedule();
  1286. set_user_nice(current, 0);
  1287. sp.sched_priority = prio;
  1288. sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
  1289. del_timer(&yield_timer);
  1290. }
  1291. /*
  1292. * Handle cases where the rcu_cpu_kthread() ends up on the wrong CPU.
  1293. * This can happen while the corresponding CPU is either coming online
  1294. * or going offline. We cannot wait until the CPU is fully online
  1295. * before starting the kthread, because the various notifier functions
  1296. * can wait for RCU grace periods. So we park rcu_cpu_kthread() until
  1297. * the corresponding CPU is online.
  1298. *
  1299. * Return 1 if the kthread needs to stop, 0 otherwise.
  1300. *
  1301. * Caller must disable bh. This function can momentarily enable it.
  1302. */
  1303. static int rcu_cpu_kthread_should_stop(int cpu)
  1304. {
  1305. while (cpu_is_offline(cpu) ||
  1306. !cpumask_equal(&current->cpus_allowed, cpumask_of(cpu)) ||
  1307. smp_processor_id() != cpu) {
  1308. if (kthread_should_stop())
  1309. return 1;
  1310. per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
  1311. per_cpu(rcu_cpu_kthread_cpu, cpu) = raw_smp_processor_id();
  1312. local_bh_enable();
  1313. schedule_timeout_uninterruptible(1);
  1314. if (!cpumask_equal(&current->cpus_allowed, cpumask_of(cpu)))
  1315. set_cpus_allowed_ptr(current, cpumask_of(cpu));
  1316. local_bh_disable();
  1317. }
  1318. per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
  1319. return 0;
  1320. }
  1321. /*
  1322. * Per-CPU kernel thread that invokes RCU callbacks. This replaces the
  1323. * RCU softirq used in flavors and configurations of RCU that do not
  1324. * support RCU priority boosting.
  1325. */
  1326. static int rcu_cpu_kthread(void *arg)
  1327. {
  1328. int cpu = (int)(long)arg;
  1329. unsigned long flags;
  1330. int spincnt = 0;
  1331. unsigned int *statusp = &per_cpu(rcu_cpu_kthread_status, cpu);
  1332. char work;
  1333. char *workp = &per_cpu(rcu_cpu_has_work, cpu);
  1334. trace_rcu_utilization("Start CPU kthread@init");
  1335. for (;;) {
  1336. *statusp = RCU_KTHREAD_WAITING;
  1337. trace_rcu_utilization("End CPU kthread@rcu_wait");
  1338. rcu_wait(*workp != 0 || kthread_should_stop());
  1339. trace_rcu_utilization("Start CPU kthread@rcu_wait");
  1340. local_bh_disable();
  1341. if (rcu_cpu_kthread_should_stop(cpu)) {
  1342. local_bh_enable();
  1343. break;
  1344. }
  1345. *statusp = RCU_KTHREAD_RUNNING;
  1346. per_cpu(rcu_cpu_kthread_loops, cpu)++;
  1347. local_irq_save(flags);
  1348. work = *workp;
  1349. *workp = 0;
  1350. local_irq_restore(flags);
  1351. if (work)
  1352. rcu_kthread_do_work();
  1353. local_bh_enable();
  1354. if (*workp != 0)
  1355. spincnt++;
  1356. else
  1357. spincnt = 0;
  1358. if (spincnt > 10) {
  1359. *statusp = RCU_KTHREAD_YIELDING;
  1360. trace_rcu_utilization("End CPU kthread@rcu_yield");
  1361. rcu_yield(rcu_cpu_kthread_timer, (unsigned long)cpu);
  1362. trace_rcu_utilization("Start CPU kthread@rcu_yield");
  1363. spincnt = 0;
  1364. }
  1365. }
  1366. *statusp = RCU_KTHREAD_STOPPED;
  1367. trace_rcu_utilization("End CPU kthread@term");
  1368. return 0;
  1369. }
  1370. /*
  1371. * Spawn a per-CPU kthread, setting up affinity and priority.
  1372. * Because the CPU hotplug lock is held, no other CPU will be attempting
  1373. * to manipulate rcu_cpu_kthread_task. There might be another CPU
  1374. * attempting to access it during boot, but the locking in kthread_bind()
  1375. * will enforce sufficient ordering.
  1376. *
  1377. * Please note that we cannot simply refuse to wake up the per-CPU
  1378. * kthread because kthreads are created in TASK_UNINTERRUPTIBLE state,
  1379. * which can result in softlockup complaints if the task ends up being
  1380. * idle for more than a couple of minutes.
  1381. *
  1382. * However, please note also that we cannot bind the per-CPU kthread to its
  1383. * CPU until that CPU is fully online. We also cannot wait until the
  1384. * CPU is fully online before we create its per-CPU kthread, as this would
  1385. * deadlock the system when CPU notifiers tried waiting for grace
  1386. * periods. So we bind the per-CPU kthread to its CPU only if the CPU
  1387. * is online. If its CPU is not yet fully online, then the code in
  1388. * rcu_cpu_kthread() will wait until it is fully online, and then do
  1389. * the binding.
  1390. */
  1391. static int __cpuinit rcu_spawn_one_cpu_kthread(int cpu)
  1392. {
  1393. struct sched_param sp;
  1394. struct task_struct *t;
  1395. if (!rcu_scheduler_fully_active ||
  1396. per_cpu(rcu_cpu_kthread_task, cpu) != NULL)
  1397. return 0;
  1398. t = kthread_create_on_node(rcu_cpu_kthread,
  1399. (void *)(long)cpu,
  1400. cpu_to_node(cpu),
  1401. "rcuc/%d", cpu);
  1402. if (IS_ERR(t))
  1403. return PTR_ERR(t);
  1404. if (cpu_online(cpu))
  1405. kthread_bind(t, cpu);
  1406. per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
  1407. WARN_ON_ONCE(per_cpu(rcu_cpu_kthread_task, cpu) != NULL);
  1408. sp.sched_priority = RCU_KTHREAD_PRIO;
  1409. sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
  1410. per_cpu(rcu_cpu_kthread_task, cpu) = t;
  1411. wake_up_process(t); /* Get to TASK_INTERRUPTIBLE quickly. */
  1412. return 0;
  1413. }
  1414. /*
  1415. * Per-rcu_node kthread, which is in charge of waking up the per-CPU
  1416. * kthreads when needed. We ignore requests to wake up kthreads
  1417. * for offline CPUs, which is OK because force_quiescent_state()
  1418. * takes care of this case.
  1419. */
  1420. static int rcu_node_kthread(void *arg)
  1421. {
  1422. int cpu;
  1423. unsigned long flags;
  1424. unsigned long mask;
  1425. struct rcu_node *rnp = (struct rcu_node *)arg;
  1426. struct sched_param sp;
  1427. struct task_struct *t;
  1428. for (;;) {
  1429. rnp->node_kthread_status = RCU_KTHREAD_WAITING;
  1430. rcu_wait(atomic_read(&rnp->wakemask) != 0);
  1431. rnp->node_kthread_status = RCU_KTHREAD_RUNNING;
  1432. raw_spin_lock_irqsave(&rnp->lock, flags);
  1433. mask = atomic_xchg(&rnp->wakemask, 0);
  1434. rcu_initiate_boost(rnp, flags); /* releases rnp->lock. */
  1435. for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1) {
  1436. if ((mask & 0x1) == 0)
  1437. continue;
  1438. preempt_disable();
  1439. t = per_cpu(rcu_cpu_kthread_task, cpu);
  1440. if (!cpu_online(cpu) || t == NULL) {
  1441. preempt_enable();
  1442. continue;
  1443. }
  1444. per_cpu(rcu_cpu_has_work, cpu) = 1;
  1445. sp.sched_priority = RCU_KTHREAD_PRIO;
  1446. sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
  1447. preempt_enable();
  1448. }
  1449. }
  1450. /* NOTREACHED */
  1451. rnp->node_kthread_status = RCU_KTHREAD_STOPPED;
  1452. return 0;
  1453. }
  1454. /*
  1455. * Set the per-rcu_node kthread's affinity to cover all CPUs that are
  1456. * served by the rcu_node in question. The CPU hotplug lock is still
  1457. * held, so the value of rnp->qsmaskinit will be stable.
  1458. *
  1459. * We don't include outgoingcpu in the affinity set, use -1 if there is
  1460. * no outgoing CPU. If there are no CPUs left in the affinity set,
  1461. * this function allows the kthread to execute on any CPU.
  1462. */
  1463. static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
  1464. {
  1465. cpumask_var_t cm;
  1466. int cpu;
  1467. unsigned long mask = rnp->qsmaskinit;
  1468. if (rnp->node_kthread_task == NULL)
  1469. return;
  1470. if (!alloc_cpumask_var(&cm, GFP_KERNEL))
  1471. return;
  1472. cpumask_clear(cm);
  1473. for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++, mask >>= 1)
  1474. if ((mask & 0x1) && cpu != outgoingcpu)
  1475. cpumask_set_cpu(cpu, cm);
  1476. if (cpumask_weight(cm) == 0) {
  1477. cpumask_setall(cm);
  1478. for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++)
  1479. cpumask_clear_cpu(cpu, cm);
  1480. WARN_ON_ONCE(cpumask_weight(cm) == 0);
  1481. }
  1482. set_cpus_allowed_ptr(rnp->node_kthread_task, cm);
  1483. rcu_boost_kthread_setaffinity(rnp, cm);
  1484. free_cpumask_var(cm);
  1485. }
  1486. /*
  1487. * Spawn a per-rcu_node kthread, setting priority and affinity.
  1488. * Called during boot before online/offline can happen, or, if
  1489. * during runtime, with the main CPU-hotplug locks held. So only
  1490. * one of these can be executing at a time.
  1491. */
  1492. static int __cpuinit rcu_spawn_one_node_kthread(struct rcu_state *rsp,
  1493. struct rcu_node *rnp)
  1494. {
  1495. unsigned long flags;
  1496. int rnp_index = rnp - &rsp->node[0];
  1497. struct sched_param sp;
  1498. struct task_struct *t;
  1499. if (!rcu_scheduler_fully_active ||
  1500. rnp->qsmaskinit == 0)
  1501. return 0;
  1502. if (rnp->node_kthread_task == NULL) {
  1503. t = kthread_create(rcu_node_kthread, (void *)rnp,
  1504. "rcun/%d", rnp_index);
  1505. if (IS_ERR(t))
  1506. return PTR_ERR(t);
  1507. raw_spin_lock_irqsave(&rnp->lock, flags);
  1508. rnp->node_kthread_task = t;
  1509. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  1510. sp.sched_priority = 99;
  1511. sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
  1512. wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
  1513. }
  1514. return rcu_spawn_one_boost_kthread(rsp, rnp, rnp_index);
  1515. }
  1516. /*
  1517. * Spawn all kthreads -- called as soon as the scheduler is running.
  1518. */
  1519. static int __init rcu_spawn_kthreads(void)
  1520. {
  1521. int cpu;
  1522. struct rcu_node *rnp;
  1523. rcu_scheduler_fully_active = 1;
  1524. for_each_possible_cpu(cpu) {
  1525. per_cpu(rcu_cpu_has_work, cpu) = 0;
  1526. if (cpu_online(cpu))
  1527. (void)rcu_spawn_one_cpu_kthread(cpu);
  1528. }
  1529. rnp = rcu_get_root(rcu_state);
  1530. (void)rcu_spawn_one_node_kthread(rcu_state, rnp);
  1531. if (NUM_RCU_NODES > 1) {
  1532. rcu_for_each_leaf_node(rcu_state, rnp)
  1533. (void)rcu_spawn_one_node_kthread(rcu_state, rnp);
  1534. }
  1535. return 0;
  1536. }
  1537. early_initcall(rcu_spawn_kthreads);
  1538. static void __cpuinit rcu_prepare_kthreads(int cpu)
  1539. {
  1540. struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
  1541. struct rcu_node *rnp = rdp->mynode;
  1542. /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
  1543. if (rcu_scheduler_fully_active) {
  1544. (void)rcu_spawn_one_cpu_kthread(cpu);
  1545. if (rnp->node_kthread_task == NULL)
  1546. (void)rcu_spawn_one_node_kthread(rcu_state, rnp);
  1547. }
  1548. }
  1549. #else /* #ifdef CONFIG_RCU_BOOST */
  1550. static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
  1551. {
  1552. raw_spin_unlock_irqrestore(&rnp->lock, flags);
  1553. }
  1554. static void invoke_rcu_callbacks_kthread(void)
  1555. {
  1556. WARN_ON_ONCE(1);
  1557. }
  1558. static bool rcu_is_callbacks_kthread(void)
  1559. {
  1560. return false;
  1561. }
  1562. static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
  1563. {
  1564. }
  1565. #ifdef CONFIG_HOTPLUG_CPU
  1566. static void rcu_stop_cpu_kthread(int cpu)
  1567. {
  1568. }
  1569. #endif /* #ifdef CONFIG_HOTPLUG_CPU */
  1570. static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
  1571. {
  1572. }
  1573. static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
  1574. {
  1575. }
  1576. static int __init rcu_scheduler_really_started(void)
  1577. {
  1578. rcu_scheduler_fully_active = 1;
  1579. return 0;
  1580. }
  1581. early_initcall(rcu_scheduler_really_started);
  1582. static void __cpuinit rcu_prepare_kthreads(int cpu)
  1583. {
  1584. }
  1585. #endif /* #else #ifdef CONFIG_RCU_BOOST */
  1586. #if !defined(CONFIG_RCU_FAST_NO_HZ)
  1587. /*
  1588. * Check to see if any future RCU-related work will need to be done
  1589. * by the current CPU, even if none need be done immediately, returning
  1590. * 1 if so. This function is part of the RCU implementation; it is -not-
  1591. * an exported member of the RCU API.
  1592. *
  1593. * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
  1594. * any flavor of RCU.
  1595. */
  1596. int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
  1597. {
  1598. *delta_jiffies = ULONG_MAX;
  1599. return rcu_cpu_has_callbacks(cpu);
  1600. }
  1601. /*
  1602. * Because we do not have RCU_FAST_NO_HZ, don't bother initializing for it.
  1603. */
  1604. static void rcu_prepare_for_idle_init(int cpu)
  1605. {
  1606. }
  1607. /*
  1608. * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
  1609. * after it.
  1610. */
  1611. static void rcu_cleanup_after_idle(int cpu)
  1612. {
  1613. }
  1614. /*
  1615. * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
  1616. * is nothing.
  1617. */
  1618. static void rcu_prepare_for_idle(int cpu)
  1619. {
  1620. }
  1621. /*
  1622. * Don't bother keeping a running count of the number of RCU callbacks
  1623. * posted because CONFIG_RCU_FAST_NO_HZ=n.
  1624. */
  1625. static void rcu_idle_count_callbacks_posted(void)
  1626. {
  1627. }
  1628. #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
  1629. /*
  1630. * This code is invoked when a CPU goes idle, at which point we want
  1631. * to have the CPU do everything required for RCU so that it can enter
  1632. * the energy-efficient dyntick-idle mode. This is handled by a
  1633. * state machine implemented by rcu_prepare_for_idle() below.
  1634. *
  1635. * The following three proprocessor symbols control this state machine:
  1636. *
  1637. * RCU_IDLE_FLUSHES gives the maximum number of times that we will attempt
  1638. * to satisfy RCU. Beyond this point, it is better to incur a periodic
  1639. * scheduling-clock interrupt than to loop through the state machine
  1640. * at full power.
  1641. * RCU_IDLE_OPT_FLUSHES gives the number of RCU_IDLE_FLUSHES that are
  1642. * optional if RCU does not need anything immediately from this
  1643. * CPU, even if this CPU still has RCU callbacks queued. The first
  1644. * times through the state machine are mandatory: we need to give
  1645. * the state machine a chance to communicate a quiescent state
  1646. * to the RCU core.
  1647. * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
  1648. * to sleep in dyntick-idle mode with RCU callbacks pending. This
  1649. * is sized to be roughly one RCU grace period. Those energy-efficiency
  1650. * benchmarkers who might otherwise be tempted to set this to a large
  1651. * number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
  1652. * system. And if you are -that- concerned about energy efficiency,
  1653. * just power the system down and be done with it!
  1654. * RCU_IDLE_LAZY_GP_DELAY gives the number of jiffies that a CPU is
  1655. * permitted to sleep in dyntick-idle mode with only lazy RCU
  1656. * callbacks pending. Setting this too high can OOM your system.
  1657. *
  1658. * The values below work well in practice. If future workloads require
  1659. * adjustment, they can be converted into kernel config parameters, though
  1660. * making the state machine smarter might be a better option.
  1661. */
  1662. #define RCU_IDLE_FLUSHES 5 /* Number of dyntick-idle tries. */
  1663. #define RCU_IDLE_OPT_FLUSHES 3 /* Optional dyntick-idle tries. */
  1664. #define RCU_IDLE_GP_DELAY 6 /* Roughly one grace period. */
  1665. #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
  1666. /*
  1667. * Does the specified flavor of RCU have non-lazy callbacks pending on
  1668. * the specified CPU? Both RCU flavor and CPU are specified by the
  1669. * rcu_data structure.
  1670. */
  1671. static bool __rcu_cpu_has_nonlazy_callbacks(struct rcu_data *rdp)
  1672. {
  1673. return rdp->qlen != rdp->qlen_lazy;
  1674. }
  1675. #ifdef CONFIG_TREE_PREEMPT_RCU
  1676. /*
  1677. * Are there non-lazy RCU-preempt callbacks? (There cannot be if there
  1678. * is no RCU-preempt in the kernel.)
  1679. */
  1680. static bool rcu_preempt_cpu_has_nonlazy_callbacks(int cpu)
  1681. {
  1682. struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
  1683. return __rcu_cpu_has_nonlazy_callbacks(rdp);
  1684. }
  1685. #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
  1686. static bool rcu_preempt_cpu_has_nonlazy_callbacks(int cpu)
  1687. {
  1688. return 0;
  1689. }
  1690. #endif /* else #ifdef CONFIG_TREE_PREEMPT_RCU */
  1691. /*
  1692. * Does any flavor of RCU have non-lazy callbacks on the specified CPU?
  1693. */
  1694. static bool rcu_cpu_has_nonlazy_callbacks(int cpu)
  1695. {
  1696. return __rcu_cpu_has_nonlazy_callbacks(&per_cpu(rcu_sched_data, cpu)) ||
  1697. __rcu_cpu_has_nonlazy_callbacks(&per_cpu(rcu_bh_data, cpu)) ||
  1698. rcu_preempt_cpu_has_nonlazy_callbacks(cpu);
  1699. }
  1700. /*
  1701. * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
  1702. * callbacks on this CPU, (2) this CPU has not yet attempted to enter
  1703. * dyntick-idle mode, or (3) this CPU is in the process of attempting to
  1704. * enter dyntick-idle mode. Otherwise, if we have recently tried and failed
  1705. * to enter dyntick-idle mode, we refuse to try to enter it. After all,
  1706. * it is better to incur scheduling-clock interrupts than to spin
  1707. * continuously for the same time duration!
  1708. *
  1709. * The delta_jiffies argument is used to store the time when RCU is
  1710. * going to need the CPU again if it still has callbacks. The reason
  1711. * for this is that rcu_prepare_for_idle() might need to post a timer,
  1712. * but if so, it will do so after tick_nohz_stop_sched_tick() has set
  1713. * the wakeup time for this CPU. This means that RCU's timer can be
  1714. * delayed until the wakeup time, which defeats the purpose of posting
  1715. * a timer.
  1716. */
  1717. int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
  1718. {
  1719. struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
  1720. /* Flag a new idle sojourn to the idle-entry state machine. */
  1721. rdtp->idle_first_pass = 1;
  1722. /* If no callbacks, RCU doesn't need the CPU. */
  1723. if (!rcu_cpu_has_callbacks(cpu)) {
  1724. *delta_jiffies = ULONG_MAX;
  1725. return 0;
  1726. }
  1727. if (rdtp->dyntick_holdoff == jiffies) {
  1728. /* RCU recently tried and failed, so don't try again. */
  1729. *delta_jiffies = 1;
  1730. return 1;
  1731. }
  1732. /* Set up for the possibility that RCU will post a timer. */
  1733. if (rcu_cpu_has_nonlazy_callbacks(cpu))
  1734. *delta_jiffies = RCU_IDLE_GP_DELAY;
  1735. else
  1736. *delta_jiffies = RCU_IDLE_LAZY_GP_DELAY;
  1737. return 0;
  1738. }
  1739. /*
  1740. * Handler for smp_call_function_single(). The only point of this
  1741. * handler is to wake the CPU up, so the handler does only tracing.
  1742. */
  1743. void rcu_idle_demigrate(void *unused)
  1744. {
  1745. trace_rcu_prep_idle("Demigrate");
  1746. }
  1747. /*
  1748. * Timer handler used to force CPU to start pushing its remaining RCU
  1749. * callbacks in the case where it entered dyntick-idle mode with callbacks
  1750. * pending. The hander doesn't really need to do anything because the
  1751. * real work is done upon re-entry to idle, or by the next scheduling-clock
  1752. * interrupt should idle not be re-entered.
  1753. *
  1754. * One special case: the timer gets migrated without awakening the CPU
  1755. * on which the timer was scheduled on. In this case, we must wake up
  1756. * that CPU. We do so with smp_call_function_single().
  1757. */
  1758. static void rcu_idle_gp_timer_func(unsigned long cpu_in)
  1759. {
  1760. int cpu = (int)cpu_in;
  1761. trace_rcu_prep_idle("Timer");
  1762. if (cpu != smp_processor_id())
  1763. smp_call_function_single(cpu, rcu_idle_demigrate, NULL, 0);
  1764. else
  1765. WARN_ON_ONCE(1); /* Getting here can hang the system... */
  1766. }
  1767. /*
  1768. * Initialize the timer used to pull CPUs out of dyntick-idle mode.
  1769. */
  1770. static void rcu_prepare_for_idle_init(int cpu)
  1771. {
  1772. struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
  1773. rdtp->dyntick_holdoff = jiffies - 1;
  1774. setup_timer(&rdtp->idle_gp_timer, rcu_idle_gp_timer_func, cpu);
  1775. rdtp->idle_gp_timer_expires = jiffies - 1;
  1776. rdtp->idle_first_pass = 1;
  1777. }
  1778. /*
  1779. * Clean up for exit from idle. Because we are exiting from idle, there
  1780. * is no longer any point to ->idle_gp_timer, so cancel it. This will
  1781. * do nothing if this timer is not active, so just cancel it unconditionally.
  1782. */
  1783. static void rcu_cleanup_after_idle(int cpu)
  1784. {
  1785. struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
  1786. del_timer(&rdtp->idle_gp_timer);
  1787. trace_rcu_prep_idle("Cleanup after idle");
  1788. }
  1789. /*
  1790. * Check to see if any RCU-related work can be done by the current CPU,
  1791. * and if so, schedule a softirq to get it done. This function is part
  1792. * of the RCU implementation; it is -not- an exported member of the RCU API.
  1793. *
  1794. * The idea is for the current CPU to clear out all work required by the
  1795. * RCU core for the current grace period, so that this CPU can be permitted
  1796. * to enter dyntick-idle mode. In some cases, it will need to be awakened
  1797. * at the end of the grace period by whatever CPU ends the grace period.
  1798. * This allows CPUs to go dyntick-idle more quickly, and to reduce the
  1799. * number of wakeups by a modest integer factor.
  1800. *
  1801. * Because it is not legal to invoke rcu_process_callbacks() with irqs
  1802. * disabled, we do one pass of force_quiescent_state(), then do a
  1803. * invoke_rcu_core() to cause rcu_process_callbacks() to be invoked
  1804. * later. The ->dyntick_drain field controls the sequencing.
  1805. *
  1806. * The caller must have disabled interrupts.
  1807. */
  1808. static void rcu_prepare_for_idle(int cpu)
  1809. {
  1810. struct timer_list *tp;
  1811. struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
  1812. /*
  1813. * If this is an idle re-entry, for example, due to use of
  1814. * RCU_NONIDLE() or the new idle-loop tracing API within the idle
  1815. * loop, then don't take any state-machine actions, unless the
  1816. * momentary exit from idle queued additional non-lazy callbacks.
  1817. * Instead, repost the ->idle_gp_timer if this CPU has callbacks
  1818. * pending.
  1819. */
  1820. if (!rdtp->idle_first_pass &&
  1821. (rdtp->nonlazy_posted == rdtp->nonlazy_posted_snap)) {
  1822. if (rcu_cpu_has_callbacks(cpu)) {
  1823. tp = &rdtp->idle_gp_timer;
  1824. mod_timer_pinned(tp, rdtp->idle_gp_timer_expires);
  1825. }
  1826. return;
  1827. }
  1828. rdtp->idle_first_pass = 0;
  1829. rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted - 1;
  1830. /*
  1831. * If there are no callbacks on this CPU, enter dyntick-idle mode.
  1832. * Also reset state to avoid prejudicing later attempts.
  1833. */
  1834. if (!rcu_cpu_has_callbacks(cpu)) {
  1835. rdtp->dyntick_holdoff = jiffies - 1;
  1836. rdtp->dyntick_drain = 0;
  1837. trace_rcu_prep_idle("No callbacks");
  1838. return;
  1839. }
  1840. /*
  1841. * If in holdoff mode, just return. We will presumably have
  1842. * refrained from disabling the scheduling-clock tick.
  1843. */
  1844. if (rdtp->dyntick_holdoff == jiffies) {
  1845. trace_rcu_prep_idle("In holdoff");
  1846. return;
  1847. }
  1848. /* Check and update the ->dyntick_drain sequencing. */
  1849. if (rdtp->dyntick_drain <= 0) {
  1850. /* First time through, initialize the counter. */
  1851. rdtp->dyntick_drain = RCU_IDLE_FLUSHES;
  1852. } else if (rdtp->dyntick_drain <= RCU_IDLE_OPT_FLUSHES &&
  1853. !rcu_pending(cpu) &&
  1854. !local_softirq_pending()) {
  1855. /* Can we go dyntick-idle despite still having callbacks? */
  1856. rdtp->dyntick_drain = 0;
  1857. rdtp->dyntick_holdoff = jiffies;
  1858. if (rcu_cpu_has_nonlazy_callbacks(cpu)) {
  1859. trace_rcu_prep_idle("Dyntick with callbacks");
  1860. rdtp->idle_gp_timer_expires =
  1861. jiffies + RCU_IDLE_GP_DELAY;
  1862. } else {
  1863. rdtp->idle_gp_timer_expires =
  1864. jiffies + RCU_IDLE_LAZY_GP_DELAY;
  1865. trace_rcu_prep_idle("Dyntick with lazy callbacks");
  1866. }
  1867. tp = &rdtp->idle_gp_timer;
  1868. mod_timer_pinned(tp, rdtp->idle_gp_timer_expires);
  1869. rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted;
  1870. return; /* Nothing more to do immediately. */
  1871. } else if (--(rdtp->dyntick_drain) <= 0) {
  1872. /* We have hit the limit, so time to give up. */
  1873. rdtp->dyntick_holdoff = jiffies;
  1874. trace_rcu_prep_idle("Begin holdoff");
  1875. invoke_rcu_core(); /* Force the CPU out of dyntick-idle. */
  1876. return;
  1877. }
  1878. /*
  1879. * Do one step of pushing the remaining RCU callbacks through
  1880. * the RCU core state machine.
  1881. */
  1882. #ifdef CONFIG_TREE_PREEMPT_RCU
  1883. if (per_cpu(rcu_preempt_data, cpu).nxtlist) {
  1884. rcu_preempt_qs(cpu);
  1885. force_quiescent_state(&rcu_preempt_state, 0);
  1886. }
  1887. #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
  1888. if (per_cpu(rcu_sched_data, cpu).nxtlist) {
  1889. rcu_sched_qs(cpu);
  1890. force_quiescent_state(&rcu_sched_state, 0);
  1891. }
  1892. if (per_cpu(rcu_bh_data, cpu).nxtlist) {
  1893. rcu_bh_qs(cpu);
  1894. force_quiescent_state(&rcu_bh_state, 0);
  1895. }
  1896. /*
  1897. * If RCU callbacks are still pending, RCU still needs this CPU.
  1898. * So try forcing the callbacks through the grace period.
  1899. */
  1900. if (rcu_cpu_has_callbacks(cpu)) {
  1901. trace_rcu_prep_idle("More callbacks");
  1902. invoke_rcu_core();
  1903. } else
  1904. trace_rcu_prep_idle("Callbacks drained");
  1905. }
  1906. /*
  1907. * Keep a running count of the number of non-lazy callbacks posted
  1908. * on this CPU. This running counter (which is never decremented) allows
  1909. * rcu_prepare_for_idle() to detect when something out of the idle loop
  1910. * posts a callback, even if an equal number of callbacks are invoked.
  1911. * Of course, callbacks should only be posted from within a trace event
  1912. * designed to be called from idle or from within RCU_NONIDLE().
  1913. */
  1914. static void rcu_idle_count_callbacks_posted(void)
  1915. {
  1916. __this_cpu_add(rcu_dynticks.nonlazy_posted, 1);
  1917. }
  1918. #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
  1919. #ifdef CONFIG_RCU_CPU_STALL_INFO
  1920. #ifdef CONFIG_RCU_FAST_NO_HZ
  1921. static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
  1922. {
  1923. struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
  1924. struct timer_list *tltp = &rdtp->idle_gp_timer;
  1925. sprintf(cp, "drain=%d %c timer=%lu",
  1926. rdtp->dyntick_drain,
  1927. rdtp->dyntick_holdoff == jiffies ? 'H' : '.',
  1928. timer_pending(tltp) ? tltp->expires - jiffies : -1);
  1929. }
  1930. #else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
  1931. static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
  1932. {
  1933. }
  1934. #endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
  1935. /* Initiate the stall-info list. */
  1936. static void print_cpu_stall_info_begin(void)
  1937. {
  1938. printk(KERN_CONT "\n");
  1939. }
  1940. /*
  1941. * Print out diagnostic information for the specified stalled CPU.
  1942. *
  1943. * If the specified CPU is aware of the current RCU grace period
  1944. * (flavor specified by rsp), then print the number of scheduling
  1945. * clock interrupts the CPU has taken during the time that it has
  1946. * been aware. Otherwise, print the number of RCU grace periods
  1947. * that this CPU is ignorant of, for example, "1" if the CPU was
  1948. * aware of the previous grace period.
  1949. *
  1950. * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
  1951. */
  1952. static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
  1953. {
  1954. char fast_no_hz[72];
  1955. struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
  1956. struct rcu_dynticks *rdtp = rdp->dynticks;
  1957. char *ticks_title;
  1958. unsigned long ticks_value;
  1959. if (rsp->gpnum == rdp->gpnum) {
  1960. ticks_title = "ticks this GP";
  1961. ticks_value = rdp->ticks_this_gp;
  1962. } else {
  1963. ticks_title = "GPs behind";
  1964. ticks_value = rsp->gpnum - rdp->gpnum;
  1965. }
  1966. print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
  1967. printk(KERN_ERR "\t%d: (%lu %s) idle=%03x/%llx/%d %s\n",
  1968. cpu, ticks_value, ticks_title,
  1969. atomic_read(&rdtp->dynticks) & 0xfff,
  1970. rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
  1971. fast_no_hz);
  1972. }
  1973. /* Terminate the stall-info list. */
  1974. static void print_cpu_stall_info_end(void)
  1975. {
  1976. printk(KERN_ERR "\t");
  1977. }
  1978. /* Zero ->ticks_this_gp for all flavors of RCU. */
  1979. static void zero_cpu_stall_ticks(struct rcu_data *rdp)
  1980. {
  1981. rdp->ticks_this_gp = 0;
  1982. }
  1983. /* Increment ->ticks_this_gp for all flavors of RCU. */
  1984. static void increment_cpu_stall_ticks(void)
  1985. {
  1986. __get_cpu_var(rcu_sched_data).ticks_this_gp++;
  1987. __get_cpu_var(rcu_bh_data).ticks_this_gp++;
  1988. #ifdef CONFIG_TREE_PREEMPT_RCU
  1989. __get_cpu_var(rcu_preempt_data).ticks_this_gp++;
  1990. #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
  1991. }
  1992. #else /* #ifdef CONFIG_RCU_CPU_STALL_INFO */
  1993. static void print_cpu_stall_info_begin(void)
  1994. {
  1995. printk(KERN_CONT " {");
  1996. }
  1997. static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
  1998. {
  1999. printk(KERN_CONT " %d", cpu);
  2000. }
  2001. static void print_cpu_stall_info_end(void)
  2002. {
  2003. printk(KERN_CONT "} ");
  2004. }
  2005. static void zero_cpu_stall_ticks(struct rcu_data *rdp)
  2006. {
  2007. }
  2008. static void increment_cpu_stall_ticks(void)
  2009. {
  2010. }
  2011. #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_INFO */