softirq.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778
  1. /*
  2. * linux/kernel/softirq.c
  3. *
  4. * Copyright (C) 1992 Linus Torvalds
  5. *
  6. * Distribute under GPLv2.
  7. *
  8. * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
  9. */
  10. #include <linux/export.h>
  11. #include <linux/kernel_stat.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/init.h>
  14. #include <linux/mm.h>
  15. #include <linux/notifier.h>
  16. #include <linux/percpu.h>
  17. #include <linux/cpu.h>
  18. #include <linux/freezer.h>
  19. #include <linux/kthread.h>
  20. #include <linux/rcupdate.h>
  21. #include <linux/ftrace.h>
  22. #include <linux/smp.h>
  23. #include <linux/smpboot.h>
  24. #include <linux/tick.h>
  25. #define CREATE_TRACE_POINTS
  26. #include <trace/events/irq.h>
  27. /*
  28. - No shared variables, all the data are CPU local.
  29. - If a softirq needs serialization, let it serialize itself
  30. by its own spinlocks.
  31. - Even if softirq is serialized, only local cpu is marked for
  32. execution. Hence, we get something sort of weak cpu binding.
  33. Though it is still not clear, will it result in better locality
  34. or will not.
  35. Examples:
  36. - NET RX softirq. It is multithreaded and does not require
  37. any global serialization.
  38. - NET TX softirq. It kicks software netdevice queues, hence
  39. it is logically serialized per device, but this serialization
  40. is invisible to common code.
  41. - Tasklets: serialized wrt itself.
  42. */
  43. #ifndef __ARCH_IRQ_STAT
  44. irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  45. EXPORT_SYMBOL(irq_stat);
  46. #endif
  47. static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  48. DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  49. char *softirq_to_name[NR_SOFTIRQS] = {
  50. "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  51. "TASKLET", "SCHED", "HRTIMER", "RCU"
  52. };
  53. /*
  54. * we cannot loop indefinitely here to avoid userspace starvation,
  55. * but we also don't want to introduce a worst case 1/HZ latency
  56. * to the pending events, so lets the scheduler to balance
  57. * the softirq load for us.
  58. */
  59. static void wakeup_softirqd(void)
  60. {
  61. /* Interrupts are disabled: no need to stop preemption */
  62. struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  63. if (tsk && tsk->state != TASK_RUNNING)
  64. wake_up_process(tsk);
  65. }
  66. /*
  67. * preempt_count and SOFTIRQ_OFFSET usage:
  68. * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  69. * softirq processing.
  70. * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  71. * on local_bh_disable or local_bh_enable.
  72. * This lets us distinguish between whether we are currently processing
  73. * softirq and whether we just have bh disabled.
  74. */
  75. /*
  76. * This one is for softirq.c-internal use,
  77. * where hardirqs are disabled legitimately:
  78. */
  79. #ifdef CONFIG_TRACE_IRQFLAGS
  80. static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  81. {
  82. unsigned long flags;
  83. WARN_ON_ONCE(in_irq());
  84. raw_local_irq_save(flags);
  85. /*
  86. * The preempt tracer hooks into preempt_count_add and will break
  87. * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
  88. * is set and before current->softirq_enabled is cleared.
  89. * We must manually increment preempt_count here and manually
  90. * call the trace_preempt_off later.
  91. */
  92. __preempt_count_add(cnt);
  93. /*
  94. * Were softirqs turned off above:
  95. */
  96. if (softirq_count() == cnt)
  97. trace_softirqs_off(ip);
  98. raw_local_irq_restore(flags);
  99. if (preempt_count() == cnt)
  100. trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
  101. }
  102. #else /* !CONFIG_TRACE_IRQFLAGS */
  103. static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
  104. {
  105. preempt_count_add(cnt);
  106. barrier();
  107. }
  108. #endif /* CONFIG_TRACE_IRQFLAGS */
  109. void local_bh_disable(void)
  110. {
  111. __local_bh_disable(_RET_IP_, SOFTIRQ_DISABLE_OFFSET);
  112. }
  113. EXPORT_SYMBOL(local_bh_disable);
  114. static void __local_bh_enable(unsigned int cnt)
  115. {
  116. WARN_ON_ONCE(!irqs_disabled());
  117. if (softirq_count() == cnt)
  118. trace_softirqs_on(_RET_IP_);
  119. preempt_count_sub(cnt);
  120. }
  121. /*
  122. * Special-case - softirqs can safely be enabled in
  123. * cond_resched_softirq(), or by __do_softirq(),
  124. * without processing still-pending softirqs:
  125. */
  126. void _local_bh_enable(void)
  127. {
  128. WARN_ON_ONCE(in_irq());
  129. __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
  130. }
  131. EXPORT_SYMBOL(_local_bh_enable);
  132. static inline void _local_bh_enable_ip(unsigned long ip)
  133. {
  134. WARN_ON_ONCE(in_irq() || irqs_disabled());
  135. #ifdef CONFIG_TRACE_IRQFLAGS
  136. local_irq_disable();
  137. #endif
  138. /*
  139. * Are softirqs going to be turned on now:
  140. */
  141. if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
  142. trace_softirqs_on(ip);
  143. /*
  144. * Keep preemption disabled until we are done with
  145. * softirq processing:
  146. */
  147. preempt_count_sub(SOFTIRQ_DISABLE_OFFSET - 1);
  148. if (unlikely(!in_interrupt() && local_softirq_pending())) {
  149. /*
  150. * Run softirq if any pending. And do it in its own stack
  151. * as we may be calling this deep in a task call stack already.
  152. */
  153. do_softirq();
  154. }
  155. preempt_count_dec();
  156. #ifdef CONFIG_TRACE_IRQFLAGS
  157. local_irq_enable();
  158. #endif
  159. preempt_check_resched();
  160. }
  161. void local_bh_enable(void)
  162. {
  163. _local_bh_enable_ip(_RET_IP_);
  164. }
  165. EXPORT_SYMBOL(local_bh_enable);
  166. void local_bh_enable_ip(unsigned long ip)
  167. {
  168. _local_bh_enable_ip(ip);
  169. }
  170. EXPORT_SYMBOL(local_bh_enable_ip);
  171. /*
  172. * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
  173. * but break the loop if need_resched() is set or after 2 ms.
  174. * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
  175. * certain cases, such as stop_machine(), jiffies may cease to
  176. * increment and so we need the MAX_SOFTIRQ_RESTART limit as
  177. * well to make sure we eventually return from this method.
  178. *
  179. * These limits have been established via experimentation.
  180. * The two things to balance is latency against fairness -
  181. * we want to handle softirqs as soon as possible, but they
  182. * should not be able to lock up the box.
  183. */
  184. #define MAX_SOFTIRQ_TIME msecs_to_jiffies(2)
  185. #define MAX_SOFTIRQ_RESTART 10
  186. asmlinkage void __do_softirq(void)
  187. {
  188. struct softirq_action *h;
  189. __u32 pending;
  190. unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
  191. int cpu;
  192. unsigned long old_flags = current->flags;
  193. int max_restart = MAX_SOFTIRQ_RESTART;
  194. /*
  195. * Mask out PF_MEMALLOC s current task context is borrowed for the
  196. * softirq. A softirq handled such as network RX might set PF_MEMALLOC
  197. * again if the socket is related to swap
  198. */
  199. current->flags &= ~PF_MEMALLOC;
  200. pending = local_softirq_pending();
  201. account_irq_enter_time(current);
  202. __local_bh_disable(_RET_IP_, SOFTIRQ_OFFSET);
  203. lockdep_softirq_enter();
  204. cpu = smp_processor_id();
  205. restart:
  206. /* Reset the pending bitmask before enabling irqs */
  207. set_softirq_pending(0);
  208. local_irq_enable();
  209. h = softirq_vec;
  210. do {
  211. if (pending & 1) {
  212. unsigned int vec_nr = h - softirq_vec;
  213. int prev_count = preempt_count();
  214. kstat_incr_softirqs_this_cpu(vec_nr);
  215. trace_softirq_entry(vec_nr);
  216. h->action(h);
  217. trace_softirq_exit(vec_nr);
  218. if (unlikely(prev_count != preempt_count())) {
  219. printk(KERN_ERR "huh, entered softirq %u %s %p"
  220. "with preempt_count %08x,"
  221. " exited with %08x?\n", vec_nr,
  222. softirq_to_name[vec_nr], h->action,
  223. prev_count, preempt_count());
  224. preempt_count_set(prev_count);
  225. }
  226. rcu_bh_qs(cpu);
  227. }
  228. h++;
  229. pending >>= 1;
  230. } while (pending);
  231. local_irq_disable();
  232. pending = local_softirq_pending();
  233. if (pending) {
  234. if (time_before(jiffies, end) && !need_resched() &&
  235. --max_restart)
  236. goto restart;
  237. wakeup_softirqd();
  238. }
  239. lockdep_softirq_exit();
  240. account_irq_exit_time(current);
  241. __local_bh_enable(SOFTIRQ_OFFSET);
  242. WARN_ON_ONCE(in_interrupt());
  243. tsk_restore_flags(current, old_flags, PF_MEMALLOC);
  244. }
  245. asmlinkage void do_softirq(void)
  246. {
  247. __u32 pending;
  248. unsigned long flags;
  249. if (in_interrupt())
  250. return;
  251. local_irq_save(flags);
  252. pending = local_softirq_pending();
  253. if (pending)
  254. do_softirq_own_stack();
  255. local_irq_restore(flags);
  256. }
  257. /*
  258. * Enter an interrupt context.
  259. */
  260. void irq_enter(void)
  261. {
  262. int cpu = smp_processor_id();
  263. rcu_irq_enter();
  264. if (is_idle_task(current) && !in_interrupt()) {
  265. /*
  266. * Prevent raise_softirq from needlessly waking up ksoftirqd
  267. * here, as softirq will be serviced on return from interrupt.
  268. */
  269. local_bh_disable();
  270. tick_check_idle(cpu);
  271. _local_bh_enable();
  272. }
  273. __irq_enter();
  274. }
  275. static inline void invoke_softirq(void)
  276. {
  277. if (!force_irqthreads) {
  278. #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
  279. /*
  280. * We can safely execute softirq on the current stack if
  281. * it is the irq stack, because it should be near empty
  282. * at this stage.
  283. */
  284. __do_softirq();
  285. #else
  286. /*
  287. * Otherwise, irq_exit() is called on the task stack that can
  288. * be potentially deep already. So call softirq in its own stack
  289. * to prevent from any overrun.
  290. */
  291. do_softirq_own_stack();
  292. #endif
  293. } else {
  294. wakeup_softirqd();
  295. }
  296. }
  297. static inline void tick_irq_exit(void)
  298. {
  299. #ifdef CONFIG_NO_HZ_COMMON
  300. int cpu = smp_processor_id();
  301. /* Make sure that timer wheel updates are propagated */
  302. if ((idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
  303. if (!in_interrupt())
  304. tick_nohz_irq_exit();
  305. }
  306. #endif
  307. }
  308. /*
  309. * Exit an interrupt context. Process softirqs if needed and possible:
  310. */
  311. void irq_exit(void)
  312. {
  313. #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
  314. local_irq_disable();
  315. #else
  316. WARN_ON_ONCE(!irqs_disabled());
  317. #endif
  318. account_irq_exit_time(current);
  319. trace_hardirq_exit();
  320. preempt_count_sub(HARDIRQ_OFFSET);
  321. if (!in_interrupt() && local_softirq_pending())
  322. invoke_softirq();
  323. tick_irq_exit();
  324. rcu_irq_exit();
  325. }
  326. /*
  327. * This function must run with irqs disabled!
  328. */
  329. inline void raise_softirq_irqoff(unsigned int nr)
  330. {
  331. __raise_softirq_irqoff(nr);
  332. /*
  333. * If we're in an interrupt or softirq, we're done
  334. * (this also catches softirq-disabled code). We will
  335. * actually run the softirq once we return from
  336. * the irq or softirq.
  337. *
  338. * Otherwise we wake up ksoftirqd to make sure we
  339. * schedule the softirq soon.
  340. */
  341. if (!in_interrupt())
  342. wakeup_softirqd();
  343. }
  344. void raise_softirq(unsigned int nr)
  345. {
  346. unsigned long flags;
  347. local_irq_save(flags);
  348. raise_softirq_irqoff(nr);
  349. local_irq_restore(flags);
  350. }
  351. void __raise_softirq_irqoff(unsigned int nr)
  352. {
  353. trace_softirq_raise(nr);
  354. or_softirq_pending(1UL << nr);
  355. }
  356. void open_softirq(int nr, void (*action)(struct softirq_action *))
  357. {
  358. softirq_vec[nr].action = action;
  359. }
  360. /*
  361. * Tasklets
  362. */
  363. struct tasklet_head
  364. {
  365. struct tasklet_struct *head;
  366. struct tasklet_struct **tail;
  367. };
  368. static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
  369. static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
  370. void __tasklet_schedule(struct tasklet_struct *t)
  371. {
  372. unsigned long flags;
  373. local_irq_save(flags);
  374. t->next = NULL;
  375. *__this_cpu_read(tasklet_vec.tail) = t;
  376. __this_cpu_write(tasklet_vec.tail, &(t->next));
  377. raise_softirq_irqoff(TASKLET_SOFTIRQ);
  378. local_irq_restore(flags);
  379. }
  380. EXPORT_SYMBOL(__tasklet_schedule);
  381. void __tasklet_hi_schedule(struct tasklet_struct *t)
  382. {
  383. unsigned long flags;
  384. local_irq_save(flags);
  385. t->next = NULL;
  386. *__this_cpu_read(tasklet_hi_vec.tail) = t;
  387. __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
  388. raise_softirq_irqoff(HI_SOFTIRQ);
  389. local_irq_restore(flags);
  390. }
  391. EXPORT_SYMBOL(__tasklet_hi_schedule);
  392. void __tasklet_hi_schedule_first(struct tasklet_struct *t)
  393. {
  394. BUG_ON(!irqs_disabled());
  395. t->next = __this_cpu_read(tasklet_hi_vec.head);
  396. __this_cpu_write(tasklet_hi_vec.head, t);
  397. __raise_softirq_irqoff(HI_SOFTIRQ);
  398. }
  399. EXPORT_SYMBOL(__tasklet_hi_schedule_first);
  400. static void tasklet_action(struct softirq_action *a)
  401. {
  402. struct tasklet_struct *list;
  403. local_irq_disable();
  404. list = __this_cpu_read(tasklet_vec.head);
  405. __this_cpu_write(tasklet_vec.head, NULL);
  406. __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
  407. local_irq_enable();
  408. while (list) {
  409. struct tasklet_struct *t = list;
  410. list = list->next;
  411. if (tasklet_trylock(t)) {
  412. if (!atomic_read(&t->count)) {
  413. if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
  414. BUG();
  415. t->func(t->data);
  416. tasklet_unlock(t);
  417. continue;
  418. }
  419. tasklet_unlock(t);
  420. }
  421. local_irq_disable();
  422. t->next = NULL;
  423. *__this_cpu_read(tasklet_vec.tail) = t;
  424. __this_cpu_write(tasklet_vec.tail, &(t->next));
  425. __raise_softirq_irqoff(TASKLET_SOFTIRQ);
  426. local_irq_enable();
  427. }
  428. }
  429. static void tasklet_hi_action(struct softirq_action *a)
  430. {
  431. struct tasklet_struct *list;
  432. local_irq_disable();
  433. list = __this_cpu_read(tasklet_hi_vec.head);
  434. __this_cpu_write(tasklet_hi_vec.head, NULL);
  435. __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
  436. local_irq_enable();
  437. while (list) {
  438. struct tasklet_struct *t = list;
  439. list = list->next;
  440. if (tasklet_trylock(t)) {
  441. if (!atomic_read(&t->count)) {
  442. if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
  443. BUG();
  444. t->func(t->data);
  445. tasklet_unlock(t);
  446. continue;
  447. }
  448. tasklet_unlock(t);
  449. }
  450. local_irq_disable();
  451. t->next = NULL;
  452. *__this_cpu_read(tasklet_hi_vec.tail) = t;
  453. __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
  454. __raise_softirq_irqoff(HI_SOFTIRQ);
  455. local_irq_enable();
  456. }
  457. }
  458. void tasklet_init(struct tasklet_struct *t,
  459. void (*func)(unsigned long), unsigned long data)
  460. {
  461. t->next = NULL;
  462. t->state = 0;
  463. atomic_set(&t->count, 0);
  464. t->func = func;
  465. t->data = data;
  466. }
  467. EXPORT_SYMBOL(tasklet_init);
  468. void tasklet_kill(struct tasklet_struct *t)
  469. {
  470. if (in_interrupt())
  471. printk("Attempt to kill tasklet from interrupt\n");
  472. while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
  473. do {
  474. yield();
  475. } while (test_bit(TASKLET_STATE_SCHED, &t->state));
  476. }
  477. tasklet_unlock_wait(t);
  478. clear_bit(TASKLET_STATE_SCHED, &t->state);
  479. }
  480. EXPORT_SYMBOL(tasklet_kill);
  481. /*
  482. * tasklet_hrtimer
  483. */
  484. /*
  485. * The trampoline is called when the hrtimer expires. It schedules a tasklet
  486. * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
  487. * hrtimer callback, but from softirq context.
  488. */
  489. static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
  490. {
  491. struct tasklet_hrtimer *ttimer =
  492. container_of(timer, struct tasklet_hrtimer, timer);
  493. tasklet_hi_schedule(&ttimer->tasklet);
  494. return HRTIMER_NORESTART;
  495. }
  496. /*
  497. * Helper function which calls the hrtimer callback from
  498. * tasklet/softirq context
  499. */
  500. static void __tasklet_hrtimer_trampoline(unsigned long data)
  501. {
  502. struct tasklet_hrtimer *ttimer = (void *)data;
  503. enum hrtimer_restart restart;
  504. restart = ttimer->function(&ttimer->timer);
  505. if (restart != HRTIMER_NORESTART)
  506. hrtimer_restart(&ttimer->timer);
  507. }
  508. /**
  509. * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
  510. * @ttimer: tasklet_hrtimer which is initialized
  511. * @function: hrtimer callback function which gets called from softirq context
  512. * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
  513. * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
  514. */
  515. void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
  516. enum hrtimer_restart (*function)(struct hrtimer *),
  517. clockid_t which_clock, enum hrtimer_mode mode)
  518. {
  519. hrtimer_init(&ttimer->timer, which_clock, mode);
  520. ttimer->timer.function = __hrtimer_tasklet_trampoline;
  521. tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
  522. (unsigned long)ttimer);
  523. ttimer->function = function;
  524. }
  525. EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
  526. void __init softirq_init(void)
  527. {
  528. int cpu;
  529. for_each_possible_cpu(cpu) {
  530. per_cpu(tasklet_vec, cpu).tail =
  531. &per_cpu(tasklet_vec, cpu).head;
  532. per_cpu(tasklet_hi_vec, cpu).tail =
  533. &per_cpu(tasklet_hi_vec, cpu).head;
  534. }
  535. open_softirq(TASKLET_SOFTIRQ, tasklet_action);
  536. open_softirq(HI_SOFTIRQ, tasklet_hi_action);
  537. }
  538. static int ksoftirqd_should_run(unsigned int cpu)
  539. {
  540. return local_softirq_pending();
  541. }
  542. static void run_ksoftirqd(unsigned int cpu)
  543. {
  544. local_irq_disable();
  545. if (local_softirq_pending()) {
  546. /*
  547. * We can safely run softirq on inline stack, as we are not deep
  548. * in the task stack here.
  549. */
  550. __do_softirq();
  551. rcu_note_context_switch(cpu);
  552. local_irq_enable();
  553. cond_resched();
  554. return;
  555. }
  556. local_irq_enable();
  557. }
  558. #ifdef CONFIG_HOTPLUG_CPU
  559. /*
  560. * tasklet_kill_immediate is called to remove a tasklet which can already be
  561. * scheduled for execution on @cpu.
  562. *
  563. * Unlike tasklet_kill, this function removes the tasklet
  564. * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
  565. *
  566. * When this function is called, @cpu must be in the CPU_DEAD state.
  567. */
  568. void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
  569. {
  570. struct tasklet_struct **i;
  571. BUG_ON(cpu_online(cpu));
  572. BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
  573. if (!test_bit(TASKLET_STATE_SCHED, &t->state))
  574. return;
  575. /* CPU is dead, so no lock needed. */
  576. for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
  577. if (*i == t) {
  578. *i = t->next;
  579. /* If this was the tail element, move the tail ptr */
  580. if (*i == NULL)
  581. per_cpu(tasklet_vec, cpu).tail = i;
  582. return;
  583. }
  584. }
  585. BUG();
  586. }
  587. static void takeover_tasklets(unsigned int cpu)
  588. {
  589. /* CPU is dead, so no lock needed. */
  590. local_irq_disable();
  591. /* Find end, append list for that CPU. */
  592. if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
  593. *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
  594. this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
  595. per_cpu(tasklet_vec, cpu).head = NULL;
  596. per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
  597. }
  598. raise_softirq_irqoff(TASKLET_SOFTIRQ);
  599. if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
  600. *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
  601. __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
  602. per_cpu(tasklet_hi_vec, cpu).head = NULL;
  603. per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
  604. }
  605. raise_softirq_irqoff(HI_SOFTIRQ);
  606. local_irq_enable();
  607. }
  608. #endif /* CONFIG_HOTPLUG_CPU */
  609. static int cpu_callback(struct notifier_block *nfb,
  610. unsigned long action,
  611. void *hcpu)
  612. {
  613. switch (action) {
  614. #ifdef CONFIG_HOTPLUG_CPU
  615. case CPU_DEAD:
  616. case CPU_DEAD_FROZEN:
  617. takeover_tasklets((unsigned long)hcpu);
  618. break;
  619. #endif /* CONFIG_HOTPLUG_CPU */
  620. }
  621. return NOTIFY_OK;
  622. }
  623. static struct notifier_block cpu_nfb = {
  624. .notifier_call = cpu_callback
  625. };
  626. static struct smp_hotplug_thread softirq_threads = {
  627. .store = &ksoftirqd,
  628. .thread_should_run = ksoftirqd_should_run,
  629. .thread_fn = run_ksoftirqd,
  630. .thread_comm = "ksoftirqd/%u",
  631. };
  632. static __init int spawn_ksoftirqd(void)
  633. {
  634. register_cpu_notifier(&cpu_nfb);
  635. BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
  636. return 0;
  637. }
  638. early_initcall(spawn_ksoftirqd);
  639. /*
  640. * [ These __weak aliases are kept in a separate compilation unit, so that
  641. * GCC does not inline them incorrectly. ]
  642. */
  643. int __init __weak early_irq_init(void)
  644. {
  645. return 0;
  646. }
  647. int __init __weak arch_probe_nr_irqs(void)
  648. {
  649. return NR_IRQS_LEGACY;
  650. }
  651. int __init __weak arch_early_irq_init(void)
  652. {
  653. return 0;
  654. }