nmi.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545
  1. /*
  2. * Copyright (C) 1991, 1992 Linus Torvalds
  3. * Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
  4. * Copyright (C) 2011 Don Zickus Red Hat, Inc.
  5. *
  6. * Pentium III FXSR, SSE support
  7. * Gareth Hughes <gareth@valinux.com>, May 2000
  8. */
  9. /*
  10. * Handle hardware traps and faults.
  11. */
  12. #include <linux/spinlock.h>
  13. #include <linux/kprobes.h>
  14. #include <linux/kdebug.h>
  15. #include <linux/nmi.h>
  16. #include <linux/debugfs.h>
  17. #include <linux/delay.h>
  18. #include <linux/hardirq.h>
  19. #include <linux/slab.h>
  20. #include <linux/export.h>
  21. #if defined(CONFIG_EDAC)
  22. #include <linux/edac.h>
  23. #endif
  24. #include <linux/atomic.h>
  25. #include <asm/traps.h>
  26. #include <asm/mach_traps.h>
  27. #include <asm/nmi.h>
  28. #include <asm/x86_init.h>
  29. #define CREATE_TRACE_POINTS
  30. #include <trace/events/nmi.h>
  31. struct nmi_desc {
  32. spinlock_t lock;
  33. struct list_head head;
  34. };
  35. static struct nmi_desc nmi_desc[NMI_MAX] =
  36. {
  37. {
  38. .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[0].lock),
  39. .head = LIST_HEAD_INIT(nmi_desc[0].head),
  40. },
  41. {
  42. .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[1].lock),
  43. .head = LIST_HEAD_INIT(nmi_desc[1].head),
  44. },
  45. {
  46. .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[2].lock),
  47. .head = LIST_HEAD_INIT(nmi_desc[2].head),
  48. },
  49. {
  50. .lock = __SPIN_LOCK_UNLOCKED(&nmi_desc[3].lock),
  51. .head = LIST_HEAD_INIT(nmi_desc[3].head),
  52. },
  53. };
  54. struct nmi_stats {
  55. unsigned int normal;
  56. unsigned int unknown;
  57. unsigned int external;
  58. unsigned int swallow;
  59. };
  60. static DEFINE_PER_CPU(struct nmi_stats, nmi_stats);
  61. static int ignore_nmis;
  62. int unknown_nmi_panic;
  63. /*
  64. * Prevent NMI reason port (0x61) being accessed simultaneously, can
  65. * only be used in NMI handler.
  66. */
  67. static DEFINE_RAW_SPINLOCK(nmi_reason_lock);
  68. static int __init setup_unknown_nmi_panic(char *str)
  69. {
  70. unknown_nmi_panic = 1;
  71. return 1;
  72. }
  73. __setup("unknown_nmi_panic", setup_unknown_nmi_panic);
  74. #define nmi_to_desc(type) (&nmi_desc[type])
  75. static u64 nmi_longest_ns = 1 * NSEC_PER_MSEC;
  76. static int __init nmi_warning_debugfs(void)
  77. {
  78. debugfs_create_u64("nmi_longest_ns", 0644,
  79. arch_debugfs_dir, &nmi_longest_ns);
  80. return 0;
  81. }
  82. fs_initcall(nmi_warning_debugfs);
  83. static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
  84. {
  85. struct nmi_desc *desc = nmi_to_desc(type);
  86. struct nmiaction *a;
  87. int handled=0;
  88. rcu_read_lock();
  89. /*
  90. * NMIs are edge-triggered, which means if you have enough
  91. * of them concurrently, you can lose some because only one
  92. * can be latched at any given time. Walk the whole list
  93. * to handle those situations.
  94. */
  95. list_for_each_entry_rcu(a, &desc->head, list) {
  96. u64 before, delta, whole_msecs;
  97. int remainder_ns, decimal_msecs, thishandled;
  98. before = local_clock();
  99. thishandled = a->handler(type, regs);
  100. handled += thishandled;
  101. delta = local_clock() - before;
  102. trace_nmi_handler(a->handler, (int)delta, thishandled);
  103. if (delta < nmi_longest_ns)
  104. continue;
  105. nmi_longest_ns = delta;
  106. whole_msecs = delta;
  107. remainder_ns = do_div(whole_msecs, (1000 * 1000));
  108. decimal_msecs = remainder_ns / 1000;
  109. printk_ratelimited(KERN_INFO
  110. "INFO: NMI handler (%ps) took too long to run: "
  111. "%lld.%03d msecs\n", a->handler, whole_msecs,
  112. decimal_msecs);
  113. }
  114. rcu_read_unlock();
  115. /* return total number of NMI events handled */
  116. return handled;
  117. }
  118. int __register_nmi_handler(unsigned int type, struct nmiaction *action)
  119. {
  120. struct nmi_desc *desc = nmi_to_desc(type);
  121. unsigned long flags;
  122. if (!action->handler)
  123. return -EINVAL;
  124. spin_lock_irqsave(&desc->lock, flags);
  125. /*
  126. * most handlers of type NMI_UNKNOWN never return because
  127. * they just assume the NMI is theirs. Just a sanity check
  128. * to manage expectations
  129. */
  130. WARN_ON_ONCE(type == NMI_UNKNOWN && !list_empty(&desc->head));
  131. WARN_ON_ONCE(type == NMI_SERR && !list_empty(&desc->head));
  132. WARN_ON_ONCE(type == NMI_IO_CHECK && !list_empty(&desc->head));
  133. /*
  134. * some handlers need to be executed first otherwise a fake
  135. * event confuses some handlers (kdump uses this flag)
  136. */
  137. if (action->flags & NMI_FLAG_FIRST)
  138. list_add_rcu(&action->list, &desc->head);
  139. else
  140. list_add_tail_rcu(&action->list, &desc->head);
  141. spin_unlock_irqrestore(&desc->lock, flags);
  142. return 0;
  143. }
  144. EXPORT_SYMBOL(__register_nmi_handler);
  145. void unregister_nmi_handler(unsigned int type, const char *name)
  146. {
  147. struct nmi_desc *desc = nmi_to_desc(type);
  148. struct nmiaction *n;
  149. unsigned long flags;
  150. spin_lock_irqsave(&desc->lock, flags);
  151. list_for_each_entry_rcu(n, &desc->head, list) {
  152. /*
  153. * the name passed in to describe the nmi handler
  154. * is used as the lookup key
  155. */
  156. if (!strcmp(n->name, name)) {
  157. WARN(in_nmi(),
  158. "Trying to free NMI (%s) from NMI context!\n", n->name);
  159. list_del_rcu(&n->list);
  160. break;
  161. }
  162. }
  163. spin_unlock_irqrestore(&desc->lock, flags);
  164. synchronize_rcu();
  165. }
  166. EXPORT_SYMBOL_GPL(unregister_nmi_handler);
  167. static __kprobes void
  168. pci_serr_error(unsigned char reason, struct pt_regs *regs)
  169. {
  170. /* check to see if anyone registered against these types of errors */
  171. if (nmi_handle(NMI_SERR, regs, false))
  172. return;
  173. pr_emerg("NMI: PCI system error (SERR) for reason %02x on CPU %d.\n",
  174. reason, smp_processor_id());
  175. /*
  176. * On some machines, PCI SERR line is used to report memory
  177. * errors. EDAC makes use of it.
  178. */
  179. #if defined(CONFIG_EDAC)
  180. if (edac_handler_set()) {
  181. edac_atomic_assert_error();
  182. return;
  183. }
  184. #endif
  185. if (panic_on_unrecovered_nmi)
  186. panic("NMI: Not continuing");
  187. pr_emerg("Dazed and confused, but trying to continue\n");
  188. /* Clear and disable the PCI SERR error line. */
  189. reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
  190. outb(reason, NMI_REASON_PORT);
  191. }
  192. static __kprobes void
  193. io_check_error(unsigned char reason, struct pt_regs *regs)
  194. {
  195. unsigned long i;
  196. /* check to see if anyone registered against these types of errors */
  197. if (nmi_handle(NMI_IO_CHECK, regs, false))
  198. return;
  199. pr_emerg(
  200. "NMI: IOCK error (debug interrupt?) for reason %02x on CPU %d.\n",
  201. reason, smp_processor_id());
  202. show_regs(regs);
  203. if (panic_on_io_nmi)
  204. panic("NMI IOCK error: Not continuing");
  205. /* Re-enable the IOCK line, wait for a few seconds */
  206. reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_IOCHK;
  207. outb(reason, NMI_REASON_PORT);
  208. i = 20000;
  209. while (--i) {
  210. touch_nmi_watchdog();
  211. udelay(100);
  212. }
  213. reason &= ~NMI_REASON_CLEAR_IOCHK;
  214. outb(reason, NMI_REASON_PORT);
  215. }
  216. static __kprobes void
  217. unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
  218. {
  219. int handled;
  220. /*
  221. * Use 'false' as back-to-back NMIs are dealt with one level up.
  222. * Of course this makes having multiple 'unknown' handlers useless
  223. * as only the first one is ever run (unless it can actually determine
  224. * if it caused the NMI)
  225. */
  226. handled = nmi_handle(NMI_UNKNOWN, regs, false);
  227. if (handled) {
  228. __this_cpu_add(nmi_stats.unknown, handled);
  229. return;
  230. }
  231. __this_cpu_add(nmi_stats.unknown, 1);
  232. pr_emerg("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
  233. reason, smp_processor_id());
  234. pr_emerg("Do you have a strange power saving mode enabled?\n");
  235. if (unknown_nmi_panic || panic_on_unrecovered_nmi)
  236. panic("NMI: Not continuing");
  237. pr_emerg("Dazed and confused, but trying to continue\n");
  238. }
  239. static DEFINE_PER_CPU(bool, swallow_nmi);
  240. static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
  241. static __kprobes void default_do_nmi(struct pt_regs *regs)
  242. {
  243. unsigned char reason = 0;
  244. int handled;
  245. bool b2b = false;
  246. /*
  247. * CPU-specific NMI must be processed before non-CPU-specific
  248. * NMI, otherwise we may lose it, because the CPU-specific
  249. * NMI can not be detected/processed on other CPUs.
  250. */
  251. /*
  252. * Back-to-back NMIs are interesting because they can either
  253. * be two NMI or more than two NMIs (any thing over two is dropped
  254. * due to NMI being edge-triggered). If this is the second half
  255. * of the back-to-back NMI, assume we dropped things and process
  256. * more handlers. Otherwise reset the 'swallow' NMI behaviour
  257. */
  258. if (regs->ip == __this_cpu_read(last_nmi_rip))
  259. b2b = true;
  260. else
  261. __this_cpu_write(swallow_nmi, false);
  262. __this_cpu_write(last_nmi_rip, regs->ip);
  263. handled = nmi_handle(NMI_LOCAL, regs, b2b);
  264. __this_cpu_add(nmi_stats.normal, handled);
  265. if (handled) {
  266. /*
  267. * There are cases when a NMI handler handles multiple
  268. * events in the current NMI. One of these events may
  269. * be queued for in the next NMI. Because the event is
  270. * already handled, the next NMI will result in an unknown
  271. * NMI. Instead lets flag this for a potential NMI to
  272. * swallow.
  273. */
  274. if (handled > 1)
  275. __this_cpu_write(swallow_nmi, true);
  276. return;
  277. }
  278. /* Non-CPU-specific NMI: NMI sources can be processed on any CPU */
  279. raw_spin_lock(&nmi_reason_lock);
  280. reason = x86_platform.get_nmi_reason();
  281. if (reason & NMI_REASON_MASK) {
  282. if (reason & NMI_REASON_SERR)
  283. pci_serr_error(reason, regs);
  284. else if (reason & NMI_REASON_IOCHK)
  285. io_check_error(reason, regs);
  286. #ifdef CONFIG_X86_32
  287. /*
  288. * Reassert NMI in case it became active
  289. * meanwhile as it's edge-triggered:
  290. */
  291. reassert_nmi();
  292. #endif
  293. __this_cpu_add(nmi_stats.external, 1);
  294. raw_spin_unlock(&nmi_reason_lock);
  295. return;
  296. }
  297. raw_spin_unlock(&nmi_reason_lock);
  298. /*
  299. * Only one NMI can be latched at a time. To handle
  300. * this we may process multiple nmi handlers at once to
  301. * cover the case where an NMI is dropped. The downside
  302. * to this approach is we may process an NMI prematurely,
  303. * while its real NMI is sitting latched. This will cause
  304. * an unknown NMI on the next run of the NMI processing.
  305. *
  306. * We tried to flag that condition above, by setting the
  307. * swallow_nmi flag when we process more than one event.
  308. * This condition is also only present on the second half
  309. * of a back-to-back NMI, so we flag that condition too.
  310. *
  311. * If both are true, we assume we already processed this
  312. * NMI previously and we swallow it. Otherwise we reset
  313. * the logic.
  314. *
  315. * There are scenarios where we may accidentally swallow
  316. * a 'real' unknown NMI. For example, while processing
  317. * a perf NMI another perf NMI comes in along with a
  318. * 'real' unknown NMI. These two NMIs get combined into
  319. * one (as descibed above). When the next NMI gets
  320. * processed, it will be flagged by perf as handled, but
  321. * noone will know that there was a 'real' unknown NMI sent
  322. * also. As a result it gets swallowed. Or if the first
  323. * perf NMI returns two events handled then the second
  324. * NMI will get eaten by the logic below, again losing a
  325. * 'real' unknown NMI. But this is the best we can do
  326. * for now.
  327. */
  328. if (b2b && __this_cpu_read(swallow_nmi))
  329. __this_cpu_add(nmi_stats.swallow, 1);
  330. else
  331. unknown_nmi_error(reason, regs);
  332. }
  333. /*
  334. * NMIs can hit breakpoints which will cause it to lose its
  335. * NMI context with the CPU when the breakpoint does an iret.
  336. */
  337. #ifdef CONFIG_X86_32
  338. /*
  339. * For i386, NMIs use the same stack as the kernel, and we can
  340. * add a workaround to the iret problem in C (preventing nested
  341. * NMIs if an NMI takes a trap). Simply have 3 states the NMI
  342. * can be in:
  343. *
  344. * 1) not running
  345. * 2) executing
  346. * 3) latched
  347. *
  348. * When no NMI is in progress, it is in the "not running" state.
  349. * When an NMI comes in, it goes into the "executing" state.
  350. * Normally, if another NMI is triggered, it does not interrupt
  351. * the running NMI and the HW will simply latch it so that when
  352. * the first NMI finishes, it will restart the second NMI.
  353. * (Note, the latch is binary, thus multiple NMIs triggering,
  354. * when one is running, are ignored. Only one NMI is restarted.)
  355. *
  356. * If an NMI hits a breakpoint that executes an iret, another
  357. * NMI can preempt it. We do not want to allow this new NMI
  358. * to run, but we want to execute it when the first one finishes.
  359. * We set the state to "latched", and the exit of the first NMI will
  360. * perform a dec_return, if the result is zero (NOT_RUNNING), then
  361. * it will simply exit the NMI handler. If not, the dec_return
  362. * would have set the state to NMI_EXECUTING (what we want it to
  363. * be when we are running). In this case, we simply jump back
  364. * to rerun the NMI handler again, and restart the 'latched' NMI.
  365. *
  366. * No trap (breakpoint or page fault) should be hit before nmi_restart,
  367. * thus there is no race between the first check of state for NOT_RUNNING
  368. * and setting it to NMI_EXECUTING. The HW will prevent nested NMIs
  369. * at this point.
  370. *
  371. * In case the NMI takes a page fault, we need to save off the CR2
  372. * because the NMI could have preempted another page fault and corrupt
  373. * the CR2 that is about to be read. As nested NMIs must be restarted
  374. * and they can not take breakpoints or page faults, the update of the
  375. * CR2 must be done before converting the nmi state back to NOT_RUNNING.
  376. * Otherwise, there would be a race of another nested NMI coming in
  377. * after setting state to NOT_RUNNING but before updating the nmi_cr2.
  378. */
  379. enum nmi_states {
  380. NMI_NOT_RUNNING = 0,
  381. NMI_EXECUTING,
  382. NMI_LATCHED,
  383. };
  384. static DEFINE_PER_CPU(enum nmi_states, nmi_state);
  385. static DEFINE_PER_CPU(unsigned long, nmi_cr2);
  386. #define nmi_nesting_preprocess(regs) \
  387. do { \
  388. if (this_cpu_read(nmi_state) != NMI_NOT_RUNNING) { \
  389. this_cpu_write(nmi_state, NMI_LATCHED); \
  390. return; \
  391. } \
  392. this_cpu_write(nmi_state, NMI_EXECUTING); \
  393. this_cpu_write(nmi_cr2, read_cr2()); \
  394. } while (0); \
  395. nmi_restart:
  396. #define nmi_nesting_postprocess() \
  397. do { \
  398. if (unlikely(this_cpu_read(nmi_cr2) != read_cr2())) \
  399. write_cr2(this_cpu_read(nmi_cr2)); \
  400. if (this_cpu_dec_return(nmi_state)) \
  401. goto nmi_restart; \
  402. } while (0)
  403. #else /* x86_64 */
  404. /*
  405. * In x86_64 things are a bit more difficult. This has the same problem
  406. * where an NMI hitting a breakpoint that calls iret will remove the
  407. * NMI context, allowing a nested NMI to enter. What makes this more
  408. * difficult is that both NMIs and breakpoints have their own stack.
  409. * When a new NMI or breakpoint is executed, the stack is set to a fixed
  410. * point. If an NMI is nested, it will have its stack set at that same
  411. * fixed address that the first NMI had, and will start corrupting the
  412. * stack. This is handled in entry_64.S, but the same problem exists with
  413. * the breakpoint stack.
  414. *
  415. * If a breakpoint is being processed, and the debug stack is being used,
  416. * if an NMI comes in and also hits a breakpoint, the stack pointer
  417. * will be set to the same fixed address as the breakpoint that was
  418. * interrupted, causing that stack to be corrupted. To handle this case,
  419. * check if the stack that was interrupted is the debug stack, and if
  420. * so, change the IDT so that new breakpoints will use the current stack
  421. * and not switch to the fixed address. On return of the NMI, switch back
  422. * to the original IDT.
  423. */
  424. static DEFINE_PER_CPU(int, update_debug_stack);
  425. static inline void nmi_nesting_preprocess(struct pt_regs *regs)
  426. {
  427. /*
  428. * If we interrupted a breakpoint, it is possible that
  429. * the nmi handler will have breakpoints too. We need to
  430. * change the IDT such that breakpoints that happen here
  431. * continue to use the NMI stack.
  432. */
  433. if (unlikely(is_debug_stack(regs->sp))) {
  434. debug_stack_set_zero();
  435. this_cpu_write(update_debug_stack, 1);
  436. }
  437. }
  438. static inline void nmi_nesting_postprocess(void)
  439. {
  440. if (unlikely(this_cpu_read(update_debug_stack))) {
  441. debug_stack_reset();
  442. this_cpu_write(update_debug_stack, 0);
  443. }
  444. }
  445. #endif
  446. dotraplinkage notrace __kprobes void
  447. do_nmi(struct pt_regs *regs, long error_code)
  448. {
  449. nmi_nesting_preprocess(regs);
  450. nmi_enter();
  451. inc_irq_stat(__nmi_count);
  452. if (!ignore_nmis)
  453. default_do_nmi(regs);
  454. nmi_exit();
  455. /* On i386, may loop back to preprocess */
  456. nmi_nesting_postprocess();
  457. }
  458. void stop_nmi(void)
  459. {
  460. ignore_nmis++;
  461. }
  462. void restart_nmi(void)
  463. {
  464. ignore_nmis--;
  465. }
  466. /* reset the back-to-back NMI logic */
  467. void local_touch_nmi(void)
  468. {
  469. __this_cpu_write(last_nmi_rip, 0);
  470. }
  471. EXPORT_SYMBOL_GPL(local_touch_nmi);