fault.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167
  1. /*
  2. * Copyright (C) 1995 Linus Torvalds
  3. * Copyright (C) 2001, 2002 Andi Kleen, SuSE Labs.
  4. */
  5. #include <linux/interrupt.h>
  6. #include <linux/mmiotrace.h>
  7. #include <linux/bootmem.h>
  8. #include <linux/compiler.h>
  9. #include <linux/highmem.h>
  10. #include <linux/kprobes.h>
  11. #include <linux/uaccess.h>
  12. #include <linux/vmalloc.h>
  13. #include <linux/vt_kern.h>
  14. #include <linux/signal.h>
  15. #include <linux/kernel.h>
  16. #include <linux/ptrace.h>
  17. #include <linux/string.h>
  18. #include <linux/module.h>
  19. #include <linux/kdebug.h>
  20. #include <linux/errno.h>
  21. #include <linux/magic.h>
  22. #include <linux/sched.h>
  23. #include <linux/types.h>
  24. #include <linux/init.h>
  25. #include <linux/mman.h>
  26. #include <linux/tty.h>
  27. #include <linux/smp.h>
  28. #include <linux/mm.h>
  29. #include <asm-generic/sections.h>
  30. #include <asm/tlbflush.h>
  31. #include <asm/pgalloc.h>
  32. #include <asm/segment.h>
  33. #include <asm/system.h>
  34. #include <asm/proto.h>
  35. #include <asm/traps.h>
  36. #include <asm/desc.h>
  37. /*
  38. * Page fault error code bits:
  39. *
  40. * bit 0 == 0: no page found 1: protection fault
  41. * bit 1 == 0: read access 1: write access
  42. * bit 2 == 0: kernel-mode access 1: user-mode access
  43. * bit 3 == 1: use of reserved bit detected
  44. * bit 4 == 1: fault was an instruction fetch
  45. */
  46. enum x86_pf_error_code {
  47. PF_PROT = 1 << 0,
  48. PF_WRITE = 1 << 1,
  49. PF_USER = 1 << 2,
  50. PF_RSVD = 1 << 3,
  51. PF_INSTR = 1 << 4,
  52. };
  53. /*
  54. * (returns 0 if mmiotrace is disabled)
  55. */
  56. static inline int kmmio_fault(struct pt_regs *regs, unsigned long addr)
  57. {
  58. if (unlikely(is_kmmio_active()))
  59. if (kmmio_handler(regs, addr) == 1)
  60. return -1;
  61. return 0;
  62. }
  63. static inline int notify_page_fault(struct pt_regs *regs)
  64. {
  65. int ret = 0;
  66. /* kprobe_running() needs smp_processor_id() */
  67. if (kprobes_built_in() && !user_mode_vm(regs)) {
  68. preempt_disable();
  69. if (kprobe_running() && kprobe_fault_handler(regs, 14))
  70. ret = 1;
  71. preempt_enable();
  72. }
  73. return ret;
  74. }
  75. /*
  76. * Prefetch quirks:
  77. *
  78. * 32-bit mode:
  79. *
  80. * Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
  81. * Check that here and ignore it.
  82. *
  83. * 64-bit mode:
  84. *
  85. * Sometimes the CPU reports invalid exceptions on prefetch.
  86. * Check that here and ignore it.
  87. *
  88. * Opcode checker based on code by Richard Brunner.
  89. */
  90. static inline int
  91. check_prefetch_opcode(struct pt_regs *regs, unsigned char *instr,
  92. unsigned char opcode, int *prefetch)
  93. {
  94. unsigned char instr_hi = opcode & 0xf0;
  95. unsigned char instr_lo = opcode & 0x0f;
  96. switch (instr_hi) {
  97. case 0x20:
  98. case 0x30:
  99. /*
  100. * Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes.
  101. * In X86_64 long mode, the CPU will signal invalid
  102. * opcode if some of these prefixes are present so
  103. * X86_64 will never get here anyway
  104. */
  105. return ((instr_lo & 7) == 0x6);
  106. #ifdef CONFIG_X86_64
  107. case 0x40:
  108. /*
  109. * In AMD64 long mode 0x40..0x4F are valid REX prefixes
  110. * Need to figure out under what instruction mode the
  111. * instruction was issued. Could check the LDT for lm,
  112. * but for now it's good enough to assume that long
  113. * mode only uses well known segments or kernel.
  114. */
  115. return (!user_mode(regs)) || (regs->cs == __USER_CS);
  116. #endif
  117. case 0x60:
  118. /* 0x64 thru 0x67 are valid prefixes in all modes. */
  119. return (instr_lo & 0xC) == 0x4;
  120. case 0xF0:
  121. /* 0xF0, 0xF2, 0xF3 are valid prefixes in all modes. */
  122. return !instr_lo || (instr_lo>>1) == 1;
  123. case 0x00:
  124. /* Prefetch instruction is 0x0F0D or 0x0F18 */
  125. if (probe_kernel_address(instr, opcode))
  126. return 0;
  127. *prefetch = (instr_lo == 0xF) &&
  128. (opcode == 0x0D || opcode == 0x18);
  129. return 0;
  130. default:
  131. return 0;
  132. }
  133. }
  134. static int
  135. is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
  136. {
  137. unsigned char *max_instr;
  138. unsigned char *instr;
  139. int prefetch = 0;
  140. /*
  141. * If it was a exec (instruction fetch) fault on NX page, then
  142. * do not ignore the fault:
  143. */
  144. if (error_code & PF_INSTR)
  145. return 0;
  146. instr = (void *)convert_ip_to_linear(current, regs);
  147. max_instr = instr + 15;
  148. if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
  149. return 0;
  150. while (instr < max_instr) {
  151. unsigned char opcode;
  152. if (probe_kernel_address(instr, opcode))
  153. break;
  154. instr++;
  155. if (!check_prefetch_opcode(regs, instr, opcode, &prefetch))
  156. break;
  157. }
  158. return prefetch;
  159. }
  160. static void
  161. force_sig_info_fault(int si_signo, int si_code, unsigned long address,
  162. struct task_struct *tsk)
  163. {
  164. siginfo_t info;
  165. info.si_signo = si_signo;
  166. info.si_errno = 0;
  167. info.si_code = si_code;
  168. info.si_addr = (void __user *)address;
  169. force_sig_info(si_signo, &info, tsk);
  170. }
  171. DEFINE_SPINLOCK(pgd_lock);
  172. LIST_HEAD(pgd_list);
  173. #ifdef CONFIG_X86_32
  174. static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
  175. {
  176. unsigned index = pgd_index(address);
  177. pgd_t *pgd_k;
  178. pud_t *pud, *pud_k;
  179. pmd_t *pmd, *pmd_k;
  180. pgd += index;
  181. pgd_k = init_mm.pgd + index;
  182. if (!pgd_present(*pgd_k))
  183. return NULL;
  184. /*
  185. * set_pgd(pgd, *pgd_k); here would be useless on PAE
  186. * and redundant with the set_pmd() on non-PAE. As would
  187. * set_pud.
  188. */
  189. pud = pud_offset(pgd, address);
  190. pud_k = pud_offset(pgd_k, address);
  191. if (!pud_present(*pud_k))
  192. return NULL;
  193. pmd = pmd_offset(pud, address);
  194. pmd_k = pmd_offset(pud_k, address);
  195. if (!pmd_present(*pmd_k))
  196. return NULL;
  197. if (!pmd_present(*pmd)) {
  198. set_pmd(pmd, *pmd_k);
  199. arch_flush_lazy_mmu_mode();
  200. } else {
  201. BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
  202. }
  203. return pmd_k;
  204. }
  205. void vmalloc_sync_all(void)
  206. {
  207. unsigned long address;
  208. if (SHARED_KERNEL_PMD)
  209. return;
  210. for (address = VMALLOC_START & PMD_MASK;
  211. address >= TASK_SIZE && address < FIXADDR_TOP;
  212. address += PMD_SIZE) {
  213. unsigned long flags;
  214. struct page *page;
  215. spin_lock_irqsave(&pgd_lock, flags);
  216. list_for_each_entry(page, &pgd_list, lru) {
  217. if (!vmalloc_sync_one(page_address(page), address))
  218. break;
  219. }
  220. spin_unlock_irqrestore(&pgd_lock, flags);
  221. }
  222. }
  223. /*
  224. * 32-bit:
  225. *
  226. * Handle a fault on the vmalloc or module mapping area
  227. */
  228. static noinline int vmalloc_fault(unsigned long address)
  229. {
  230. unsigned long pgd_paddr;
  231. pmd_t *pmd_k;
  232. pte_t *pte_k;
  233. /* Make sure we are in vmalloc area: */
  234. if (!(address >= VMALLOC_START && address < VMALLOC_END))
  235. return -1;
  236. /*
  237. * Synchronize this task's top level page-table
  238. * with the 'reference' page table.
  239. *
  240. * Do _not_ use "current" here. We might be inside
  241. * an interrupt in the middle of a task switch..
  242. */
  243. pgd_paddr = read_cr3();
  244. pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
  245. if (!pmd_k)
  246. return -1;
  247. pte_k = pte_offset_kernel(pmd_k, address);
  248. if (!pte_present(*pte_k))
  249. return -1;
  250. return 0;
  251. }
  252. /*
  253. * Did it hit the DOS screen memory VA from vm86 mode?
  254. */
  255. static inline void
  256. check_v8086_mode(struct pt_regs *regs, unsigned long address,
  257. struct task_struct *tsk)
  258. {
  259. unsigned long bit;
  260. if (!v8086_mode(regs))
  261. return;
  262. bit = (address - 0xA0000) >> PAGE_SHIFT;
  263. if (bit < 32)
  264. tsk->thread.screen_bitmap |= 1 << bit;
  265. }
  266. static void dump_pagetable(unsigned long address)
  267. {
  268. __typeof__(pte_val(__pte(0))) page;
  269. page = read_cr3();
  270. page = ((__typeof__(page) *) __va(page))[address >> PGDIR_SHIFT];
  271. #ifdef CONFIG_X86_PAE
  272. printk("*pdpt = %016Lx ", page);
  273. if ((page >> PAGE_SHIFT) < max_low_pfn
  274. && page & _PAGE_PRESENT) {
  275. page &= PAGE_MASK;
  276. page = ((__typeof__(page) *) __va(page))[(address >> PMD_SHIFT)
  277. & (PTRS_PER_PMD - 1)];
  278. printk(KERN_CONT "*pde = %016Lx ", page);
  279. page &= ~_PAGE_NX;
  280. }
  281. #else
  282. printk("*pde = %08lx ", page);
  283. #endif
  284. /*
  285. * We must not directly access the pte in the highpte
  286. * case if the page table is located in highmem.
  287. * And let's rather not kmap-atomic the pte, just in case
  288. * it's allocated already:
  289. */
  290. if ((page >> PAGE_SHIFT) < max_low_pfn
  291. && (page & _PAGE_PRESENT)
  292. && !(page & _PAGE_PSE)) {
  293. page &= PAGE_MASK;
  294. page = ((__typeof__(page) *) __va(page))[(address >> PAGE_SHIFT)
  295. & (PTRS_PER_PTE - 1)];
  296. printk("*pte = %0*Lx ", sizeof(page)*2, (u64)page);
  297. }
  298. printk("\n");
  299. }
  300. #else /* CONFIG_X86_64: */
  301. void vmalloc_sync_all(void)
  302. {
  303. unsigned long address;
  304. for (address = VMALLOC_START & PGDIR_MASK; address <= VMALLOC_END;
  305. address += PGDIR_SIZE) {
  306. const pgd_t *pgd_ref = pgd_offset_k(address);
  307. unsigned long flags;
  308. struct page *page;
  309. if (pgd_none(*pgd_ref))
  310. continue;
  311. spin_lock_irqsave(&pgd_lock, flags);
  312. list_for_each_entry(page, &pgd_list, lru) {
  313. pgd_t *pgd;
  314. pgd = (pgd_t *)page_address(page) + pgd_index(address);
  315. if (pgd_none(*pgd))
  316. set_pgd(pgd, *pgd_ref);
  317. else
  318. BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
  319. }
  320. spin_unlock_irqrestore(&pgd_lock, flags);
  321. }
  322. }
  323. /*
  324. * 64-bit:
  325. *
  326. * Handle a fault on the vmalloc area
  327. *
  328. * This assumes no large pages in there.
  329. */
  330. static noinline int vmalloc_fault(unsigned long address)
  331. {
  332. pgd_t *pgd, *pgd_ref;
  333. pud_t *pud, *pud_ref;
  334. pmd_t *pmd, *pmd_ref;
  335. pte_t *pte, *pte_ref;
  336. /* Make sure we are in vmalloc area: */
  337. if (!(address >= VMALLOC_START && address < VMALLOC_END))
  338. return -1;
  339. /*
  340. * Copy kernel mappings over when needed. This can also
  341. * happen within a race in page table update. In the later
  342. * case just flush:
  343. */
  344. pgd = pgd_offset(current->active_mm, address);
  345. pgd_ref = pgd_offset_k(address);
  346. if (pgd_none(*pgd_ref))
  347. return -1;
  348. if (pgd_none(*pgd))
  349. set_pgd(pgd, *pgd_ref);
  350. else
  351. BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
  352. /*
  353. * Below here mismatches are bugs because these lower tables
  354. * are shared:
  355. */
  356. pud = pud_offset(pgd, address);
  357. pud_ref = pud_offset(pgd_ref, address);
  358. if (pud_none(*pud_ref))
  359. return -1;
  360. if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
  361. BUG();
  362. pmd = pmd_offset(pud, address);
  363. pmd_ref = pmd_offset(pud_ref, address);
  364. if (pmd_none(*pmd_ref))
  365. return -1;
  366. if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
  367. BUG();
  368. pte_ref = pte_offset_kernel(pmd_ref, address);
  369. if (!pte_present(*pte_ref))
  370. return -1;
  371. pte = pte_offset_kernel(pmd, address);
  372. /*
  373. * Don't use pte_page here, because the mappings can point
  374. * outside mem_map, and the NUMA hash lookup cannot handle
  375. * that:
  376. */
  377. if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
  378. BUG();
  379. return 0;
  380. }
  381. static const char errata93_warning[] =
  382. KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
  383. KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
  384. KERN_ERR "******* Please consider a BIOS update.\n"
  385. KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
  386. /*
  387. * No vm86 mode in 64-bit mode:
  388. */
  389. static inline void
  390. check_v8086_mode(struct pt_regs *regs, unsigned long address,
  391. struct task_struct *tsk)
  392. {
  393. }
  394. static int bad_address(void *p)
  395. {
  396. unsigned long dummy;
  397. return probe_kernel_address((unsigned long *)p, dummy);
  398. }
  399. static void dump_pagetable(unsigned long address)
  400. {
  401. pgd_t *pgd;
  402. pud_t *pud;
  403. pmd_t *pmd;
  404. pte_t *pte;
  405. pgd = (pgd_t *)read_cr3();
  406. pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
  407. pgd += pgd_index(address);
  408. if (bad_address(pgd))
  409. goto bad;
  410. printk("PGD %lx ", pgd_val(*pgd));
  411. if (!pgd_present(*pgd))
  412. goto out;
  413. pud = pud_offset(pgd, address);
  414. if (bad_address(pud))
  415. goto bad;
  416. printk("PUD %lx ", pud_val(*pud));
  417. if (!pud_present(*pud) || pud_large(*pud))
  418. goto out;
  419. pmd = pmd_offset(pud, address);
  420. if (bad_address(pmd))
  421. goto bad;
  422. printk("PMD %lx ", pmd_val(*pmd));
  423. if (!pmd_present(*pmd) || pmd_large(*pmd))
  424. goto out;
  425. pte = pte_offset_kernel(pmd, address);
  426. if (bad_address(pte))
  427. goto bad;
  428. printk("PTE %lx", pte_val(*pte));
  429. out:
  430. printk("\n");
  431. return;
  432. bad:
  433. printk("BAD\n");
  434. }
  435. #endif /* CONFIG_X86_64 */
  436. /*
  437. * Workaround for K8 erratum #93 & buggy BIOS.
  438. *
  439. * BIOS SMM functions are required to use a specific workaround
  440. * to avoid corruption of the 64bit RIP register on C stepping K8.
  441. *
  442. * A lot of BIOS that didn't get tested properly miss this.
  443. *
  444. * The OS sees this as a page fault with the upper 32bits of RIP cleared.
  445. * Try to work around it here.
  446. *
  447. * Note we only handle faults in kernel here.
  448. * Does nothing on 32-bit.
  449. */
  450. static int is_errata93(struct pt_regs *regs, unsigned long address)
  451. {
  452. #ifdef CONFIG_X86_64
  453. static int once;
  454. if (address != regs->ip)
  455. return 0;
  456. if ((address >> 32) != 0)
  457. return 0;
  458. address |= 0xffffffffUL << 32;
  459. if ((address >= (u64)_stext && address <= (u64)_etext) ||
  460. (address >= MODULES_VADDR && address <= MODULES_END)) {
  461. if (!once) {
  462. printk(errata93_warning);
  463. once = 1;
  464. }
  465. regs->ip = address;
  466. return 1;
  467. }
  468. #endif
  469. return 0;
  470. }
  471. /*
  472. * Work around K8 erratum #100 K8 in compat mode occasionally jumps
  473. * to illegal addresses >4GB.
  474. *
  475. * We catch this in the page fault handler because these addresses
  476. * are not reachable. Just detect this case and return. Any code
  477. * segment in LDT is compatibility mode.
  478. */
  479. static int is_errata100(struct pt_regs *regs, unsigned long address)
  480. {
  481. #ifdef CONFIG_X86_64
  482. if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) && (address >> 32))
  483. return 1;
  484. #endif
  485. return 0;
  486. }
  487. static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
  488. {
  489. #ifdef CONFIG_X86_F00F_BUG
  490. unsigned long nr;
  491. /*
  492. * Pentium F0 0F C7 C8 bug workaround:
  493. */
  494. if (boot_cpu_data.f00f_bug) {
  495. nr = (address - idt_descr.address) >> 3;
  496. if (nr == 6) {
  497. do_invalid_op(regs, 0);
  498. return 1;
  499. }
  500. }
  501. #endif
  502. return 0;
  503. }
  504. static const char nx_warning[] = KERN_CRIT
  505. "kernel tried to execute NX-protected page - exploit attempt? (uid: %d)\n";
  506. static void
  507. show_fault_oops(struct pt_regs *regs, unsigned long error_code,
  508. unsigned long address)
  509. {
  510. if (!oops_may_print())
  511. return;
  512. if (error_code & PF_INSTR) {
  513. unsigned int level;
  514. pte_t *pte = lookup_address(address, &level);
  515. if (pte && pte_present(*pte) && !pte_exec(*pte))
  516. printk(nx_warning, current_uid());
  517. }
  518. printk(KERN_ALERT "BUG: unable to handle kernel ");
  519. if (address < PAGE_SIZE)
  520. printk(KERN_CONT "NULL pointer dereference");
  521. else
  522. printk(KERN_CONT "paging request");
  523. printk(KERN_CONT " at %p\n", (void *) address);
  524. printk(KERN_ALERT "IP:");
  525. printk_address(regs->ip, 1);
  526. dump_pagetable(address);
  527. }
  528. static noinline void
  529. pgtable_bad(struct pt_regs *regs, unsigned long error_code,
  530. unsigned long address)
  531. {
  532. struct task_struct *tsk;
  533. unsigned long flags;
  534. int sig;
  535. flags = oops_begin();
  536. tsk = current;
  537. sig = SIGKILL;
  538. printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
  539. tsk->comm, address);
  540. dump_pagetable(address);
  541. tsk->thread.cr2 = address;
  542. tsk->thread.trap_no = 14;
  543. tsk->thread.error_code = error_code;
  544. if (__die("Bad pagetable", regs, error_code))
  545. sig = 0;
  546. oops_end(flags, regs, sig);
  547. }
  548. static noinline void
  549. no_context(struct pt_regs *regs, unsigned long error_code,
  550. unsigned long address)
  551. {
  552. struct task_struct *tsk = current;
  553. unsigned long *stackend;
  554. #ifdef CONFIG_X86_64
  555. unsigned long flags;
  556. int sig;
  557. #endif
  558. /* Are we prepared to handle this kernel fault? */
  559. if (fixup_exception(regs))
  560. return;
  561. /*
  562. * 32-bit:
  563. *
  564. * Valid to do another page fault here, because if this fault
  565. * had been triggered by is_prefetch fixup_exception would have
  566. * handled it.
  567. *
  568. * 64-bit:
  569. *
  570. * Hall of shame of CPU/BIOS bugs.
  571. */
  572. if (is_prefetch(regs, error_code, address))
  573. return;
  574. if (is_errata93(regs, address))
  575. return;
  576. /*
  577. * Oops. The kernel tried to access some bad page. We'll have to
  578. * terminate things with extreme prejudice:
  579. */
  580. #ifdef CONFIG_X86_32
  581. bust_spinlocks(1);
  582. #else
  583. flags = oops_begin();
  584. #endif
  585. show_fault_oops(regs, error_code, address);
  586. stackend = end_of_stack(tsk);
  587. if (*stackend != STACK_END_MAGIC)
  588. printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
  589. tsk->thread.cr2 = address;
  590. tsk->thread.trap_no = 14;
  591. tsk->thread.error_code = error_code;
  592. #ifdef CONFIG_X86_32
  593. die("Oops", regs, error_code);
  594. bust_spinlocks(0);
  595. do_exit(SIGKILL);
  596. #else
  597. sig = SIGKILL;
  598. if (__die("Oops", regs, error_code))
  599. sig = 0;
  600. /* Executive summary in case the body of the oops scrolled away */
  601. printk(KERN_EMERG "CR2: %016lx\n", address);
  602. oops_end(flags, regs, sig);
  603. #endif
  604. }
  605. /*
  606. * Print out info about fatal segfaults, if the show_unhandled_signals
  607. * sysctl is set:
  608. */
  609. static inline void
  610. show_signal_msg(struct pt_regs *regs, unsigned long error_code,
  611. unsigned long address, struct task_struct *tsk)
  612. {
  613. if (!unhandled_signal(tsk, SIGSEGV))
  614. return;
  615. if (!printk_ratelimit())
  616. return;
  617. printk(KERN_CONT "%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
  618. task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
  619. tsk->comm, task_pid_nr(tsk), address,
  620. (void *)regs->ip, (void *)regs->sp, error_code);
  621. print_vma_addr(KERN_CONT " in ", regs->ip);
  622. printk(KERN_CONT "\n");
  623. }
  624. static void
  625. __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
  626. unsigned long address, int si_code)
  627. {
  628. struct task_struct *tsk = current;
  629. /* User mode accesses just cause a SIGSEGV */
  630. if (error_code & PF_USER) {
  631. /*
  632. * It's possible to have interrupts off here:
  633. */
  634. local_irq_enable();
  635. /*
  636. * Valid to do another page fault here because this one came
  637. * from user space:
  638. */
  639. if (is_prefetch(regs, error_code, address))
  640. return;
  641. if (is_errata100(regs, address))
  642. return;
  643. if (unlikely(show_unhandled_signals))
  644. show_signal_msg(regs, error_code, address, tsk);
  645. /* Kernel addresses are always protection faults: */
  646. tsk->thread.cr2 = address;
  647. tsk->thread.error_code = error_code | (address >= TASK_SIZE);
  648. tsk->thread.trap_no = 14;
  649. force_sig_info_fault(SIGSEGV, si_code, address, tsk);
  650. return;
  651. }
  652. if (is_f00f_bug(regs, address))
  653. return;
  654. no_context(regs, error_code, address);
  655. }
  656. static noinline void
  657. bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code,
  658. unsigned long address)
  659. {
  660. __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
  661. }
  662. static void
  663. __bad_area(struct pt_regs *regs, unsigned long error_code,
  664. unsigned long address, int si_code)
  665. {
  666. struct mm_struct *mm = current->mm;
  667. /*
  668. * Something tried to access memory that isn't in our memory map..
  669. * Fix it, but check if it's kernel or user first..
  670. */
  671. up_read(&mm->mmap_sem);
  672. __bad_area_nosemaphore(regs, error_code, address, si_code);
  673. }
  674. static noinline void
  675. bad_area(struct pt_regs *regs, unsigned long error_code, unsigned long address)
  676. {
  677. __bad_area(regs, error_code, address, SEGV_MAPERR);
  678. }
  679. static noinline void
  680. bad_area_access_error(struct pt_regs *regs, unsigned long error_code,
  681. unsigned long address)
  682. {
  683. __bad_area(regs, error_code, address, SEGV_ACCERR);
  684. }
  685. /* TODO: fixup for "mm-invoke-oom-killer-from-page-fault.patch" */
  686. static void
  687. out_of_memory(struct pt_regs *regs, unsigned long error_code,
  688. unsigned long address)
  689. {
  690. /*
  691. * We ran out of memory, call the OOM killer, and return the userspace
  692. * (which will retry the fault, or kill us if we got oom-killed):
  693. */
  694. up_read(&current->mm->mmap_sem);
  695. pagefault_out_of_memory();
  696. }
  697. static void
  698. do_sigbus(struct pt_regs *regs, unsigned long error_code, unsigned long address)
  699. {
  700. struct task_struct *tsk = current;
  701. struct mm_struct *mm = tsk->mm;
  702. up_read(&mm->mmap_sem);
  703. /* Kernel mode? Handle exceptions or die: */
  704. if (!(error_code & PF_USER))
  705. no_context(regs, error_code, address);
  706. #ifdef CONFIG_X86_32
  707. /* User space => ok to do another page fault: */
  708. if (is_prefetch(regs, error_code, address))
  709. return;
  710. #endif
  711. tsk->thread.cr2 = address;
  712. tsk->thread.error_code = error_code;
  713. tsk->thread.trap_no = 14;
  714. force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
  715. }
  716. static noinline void
  717. mm_fault_error(struct pt_regs *regs, unsigned long error_code,
  718. unsigned long address, unsigned int fault)
  719. {
  720. if (fault & VM_FAULT_OOM) {
  721. out_of_memory(regs, error_code, address);
  722. } else {
  723. if (fault & VM_FAULT_SIGBUS)
  724. do_sigbus(regs, error_code, address);
  725. else
  726. BUG();
  727. }
  728. }
  729. static int spurious_fault_check(unsigned long error_code, pte_t *pte)
  730. {
  731. if ((error_code & PF_WRITE) && !pte_write(*pte))
  732. return 0;
  733. if ((error_code & PF_INSTR) && !pte_exec(*pte))
  734. return 0;
  735. return 1;
  736. }
  737. /*
  738. * Handle a spurious fault caused by a stale TLB entry.
  739. *
  740. * This allows us to lazily refresh the TLB when increasing the
  741. * permissions of a kernel page (RO -> RW or NX -> X). Doing it
  742. * eagerly is very expensive since that implies doing a full
  743. * cross-processor TLB flush, even if no stale TLB entries exist
  744. * on other processors.
  745. *
  746. * There are no security implications to leaving a stale TLB when
  747. * increasing the permissions on a page.
  748. */
  749. static noinline int
  750. spurious_fault(unsigned long error_code, unsigned long address)
  751. {
  752. pgd_t *pgd;
  753. pud_t *pud;
  754. pmd_t *pmd;
  755. pte_t *pte;
  756. int ret;
  757. /* Reserved-bit violation or user access to kernel space? */
  758. if (error_code & (PF_USER | PF_RSVD))
  759. return 0;
  760. pgd = init_mm.pgd + pgd_index(address);
  761. if (!pgd_present(*pgd))
  762. return 0;
  763. pud = pud_offset(pgd, address);
  764. if (!pud_present(*pud))
  765. return 0;
  766. if (pud_large(*pud))
  767. return spurious_fault_check(error_code, (pte_t *) pud);
  768. pmd = pmd_offset(pud, address);
  769. if (!pmd_present(*pmd))
  770. return 0;
  771. if (pmd_large(*pmd))
  772. return spurious_fault_check(error_code, (pte_t *) pmd);
  773. pte = pte_offset_kernel(pmd, address);
  774. if (!pte_present(*pte))
  775. return 0;
  776. ret = spurious_fault_check(error_code, pte);
  777. if (!ret)
  778. return 0;
  779. /*
  780. * Make sure we have permissions in PMD.
  781. * If not, then there's a bug in the page tables:
  782. */
  783. ret = spurious_fault_check(error_code, (pte_t *) pmd);
  784. WARN_ONCE(!ret, "PMD has incorrect permission bits\n");
  785. return ret;
  786. }
  787. int show_unhandled_signals = 1;
  788. static inline int
  789. access_error(unsigned long error_code, int write, struct vm_area_struct *vma)
  790. {
  791. if (write) {
  792. /* write, present and write, not present: */
  793. if (unlikely(!(vma->vm_flags & VM_WRITE)))
  794. return 1;
  795. return 0;
  796. }
  797. /* read, present: */
  798. if (unlikely(error_code & PF_PROT))
  799. return 1;
  800. /* read, not present: */
  801. if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
  802. return 1;
  803. return 0;
  804. }
  805. static int fault_in_kernel_space(unsigned long address)
  806. {
  807. #ifdef CONFIG_X86_32
  808. return address >= TASK_SIZE;
  809. #else
  810. return address >= TASK_SIZE64;
  811. #endif
  812. }
  813. /*
  814. * This routine handles page faults. It determines the address,
  815. * and the problem, and then passes it off to one of the appropriate
  816. * routines.
  817. */
  818. #ifdef CONFIG_X86_64
  819. asmlinkage
  820. #endif
  821. void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
  822. {
  823. struct vm_area_struct *vma;
  824. struct task_struct *tsk;
  825. unsigned long address;
  826. struct mm_struct *mm;
  827. int write;
  828. int fault;
  829. tsk = current;
  830. mm = tsk->mm;
  831. prefetchw(&mm->mmap_sem);
  832. /* Get the faulting address: */
  833. address = read_cr2();
  834. if (unlikely(kmmio_fault(regs, address)))
  835. return;
  836. /*
  837. * We fault-in kernel-space virtual memory on-demand. The
  838. * 'reference' page table is init_mm.pgd.
  839. *
  840. * NOTE! We MUST NOT take any locks for this case. We may
  841. * be in an interrupt or a critical region, and should
  842. * only copy the information from the master page table,
  843. * nothing more.
  844. *
  845. * This verifies that the fault happens in kernel space
  846. * (error_code & 4) == 0, and that the fault was not a
  847. * protection error (error_code & 9) == 0.
  848. */
  849. if (unlikely(fault_in_kernel_space(address))) {
  850. if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
  851. vmalloc_fault(address) >= 0)
  852. return;
  853. /* Can handle a stale RO->RW TLB: */
  854. if (spurious_fault(error_code, address))
  855. return;
  856. /* kprobes don't want to hook the spurious faults: */
  857. if (notify_page_fault(regs))
  858. return;
  859. /*
  860. * Don't take the mm semaphore here. If we fixup a prefetch
  861. * fault we could otherwise deadlock:
  862. */
  863. bad_area_nosemaphore(regs, error_code, address);
  864. return;
  865. }
  866. /* kprobes don't want to hook the spurious faults: */
  867. if (unlikely(notify_page_fault(regs)))
  868. return;
  869. /*
  870. * It's safe to allow irq's after cr2 has been saved and the
  871. * vmalloc fault has been handled.
  872. *
  873. * User-mode registers count as a user access even for any
  874. * potential system fault or CPU buglet:
  875. */
  876. if (user_mode_vm(regs)) {
  877. local_irq_enable();
  878. error_code |= PF_USER;
  879. } else {
  880. if (regs->flags & X86_EFLAGS_IF)
  881. local_irq_enable();
  882. }
  883. if (unlikely(error_code & PF_RSVD))
  884. pgtable_bad(regs, error_code, address);
  885. /*
  886. * If we're in an interrupt, have no user context or are running
  887. * in an atomic region then we must not take the fault:
  888. */
  889. if (unlikely(in_atomic() || !mm)) {
  890. bad_area_nosemaphore(regs, error_code, address);
  891. return;
  892. }
  893. /*
  894. * When running in the kernel we expect faults to occur only to
  895. * addresses in user space. All other faults represent errors in
  896. * the kernel and should generate an OOPS. Unfortunately, in the
  897. * case of an erroneous fault occurring in a code path which already
  898. * holds mmap_sem we will deadlock attempting to validate the fault
  899. * against the address space. Luckily the kernel only validly
  900. * references user space from well defined areas of code, which are
  901. * listed in the exceptions table.
  902. *
  903. * As the vast majority of faults will be valid we will only perform
  904. * the source reference check when there is a possibility of a
  905. * deadlock. Attempt to lock the address space, if we cannot we then
  906. * validate the source. If this is invalid we can skip the address
  907. * space check, thus avoiding the deadlock:
  908. */
  909. if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
  910. if ((error_code & PF_USER) == 0 &&
  911. !search_exception_tables(regs->ip)) {
  912. bad_area_nosemaphore(regs, error_code, address);
  913. return;
  914. }
  915. down_read(&mm->mmap_sem);
  916. } else {
  917. /*
  918. * The above down_read_trylock() might have succeeded in
  919. * which case we'll have missed the might_sleep() from
  920. * down_read():
  921. */
  922. might_sleep();
  923. }
  924. vma = find_vma(mm, address);
  925. if (unlikely(!vma)) {
  926. bad_area(regs, error_code, address);
  927. return;
  928. }
  929. if (likely(vma->vm_start <= address))
  930. goto good_area;
  931. if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
  932. bad_area(regs, error_code, address);
  933. return;
  934. }
  935. if (error_code & PF_USER) {
  936. /*
  937. * Accessing the stack below %sp is always a bug.
  938. * The large cushion allows instructions like enter
  939. * and pusha to work. ("enter $65535, $31" pushes
  940. * 32 pointers and then decrements %sp by 65535.)
  941. */
  942. if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
  943. bad_area(regs, error_code, address);
  944. return;
  945. }
  946. }
  947. if (unlikely(expand_stack(vma, address))) {
  948. bad_area(regs, error_code, address);
  949. return;
  950. }
  951. /*
  952. * Ok, we have a good vm_area for this memory access, so
  953. * we can handle it..
  954. */
  955. good_area:
  956. write = error_code & PF_WRITE;
  957. if (unlikely(access_error(error_code, write, vma))) {
  958. bad_area_access_error(regs, error_code, address);
  959. return;
  960. }
  961. /*
  962. * If for any reason at all we couldn't handle the fault,
  963. * make sure we exit gracefully rather than endlessly redo
  964. * the fault:
  965. */
  966. fault = handle_mm_fault(mm, vma, address, write);
  967. if (unlikely(fault & VM_FAULT_ERROR)) {
  968. mm_fault_error(regs, error_code, address, fault);
  969. return;
  970. }
  971. if (fault & VM_FAULT_MAJOR)
  972. tsk->maj_flt++;
  973. else
  974. tsk->min_flt++;
  975. check_v8086_mode(regs, address, tsk);
  976. up_read(&mm->mmap_sem);
  977. }