fault.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628
  1. /*
  2. * linux/arch/i386/mm/fault.c
  3. *
  4. * Copyright (C) 1995 Linus Torvalds
  5. */
  6. #include <linux/signal.h>
  7. #include <linux/sched.h>
  8. #include <linux/kernel.h>
  9. #include <linux/errno.h>
  10. #include <linux/string.h>
  11. #include <linux/types.h>
  12. #include <linux/ptrace.h>
  13. #include <linux/mman.h>
  14. #include <linux/mm.h>
  15. #include <linux/smp.h>
  16. #include <linux/smp_lock.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/init.h>
  19. #include <linux/tty.h>
  20. #include <linux/vt_kern.h> /* For unblank_screen() */
  21. #include <linux/highmem.h>
  22. #include <linux/module.h>
  23. #include <linux/kprobes.h>
  24. #include <linux/uaccess.h>
  25. #include <asm/system.h>
  26. #include <asm/desc.h>
  27. #include <asm/kdebug.h>
  28. #include <asm/segment.h>
  29. extern void die(const char *,struct pt_regs *,long);
  30. static ATOMIC_NOTIFIER_HEAD(notify_page_fault_chain);
  31. int register_page_fault_notifier(struct notifier_block *nb)
  32. {
  33. vmalloc_sync_all();
  34. return atomic_notifier_chain_register(&notify_page_fault_chain, nb);
  35. }
  36. EXPORT_SYMBOL_GPL(register_page_fault_notifier);
  37. int unregister_page_fault_notifier(struct notifier_block *nb)
  38. {
  39. return atomic_notifier_chain_unregister(&notify_page_fault_chain, nb);
  40. }
  41. EXPORT_SYMBOL_GPL(unregister_page_fault_notifier);
  42. static inline int notify_page_fault(enum die_val val, const char *str,
  43. struct pt_regs *regs, long err, int trap, int sig)
  44. {
  45. struct die_args args = {
  46. .regs = regs,
  47. .str = str,
  48. .err = err,
  49. .trapnr = trap,
  50. .signr = sig
  51. };
  52. return atomic_notifier_call_chain(&notify_page_fault_chain, val, &args);
  53. }
  54. /*
  55. * Return EIP plus the CS segment base. The segment limit is also
  56. * adjusted, clamped to the kernel/user address space (whichever is
  57. * appropriate), and returned in *eip_limit.
  58. *
  59. * The segment is checked, because it might have been changed by another
  60. * task between the original faulting instruction and here.
  61. *
  62. * If CS is no longer a valid code segment, or if EIP is beyond the
  63. * limit, or if it is a kernel address when CS is not a kernel segment,
  64. * then the returned value will be greater than *eip_limit.
  65. *
  66. * This is slow, but is very rarely executed.
  67. */
  68. static inline unsigned long get_segment_eip(struct pt_regs *regs,
  69. unsigned long *eip_limit)
  70. {
  71. unsigned long eip = regs->eip;
  72. unsigned seg = regs->xcs & 0xffff;
  73. u32 seg_ar, seg_limit, base, *desc;
  74. /* Unlikely, but must come before segment checks. */
  75. if (unlikely(regs->eflags & VM_MASK)) {
  76. base = seg << 4;
  77. *eip_limit = base + 0xffff;
  78. return base + (eip & 0xffff);
  79. }
  80. /* The standard kernel/user address space limit. */
  81. *eip_limit = user_mode(regs) ? USER_DS.seg : KERNEL_DS.seg;
  82. /* By far the most common cases. */
  83. if (likely(SEGMENT_IS_FLAT_CODE(seg)))
  84. return eip;
  85. /* Check the segment exists, is within the current LDT/GDT size,
  86. that kernel/user (ring 0..3) has the appropriate privilege,
  87. that it's a code segment, and get the limit. */
  88. __asm__ ("larl %3,%0; lsll %3,%1"
  89. : "=&r" (seg_ar), "=r" (seg_limit) : "0" (0), "rm" (seg));
  90. if ((~seg_ar & 0x9800) || eip > seg_limit) {
  91. *eip_limit = 0;
  92. return 1; /* So that returned eip > *eip_limit. */
  93. }
  94. /* Get the GDT/LDT descriptor base.
  95. When you look for races in this code remember that
  96. LDT and other horrors are only used in user space. */
  97. if (seg & (1<<2)) {
  98. /* Must lock the LDT while reading it. */
  99. down(&current->mm->context.sem);
  100. desc = current->mm->context.ldt;
  101. desc = (void *)desc + (seg & ~7);
  102. } else {
  103. /* Must disable preemption while reading the GDT. */
  104. desc = (u32 *)get_cpu_gdt_table(get_cpu());
  105. desc = (void *)desc + (seg & ~7);
  106. }
  107. /* Decode the code segment base from the descriptor */
  108. base = get_desc_base((unsigned long *)desc);
  109. if (seg & (1<<2)) {
  110. up(&current->mm->context.sem);
  111. } else
  112. put_cpu();
  113. /* Adjust EIP and segment limit, and clamp at the kernel limit.
  114. It's legitimate for segments to wrap at 0xffffffff. */
  115. seg_limit += base;
  116. if (seg_limit < *eip_limit && seg_limit >= base)
  117. *eip_limit = seg_limit;
  118. return eip + base;
  119. }
  120. /*
  121. * Sometimes AMD Athlon/Opteron CPUs report invalid exceptions on prefetch.
  122. * Check that here and ignore it.
  123. */
  124. static int __is_prefetch(struct pt_regs *regs, unsigned long addr)
  125. {
  126. unsigned long limit;
  127. unsigned char *instr = (unsigned char *)get_segment_eip (regs, &limit);
  128. int scan_more = 1;
  129. int prefetch = 0;
  130. int i;
  131. for (i = 0; scan_more && i < 15; i++) {
  132. unsigned char opcode;
  133. unsigned char instr_hi;
  134. unsigned char instr_lo;
  135. if (instr > (unsigned char *)limit)
  136. break;
  137. if (probe_kernel_address(instr, opcode))
  138. break;
  139. instr_hi = opcode & 0xf0;
  140. instr_lo = opcode & 0x0f;
  141. instr++;
  142. switch (instr_hi) {
  143. case 0x20:
  144. case 0x30:
  145. /* Values 0x26,0x2E,0x36,0x3E are valid x86 prefixes. */
  146. scan_more = ((instr_lo & 7) == 0x6);
  147. break;
  148. case 0x60:
  149. /* 0x64 thru 0x67 are valid prefixes in all modes. */
  150. scan_more = (instr_lo & 0xC) == 0x4;
  151. break;
  152. case 0xF0:
  153. /* 0xF0, 0xF2, and 0xF3 are valid prefixes */
  154. scan_more = !instr_lo || (instr_lo>>1) == 1;
  155. break;
  156. case 0x00:
  157. /* Prefetch instruction is 0x0F0D or 0x0F18 */
  158. scan_more = 0;
  159. if (instr > (unsigned char *)limit)
  160. break;
  161. if (probe_kernel_address(instr, opcode))
  162. break;
  163. prefetch = (instr_lo == 0xF) &&
  164. (opcode == 0x0D || opcode == 0x18);
  165. break;
  166. default:
  167. scan_more = 0;
  168. break;
  169. }
  170. }
  171. return prefetch;
  172. }
  173. static inline int is_prefetch(struct pt_regs *regs, unsigned long addr,
  174. unsigned long error_code)
  175. {
  176. if (unlikely(boot_cpu_data.x86_vendor == X86_VENDOR_AMD &&
  177. boot_cpu_data.x86 >= 6)) {
  178. /* Catch an obscure case of prefetch inside an NX page. */
  179. if (nx_enabled && (error_code & 16))
  180. return 0;
  181. return __is_prefetch(regs, addr);
  182. }
  183. return 0;
  184. }
  185. static noinline void force_sig_info_fault(int si_signo, int si_code,
  186. unsigned long address, struct task_struct *tsk)
  187. {
  188. siginfo_t info;
  189. info.si_signo = si_signo;
  190. info.si_errno = 0;
  191. info.si_code = si_code;
  192. info.si_addr = (void __user *)address;
  193. force_sig_info(si_signo, &info, tsk);
  194. }
  195. fastcall void do_invalid_op(struct pt_regs *, unsigned long);
  196. static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
  197. {
  198. unsigned index = pgd_index(address);
  199. pgd_t *pgd_k;
  200. pud_t *pud, *pud_k;
  201. pmd_t *pmd, *pmd_k;
  202. pgd += index;
  203. pgd_k = init_mm.pgd + index;
  204. if (!pgd_present(*pgd_k))
  205. return NULL;
  206. /*
  207. * set_pgd(pgd, *pgd_k); here would be useless on PAE
  208. * and redundant with the set_pmd() on non-PAE. As would
  209. * set_pud.
  210. */
  211. pud = pud_offset(pgd, address);
  212. pud_k = pud_offset(pgd_k, address);
  213. if (!pud_present(*pud_k))
  214. return NULL;
  215. pmd = pmd_offset(pud, address);
  216. pmd_k = pmd_offset(pud_k, address);
  217. if (!pmd_present(*pmd_k))
  218. return NULL;
  219. if (!pmd_present(*pmd))
  220. set_pmd(pmd, *pmd_k);
  221. else
  222. BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
  223. return pmd_k;
  224. }
  225. /*
  226. * Handle a fault on the vmalloc or module mapping area
  227. *
  228. * This assumes no large pages in there.
  229. */
  230. static inline int vmalloc_fault(unsigned long address)
  231. {
  232. unsigned long pgd_paddr;
  233. pmd_t *pmd_k;
  234. pte_t *pte_k;
  235. /*
  236. * Synchronize this task's top level page-table
  237. * with the 'reference' page table.
  238. *
  239. * Do _not_ use "current" here. We might be inside
  240. * an interrupt in the middle of a task switch..
  241. */
  242. pgd_paddr = read_cr3();
  243. pmd_k = vmalloc_sync_one(__va(pgd_paddr), address);
  244. if (!pmd_k)
  245. return -1;
  246. pte_k = pte_offset_kernel(pmd_k, address);
  247. if (!pte_present(*pte_k))
  248. return -1;
  249. return 0;
  250. }
  251. /*
  252. * This routine handles page faults. It determines the address,
  253. * and the problem, and then passes it off to one of the appropriate
  254. * routines.
  255. *
  256. * error_code:
  257. * bit 0 == 0 means no page found, 1 means protection fault
  258. * bit 1 == 0 means read, 1 means write
  259. * bit 2 == 0 means kernel, 1 means user-mode
  260. * bit 3 == 1 means use of reserved bit detected
  261. * bit 4 == 1 means fault was an instruction fetch
  262. */
  263. fastcall void __kprobes do_page_fault(struct pt_regs *regs,
  264. unsigned long error_code)
  265. {
  266. struct task_struct *tsk;
  267. struct mm_struct *mm;
  268. struct vm_area_struct * vma;
  269. unsigned long address;
  270. unsigned long page;
  271. int write, si_code;
  272. /* get the address */
  273. address = read_cr2();
  274. tsk = current;
  275. si_code = SEGV_MAPERR;
  276. /*
  277. * We fault-in kernel-space virtual memory on-demand. The
  278. * 'reference' page table is init_mm.pgd.
  279. *
  280. * NOTE! We MUST NOT take any locks for this case. We may
  281. * be in an interrupt or a critical region, and should
  282. * only copy the information from the master page table,
  283. * nothing more.
  284. *
  285. * This verifies that the fault happens in kernel space
  286. * (error_code & 4) == 0, and that the fault was not a
  287. * protection error (error_code & 9) == 0.
  288. */
  289. if (unlikely(address >= TASK_SIZE)) {
  290. if (!(error_code & 0x0000000d) && vmalloc_fault(address) >= 0)
  291. return;
  292. if (notify_page_fault(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
  293. SIGSEGV) == NOTIFY_STOP)
  294. return;
  295. /*
  296. * Don't take the mm semaphore here. If we fixup a prefetch
  297. * fault we could otherwise deadlock.
  298. */
  299. goto bad_area_nosemaphore;
  300. }
  301. if (notify_page_fault(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
  302. SIGSEGV) == NOTIFY_STOP)
  303. return;
  304. /* It's safe to allow irq's after cr2 has been saved and the vmalloc
  305. fault has been handled. */
  306. if (regs->eflags & (X86_EFLAGS_IF|VM_MASK))
  307. local_irq_enable();
  308. mm = tsk->mm;
  309. /*
  310. * If we're in an interrupt, have no user context or are running in an
  311. * atomic region then we must not take the fault..
  312. */
  313. if (in_atomic() || !mm)
  314. goto bad_area_nosemaphore;
  315. /* When running in the kernel we expect faults to occur only to
  316. * addresses in user space. All other faults represent errors in the
  317. * kernel and should generate an OOPS. Unfortunatly, in the case of an
  318. * erroneous fault occurring in a code path which already holds mmap_sem
  319. * we will deadlock attempting to validate the fault against the
  320. * address space. Luckily the kernel only validly references user
  321. * space from well defined areas of code, which are listed in the
  322. * exceptions table.
  323. *
  324. * As the vast majority of faults will be valid we will only perform
  325. * the source reference check when there is a possibilty of a deadlock.
  326. * Attempt to lock the address space, if we cannot we then validate the
  327. * source. If this is invalid we can skip the address space check,
  328. * thus avoiding the deadlock.
  329. */
  330. if (!down_read_trylock(&mm->mmap_sem)) {
  331. if ((error_code & 4) == 0 &&
  332. !search_exception_tables(regs->eip))
  333. goto bad_area_nosemaphore;
  334. down_read(&mm->mmap_sem);
  335. }
  336. vma = find_vma(mm, address);
  337. if (!vma)
  338. goto bad_area;
  339. if (vma->vm_start <= address)
  340. goto good_area;
  341. if (!(vma->vm_flags & VM_GROWSDOWN))
  342. goto bad_area;
  343. if (error_code & 4) {
  344. /*
  345. * Accessing the stack below %esp is always a bug.
  346. * The large cushion allows instructions like enter
  347. * and pusha to work. ("enter $65535,$31" pushes
  348. * 32 pointers and then decrements %esp by 65535.)
  349. */
  350. if (address + 65536 + 32 * sizeof(unsigned long) < regs->esp)
  351. goto bad_area;
  352. }
  353. if (expand_stack(vma, address))
  354. goto bad_area;
  355. /*
  356. * Ok, we have a good vm_area for this memory access, so
  357. * we can handle it..
  358. */
  359. good_area:
  360. si_code = SEGV_ACCERR;
  361. write = 0;
  362. switch (error_code & 3) {
  363. default: /* 3: write, present */
  364. /* fall through */
  365. case 2: /* write, not present */
  366. if (!(vma->vm_flags & VM_WRITE))
  367. goto bad_area;
  368. write++;
  369. break;
  370. case 1: /* read, present */
  371. goto bad_area;
  372. case 0: /* read, not present */
  373. if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
  374. goto bad_area;
  375. }
  376. survive:
  377. /*
  378. * If for any reason at all we couldn't handle the fault,
  379. * make sure we exit gracefully rather than endlessly redo
  380. * the fault.
  381. */
  382. switch (handle_mm_fault(mm, vma, address, write)) {
  383. case VM_FAULT_MINOR:
  384. tsk->min_flt++;
  385. break;
  386. case VM_FAULT_MAJOR:
  387. tsk->maj_flt++;
  388. break;
  389. case VM_FAULT_SIGBUS:
  390. goto do_sigbus;
  391. case VM_FAULT_OOM:
  392. goto out_of_memory;
  393. default:
  394. BUG();
  395. }
  396. /*
  397. * Did it hit the DOS screen memory VA from vm86 mode?
  398. */
  399. if (regs->eflags & VM_MASK) {
  400. unsigned long bit = (address - 0xA0000) >> PAGE_SHIFT;
  401. if (bit < 32)
  402. tsk->thread.screen_bitmap |= 1 << bit;
  403. }
  404. up_read(&mm->mmap_sem);
  405. return;
  406. /*
  407. * Something tried to access memory that isn't in our memory map..
  408. * Fix it, but check if it's kernel or user first..
  409. */
  410. bad_area:
  411. up_read(&mm->mmap_sem);
  412. bad_area_nosemaphore:
  413. /* User mode accesses just cause a SIGSEGV */
  414. if (error_code & 4) {
  415. /*
  416. * Valid to do another page fault here because this one came
  417. * from user space.
  418. */
  419. if (is_prefetch(regs, address, error_code))
  420. return;
  421. tsk->thread.cr2 = address;
  422. /* Kernel addresses are always protection faults */
  423. tsk->thread.error_code = error_code | (address >= TASK_SIZE);
  424. tsk->thread.trap_no = 14;
  425. force_sig_info_fault(SIGSEGV, si_code, address, tsk);
  426. return;
  427. }
  428. #ifdef CONFIG_X86_F00F_BUG
  429. /*
  430. * Pentium F0 0F C7 C8 bug workaround.
  431. */
  432. if (boot_cpu_data.f00f_bug) {
  433. unsigned long nr;
  434. nr = (address - idt_descr.address) >> 3;
  435. if (nr == 6) {
  436. do_invalid_op(regs, 0);
  437. return;
  438. }
  439. }
  440. #endif
  441. no_context:
  442. /* Are we prepared to handle this kernel fault? */
  443. if (fixup_exception(regs))
  444. return;
  445. /*
  446. * Valid to do another page fault here, because if this fault
  447. * had been triggered by is_prefetch fixup_exception would have
  448. * handled it.
  449. */
  450. if (is_prefetch(regs, address, error_code))
  451. return;
  452. /*
  453. * Oops. The kernel tried to access some bad page. We'll have to
  454. * terminate things with extreme prejudice.
  455. */
  456. bust_spinlocks(1);
  457. if (oops_may_print()) {
  458. #ifdef CONFIG_X86_PAE
  459. if (error_code & 16) {
  460. pte_t *pte = lookup_address(address);
  461. if (pte && pte_present(*pte) && !pte_exec_kernel(*pte))
  462. printk(KERN_CRIT "kernel tried to execute "
  463. "NX-protected page - exploit attempt? "
  464. "(uid: %d)\n", current->uid);
  465. }
  466. #endif
  467. if (address < PAGE_SIZE)
  468. printk(KERN_ALERT "BUG: unable to handle kernel NULL "
  469. "pointer dereference");
  470. else
  471. printk(KERN_ALERT "BUG: unable to handle kernel paging"
  472. " request");
  473. printk(" at virtual address %08lx\n",address);
  474. printk(KERN_ALERT " printing eip:\n");
  475. printk("%08lx\n", regs->eip);
  476. }
  477. page = read_cr3();
  478. page = ((unsigned long *) __va(page))[address >> 22];
  479. if (oops_may_print())
  480. printk(KERN_ALERT "*pde = %08lx\n", page);
  481. /*
  482. * We must not directly access the pte in the highpte
  483. * case, the page table might be allocated in highmem.
  484. * And lets rather not kmap-atomic the pte, just in case
  485. * it's allocated already.
  486. */
  487. #ifndef CONFIG_HIGHPTE
  488. if ((page & 1) && oops_may_print()) {
  489. page &= PAGE_MASK;
  490. address &= 0x003ff000;
  491. page = ((unsigned long *) __va(page))[address >> PAGE_SHIFT];
  492. printk(KERN_ALERT "*pte = %08lx\n", page);
  493. }
  494. #endif
  495. tsk->thread.cr2 = address;
  496. tsk->thread.trap_no = 14;
  497. tsk->thread.error_code = error_code;
  498. die("Oops", regs, error_code);
  499. bust_spinlocks(0);
  500. do_exit(SIGKILL);
  501. /*
  502. * We ran out of memory, or some other thing happened to us that made
  503. * us unable to handle the page fault gracefully.
  504. */
  505. out_of_memory:
  506. up_read(&mm->mmap_sem);
  507. if (is_init(tsk)) {
  508. yield();
  509. down_read(&mm->mmap_sem);
  510. goto survive;
  511. }
  512. printk("VM: killing process %s\n", tsk->comm);
  513. if (error_code & 4)
  514. do_exit(SIGKILL);
  515. goto no_context;
  516. do_sigbus:
  517. up_read(&mm->mmap_sem);
  518. /* Kernel mode? Handle exceptions or die */
  519. if (!(error_code & 4))
  520. goto no_context;
  521. /* User space => ok to do another page fault */
  522. if (is_prefetch(regs, address, error_code))
  523. return;
  524. tsk->thread.cr2 = address;
  525. tsk->thread.error_code = error_code;
  526. tsk->thread.trap_no = 14;
  527. force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
  528. }
  529. #ifndef CONFIG_X86_PAE
  530. void vmalloc_sync_all(void)
  531. {
  532. /*
  533. * Note that races in the updates of insync and start aren't
  534. * problematic: insync can only get set bits added, and updates to
  535. * start are only improving performance (without affecting correctness
  536. * if undone).
  537. */
  538. static DECLARE_BITMAP(insync, PTRS_PER_PGD);
  539. static unsigned long start = TASK_SIZE;
  540. unsigned long address;
  541. BUILD_BUG_ON(TASK_SIZE & ~PGDIR_MASK);
  542. for (address = start; address >= TASK_SIZE; address += PGDIR_SIZE) {
  543. if (!test_bit(pgd_index(address), insync)) {
  544. unsigned long flags;
  545. struct page *page;
  546. spin_lock_irqsave(&pgd_lock, flags);
  547. for (page = pgd_list; page; page =
  548. (struct page *)page->index)
  549. if (!vmalloc_sync_one(page_address(page),
  550. address)) {
  551. BUG_ON(page != pgd_list);
  552. break;
  553. }
  554. spin_unlock_irqrestore(&pgd_lock, flags);
  555. if (!page)
  556. set_bit(pgd_index(address), insync);
  557. }
  558. if (address == start && test_bit(pgd_index(address), insync))
  559. start = address + PGDIR_SIZE;
  560. }
  561. }
  562. #endif