init.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777
  1. /*
  2. * linux/arch/i386/mm/init.c
  3. *
  4. * Copyright (C) 1995 Linus Torvalds
  5. *
  6. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  7. */
  8. #include <linux/config.h>
  9. #include <linux/module.h>
  10. #include <linux/signal.h>
  11. #include <linux/sched.h>
  12. #include <linux/kernel.h>
  13. #include <linux/errno.h>
  14. #include <linux/string.h>
  15. #include <linux/types.h>
  16. #include <linux/ptrace.h>
  17. #include <linux/mman.h>
  18. #include <linux/mm.h>
  19. #include <linux/hugetlb.h>
  20. #include <linux/swap.h>
  21. #include <linux/smp.h>
  22. #include <linux/init.h>
  23. #include <linux/highmem.h>
  24. #include <linux/pagemap.h>
  25. #include <linux/poison.h>
  26. #include <linux/bootmem.h>
  27. #include <linux/slab.h>
  28. #include <linux/proc_fs.h>
  29. #include <linux/efi.h>
  30. #include <linux/memory_hotplug.h>
  31. #include <linux/initrd.h>
  32. #include <linux/cpumask.h>
  33. #include <asm/processor.h>
  34. #include <asm/system.h>
  35. #include <asm/uaccess.h>
  36. #include <asm/pgtable.h>
  37. #include <asm/dma.h>
  38. #include <asm/fixmap.h>
  39. #include <asm/e820.h>
  40. #include <asm/apic.h>
  41. #include <asm/tlb.h>
  42. #include <asm/tlbflush.h>
  43. #include <asm/sections.h>
  44. unsigned int __VMALLOC_RESERVE = 128 << 20;
  45. DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
  46. unsigned long highstart_pfn, highend_pfn;
  47. static int noinline do_test_wp_bit(void);
  48. /*
  49. * Creates a middle page table and puts a pointer to it in the
  50. * given global directory entry. This only returns the gd entry
  51. * in non-PAE compilation mode, since the middle layer is folded.
  52. */
  53. static pmd_t * __init one_md_table_init(pgd_t *pgd)
  54. {
  55. pud_t *pud;
  56. pmd_t *pmd_table;
  57. #ifdef CONFIG_X86_PAE
  58. pmd_table = (pmd_t *) alloc_bootmem_low_pages(PAGE_SIZE);
  59. set_pgd(pgd, __pgd(__pa(pmd_table) | _PAGE_PRESENT));
  60. pud = pud_offset(pgd, 0);
  61. if (pmd_table != pmd_offset(pud, 0))
  62. BUG();
  63. #else
  64. pud = pud_offset(pgd, 0);
  65. pmd_table = pmd_offset(pud, 0);
  66. #endif
  67. return pmd_table;
  68. }
  69. /*
  70. * Create a page table and place a pointer to it in a middle page
  71. * directory entry.
  72. */
  73. static pte_t * __init one_page_table_init(pmd_t *pmd)
  74. {
  75. if (pmd_none(*pmd)) {
  76. pte_t *page_table = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
  77. set_pmd(pmd, __pmd(__pa(page_table) | _PAGE_TABLE));
  78. if (page_table != pte_offset_kernel(pmd, 0))
  79. BUG();
  80. return page_table;
  81. }
  82. return pte_offset_kernel(pmd, 0);
  83. }
  84. /*
  85. * This function initializes a certain range of kernel virtual memory
  86. * with new bootmem page tables, everywhere page tables are missing in
  87. * the given range.
  88. */
  89. /*
  90. * NOTE: The pagetables are allocated contiguous on the physical space
  91. * so we can cache the place of the first one and move around without
  92. * checking the pgd every time.
  93. */
  94. static void __init page_table_range_init (unsigned long start, unsigned long end, pgd_t *pgd_base)
  95. {
  96. pgd_t *pgd;
  97. pud_t *pud;
  98. pmd_t *pmd;
  99. int pgd_idx, pmd_idx;
  100. unsigned long vaddr;
  101. vaddr = start;
  102. pgd_idx = pgd_index(vaddr);
  103. pmd_idx = pmd_index(vaddr);
  104. pgd = pgd_base + pgd_idx;
  105. for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
  106. if (pgd_none(*pgd))
  107. one_md_table_init(pgd);
  108. pud = pud_offset(pgd, vaddr);
  109. pmd = pmd_offset(pud, vaddr);
  110. for (; (pmd_idx < PTRS_PER_PMD) && (vaddr != end); pmd++, pmd_idx++) {
  111. if (pmd_none(*pmd))
  112. one_page_table_init(pmd);
  113. vaddr += PMD_SIZE;
  114. }
  115. pmd_idx = 0;
  116. }
  117. }
  118. static inline int is_kernel_text(unsigned long addr)
  119. {
  120. if (addr >= PAGE_OFFSET && addr <= (unsigned long)__init_end)
  121. return 1;
  122. return 0;
  123. }
  124. /*
  125. * This maps the physical memory to kernel virtual address space, a total
  126. * of max_low_pfn pages, by creating page tables starting from address
  127. * PAGE_OFFSET.
  128. */
  129. static void __init kernel_physical_mapping_init(pgd_t *pgd_base)
  130. {
  131. unsigned long pfn;
  132. pgd_t *pgd;
  133. pmd_t *pmd;
  134. pte_t *pte;
  135. int pgd_idx, pmd_idx, pte_ofs;
  136. pgd_idx = pgd_index(PAGE_OFFSET);
  137. pgd = pgd_base + pgd_idx;
  138. pfn = 0;
  139. for (; pgd_idx < PTRS_PER_PGD; pgd++, pgd_idx++) {
  140. pmd = one_md_table_init(pgd);
  141. if (pfn >= max_low_pfn)
  142. continue;
  143. for (pmd_idx = 0; pmd_idx < PTRS_PER_PMD && pfn < max_low_pfn; pmd++, pmd_idx++) {
  144. unsigned int address = pfn * PAGE_SIZE + PAGE_OFFSET;
  145. /* Map with big pages if possible, otherwise create normal page tables. */
  146. if (cpu_has_pse) {
  147. unsigned int address2 = (pfn + PTRS_PER_PTE - 1) * PAGE_SIZE + PAGE_OFFSET + PAGE_SIZE-1;
  148. if (is_kernel_text(address) || is_kernel_text(address2))
  149. set_pmd(pmd, pfn_pmd(pfn, PAGE_KERNEL_LARGE_EXEC));
  150. else
  151. set_pmd(pmd, pfn_pmd(pfn, PAGE_KERNEL_LARGE));
  152. pfn += PTRS_PER_PTE;
  153. } else {
  154. pte = one_page_table_init(pmd);
  155. for (pte_ofs = 0; pte_ofs < PTRS_PER_PTE && pfn < max_low_pfn; pte++, pfn++, pte_ofs++) {
  156. if (is_kernel_text(address))
  157. set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
  158. else
  159. set_pte(pte, pfn_pte(pfn, PAGE_KERNEL));
  160. }
  161. }
  162. }
  163. }
  164. }
  165. static inline int page_kills_ppro(unsigned long pagenr)
  166. {
  167. if (pagenr >= 0x70000 && pagenr <= 0x7003F)
  168. return 1;
  169. return 0;
  170. }
  171. extern int is_available_memory(efi_memory_desc_t *);
  172. int page_is_ram(unsigned long pagenr)
  173. {
  174. int i;
  175. unsigned long addr, end;
  176. if (efi_enabled) {
  177. efi_memory_desc_t *md;
  178. void *p;
  179. for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
  180. md = p;
  181. if (!is_available_memory(md))
  182. continue;
  183. addr = (md->phys_addr+PAGE_SIZE-1) >> PAGE_SHIFT;
  184. end = (md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT)) >> PAGE_SHIFT;
  185. if ((pagenr >= addr) && (pagenr < end))
  186. return 1;
  187. }
  188. return 0;
  189. }
  190. for (i = 0; i < e820.nr_map; i++) {
  191. if (e820.map[i].type != E820_RAM) /* not usable memory */
  192. continue;
  193. /*
  194. * !!!FIXME!!! Some BIOSen report areas as RAM that
  195. * are not. Notably the 640->1Mb area. We need a sanity
  196. * check here.
  197. */
  198. addr = (e820.map[i].addr+PAGE_SIZE-1) >> PAGE_SHIFT;
  199. end = (e820.map[i].addr+e820.map[i].size) >> PAGE_SHIFT;
  200. if ((pagenr >= addr) && (pagenr < end))
  201. return 1;
  202. }
  203. return 0;
  204. }
  205. #ifdef CONFIG_HIGHMEM
  206. pte_t *kmap_pte;
  207. pgprot_t kmap_prot;
  208. #define kmap_get_fixmap_pte(vaddr) \
  209. pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), vaddr), (vaddr)), (vaddr))
  210. static void __init kmap_init(void)
  211. {
  212. unsigned long kmap_vstart;
  213. /* cache the first kmap pte */
  214. kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN);
  215. kmap_pte = kmap_get_fixmap_pte(kmap_vstart);
  216. kmap_prot = PAGE_KERNEL;
  217. }
  218. static void __init permanent_kmaps_init(pgd_t *pgd_base)
  219. {
  220. pgd_t *pgd;
  221. pud_t *pud;
  222. pmd_t *pmd;
  223. pte_t *pte;
  224. unsigned long vaddr;
  225. vaddr = PKMAP_BASE;
  226. page_table_range_init(vaddr, vaddr + PAGE_SIZE*LAST_PKMAP, pgd_base);
  227. pgd = swapper_pg_dir + pgd_index(vaddr);
  228. pud = pud_offset(pgd, vaddr);
  229. pmd = pmd_offset(pud, vaddr);
  230. pte = pte_offset_kernel(pmd, vaddr);
  231. pkmap_page_table = pte;
  232. }
  233. static void __meminit free_new_highpage(struct page *page)
  234. {
  235. init_page_count(page);
  236. __free_page(page);
  237. totalhigh_pages++;
  238. }
  239. void __init add_one_highpage_init(struct page *page, int pfn, int bad_ppro)
  240. {
  241. if (page_is_ram(pfn) && !(bad_ppro && page_kills_ppro(pfn))) {
  242. ClearPageReserved(page);
  243. free_new_highpage(page);
  244. } else
  245. SetPageReserved(page);
  246. }
  247. static int add_one_highpage_hotplug(struct page *page, unsigned long pfn)
  248. {
  249. free_new_highpage(page);
  250. totalram_pages++;
  251. #ifdef CONFIG_FLATMEM
  252. max_mapnr = max(pfn, max_mapnr);
  253. #endif
  254. num_physpages++;
  255. return 0;
  256. }
  257. /*
  258. * Not currently handling the NUMA case.
  259. * Assuming single node and all memory that
  260. * has been added dynamically that would be
  261. * onlined here is in HIGHMEM
  262. */
  263. void online_page(struct page *page)
  264. {
  265. ClearPageReserved(page);
  266. add_one_highpage_hotplug(page, page_to_pfn(page));
  267. }
  268. #ifdef CONFIG_NUMA
  269. extern void set_highmem_pages_init(int);
  270. #else
  271. static void __init set_highmem_pages_init(int bad_ppro)
  272. {
  273. int pfn;
  274. for (pfn = highstart_pfn; pfn < highend_pfn; pfn++)
  275. add_one_highpage_init(pfn_to_page(pfn), pfn, bad_ppro);
  276. totalram_pages += totalhigh_pages;
  277. }
  278. #endif /* CONFIG_FLATMEM */
  279. #else
  280. #define kmap_init() do { } while (0)
  281. #define permanent_kmaps_init(pgd_base) do { } while (0)
  282. #define set_highmem_pages_init(bad_ppro) do { } while (0)
  283. #endif /* CONFIG_HIGHMEM */
  284. unsigned long long __PAGE_KERNEL = _PAGE_KERNEL;
  285. EXPORT_SYMBOL(__PAGE_KERNEL);
  286. unsigned long long __PAGE_KERNEL_EXEC = _PAGE_KERNEL_EXEC;
  287. #ifdef CONFIG_NUMA
  288. extern void __init remap_numa_kva(void);
  289. #else
  290. #define remap_numa_kva() do {} while (0)
  291. #endif
  292. static void __init pagetable_init (void)
  293. {
  294. unsigned long vaddr;
  295. pgd_t *pgd_base = swapper_pg_dir;
  296. #ifdef CONFIG_X86_PAE
  297. int i;
  298. /* Init entries of the first-level page table to the zero page */
  299. for (i = 0; i < PTRS_PER_PGD; i++)
  300. set_pgd(pgd_base + i, __pgd(__pa(empty_zero_page) | _PAGE_PRESENT));
  301. #endif
  302. /* Enable PSE if available */
  303. if (cpu_has_pse) {
  304. set_in_cr4(X86_CR4_PSE);
  305. }
  306. /* Enable PGE if available */
  307. if (cpu_has_pge) {
  308. set_in_cr4(X86_CR4_PGE);
  309. __PAGE_KERNEL |= _PAGE_GLOBAL;
  310. __PAGE_KERNEL_EXEC |= _PAGE_GLOBAL;
  311. }
  312. kernel_physical_mapping_init(pgd_base);
  313. remap_numa_kva();
  314. /*
  315. * Fixed mappings, only the page table structure has to be
  316. * created - mappings will be set by set_fixmap():
  317. */
  318. vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
  319. page_table_range_init(vaddr, 0, pgd_base);
  320. permanent_kmaps_init(pgd_base);
  321. #ifdef CONFIG_X86_PAE
  322. /*
  323. * Add low memory identity-mappings - SMP needs it when
  324. * starting up on an AP from real-mode. In the non-PAE
  325. * case we already have these mappings through head.S.
  326. * All user-space mappings are explicitly cleared after
  327. * SMP startup.
  328. */
  329. set_pgd(&pgd_base[0], pgd_base[USER_PTRS_PER_PGD]);
  330. #endif
  331. }
  332. #if defined(CONFIG_SOFTWARE_SUSPEND) || defined(CONFIG_ACPI_SLEEP)
  333. /*
  334. * Swap suspend & friends need this for resume because things like the intel-agp
  335. * driver might have split up a kernel 4MB mapping.
  336. */
  337. char __nosavedata swsusp_pg_dir[PAGE_SIZE]
  338. __attribute__ ((aligned (PAGE_SIZE)));
  339. static inline void save_pg_dir(void)
  340. {
  341. memcpy(swsusp_pg_dir, swapper_pg_dir, PAGE_SIZE);
  342. }
  343. #else
  344. static inline void save_pg_dir(void)
  345. {
  346. }
  347. #endif
  348. void zap_low_mappings (void)
  349. {
  350. int i;
  351. save_pg_dir();
  352. /*
  353. * Zap initial low-memory mappings.
  354. *
  355. * Note that "pgd_clear()" doesn't do it for
  356. * us, because pgd_clear() is a no-op on i386.
  357. */
  358. for (i = 0; i < USER_PTRS_PER_PGD; i++)
  359. #ifdef CONFIG_X86_PAE
  360. set_pgd(swapper_pg_dir+i, __pgd(1 + __pa(empty_zero_page)));
  361. #else
  362. set_pgd(swapper_pg_dir+i, __pgd(0));
  363. #endif
  364. flush_tlb_all();
  365. }
  366. static int disable_nx __initdata = 0;
  367. u64 __supported_pte_mask __read_mostly = ~_PAGE_NX;
  368. /*
  369. * noexec = on|off
  370. *
  371. * Control non executable mappings.
  372. *
  373. * on Enable
  374. * off Disable
  375. */
  376. void __init noexec_setup(const char *str)
  377. {
  378. if (!strncmp(str, "on",2) && cpu_has_nx) {
  379. __supported_pte_mask |= _PAGE_NX;
  380. disable_nx = 0;
  381. } else if (!strncmp(str,"off",3)) {
  382. disable_nx = 1;
  383. __supported_pte_mask &= ~_PAGE_NX;
  384. }
  385. }
  386. int nx_enabled = 0;
  387. #ifdef CONFIG_X86_PAE
  388. static void __init set_nx(void)
  389. {
  390. unsigned int v[4], l, h;
  391. if (cpu_has_pae && (cpuid_eax(0x80000000) > 0x80000001)) {
  392. cpuid(0x80000001, &v[0], &v[1], &v[2], &v[3]);
  393. if ((v[3] & (1 << 20)) && !disable_nx) {
  394. rdmsr(MSR_EFER, l, h);
  395. l |= EFER_NX;
  396. wrmsr(MSR_EFER, l, h);
  397. nx_enabled = 1;
  398. __supported_pte_mask |= _PAGE_NX;
  399. }
  400. }
  401. }
  402. /*
  403. * Enables/disables executability of a given kernel page and
  404. * returns the previous setting.
  405. */
  406. int __init set_kernel_exec(unsigned long vaddr, int enable)
  407. {
  408. pte_t *pte;
  409. int ret = 1;
  410. if (!nx_enabled)
  411. goto out;
  412. pte = lookup_address(vaddr);
  413. BUG_ON(!pte);
  414. if (!pte_exec_kernel(*pte))
  415. ret = 0;
  416. if (enable)
  417. pte->pte_high &= ~(1 << (_PAGE_BIT_NX - 32));
  418. else
  419. pte->pte_high |= 1 << (_PAGE_BIT_NX - 32);
  420. __flush_tlb_all();
  421. out:
  422. return ret;
  423. }
  424. #endif
  425. /*
  426. * paging_init() sets up the page tables - note that the first 8MB are
  427. * already mapped by head.S.
  428. *
  429. * This routines also unmaps the page at virtual kernel address 0, so
  430. * that we can trap those pesky NULL-reference errors in the kernel.
  431. */
  432. void __init paging_init(void)
  433. {
  434. #ifdef CONFIG_X86_PAE
  435. set_nx();
  436. if (nx_enabled)
  437. printk("NX (Execute Disable) protection: active\n");
  438. #endif
  439. pagetable_init();
  440. load_cr3(swapper_pg_dir);
  441. #ifdef CONFIG_X86_PAE
  442. /*
  443. * We will bail out later - printk doesn't work right now so
  444. * the user would just see a hanging kernel.
  445. */
  446. if (cpu_has_pae)
  447. set_in_cr4(X86_CR4_PAE);
  448. #endif
  449. __flush_tlb_all();
  450. kmap_init();
  451. }
  452. /*
  453. * Test if the WP bit works in supervisor mode. It isn't supported on 386's
  454. * and also on some strange 486's (NexGen etc.). All 586+'s are OK. This
  455. * used to involve black magic jumps to work around some nasty CPU bugs,
  456. * but fortunately the switch to using exceptions got rid of all that.
  457. */
  458. static void __init test_wp_bit(void)
  459. {
  460. printk("Checking if this processor honours the WP bit even in supervisor mode... ");
  461. /* Any page-aligned address will do, the test is non-destructive */
  462. __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY);
  463. boot_cpu_data.wp_works_ok = do_test_wp_bit();
  464. clear_fixmap(FIX_WP_TEST);
  465. if (!boot_cpu_data.wp_works_ok) {
  466. printk("No.\n");
  467. #ifdef CONFIG_X86_WP_WORKS_OK
  468. panic("This kernel doesn't support CPU's with broken WP. Recompile it for a 386!");
  469. #endif
  470. } else {
  471. printk("Ok.\n");
  472. }
  473. }
  474. static void __init set_max_mapnr_init(void)
  475. {
  476. #ifdef CONFIG_HIGHMEM
  477. num_physpages = highend_pfn;
  478. #else
  479. num_physpages = max_low_pfn;
  480. #endif
  481. #ifdef CONFIG_FLATMEM
  482. max_mapnr = num_physpages;
  483. #endif
  484. }
  485. static struct kcore_list kcore_mem, kcore_vmalloc;
  486. void __init mem_init(void)
  487. {
  488. extern int ppro_with_ram_bug(void);
  489. int codesize, reservedpages, datasize, initsize;
  490. int tmp;
  491. int bad_ppro;
  492. #ifdef CONFIG_FLATMEM
  493. if (!mem_map)
  494. BUG();
  495. #endif
  496. bad_ppro = ppro_with_ram_bug();
  497. #ifdef CONFIG_HIGHMEM
  498. /* check that fixmap and pkmap do not overlap */
  499. if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) {
  500. printk(KERN_ERR "fixmap and kmap areas overlap - this will crash\n");
  501. printk(KERN_ERR "pkstart: %lxh pkend: %lxh fixstart %lxh\n",
  502. PKMAP_BASE, PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, FIXADDR_START);
  503. BUG();
  504. }
  505. #endif
  506. set_max_mapnr_init();
  507. #ifdef CONFIG_HIGHMEM
  508. high_memory = (void *) __va(highstart_pfn * PAGE_SIZE - 1) + 1;
  509. #else
  510. high_memory = (void *) __va(max_low_pfn * PAGE_SIZE - 1) + 1;
  511. #endif
  512. /* this will put all low memory onto the freelists */
  513. totalram_pages += free_all_bootmem();
  514. reservedpages = 0;
  515. for (tmp = 0; tmp < max_low_pfn; tmp++)
  516. /*
  517. * Only count reserved RAM pages
  518. */
  519. if (page_is_ram(tmp) && PageReserved(pfn_to_page(tmp)))
  520. reservedpages++;
  521. set_highmem_pages_init(bad_ppro);
  522. codesize = (unsigned long) &_etext - (unsigned long) &_text;
  523. datasize = (unsigned long) &_edata - (unsigned long) &_etext;
  524. initsize = (unsigned long) &__init_end - (unsigned long) &__init_begin;
  525. kclist_add(&kcore_mem, __va(0), max_low_pfn << PAGE_SHIFT);
  526. kclist_add(&kcore_vmalloc, (void *)VMALLOC_START,
  527. VMALLOC_END-VMALLOC_START);
  528. printk(KERN_INFO "Memory: %luk/%luk available (%dk kernel code, %dk reserved, %dk data, %dk init, %ldk highmem)\n",
  529. (unsigned long) nr_free_pages() << (PAGE_SHIFT-10),
  530. num_physpages << (PAGE_SHIFT-10),
  531. codesize >> 10,
  532. reservedpages << (PAGE_SHIFT-10),
  533. datasize >> 10,
  534. initsize >> 10,
  535. (unsigned long) (totalhigh_pages << (PAGE_SHIFT-10))
  536. );
  537. #ifdef CONFIG_X86_PAE
  538. if (!cpu_has_pae)
  539. panic("cannot execute a PAE-enabled kernel on a PAE-less CPU!");
  540. #endif
  541. if (boot_cpu_data.wp_works_ok < 0)
  542. test_wp_bit();
  543. /*
  544. * Subtle. SMP is doing it's boot stuff late (because it has to
  545. * fork idle threads) - but it also needs low mappings for the
  546. * protected-mode entry to work. We zap these entries only after
  547. * the WP-bit has been tested.
  548. */
  549. #ifndef CONFIG_SMP
  550. zap_low_mappings();
  551. #endif
  552. }
  553. /*
  554. * this is for the non-NUMA, single node SMP system case.
  555. * Specifically, in the case of x86, we will always add
  556. * memory to the highmem for now.
  557. */
  558. #ifdef CONFIG_MEMORY_HOTPLUG
  559. #ifndef CONFIG_NEED_MULTIPLE_NODES
  560. int arch_add_memory(int nid, u64 start, u64 size)
  561. {
  562. struct pglist_data *pgdata = &contig_page_data;
  563. struct zone *zone = pgdata->node_zones + MAX_NR_ZONES-1;
  564. unsigned long start_pfn = start >> PAGE_SHIFT;
  565. unsigned long nr_pages = size >> PAGE_SHIFT;
  566. return __add_pages(zone, start_pfn, nr_pages);
  567. }
  568. int remove_memory(u64 start, u64 size)
  569. {
  570. return -EINVAL;
  571. }
  572. #endif
  573. #endif
  574. kmem_cache_t *pgd_cache;
  575. kmem_cache_t *pmd_cache;
  576. void __init pgtable_cache_init(void)
  577. {
  578. if (PTRS_PER_PMD > 1) {
  579. pmd_cache = kmem_cache_create("pmd",
  580. PTRS_PER_PMD*sizeof(pmd_t),
  581. PTRS_PER_PMD*sizeof(pmd_t),
  582. 0,
  583. pmd_ctor,
  584. NULL);
  585. if (!pmd_cache)
  586. panic("pgtable_cache_init(): cannot create pmd cache");
  587. }
  588. pgd_cache = kmem_cache_create("pgd",
  589. PTRS_PER_PGD*sizeof(pgd_t),
  590. PTRS_PER_PGD*sizeof(pgd_t),
  591. 0,
  592. pgd_ctor,
  593. PTRS_PER_PMD == 1 ? pgd_dtor : NULL);
  594. if (!pgd_cache)
  595. panic("pgtable_cache_init(): Cannot create pgd cache");
  596. }
  597. /*
  598. * This function cannot be __init, since exceptions don't work in that
  599. * section. Put this after the callers, so that it cannot be inlined.
  600. */
  601. static int noinline do_test_wp_bit(void)
  602. {
  603. char tmp_reg;
  604. int flag;
  605. __asm__ __volatile__(
  606. " movb %0,%1 \n"
  607. "1: movb %1,%0 \n"
  608. " xorl %2,%2 \n"
  609. "2: \n"
  610. ".section __ex_table,\"a\"\n"
  611. " .align 4 \n"
  612. " .long 1b,2b \n"
  613. ".previous \n"
  614. :"=m" (*(char *)fix_to_virt(FIX_WP_TEST)),
  615. "=q" (tmp_reg),
  616. "=r" (flag)
  617. :"2" (1)
  618. :"memory");
  619. return flag;
  620. }
  621. #ifdef CONFIG_DEBUG_RODATA
  622. extern char __start_rodata, __end_rodata;
  623. void mark_rodata_ro(void)
  624. {
  625. unsigned long addr = (unsigned long)&__start_rodata;
  626. for (; addr < (unsigned long)&__end_rodata; addr += PAGE_SIZE)
  627. change_page_attr(virt_to_page(addr), 1, PAGE_KERNEL_RO);
  628. printk ("Write protecting the kernel read-only data: %luk\n",
  629. (unsigned long)(&__end_rodata - &__start_rodata) >> 10);
  630. /*
  631. * change_page_attr() requires a global_flush_tlb() call after it.
  632. * We do this after the printk so that if something went wrong in the
  633. * change, the printk gets out at least to give a better debug hint
  634. * of who is the culprit.
  635. */
  636. global_flush_tlb();
  637. }
  638. #endif
  639. void free_init_pages(char *what, unsigned long begin, unsigned long end)
  640. {
  641. unsigned long addr;
  642. for (addr = begin; addr < end; addr += PAGE_SIZE) {
  643. ClearPageReserved(virt_to_page(addr));
  644. init_page_count(virt_to_page(addr));
  645. memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE);
  646. free_page(addr);
  647. totalram_pages++;
  648. }
  649. printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
  650. }
  651. void free_initmem(void)
  652. {
  653. free_init_pages("unused kernel memory",
  654. (unsigned long)(&__init_begin),
  655. (unsigned long)(&__init_end));
  656. }
  657. #ifdef CONFIG_BLK_DEV_INITRD
  658. void free_initrd_mem(unsigned long start, unsigned long end)
  659. {
  660. free_init_pages("initrd memory", start, end);
  661. }
  662. #endif