mem.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544
  1. /*
  2. * PowerPC version
  3. * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  4. *
  5. * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
  6. * and Cort Dougan (PReP) (cort@cs.nmt.edu)
  7. * Copyright (C) 1996 Paul Mackerras
  8. * PPC44x/36-bit changes by Matt Porter (mporter@mvista.com)
  9. *
  10. * Derived from "arch/i386/mm/init.c"
  11. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  12. *
  13. * This program is free software; you can redistribute it and/or
  14. * modify it under the terms of the GNU General Public License
  15. * as published by the Free Software Foundation; either version
  16. * 2 of the License, or (at your option) any later version.
  17. *
  18. */
  19. #include <linux/export.h>
  20. #include <linux/sched.h>
  21. #include <linux/kernel.h>
  22. #include <linux/errno.h>
  23. #include <linux/string.h>
  24. #include <linux/gfp.h>
  25. #include <linux/types.h>
  26. #include <linux/mm.h>
  27. #include <linux/stddef.h>
  28. #include <linux/init.h>
  29. #include <linux/bootmem.h>
  30. #include <linux/highmem.h>
  31. #include <linux/initrd.h>
  32. #include <linux/pagemap.h>
  33. #include <linux/suspend.h>
  34. #include <linux/memblock.h>
  35. #include <linux/hugetlb.h>
  36. #include <linux/slab.h>
  37. #include <asm/pgalloc.h>
  38. #include <asm/prom.h>
  39. #include <asm/io.h>
  40. #include <asm/mmu_context.h>
  41. #include <asm/pgtable.h>
  42. #include <asm/mmu.h>
  43. #include <asm/smp.h>
  44. #include <asm/machdep.h>
  45. #include <asm/btext.h>
  46. #include <asm/tlb.h>
  47. #include <asm/sections.h>
  48. #include <asm/sparsemem.h>
  49. #include <asm/vdso.h>
  50. #include <asm/fixmap.h>
  51. #include <asm/swiotlb.h>
  52. #include <asm/rtas.h>
  53. #include "mmu_decl.h"
  54. #ifndef CPU_FTR_COHERENT_ICACHE
  55. #define CPU_FTR_COHERENT_ICACHE 0 /* XXX for now */
  56. #define CPU_FTR_NOEXECUTE 0
  57. #endif
  58. int init_bootmem_done;
  59. int mem_init_done;
  60. unsigned long long memory_limit;
  61. #ifdef CONFIG_HIGHMEM
  62. pte_t *kmap_pte;
  63. EXPORT_SYMBOL(kmap_pte);
  64. pgprot_t kmap_prot;
  65. EXPORT_SYMBOL(kmap_prot);
  66. static inline pte_t *virt_to_kpte(unsigned long vaddr)
  67. {
  68. return pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr),
  69. vaddr), vaddr), vaddr);
  70. }
  71. #endif
  72. int page_is_ram(unsigned long pfn)
  73. {
  74. #ifndef CONFIG_PPC64 /* XXX for now */
  75. return pfn < max_pfn;
  76. #else
  77. unsigned long paddr = (pfn << PAGE_SHIFT);
  78. struct memblock_region *reg;
  79. for_each_memblock(memory, reg)
  80. if (paddr >= reg->base && paddr < (reg->base + reg->size))
  81. return 1;
  82. return 0;
  83. #endif
  84. }
  85. pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
  86. unsigned long size, pgprot_t vma_prot)
  87. {
  88. if (ppc_md.phys_mem_access_prot)
  89. return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);
  90. if (!page_is_ram(pfn))
  91. vma_prot = pgprot_noncached(vma_prot);
  92. return vma_prot;
  93. }
  94. EXPORT_SYMBOL(phys_mem_access_prot);
  95. #ifdef CONFIG_MEMORY_HOTPLUG
  96. #ifdef CONFIG_NUMA
  97. int memory_add_physaddr_to_nid(u64 start)
  98. {
  99. return hot_add_scn_to_nid(start);
  100. }
  101. #endif
  102. int arch_add_memory(int nid, u64 start, u64 size)
  103. {
  104. struct pglist_data *pgdata;
  105. struct zone *zone;
  106. unsigned long start_pfn = start >> PAGE_SHIFT;
  107. unsigned long nr_pages = size >> PAGE_SHIFT;
  108. pgdata = NODE_DATA(nid);
  109. start = (unsigned long)__va(start);
  110. if (create_section_mapping(start, start + size))
  111. return -EINVAL;
  112. /* this should work for most non-highmem platforms */
  113. zone = pgdata->node_zones;
  114. return __add_pages(nid, zone, start_pfn, nr_pages);
  115. }
  116. #ifdef CONFIG_MEMORY_HOTREMOVE
  117. int arch_remove_memory(u64 start, u64 size)
  118. {
  119. unsigned long start_pfn = start >> PAGE_SHIFT;
  120. unsigned long nr_pages = size >> PAGE_SHIFT;
  121. struct zone *zone;
  122. zone = page_zone(pfn_to_page(start_pfn));
  123. return __remove_pages(zone, start_pfn, nr_pages);
  124. }
  125. #endif
  126. #endif /* CONFIG_MEMORY_HOTPLUG */
  127. /*
  128. * walk_memory_resource() needs to make sure there is no holes in a given
  129. * memory range. PPC64 does not maintain the memory layout in /proc/iomem.
  130. * Instead it maintains it in memblock.memory structures. Walk through the
  131. * memory regions, find holes and callback for contiguous regions.
  132. */
  133. int
  134. walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages,
  135. void *arg, int (*func)(unsigned long, unsigned long, void *))
  136. {
  137. struct memblock_region *reg;
  138. unsigned long end_pfn = start_pfn + nr_pages;
  139. unsigned long tstart, tend;
  140. int ret = -1;
  141. for_each_memblock(memory, reg) {
  142. tstart = max(start_pfn, memblock_region_memory_base_pfn(reg));
  143. tend = min(end_pfn, memblock_region_memory_end_pfn(reg));
  144. if (tstart >= tend)
  145. continue;
  146. ret = (*func)(tstart, tend - tstart, arg);
  147. if (ret)
  148. break;
  149. }
  150. return ret;
  151. }
  152. EXPORT_SYMBOL_GPL(walk_system_ram_range);
  153. /*
  154. * Initialize the bootmem system and give it all the memory we
  155. * have available. If we are using highmem, we only put the
  156. * lowmem into the bootmem system.
  157. */
  158. #ifndef CONFIG_NEED_MULTIPLE_NODES
  159. void __init do_init_bootmem(void)
  160. {
  161. unsigned long start, bootmap_pages;
  162. unsigned long total_pages;
  163. struct memblock_region *reg;
  164. int boot_mapsize;
  165. max_low_pfn = max_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT;
  166. total_pages = (memblock_end_of_DRAM() - memstart_addr) >> PAGE_SHIFT;
  167. #ifdef CONFIG_HIGHMEM
  168. total_pages = total_lowmem >> PAGE_SHIFT;
  169. max_low_pfn = lowmem_end_addr >> PAGE_SHIFT;
  170. #endif
  171. /*
  172. * Find an area to use for the bootmem bitmap. Calculate the size of
  173. * bitmap required as (Total Memory) / PAGE_SIZE / BITS_PER_BYTE.
  174. * Add 1 additional page in case the address isn't page-aligned.
  175. */
  176. bootmap_pages = bootmem_bootmap_pages(total_pages);
  177. start = memblock_alloc(bootmap_pages << PAGE_SHIFT, PAGE_SIZE);
  178. min_low_pfn = MEMORY_START >> PAGE_SHIFT;
  179. boot_mapsize = init_bootmem_node(NODE_DATA(0), start >> PAGE_SHIFT, min_low_pfn, max_low_pfn);
  180. /* Place all memblock_regions in the same node and merge contiguous
  181. * memblock_regions
  182. */
  183. memblock_set_node(0, (phys_addr_t)ULLONG_MAX, 0);
  184. /* Add all physical memory to the bootmem map, mark each area
  185. * present.
  186. */
  187. #ifdef CONFIG_HIGHMEM
  188. free_bootmem_with_active_regions(0, lowmem_end_addr >> PAGE_SHIFT);
  189. /* reserve the sections we're already using */
  190. for_each_memblock(reserved, reg) {
  191. unsigned long top = reg->base + reg->size - 1;
  192. if (top < lowmem_end_addr)
  193. reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
  194. else if (reg->base < lowmem_end_addr) {
  195. unsigned long trunc_size = lowmem_end_addr - reg->base;
  196. reserve_bootmem(reg->base, trunc_size, BOOTMEM_DEFAULT);
  197. }
  198. }
  199. #else
  200. free_bootmem_with_active_regions(0, max_pfn);
  201. /* reserve the sections we're already using */
  202. for_each_memblock(reserved, reg)
  203. reserve_bootmem(reg->base, reg->size, BOOTMEM_DEFAULT);
  204. #endif
  205. /* XXX need to clip this if using highmem? */
  206. sparse_memory_present_with_active_regions(0);
  207. init_bootmem_done = 1;
  208. }
  209. /* mark pages that don't exist as nosave */
  210. static int __init mark_nonram_nosave(void)
  211. {
  212. struct memblock_region *reg, *prev = NULL;
  213. for_each_memblock(memory, reg) {
  214. if (prev &&
  215. memblock_region_memory_end_pfn(prev) < memblock_region_memory_base_pfn(reg))
  216. register_nosave_region(memblock_region_memory_end_pfn(prev),
  217. memblock_region_memory_base_pfn(reg));
  218. prev = reg;
  219. }
  220. return 0;
  221. }
  222. /*
  223. * paging_init() sets up the page tables - in fact we've already done this.
  224. */
  225. void __init paging_init(void)
  226. {
  227. unsigned long long total_ram = memblock_phys_mem_size();
  228. phys_addr_t top_of_ram = memblock_end_of_DRAM();
  229. unsigned long max_zone_pfns[MAX_NR_ZONES];
  230. #ifdef CONFIG_PPC32
  231. unsigned long v = __fix_to_virt(__end_of_fixed_addresses - 1);
  232. unsigned long end = __fix_to_virt(FIX_HOLE);
  233. for (; v < end; v += PAGE_SIZE)
  234. map_page(v, 0, 0); /* XXX gross */
  235. #endif
  236. #ifdef CONFIG_HIGHMEM
  237. map_page(PKMAP_BASE, 0, 0); /* XXX gross */
  238. pkmap_page_table = virt_to_kpte(PKMAP_BASE);
  239. kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN));
  240. kmap_prot = PAGE_KERNEL;
  241. #endif /* CONFIG_HIGHMEM */
  242. printk(KERN_DEBUG "Top of RAM: 0x%llx, Total RAM: 0x%llx\n",
  243. (unsigned long long)top_of_ram, total_ram);
  244. printk(KERN_DEBUG "Memory hole size: %ldMB\n",
  245. (long int)((top_of_ram - total_ram) >> 20));
  246. memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
  247. #ifdef CONFIG_HIGHMEM
  248. max_zone_pfns[ZONE_DMA] = lowmem_end_addr >> PAGE_SHIFT;
  249. max_zone_pfns[ZONE_HIGHMEM] = top_of_ram >> PAGE_SHIFT;
  250. #else
  251. max_zone_pfns[ZONE_DMA] = top_of_ram >> PAGE_SHIFT;
  252. #endif
  253. free_area_init_nodes(max_zone_pfns);
  254. mark_nonram_nosave();
  255. }
  256. #endif /* ! CONFIG_NEED_MULTIPLE_NODES */
  257. void __init mem_init(void)
  258. {
  259. #ifdef CONFIG_SWIOTLB
  260. swiotlb_init(0);
  261. #endif
  262. high_memory = (void *) __va(max_low_pfn * PAGE_SIZE);
  263. set_max_mapnr(max_pfn);
  264. free_all_bootmem();
  265. #ifdef CONFIG_HIGHMEM
  266. {
  267. unsigned long pfn, highmem_mapnr;
  268. highmem_mapnr = lowmem_end_addr >> PAGE_SHIFT;
  269. for (pfn = highmem_mapnr; pfn < max_mapnr; ++pfn) {
  270. phys_addr_t paddr = (phys_addr_t)pfn << PAGE_SHIFT;
  271. struct page *page = pfn_to_page(pfn);
  272. if (!memblock_is_reserved(paddr))
  273. free_highmem_page(page);
  274. }
  275. }
  276. #endif /* CONFIG_HIGHMEM */
  277. #if defined(CONFIG_PPC_FSL_BOOK3E) && !defined(CONFIG_SMP)
  278. /*
  279. * If smp is enabled, next_tlbcam_idx is initialized in the cpu up
  280. * functions.... do it here for the non-smp case.
  281. */
  282. per_cpu(next_tlbcam_idx, smp_processor_id()) =
  283. (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
  284. #endif
  285. mem_init_print_info(NULL);
  286. #ifdef CONFIG_PPC32
  287. pr_info("Kernel virtual memory layout:\n");
  288. pr_info(" * 0x%08lx..0x%08lx : fixmap\n", FIXADDR_START, FIXADDR_TOP);
  289. #ifdef CONFIG_HIGHMEM
  290. pr_info(" * 0x%08lx..0x%08lx : highmem PTEs\n",
  291. PKMAP_BASE, PKMAP_ADDR(LAST_PKMAP));
  292. #endif /* CONFIG_HIGHMEM */
  293. #ifdef CONFIG_NOT_COHERENT_CACHE
  294. pr_info(" * 0x%08lx..0x%08lx : consistent mem\n",
  295. IOREMAP_TOP, IOREMAP_TOP + CONFIG_CONSISTENT_SIZE);
  296. #endif /* CONFIG_NOT_COHERENT_CACHE */
  297. pr_info(" * 0x%08lx..0x%08lx : early ioremap\n",
  298. ioremap_bot, IOREMAP_TOP);
  299. pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n",
  300. VMALLOC_START, VMALLOC_END);
  301. #endif /* CONFIG_PPC32 */
  302. mem_init_done = 1;
  303. }
  304. void free_initmem(void)
  305. {
  306. ppc_md.progress = ppc_printk_progress;
  307. free_initmem_default(POISON_FREE_INITMEM);
  308. }
  309. #ifdef CONFIG_BLK_DEV_INITRD
  310. void __init free_initrd_mem(unsigned long start, unsigned long end)
  311. {
  312. free_reserved_area((void *)start, (void *)end, -1, "initrd");
  313. }
  314. #endif
  315. /*
  316. * This is called when a page has been modified by the kernel.
  317. * It just marks the page as not i-cache clean. We do the i-cache
  318. * flush later when the page is given to a user process, if necessary.
  319. */
  320. void flush_dcache_page(struct page *page)
  321. {
  322. if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
  323. return;
  324. /* avoid an atomic op if possible */
  325. if (test_bit(PG_arch_1, &page->flags))
  326. clear_bit(PG_arch_1, &page->flags);
  327. }
  328. EXPORT_SYMBOL(flush_dcache_page);
  329. void flush_dcache_icache_page(struct page *page)
  330. {
  331. #ifdef CONFIG_HUGETLB_PAGE
  332. if (PageCompound(page)) {
  333. flush_dcache_icache_hugepage(page);
  334. return;
  335. }
  336. #endif
  337. #ifdef CONFIG_BOOKE
  338. {
  339. void *start = kmap_atomic(page);
  340. __flush_dcache_icache(start);
  341. kunmap_atomic(start);
  342. }
  343. #elif defined(CONFIG_8xx) || defined(CONFIG_PPC64)
  344. /* On 8xx there is no need to kmap since highmem is not supported */
  345. __flush_dcache_icache(page_address(page));
  346. #else
  347. __flush_dcache_icache_phys(page_to_pfn(page) << PAGE_SHIFT);
  348. #endif
  349. }
  350. EXPORT_SYMBOL(flush_dcache_icache_page);
  351. void clear_user_page(void *page, unsigned long vaddr, struct page *pg)
  352. {
  353. clear_page(page);
  354. /*
  355. * We shouldn't have to do this, but some versions of glibc
  356. * require it (ld.so assumes zero filled pages are icache clean)
  357. * - Anton
  358. */
  359. flush_dcache_page(pg);
  360. }
  361. EXPORT_SYMBOL(clear_user_page);
  362. void copy_user_page(void *vto, void *vfrom, unsigned long vaddr,
  363. struct page *pg)
  364. {
  365. copy_page(vto, vfrom);
  366. /*
  367. * We should be able to use the following optimisation, however
  368. * there are two problems.
  369. * Firstly a bug in some versions of binutils meant PLT sections
  370. * were not marked executable.
  371. * Secondly the first word in the GOT section is blrl, used
  372. * to establish the GOT address. Until recently the GOT was
  373. * not marked executable.
  374. * - Anton
  375. */
  376. #if 0
  377. if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0))
  378. return;
  379. #endif
  380. flush_dcache_page(pg);
  381. }
  382. void flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
  383. unsigned long addr, int len)
  384. {
  385. unsigned long maddr;
  386. maddr = (unsigned long) kmap(page) + (addr & ~PAGE_MASK);
  387. flush_icache_range(maddr, maddr + len);
  388. kunmap(page);
  389. }
  390. EXPORT_SYMBOL(flush_icache_user_range);
  391. /*
  392. * This is called at the end of handling a user page fault, when the
  393. * fault has been handled by updating a PTE in the linux page tables.
  394. * We use it to preload an HPTE into the hash table corresponding to
  395. * the updated linux PTE.
  396. *
  397. * This must always be called with the pte lock held.
  398. */
  399. void update_mmu_cache(struct vm_area_struct *vma, unsigned long address,
  400. pte_t *ptep)
  401. {
  402. #ifdef CONFIG_PPC_STD_MMU
  403. /*
  404. * We don't need to worry about _PAGE_PRESENT here because we are
  405. * called with either mm->page_table_lock held or ptl lock held
  406. */
  407. unsigned long access = 0, trap;
  408. /* We only want HPTEs for linux PTEs that have _PAGE_ACCESSED set */
  409. if (!pte_young(*ptep) || address >= TASK_SIZE)
  410. return;
  411. /* We try to figure out if we are coming from an instruction
  412. * access fault and pass that down to __hash_page so we avoid
  413. * double-faulting on execution of fresh text. We have to test
  414. * for regs NULL since init will get here first thing at boot
  415. *
  416. * We also avoid filling the hash if not coming from a fault
  417. */
  418. if (current->thread.regs == NULL)
  419. return;
  420. trap = TRAP(current->thread.regs);
  421. if (trap == 0x400)
  422. access |= _PAGE_EXEC;
  423. else if (trap != 0x300)
  424. return;
  425. hash_preload(vma->vm_mm, address, access, trap);
  426. #endif /* CONFIG_PPC_STD_MMU */
  427. #if (defined(CONFIG_PPC_BOOK3E_64) || defined(CONFIG_PPC_FSL_BOOK3E)) \
  428. && defined(CONFIG_HUGETLB_PAGE)
  429. if (is_vm_hugetlb_page(vma))
  430. book3e_hugetlb_preload(vma, address, *ptep);
  431. #endif
  432. }
  433. /*
  434. * System memory should not be in /proc/iomem but various tools expect it
  435. * (eg kdump).
  436. */
  437. static int add_system_ram_resources(void)
  438. {
  439. struct memblock_region *reg;
  440. for_each_memblock(memory, reg) {
  441. struct resource *res;
  442. unsigned long base = reg->base;
  443. unsigned long size = reg->size;
  444. res = kzalloc(sizeof(struct resource), GFP_KERNEL);
  445. WARN_ON(!res);
  446. if (res) {
  447. res->name = "System RAM";
  448. res->start = base;
  449. res->end = base + size - 1;
  450. res->flags = IORESOURCE_MEM;
  451. WARN_ON(request_resource(&iomem_resource, res) < 0);
  452. }
  453. }
  454. return 0;
  455. }
  456. subsys_initcall(add_system_ram_resources);
  457. #ifdef CONFIG_STRICT_DEVMEM
  458. /*
  459. * devmem_is_allowed(): check to see if /dev/mem access to a certain address
  460. * is valid. The argument is a physical page number.
  461. *
  462. * Access has to be given to non-kernel-ram areas as well, these contain the
  463. * PCI mmio resources as well as potential bios/acpi data regions.
  464. */
  465. int devmem_is_allowed(unsigned long pfn)
  466. {
  467. if (iomem_is_exclusive(pfn << PAGE_SHIFT))
  468. return 0;
  469. if (!page_is_ram(pfn))
  470. return 1;
  471. if (page_is_rtas_user_buf(pfn))
  472. return 1;
  473. return 0;
  474. }
  475. #endif /* CONFIG_STRICT_DEVMEM */