vmem.c 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376
  1. /*
  2. * arch/s390/mm/vmem.c
  3. *
  4. * Copyright IBM Corp. 2006
  5. * Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
  6. */
  7. #include <linux/bootmem.h>
  8. #include <linux/pfn.h>
  9. #include <linux/mm.h>
  10. #include <linux/module.h>
  11. #include <linux/list.h>
  12. #include <linux/hugetlb.h>
  13. #include <asm/pgalloc.h>
  14. #include <asm/pgtable.h>
  15. #include <asm/setup.h>
  16. #include <asm/tlbflush.h>
  17. #include <asm/sections.h>
  18. static DEFINE_MUTEX(vmem_mutex);
  19. struct memory_segment {
  20. struct list_head list;
  21. unsigned long start;
  22. unsigned long size;
  23. };
  24. static LIST_HEAD(mem_segs);
  25. static pud_t *vmem_pud_alloc(void)
  26. {
  27. pud_t *pud = NULL;
  28. #ifdef CONFIG_64BIT
  29. pud = vmemmap_alloc_block(PAGE_SIZE * 4, 0);
  30. if (!pud)
  31. return NULL;
  32. clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  33. #endif
  34. return pud;
  35. }
  36. static pmd_t *vmem_pmd_alloc(void)
  37. {
  38. pmd_t *pmd = NULL;
  39. #ifdef CONFIG_64BIT
  40. pmd = vmemmap_alloc_block(PAGE_SIZE * 4, 0);
  41. if (!pmd)
  42. return NULL;
  43. clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  44. #endif
  45. return pmd;
  46. }
  47. static pte_t __ref *vmem_pte_alloc(void)
  48. {
  49. pte_t *pte;
  50. if (slab_is_available())
  51. pte = (pte_t *) page_table_alloc(&init_mm);
  52. else
  53. pte = alloc_bootmem(PTRS_PER_PTE * sizeof(pte_t));
  54. if (!pte)
  55. return NULL;
  56. clear_table((unsigned long *) pte, _PAGE_TYPE_EMPTY,
  57. PTRS_PER_PTE * sizeof(pte_t));
  58. return pte;
  59. }
  60. /*
  61. * Add a physical memory range to the 1:1 mapping.
  62. */
  63. static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
  64. {
  65. unsigned long address;
  66. pgd_t *pg_dir;
  67. pud_t *pu_dir;
  68. pmd_t *pm_dir;
  69. pte_t *pt_dir;
  70. pte_t pte;
  71. int ret = -ENOMEM;
  72. for (address = start; address < start + size; address += PAGE_SIZE) {
  73. pg_dir = pgd_offset_k(address);
  74. if (pgd_none(*pg_dir)) {
  75. pu_dir = vmem_pud_alloc();
  76. if (!pu_dir)
  77. goto out;
  78. pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
  79. }
  80. pu_dir = pud_offset(pg_dir, address);
  81. if (pud_none(*pu_dir)) {
  82. pm_dir = vmem_pmd_alloc();
  83. if (!pm_dir)
  84. goto out;
  85. pud_populate_kernel(&init_mm, pu_dir, pm_dir);
  86. }
  87. pte = mk_pte_phys(address, __pgprot(ro ? _PAGE_RO : 0));
  88. pm_dir = pmd_offset(pu_dir, address);
  89. #ifdef __s390x__
  90. if (MACHINE_HAS_HPAGE && !(address & ~HPAGE_MASK) &&
  91. (address + HPAGE_SIZE <= start + size) &&
  92. (address >= HPAGE_SIZE)) {
  93. pte_val(pte) |= _SEGMENT_ENTRY_LARGE;
  94. pmd_val(*pm_dir) = pte_val(pte);
  95. address += HPAGE_SIZE - PAGE_SIZE;
  96. continue;
  97. }
  98. #endif
  99. if (pmd_none(*pm_dir)) {
  100. pt_dir = vmem_pte_alloc();
  101. if (!pt_dir)
  102. goto out;
  103. pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
  104. }
  105. pt_dir = pte_offset_kernel(pm_dir, address);
  106. *pt_dir = pte;
  107. }
  108. ret = 0;
  109. out:
  110. flush_tlb_kernel_range(start, start + size);
  111. return ret;
  112. }
  113. /*
  114. * Remove a physical memory range from the 1:1 mapping.
  115. * Currently only invalidates page table entries.
  116. */
  117. static void vmem_remove_range(unsigned long start, unsigned long size)
  118. {
  119. unsigned long address;
  120. pgd_t *pg_dir;
  121. pud_t *pu_dir;
  122. pmd_t *pm_dir;
  123. pte_t *pt_dir;
  124. pte_t pte;
  125. pte_val(pte) = _PAGE_TYPE_EMPTY;
  126. for (address = start; address < start + size; address += PAGE_SIZE) {
  127. pg_dir = pgd_offset_k(address);
  128. pu_dir = pud_offset(pg_dir, address);
  129. if (pud_none(*pu_dir))
  130. continue;
  131. pm_dir = pmd_offset(pu_dir, address);
  132. if (pmd_none(*pm_dir))
  133. continue;
  134. if (pmd_huge(*pm_dir)) {
  135. pmd_clear_kernel(pm_dir);
  136. address += HPAGE_SIZE - PAGE_SIZE;
  137. continue;
  138. }
  139. pt_dir = pte_offset_kernel(pm_dir, address);
  140. *pt_dir = pte;
  141. }
  142. flush_tlb_kernel_range(start, start + size);
  143. }
  144. /*
  145. * Add a backed mem_map array to the virtual mem_map array.
  146. */
  147. int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
  148. {
  149. unsigned long address, start_addr, end_addr;
  150. pgd_t *pg_dir;
  151. pud_t *pu_dir;
  152. pmd_t *pm_dir;
  153. pte_t *pt_dir;
  154. pte_t pte;
  155. int ret = -ENOMEM;
  156. start_addr = (unsigned long) start;
  157. end_addr = (unsigned long) (start + nr);
  158. for (address = start_addr; address < end_addr; address += PAGE_SIZE) {
  159. pg_dir = pgd_offset_k(address);
  160. if (pgd_none(*pg_dir)) {
  161. pu_dir = vmem_pud_alloc();
  162. if (!pu_dir)
  163. goto out;
  164. pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
  165. }
  166. pu_dir = pud_offset(pg_dir, address);
  167. if (pud_none(*pu_dir)) {
  168. pm_dir = vmem_pmd_alloc();
  169. if (!pm_dir)
  170. goto out;
  171. pud_populate_kernel(&init_mm, pu_dir, pm_dir);
  172. }
  173. pm_dir = pmd_offset(pu_dir, address);
  174. if (pmd_none(*pm_dir)) {
  175. pt_dir = vmem_pte_alloc();
  176. if (!pt_dir)
  177. goto out;
  178. pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
  179. }
  180. pt_dir = pte_offset_kernel(pm_dir, address);
  181. if (pte_none(*pt_dir)) {
  182. unsigned long new_page;
  183. new_page =__pa(vmemmap_alloc_block(PAGE_SIZE, 0));
  184. if (!new_page)
  185. goto out;
  186. pte = pfn_pte(new_page >> PAGE_SHIFT, PAGE_KERNEL);
  187. *pt_dir = pte;
  188. }
  189. }
  190. ret = 0;
  191. out:
  192. flush_tlb_kernel_range(start_addr, end_addr);
  193. return ret;
  194. }
  195. /*
  196. * Add memory segment to the segment list if it doesn't overlap with
  197. * an already present segment.
  198. */
  199. static int insert_memory_segment(struct memory_segment *seg)
  200. {
  201. struct memory_segment *tmp;
  202. if (seg->start + seg->size >= VMEM_MAX_PHYS ||
  203. seg->start + seg->size < seg->start)
  204. return -ERANGE;
  205. list_for_each_entry(tmp, &mem_segs, list) {
  206. if (seg->start >= tmp->start + tmp->size)
  207. continue;
  208. if (seg->start + seg->size <= tmp->start)
  209. continue;
  210. return -ENOSPC;
  211. }
  212. list_add(&seg->list, &mem_segs);
  213. return 0;
  214. }
  215. /*
  216. * Remove memory segment from the segment list.
  217. */
  218. static void remove_memory_segment(struct memory_segment *seg)
  219. {
  220. list_del(&seg->list);
  221. }
  222. static void __remove_shared_memory(struct memory_segment *seg)
  223. {
  224. remove_memory_segment(seg);
  225. vmem_remove_range(seg->start, seg->size);
  226. }
  227. int vmem_remove_mapping(unsigned long start, unsigned long size)
  228. {
  229. struct memory_segment *seg;
  230. int ret;
  231. mutex_lock(&vmem_mutex);
  232. ret = -ENOENT;
  233. list_for_each_entry(seg, &mem_segs, list) {
  234. if (seg->start == start && seg->size == size)
  235. break;
  236. }
  237. if (seg->start != start || seg->size != size)
  238. goto out;
  239. ret = 0;
  240. __remove_shared_memory(seg);
  241. kfree(seg);
  242. out:
  243. mutex_unlock(&vmem_mutex);
  244. return ret;
  245. }
  246. int vmem_add_mapping(unsigned long start, unsigned long size)
  247. {
  248. struct memory_segment *seg;
  249. int ret;
  250. mutex_lock(&vmem_mutex);
  251. ret = -ENOMEM;
  252. seg = kzalloc(sizeof(*seg), GFP_KERNEL);
  253. if (!seg)
  254. goto out;
  255. seg->start = start;
  256. seg->size = size;
  257. ret = insert_memory_segment(seg);
  258. if (ret)
  259. goto out_free;
  260. ret = vmem_add_mem(start, size, 0);
  261. if (ret)
  262. goto out_remove;
  263. goto out;
  264. out_remove:
  265. __remove_shared_memory(seg);
  266. out_free:
  267. kfree(seg);
  268. out:
  269. mutex_unlock(&vmem_mutex);
  270. return ret;
  271. }
  272. /*
  273. * map whole physical memory to virtual memory (identity mapping)
  274. * we reserve enough space in the vmalloc area for vmemmap to hotplug
  275. * additional memory segments.
  276. */
  277. void __init vmem_map_init(void)
  278. {
  279. unsigned long ro_start, ro_end;
  280. unsigned long start, end;
  281. int i;
  282. INIT_LIST_HEAD(&init_mm.context.crst_list);
  283. INIT_LIST_HEAD(&init_mm.context.pgtable_list);
  284. init_mm.context.noexec = 0;
  285. ro_start = ((unsigned long)&_stext) & PAGE_MASK;
  286. ro_end = PFN_ALIGN((unsigned long)&_eshared);
  287. for (i = 0; i < MEMORY_CHUNKS && memory_chunk[i].size > 0; i++) {
  288. start = memory_chunk[i].addr;
  289. end = memory_chunk[i].addr + memory_chunk[i].size;
  290. if (start >= ro_end || end <= ro_start)
  291. vmem_add_mem(start, end - start, 0);
  292. else if (start >= ro_start && end <= ro_end)
  293. vmem_add_mem(start, end - start, 1);
  294. else if (start >= ro_start) {
  295. vmem_add_mem(start, ro_end - start, 1);
  296. vmem_add_mem(ro_end, end - ro_end, 0);
  297. } else if (end < ro_end) {
  298. vmem_add_mem(start, ro_start - start, 0);
  299. vmem_add_mem(ro_start, end - ro_start, 1);
  300. } else {
  301. vmem_add_mem(start, ro_start - start, 0);
  302. vmem_add_mem(ro_start, ro_end - ro_start, 1);
  303. vmem_add_mem(ro_end, end - ro_end, 0);
  304. }
  305. }
  306. }
  307. /*
  308. * Convert memory chunk array to a memory segment list so there is a single
  309. * list that contains both r/w memory and shared memory segments.
  310. */
  311. static int __init vmem_convert_memory_chunk(void)
  312. {
  313. struct memory_segment *seg;
  314. int i;
  315. mutex_lock(&vmem_mutex);
  316. for (i = 0; i < MEMORY_CHUNKS; i++) {
  317. if (!memory_chunk[i].size)
  318. continue;
  319. seg = kzalloc(sizeof(*seg), GFP_KERNEL);
  320. if (!seg)
  321. panic("Out of memory...\n");
  322. seg->start = memory_chunk[i].addr;
  323. seg->size = memory_chunk[i].size;
  324. insert_memory_segment(seg);
  325. }
  326. mutex_unlock(&vmem_mutex);
  327. return 0;
  328. }
  329. core_initcall(vmem_convert_memory_chunk);