slice.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731
  1. /*
  2. * address space "slices" (meta-segments) support
  3. *
  4. * Copyright (C) 2007 Benjamin Herrenschmidt, IBM Corporation.
  5. *
  6. * Based on hugetlb implementation
  7. *
  8. * Copyright (C) 2003 David Gibson, IBM Corporation.
  9. *
  10. * This program is free software; you can redistribute it and/or modify
  11. * it under the terms of the GNU General Public License as published by
  12. * the Free Software Foundation; either version 2 of the License, or
  13. * (at your option) any later version.
  14. *
  15. * This program is distributed in the hope that it will be useful,
  16. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. * GNU General Public License for more details.
  19. *
  20. * You should have received a copy of the GNU General Public License
  21. * along with this program; if not, write to the Free Software
  22. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  23. */
  24. #undef DEBUG
  25. #include <linux/kernel.h>
  26. #include <linux/mm.h>
  27. #include <linux/pagemap.h>
  28. #include <linux/err.h>
  29. #include <linux/spinlock.h>
  30. #include <linux/export.h>
  31. #include <asm/mman.h>
  32. #include <asm/mmu.h>
  33. #include <asm/spu.h>
  34. /* some sanity checks */
  35. #if (PGTABLE_RANGE >> 43) > SLICE_MASK_SIZE
  36. #error PGTABLE_RANGE exceeds slice_mask high_slices size
  37. #endif
  38. static DEFINE_SPINLOCK(slice_convert_lock);
  39. #ifdef DEBUG
  40. int _slice_debug = 1;
  41. static void slice_print_mask(const char *label, struct slice_mask mask)
  42. {
  43. char *p, buf[16 + 3 + 64 + 1];
  44. int i;
  45. if (!_slice_debug)
  46. return;
  47. p = buf;
  48. for (i = 0; i < SLICE_NUM_LOW; i++)
  49. *(p++) = (mask.low_slices & (1 << i)) ? '1' : '0';
  50. *(p++) = ' ';
  51. *(p++) = '-';
  52. *(p++) = ' ';
  53. for (i = 0; i < SLICE_NUM_HIGH; i++)
  54. *(p++) = (mask.high_slices & (1ul << i)) ? '1' : '0';
  55. *(p++) = 0;
  56. printk(KERN_DEBUG "%s:%s\n", label, buf);
  57. }
  58. #define slice_dbg(fmt...) do { if (_slice_debug) pr_debug(fmt); } while(0)
  59. #else
  60. static void slice_print_mask(const char *label, struct slice_mask mask) {}
  61. #define slice_dbg(fmt...)
  62. #endif
  63. static struct slice_mask slice_range_to_mask(unsigned long start,
  64. unsigned long len)
  65. {
  66. unsigned long end = start + len - 1;
  67. struct slice_mask ret = { 0, 0 };
  68. if (start < SLICE_LOW_TOP) {
  69. unsigned long mend = min(end, SLICE_LOW_TOP);
  70. unsigned long mstart = min(start, SLICE_LOW_TOP);
  71. ret.low_slices = (1u << (GET_LOW_SLICE_INDEX(mend) + 1))
  72. - (1u << GET_LOW_SLICE_INDEX(mstart));
  73. }
  74. if ((start + len) > SLICE_LOW_TOP)
  75. ret.high_slices = (1ul << (GET_HIGH_SLICE_INDEX(end) + 1))
  76. - (1ul << GET_HIGH_SLICE_INDEX(start));
  77. return ret;
  78. }
  79. static int slice_area_is_free(struct mm_struct *mm, unsigned long addr,
  80. unsigned long len)
  81. {
  82. struct vm_area_struct *vma;
  83. if ((mm->task_size - len) < addr)
  84. return 0;
  85. vma = find_vma(mm, addr);
  86. return (!vma || (addr + len) <= vma->vm_start);
  87. }
  88. static int slice_low_has_vma(struct mm_struct *mm, unsigned long slice)
  89. {
  90. return !slice_area_is_free(mm, slice << SLICE_LOW_SHIFT,
  91. 1ul << SLICE_LOW_SHIFT);
  92. }
  93. static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice)
  94. {
  95. unsigned long start = slice << SLICE_HIGH_SHIFT;
  96. unsigned long end = start + (1ul << SLICE_HIGH_SHIFT);
  97. /* Hack, so that each addresses is controlled by exactly one
  98. * of the high or low area bitmaps, the first high area starts
  99. * at 4GB, not 0 */
  100. if (start == 0)
  101. start = SLICE_LOW_TOP;
  102. return !slice_area_is_free(mm, start, end - start);
  103. }
  104. static struct slice_mask slice_mask_for_free(struct mm_struct *mm)
  105. {
  106. struct slice_mask ret = { 0, 0 };
  107. unsigned long i;
  108. for (i = 0; i < SLICE_NUM_LOW; i++)
  109. if (!slice_low_has_vma(mm, i))
  110. ret.low_slices |= 1u << i;
  111. if (mm->task_size <= SLICE_LOW_TOP)
  112. return ret;
  113. for (i = 0; i < SLICE_NUM_HIGH; i++)
  114. if (!slice_high_has_vma(mm, i))
  115. ret.high_slices |= 1ul << i;
  116. return ret;
  117. }
  118. static struct slice_mask slice_mask_for_size(struct mm_struct *mm, int psize)
  119. {
  120. unsigned char *hpsizes;
  121. int index, mask_index;
  122. struct slice_mask ret = { 0, 0 };
  123. unsigned long i;
  124. u64 lpsizes;
  125. lpsizes = mm->context.low_slices_psize;
  126. for (i = 0; i < SLICE_NUM_LOW; i++)
  127. if (((lpsizes >> (i * 4)) & 0xf) == psize)
  128. ret.low_slices |= 1u << i;
  129. hpsizes = mm->context.high_slices_psize;
  130. for (i = 0; i < SLICE_NUM_HIGH; i++) {
  131. mask_index = i & 0x1;
  132. index = i >> 1;
  133. if (((hpsizes[index] >> (mask_index * 4)) & 0xf) == psize)
  134. ret.high_slices |= 1ul << i;
  135. }
  136. return ret;
  137. }
  138. static int slice_check_fit(struct slice_mask mask, struct slice_mask available)
  139. {
  140. return (mask.low_slices & available.low_slices) == mask.low_slices &&
  141. (mask.high_slices & available.high_slices) == mask.high_slices;
  142. }
  143. static void slice_flush_segments(void *parm)
  144. {
  145. struct mm_struct *mm = parm;
  146. unsigned long flags;
  147. if (mm != current->active_mm)
  148. return;
  149. /* update the paca copy of the context struct */
  150. get_paca()->context = current->active_mm->context;
  151. local_irq_save(flags);
  152. slb_flush_and_rebolt();
  153. local_irq_restore(flags);
  154. }
  155. static void slice_convert(struct mm_struct *mm, struct slice_mask mask, int psize)
  156. {
  157. int index, mask_index;
  158. /* Write the new slice psize bits */
  159. unsigned char *hpsizes;
  160. u64 lpsizes;
  161. unsigned long i, flags;
  162. slice_dbg("slice_convert(mm=%p, psize=%d)\n", mm, psize);
  163. slice_print_mask(" mask", mask);
  164. /* We need to use a spinlock here to protect against
  165. * concurrent 64k -> 4k demotion ...
  166. */
  167. spin_lock_irqsave(&slice_convert_lock, flags);
  168. lpsizes = mm->context.low_slices_psize;
  169. for (i = 0; i < SLICE_NUM_LOW; i++)
  170. if (mask.low_slices & (1u << i))
  171. lpsizes = (lpsizes & ~(0xful << (i * 4))) |
  172. (((unsigned long)psize) << (i * 4));
  173. /* Assign the value back */
  174. mm->context.low_slices_psize = lpsizes;
  175. hpsizes = mm->context.high_slices_psize;
  176. for (i = 0; i < SLICE_NUM_HIGH; i++) {
  177. mask_index = i & 0x1;
  178. index = i >> 1;
  179. if (mask.high_slices & (1ul << i))
  180. hpsizes[index] = (hpsizes[index] &
  181. ~(0xf << (mask_index * 4))) |
  182. (((unsigned long)psize) << (mask_index * 4));
  183. }
  184. slice_dbg(" lsps=%lx, hsps=%lx\n",
  185. mm->context.low_slices_psize,
  186. mm->context.high_slices_psize);
  187. spin_unlock_irqrestore(&slice_convert_lock, flags);
  188. #ifdef CONFIG_SPU_BASE
  189. spu_flush_all_slbs(mm);
  190. #endif
  191. }
  192. /*
  193. * Compute which slice addr is part of;
  194. * set *boundary_addr to the start or end boundary of that slice
  195. * (depending on 'end' parameter);
  196. * return boolean indicating if the slice is marked as available in the
  197. * 'available' slice_mark.
  198. */
  199. static bool slice_scan_available(unsigned long addr,
  200. struct slice_mask available,
  201. int end,
  202. unsigned long *boundary_addr)
  203. {
  204. unsigned long slice;
  205. if (addr < SLICE_LOW_TOP) {
  206. slice = GET_LOW_SLICE_INDEX(addr);
  207. *boundary_addr = (slice + end) << SLICE_LOW_SHIFT;
  208. return !!(available.low_slices & (1u << slice));
  209. } else {
  210. slice = GET_HIGH_SLICE_INDEX(addr);
  211. *boundary_addr = (slice + end) ?
  212. ((slice + end) << SLICE_HIGH_SHIFT) : SLICE_LOW_TOP;
  213. return !!(available.high_slices & (1u << slice));
  214. }
  215. }
  216. static unsigned long slice_find_area_bottomup(struct mm_struct *mm,
  217. unsigned long len,
  218. struct slice_mask available,
  219. int psize)
  220. {
  221. int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
  222. unsigned long addr, found, next_end;
  223. struct vm_unmapped_area_info info;
  224. info.flags = 0;
  225. info.length = len;
  226. info.align_mask = PAGE_MASK & ((1ul << pshift) - 1);
  227. info.align_offset = 0;
  228. addr = TASK_UNMAPPED_BASE;
  229. while (addr < TASK_SIZE) {
  230. info.low_limit = addr;
  231. if (!slice_scan_available(addr, available, 1, &addr))
  232. continue;
  233. next_slice:
  234. /*
  235. * At this point [info.low_limit; addr) covers
  236. * available slices only and ends at a slice boundary.
  237. * Check if we need to reduce the range, or if we can
  238. * extend it to cover the next available slice.
  239. */
  240. if (addr >= TASK_SIZE)
  241. addr = TASK_SIZE;
  242. else if (slice_scan_available(addr, available, 1, &next_end)) {
  243. addr = next_end;
  244. goto next_slice;
  245. }
  246. info.high_limit = addr;
  247. found = vm_unmapped_area(&info);
  248. if (!(found & ~PAGE_MASK))
  249. return found;
  250. }
  251. return -ENOMEM;
  252. }
  253. static unsigned long slice_find_area_topdown(struct mm_struct *mm,
  254. unsigned long len,
  255. struct slice_mask available,
  256. int psize)
  257. {
  258. int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
  259. unsigned long addr, found, prev;
  260. struct vm_unmapped_area_info info;
  261. info.flags = VM_UNMAPPED_AREA_TOPDOWN;
  262. info.length = len;
  263. info.align_mask = PAGE_MASK & ((1ul << pshift) - 1);
  264. info.align_offset = 0;
  265. addr = mm->mmap_base;
  266. while (addr > PAGE_SIZE) {
  267. info.high_limit = addr;
  268. if (!slice_scan_available(addr - 1, available, 0, &addr))
  269. continue;
  270. prev_slice:
  271. /*
  272. * At this point [addr; info.high_limit) covers
  273. * available slices only and starts at a slice boundary.
  274. * Check if we need to reduce the range, or if we can
  275. * extend it to cover the previous available slice.
  276. */
  277. if (addr < PAGE_SIZE)
  278. addr = PAGE_SIZE;
  279. else if (slice_scan_available(addr - 1, available, 0, &prev)) {
  280. addr = prev;
  281. goto prev_slice;
  282. }
  283. info.low_limit = addr;
  284. found = vm_unmapped_area(&info);
  285. if (!(found & ~PAGE_MASK))
  286. return found;
  287. }
  288. /*
  289. * A failed mmap() very likely causes application failure,
  290. * so fall back to the bottom-up function here. This scenario
  291. * can happen with large stack limits and large mmap()
  292. * allocations.
  293. */
  294. return slice_find_area_bottomup(mm, len, available, psize);
  295. }
  296. static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
  297. struct slice_mask mask, int psize,
  298. int topdown)
  299. {
  300. if (topdown)
  301. return slice_find_area_topdown(mm, len, mask, psize);
  302. else
  303. return slice_find_area_bottomup(mm, len, mask, psize);
  304. }
  305. #define or_mask(dst, src) do { \
  306. (dst).low_slices |= (src).low_slices; \
  307. (dst).high_slices |= (src).high_slices; \
  308. } while (0)
  309. #define andnot_mask(dst, src) do { \
  310. (dst).low_slices &= ~(src).low_slices; \
  311. (dst).high_slices &= ~(src).high_slices; \
  312. } while (0)
  313. #ifdef CONFIG_PPC_64K_PAGES
  314. #define MMU_PAGE_BASE MMU_PAGE_64K
  315. #else
  316. #define MMU_PAGE_BASE MMU_PAGE_4K
  317. #endif
  318. unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
  319. unsigned long flags, unsigned int psize,
  320. int topdown)
  321. {
  322. struct slice_mask mask = {0, 0};
  323. struct slice_mask good_mask;
  324. struct slice_mask potential_mask = {0,0} /* silence stupid warning */;
  325. struct slice_mask compat_mask = {0, 0};
  326. int fixed = (flags & MAP_FIXED);
  327. int pshift = max_t(int, mmu_psize_defs[psize].shift, PAGE_SHIFT);
  328. struct mm_struct *mm = current->mm;
  329. unsigned long newaddr;
  330. /* Sanity checks */
  331. BUG_ON(mm->task_size == 0);
  332. slice_dbg("slice_get_unmapped_area(mm=%p, psize=%d...\n", mm, psize);
  333. slice_dbg(" addr=%lx, len=%lx, flags=%lx, topdown=%d\n",
  334. addr, len, flags, topdown);
  335. if (len > mm->task_size)
  336. return -ENOMEM;
  337. if (len & ((1ul << pshift) - 1))
  338. return -EINVAL;
  339. if (fixed && (addr & ((1ul << pshift) - 1)))
  340. return -EINVAL;
  341. if (fixed && addr > (mm->task_size - len))
  342. return -EINVAL;
  343. /* If hint, make sure it matches our alignment restrictions */
  344. if (!fixed && addr) {
  345. addr = _ALIGN_UP(addr, 1ul << pshift);
  346. slice_dbg(" aligned addr=%lx\n", addr);
  347. /* Ignore hint if it's too large or overlaps a VMA */
  348. if (addr > mm->task_size - len ||
  349. !slice_area_is_free(mm, addr, len))
  350. addr = 0;
  351. }
  352. /* First make up a "good" mask of slices that have the right size
  353. * already
  354. */
  355. good_mask = slice_mask_for_size(mm, psize);
  356. slice_print_mask(" good_mask", good_mask);
  357. /*
  358. * Here "good" means slices that are already the right page size,
  359. * "compat" means slices that have a compatible page size (i.e.
  360. * 4k in a 64k pagesize kernel), and "free" means slices without
  361. * any VMAs.
  362. *
  363. * If MAP_FIXED:
  364. * check if fits in good | compat => OK
  365. * check if fits in good | compat | free => convert free
  366. * else bad
  367. * If have hint:
  368. * check if hint fits in good => OK
  369. * check if hint fits in good | free => convert free
  370. * Otherwise:
  371. * search in good, found => OK
  372. * search in good | free, found => convert free
  373. * search in good | compat | free, found => convert free.
  374. */
  375. #ifdef CONFIG_PPC_64K_PAGES
  376. /* If we support combo pages, we can allow 64k pages in 4k slices */
  377. if (psize == MMU_PAGE_64K) {
  378. compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K);
  379. if (fixed)
  380. or_mask(good_mask, compat_mask);
  381. }
  382. #endif
  383. /* First check hint if it's valid or if we have MAP_FIXED */
  384. if (addr != 0 || fixed) {
  385. /* Build a mask for the requested range */
  386. mask = slice_range_to_mask(addr, len);
  387. slice_print_mask(" mask", mask);
  388. /* Check if we fit in the good mask. If we do, we just return,
  389. * nothing else to do
  390. */
  391. if (slice_check_fit(mask, good_mask)) {
  392. slice_dbg(" fits good !\n");
  393. return addr;
  394. }
  395. } else {
  396. /* Now let's see if we can find something in the existing
  397. * slices for that size
  398. */
  399. newaddr = slice_find_area(mm, len, good_mask, psize, topdown);
  400. if (newaddr != -ENOMEM) {
  401. /* Found within the good mask, we don't have to setup,
  402. * we thus return directly
  403. */
  404. slice_dbg(" found area at 0x%lx\n", newaddr);
  405. return newaddr;
  406. }
  407. }
  408. /* We don't fit in the good mask, check what other slices are
  409. * empty and thus can be converted
  410. */
  411. potential_mask = slice_mask_for_free(mm);
  412. or_mask(potential_mask, good_mask);
  413. slice_print_mask(" potential", potential_mask);
  414. if ((addr != 0 || fixed) && slice_check_fit(mask, potential_mask)) {
  415. slice_dbg(" fits potential !\n");
  416. goto convert;
  417. }
  418. /* If we have MAP_FIXED and failed the above steps, then error out */
  419. if (fixed)
  420. return -EBUSY;
  421. slice_dbg(" search...\n");
  422. /* If we had a hint that didn't work out, see if we can fit
  423. * anywhere in the good area.
  424. */
  425. if (addr) {
  426. addr = slice_find_area(mm, len, good_mask, psize, topdown);
  427. if (addr != -ENOMEM) {
  428. slice_dbg(" found area at 0x%lx\n", addr);
  429. return addr;
  430. }
  431. }
  432. /* Now let's see if we can find something in the existing slices
  433. * for that size plus free slices
  434. */
  435. addr = slice_find_area(mm, len, potential_mask, psize, topdown);
  436. #ifdef CONFIG_PPC_64K_PAGES
  437. if (addr == -ENOMEM && psize == MMU_PAGE_64K) {
  438. /* retry the search with 4k-page slices included */
  439. or_mask(potential_mask, compat_mask);
  440. addr = slice_find_area(mm, len, potential_mask, psize,
  441. topdown);
  442. }
  443. #endif
  444. if (addr == -ENOMEM)
  445. return -ENOMEM;
  446. mask = slice_range_to_mask(addr, len);
  447. slice_dbg(" found potential area at 0x%lx\n", addr);
  448. slice_print_mask(" mask", mask);
  449. convert:
  450. andnot_mask(mask, good_mask);
  451. andnot_mask(mask, compat_mask);
  452. if (mask.low_slices || mask.high_slices) {
  453. slice_convert(mm, mask, psize);
  454. if (psize > MMU_PAGE_BASE)
  455. on_each_cpu(slice_flush_segments, mm, 1);
  456. }
  457. return addr;
  458. }
  459. EXPORT_SYMBOL_GPL(slice_get_unmapped_area);
  460. unsigned long arch_get_unmapped_area(struct file *filp,
  461. unsigned long addr,
  462. unsigned long len,
  463. unsigned long pgoff,
  464. unsigned long flags)
  465. {
  466. return slice_get_unmapped_area(addr, len, flags,
  467. current->mm->context.user_psize, 0);
  468. }
  469. unsigned long arch_get_unmapped_area_topdown(struct file *filp,
  470. const unsigned long addr0,
  471. const unsigned long len,
  472. const unsigned long pgoff,
  473. const unsigned long flags)
  474. {
  475. return slice_get_unmapped_area(addr0, len, flags,
  476. current->mm->context.user_psize, 1);
  477. }
  478. unsigned int get_slice_psize(struct mm_struct *mm, unsigned long addr)
  479. {
  480. unsigned char *hpsizes;
  481. int index, mask_index;
  482. if (addr < SLICE_LOW_TOP) {
  483. u64 lpsizes;
  484. lpsizes = mm->context.low_slices_psize;
  485. index = GET_LOW_SLICE_INDEX(addr);
  486. return (lpsizes >> (index * 4)) & 0xf;
  487. }
  488. hpsizes = mm->context.high_slices_psize;
  489. index = GET_HIGH_SLICE_INDEX(addr);
  490. mask_index = index & 0x1;
  491. return (hpsizes[index >> 1] >> (mask_index * 4)) & 0xf;
  492. }
  493. EXPORT_SYMBOL_GPL(get_slice_psize);
  494. /*
  495. * This is called by hash_page when it needs to do a lazy conversion of
  496. * an address space from real 64K pages to combo 4K pages (typically
  497. * when hitting a non cacheable mapping on a processor or hypervisor
  498. * that won't allow them for 64K pages).
  499. *
  500. * This is also called in init_new_context() to change back the user
  501. * psize from whatever the parent context had it set to
  502. * N.B. This may be called before mm->context.id has been set.
  503. *
  504. * This function will only change the content of the {low,high)_slice_psize
  505. * masks, it will not flush SLBs as this shall be handled lazily by the
  506. * caller.
  507. */
  508. void slice_set_user_psize(struct mm_struct *mm, unsigned int psize)
  509. {
  510. int index, mask_index;
  511. unsigned char *hpsizes;
  512. unsigned long flags, lpsizes;
  513. unsigned int old_psize;
  514. int i;
  515. slice_dbg("slice_set_user_psize(mm=%p, psize=%d)\n", mm, psize);
  516. spin_lock_irqsave(&slice_convert_lock, flags);
  517. old_psize = mm->context.user_psize;
  518. slice_dbg(" old_psize=%d\n", old_psize);
  519. if (old_psize == psize)
  520. goto bail;
  521. mm->context.user_psize = psize;
  522. wmb();
  523. lpsizes = mm->context.low_slices_psize;
  524. for (i = 0; i < SLICE_NUM_LOW; i++)
  525. if (((lpsizes >> (i * 4)) & 0xf) == old_psize)
  526. lpsizes = (lpsizes & ~(0xful << (i * 4))) |
  527. (((unsigned long)psize) << (i * 4));
  528. /* Assign the value back */
  529. mm->context.low_slices_psize = lpsizes;
  530. hpsizes = mm->context.high_slices_psize;
  531. for (i = 0; i < SLICE_NUM_HIGH; i++) {
  532. mask_index = i & 0x1;
  533. index = i >> 1;
  534. if (((hpsizes[index] >> (mask_index * 4)) & 0xf) == old_psize)
  535. hpsizes[index] = (hpsizes[index] &
  536. ~(0xf << (mask_index * 4))) |
  537. (((unsigned long)psize) << (mask_index * 4));
  538. }
  539. slice_dbg(" lsps=%lx, hsps=%lx\n",
  540. mm->context.low_slices_psize,
  541. mm->context.high_slices_psize);
  542. bail:
  543. spin_unlock_irqrestore(&slice_convert_lock, flags);
  544. }
  545. void slice_set_psize(struct mm_struct *mm, unsigned long address,
  546. unsigned int psize)
  547. {
  548. unsigned char *hpsizes;
  549. unsigned long i, flags;
  550. u64 *lpsizes;
  551. spin_lock_irqsave(&slice_convert_lock, flags);
  552. if (address < SLICE_LOW_TOP) {
  553. i = GET_LOW_SLICE_INDEX(address);
  554. lpsizes = &mm->context.low_slices_psize;
  555. *lpsizes = (*lpsizes & ~(0xful << (i * 4))) |
  556. ((unsigned long) psize << (i * 4));
  557. } else {
  558. int index, mask_index;
  559. i = GET_HIGH_SLICE_INDEX(address);
  560. hpsizes = mm->context.high_slices_psize;
  561. mask_index = i & 0x1;
  562. index = i >> 1;
  563. hpsizes[index] = (hpsizes[index] &
  564. ~(0xf << (mask_index * 4))) |
  565. (((unsigned long)psize) << (mask_index * 4));
  566. }
  567. spin_unlock_irqrestore(&slice_convert_lock, flags);
  568. #ifdef CONFIG_SPU_BASE
  569. spu_flush_all_slbs(mm);
  570. #endif
  571. }
  572. void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
  573. unsigned long len, unsigned int psize)
  574. {
  575. struct slice_mask mask = slice_range_to_mask(start, len);
  576. slice_convert(mm, mask, psize);
  577. }
  578. /*
  579. * is_hugepage_only_range() is used by generic code to verify whether
  580. * a normal mmap mapping (non hugetlbfs) is valid on a given area.
  581. *
  582. * until the generic code provides a more generic hook and/or starts
  583. * calling arch get_unmapped_area for MAP_FIXED (which our implementation
  584. * here knows how to deal with), we hijack it to keep standard mappings
  585. * away from us.
  586. *
  587. * because of that generic code limitation, MAP_FIXED mapping cannot
  588. * "convert" back a slice with no VMAs to the standard page size, only
  589. * get_unmapped_area() can. It would be possible to fix it here but I
  590. * prefer working on fixing the generic code instead.
  591. *
  592. * WARNING: This will not work if hugetlbfs isn't enabled since the
  593. * generic code will redefine that function as 0 in that. This is ok
  594. * for now as we only use slices with hugetlbfs enabled. This should
  595. * be fixed as the generic code gets fixed.
  596. */
  597. int is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
  598. unsigned long len)
  599. {
  600. struct slice_mask mask, available;
  601. unsigned int psize = mm->context.user_psize;
  602. mask = slice_range_to_mask(addr, len);
  603. available = slice_mask_for_size(mm, psize);
  604. #ifdef CONFIG_PPC_64K_PAGES
  605. /* We need to account for 4k slices too */
  606. if (psize == MMU_PAGE_64K) {
  607. struct slice_mask compat_mask;
  608. compat_mask = slice_mask_for_size(mm, MMU_PAGE_4K);
  609. or_mask(available, compat_mask);
  610. }
  611. #endif
  612. #if 0 /* too verbose */
  613. slice_dbg("is_hugepage_only_range(mm=%p, addr=%lx, len=%lx)\n",
  614. mm, addr, len);
  615. slice_print_mask(" mask", mask);
  616. slice_print_mask(" available", available);
  617. #endif
  618. return !slice_check_fit(mask, available);
  619. }