hugetlb.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837
  1. /*
  2. * Generic hugetlb support.
  3. * (C) William Irwin, April 2004
  4. */
  5. #include <linux/gfp.h>
  6. #include <linux/list.h>
  7. #include <linux/init.h>
  8. #include <linux/module.h>
  9. #include <linux/mm.h>
  10. #include <linux/sysctl.h>
  11. #include <linux/highmem.h>
  12. #include <linux/nodemask.h>
  13. #include <linux/pagemap.h>
  14. #include <linux/mempolicy.h>
  15. #include <linux/cpuset.h>
  16. #include <linux/mutex.h>
  17. #include <asm/page.h>
  18. #include <asm/pgtable.h>
  19. #include <linux/hugetlb.h>
  20. #include "internal.h"
  21. const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
  22. static unsigned long nr_huge_pages, free_huge_pages, resv_huge_pages;
  23. unsigned long max_huge_pages;
  24. static struct list_head hugepage_freelists[MAX_NUMNODES];
  25. static unsigned int nr_huge_pages_node[MAX_NUMNODES];
  26. static unsigned int free_huge_pages_node[MAX_NUMNODES];
  27. /*
  28. * Protects updates to hugepage_freelists, nr_huge_pages, and free_huge_pages
  29. */
  30. static DEFINE_SPINLOCK(hugetlb_lock);
  31. static void clear_huge_page(struct page *page, unsigned long addr)
  32. {
  33. int i;
  34. might_sleep();
  35. for (i = 0; i < (HPAGE_SIZE/PAGE_SIZE); i++) {
  36. cond_resched();
  37. clear_user_highpage(page + i, addr);
  38. }
  39. }
  40. static void copy_huge_page(struct page *dst, struct page *src,
  41. unsigned long addr)
  42. {
  43. int i;
  44. might_sleep();
  45. for (i = 0; i < HPAGE_SIZE/PAGE_SIZE; i++) {
  46. cond_resched();
  47. copy_user_highpage(dst + i, src + i, addr + i*PAGE_SIZE);
  48. }
  49. }
  50. static void enqueue_huge_page(struct page *page)
  51. {
  52. int nid = page_to_nid(page);
  53. list_add(&page->lru, &hugepage_freelists[nid]);
  54. free_huge_pages++;
  55. free_huge_pages_node[nid]++;
  56. }
  57. static struct page *dequeue_huge_page(struct vm_area_struct *vma,
  58. unsigned long address)
  59. {
  60. int nid = numa_node_id();
  61. struct page *page = NULL;
  62. struct zonelist *zonelist = huge_zonelist(vma, address);
  63. struct zone **z;
  64. for (z = zonelist->zones; *z; z++) {
  65. nid = zone_to_nid(*z);
  66. if (cpuset_zone_allowed(*z, GFP_HIGHUSER) &&
  67. !list_empty(&hugepage_freelists[nid]))
  68. break;
  69. }
  70. if (*z) {
  71. page = list_entry(hugepage_freelists[nid].next,
  72. struct page, lru);
  73. list_del(&page->lru);
  74. free_huge_pages--;
  75. free_huge_pages_node[nid]--;
  76. }
  77. return page;
  78. }
  79. static void free_huge_page(struct page *page)
  80. {
  81. BUG_ON(page_count(page));
  82. INIT_LIST_HEAD(&page->lru);
  83. spin_lock(&hugetlb_lock);
  84. enqueue_huge_page(page);
  85. spin_unlock(&hugetlb_lock);
  86. }
  87. static int alloc_fresh_huge_page(void)
  88. {
  89. static int nid = 0;
  90. struct page *page;
  91. page = alloc_pages_node(nid, GFP_HIGHUSER|__GFP_COMP|__GFP_NOWARN,
  92. HUGETLB_PAGE_ORDER);
  93. nid = next_node(nid, node_online_map);
  94. if (nid == MAX_NUMNODES)
  95. nid = first_node(node_online_map);
  96. if (page) {
  97. page[1].lru.next = (void *)free_huge_page; /* dtor */
  98. spin_lock(&hugetlb_lock);
  99. nr_huge_pages++;
  100. nr_huge_pages_node[page_to_nid(page)]++;
  101. spin_unlock(&hugetlb_lock);
  102. put_page(page); /* free it into the hugepage allocator */
  103. return 1;
  104. }
  105. return 0;
  106. }
  107. static struct page *alloc_huge_page(struct vm_area_struct *vma,
  108. unsigned long addr)
  109. {
  110. struct page *page;
  111. spin_lock(&hugetlb_lock);
  112. if (vma->vm_flags & VM_MAYSHARE)
  113. resv_huge_pages--;
  114. else if (free_huge_pages <= resv_huge_pages)
  115. goto fail;
  116. page = dequeue_huge_page(vma, addr);
  117. if (!page)
  118. goto fail;
  119. spin_unlock(&hugetlb_lock);
  120. set_page_refcounted(page);
  121. return page;
  122. fail:
  123. spin_unlock(&hugetlb_lock);
  124. return NULL;
  125. }
  126. static int __init hugetlb_init(void)
  127. {
  128. unsigned long i;
  129. if (HPAGE_SHIFT == 0)
  130. return 0;
  131. for (i = 0; i < MAX_NUMNODES; ++i)
  132. INIT_LIST_HEAD(&hugepage_freelists[i]);
  133. for (i = 0; i < max_huge_pages; ++i) {
  134. if (!alloc_fresh_huge_page())
  135. break;
  136. }
  137. max_huge_pages = free_huge_pages = nr_huge_pages = i;
  138. printk("Total HugeTLB memory allocated, %ld\n", free_huge_pages);
  139. return 0;
  140. }
  141. module_init(hugetlb_init);
  142. static int __init hugetlb_setup(char *s)
  143. {
  144. if (sscanf(s, "%lu", &max_huge_pages) <= 0)
  145. max_huge_pages = 0;
  146. return 1;
  147. }
  148. __setup("hugepages=", hugetlb_setup);
  149. #ifdef CONFIG_SYSCTL
  150. static void update_and_free_page(struct page *page)
  151. {
  152. int i;
  153. nr_huge_pages--;
  154. nr_huge_pages_node[page_to_nid(page)]--;
  155. for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) {
  156. page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced |
  157. 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved |
  158. 1 << PG_private | 1<< PG_writeback);
  159. }
  160. page[1].lru.next = NULL;
  161. set_page_refcounted(page);
  162. __free_pages(page, HUGETLB_PAGE_ORDER);
  163. }
  164. #ifdef CONFIG_HIGHMEM
  165. static void try_to_free_low(unsigned long count)
  166. {
  167. int i;
  168. for (i = 0; i < MAX_NUMNODES; ++i) {
  169. struct page *page, *next;
  170. list_for_each_entry_safe(page, next, &hugepage_freelists[i], lru) {
  171. if (PageHighMem(page))
  172. continue;
  173. list_del(&page->lru);
  174. update_and_free_page(page);
  175. free_huge_pages--;
  176. free_huge_pages_node[page_to_nid(page)]--;
  177. if (count >= nr_huge_pages)
  178. return;
  179. }
  180. }
  181. }
  182. #else
  183. static inline void try_to_free_low(unsigned long count)
  184. {
  185. }
  186. #endif
  187. static unsigned long set_max_huge_pages(unsigned long count)
  188. {
  189. while (count > nr_huge_pages) {
  190. if (!alloc_fresh_huge_page())
  191. return nr_huge_pages;
  192. }
  193. if (count >= nr_huge_pages)
  194. return nr_huge_pages;
  195. spin_lock(&hugetlb_lock);
  196. count = max(count, resv_huge_pages);
  197. try_to_free_low(count);
  198. while (count < nr_huge_pages) {
  199. struct page *page = dequeue_huge_page(NULL, 0);
  200. if (!page)
  201. break;
  202. update_and_free_page(page);
  203. }
  204. spin_unlock(&hugetlb_lock);
  205. return nr_huge_pages;
  206. }
  207. int hugetlb_sysctl_handler(struct ctl_table *table, int write,
  208. struct file *file, void __user *buffer,
  209. size_t *length, loff_t *ppos)
  210. {
  211. proc_doulongvec_minmax(table, write, file, buffer, length, ppos);
  212. max_huge_pages = set_max_huge_pages(max_huge_pages);
  213. return 0;
  214. }
  215. #endif /* CONFIG_SYSCTL */
  216. int hugetlb_report_meminfo(char *buf)
  217. {
  218. return sprintf(buf,
  219. "HugePages_Total: %5lu\n"
  220. "HugePages_Free: %5lu\n"
  221. "HugePages_Rsvd: %5lu\n"
  222. "Hugepagesize: %5lu kB\n",
  223. nr_huge_pages,
  224. free_huge_pages,
  225. resv_huge_pages,
  226. HPAGE_SIZE/1024);
  227. }
  228. int hugetlb_report_node_meminfo(int nid, char *buf)
  229. {
  230. return sprintf(buf,
  231. "Node %d HugePages_Total: %5u\n"
  232. "Node %d HugePages_Free: %5u\n",
  233. nid, nr_huge_pages_node[nid],
  234. nid, free_huge_pages_node[nid]);
  235. }
  236. /* Return the number pages of memory we physically have, in PAGE_SIZE units. */
  237. unsigned long hugetlb_total_pages(void)
  238. {
  239. return nr_huge_pages * (HPAGE_SIZE / PAGE_SIZE);
  240. }
  241. /*
  242. * We cannot handle pagefaults against hugetlb pages at all. They cause
  243. * handle_mm_fault() to try to instantiate regular-sized pages in the
  244. * hugegpage VMA. do_page_fault() is supposed to trap this, so BUG is we get
  245. * this far.
  246. */
  247. static struct page *hugetlb_nopage(struct vm_area_struct *vma,
  248. unsigned long address, int *unused)
  249. {
  250. BUG();
  251. return NULL;
  252. }
  253. struct vm_operations_struct hugetlb_vm_ops = {
  254. .nopage = hugetlb_nopage,
  255. };
  256. static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
  257. int writable)
  258. {
  259. pte_t entry;
  260. if (writable) {
  261. entry =
  262. pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
  263. } else {
  264. entry = pte_wrprotect(mk_pte(page, vma->vm_page_prot));
  265. }
  266. entry = pte_mkyoung(entry);
  267. entry = pte_mkhuge(entry);
  268. return entry;
  269. }
  270. static void set_huge_ptep_writable(struct vm_area_struct *vma,
  271. unsigned long address, pte_t *ptep)
  272. {
  273. pte_t entry;
  274. entry = pte_mkwrite(pte_mkdirty(*ptep));
  275. ptep_set_access_flags(vma, address, ptep, entry, 1);
  276. update_mmu_cache(vma, address, entry);
  277. lazy_mmu_prot_update(entry);
  278. }
  279. int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
  280. struct vm_area_struct *vma)
  281. {
  282. pte_t *src_pte, *dst_pte, entry;
  283. struct page *ptepage;
  284. unsigned long addr;
  285. int cow;
  286. cow = (vma->vm_flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
  287. for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) {
  288. src_pte = huge_pte_offset(src, addr);
  289. if (!src_pte)
  290. continue;
  291. dst_pte = huge_pte_alloc(dst, addr);
  292. if (!dst_pte)
  293. goto nomem;
  294. spin_lock(&dst->page_table_lock);
  295. spin_lock(&src->page_table_lock);
  296. if (!pte_none(*src_pte)) {
  297. if (cow)
  298. ptep_set_wrprotect(src, addr, src_pte);
  299. entry = *src_pte;
  300. ptepage = pte_page(entry);
  301. get_page(ptepage);
  302. add_mm_counter(dst, file_rss, HPAGE_SIZE / PAGE_SIZE);
  303. set_huge_pte_at(dst, addr, dst_pte, entry);
  304. }
  305. spin_unlock(&src->page_table_lock);
  306. spin_unlock(&dst->page_table_lock);
  307. }
  308. return 0;
  309. nomem:
  310. return -ENOMEM;
  311. }
  312. void __unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
  313. unsigned long end)
  314. {
  315. struct mm_struct *mm = vma->vm_mm;
  316. unsigned long address;
  317. pte_t *ptep;
  318. pte_t pte;
  319. struct page *page;
  320. struct page *tmp;
  321. /*
  322. * A page gathering list, protected by per file i_mmap_lock. The
  323. * lock is used to avoid list corruption from multiple unmapping
  324. * of the same page since we are using page->lru.
  325. */
  326. LIST_HEAD(page_list);
  327. WARN_ON(!is_vm_hugetlb_page(vma));
  328. BUG_ON(start & ~HPAGE_MASK);
  329. BUG_ON(end & ~HPAGE_MASK);
  330. spin_lock(&mm->page_table_lock);
  331. /* Update high watermark before we lower rss */
  332. update_hiwater_rss(mm);
  333. for (address = start; address < end; address += HPAGE_SIZE) {
  334. ptep = huge_pte_offset(mm, address);
  335. if (!ptep)
  336. continue;
  337. if (huge_pmd_unshare(mm, &address, ptep))
  338. continue;
  339. pte = huge_ptep_get_and_clear(mm, address, ptep);
  340. if (pte_none(pte))
  341. continue;
  342. page = pte_page(pte);
  343. list_add(&page->lru, &page_list);
  344. add_mm_counter(mm, file_rss, (int) -(HPAGE_SIZE / PAGE_SIZE));
  345. }
  346. spin_unlock(&mm->page_table_lock);
  347. flush_tlb_range(vma, start, end);
  348. list_for_each_entry_safe(page, tmp, &page_list, lru) {
  349. list_del(&page->lru);
  350. put_page(page);
  351. }
  352. }
  353. void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
  354. unsigned long end)
  355. {
  356. /*
  357. * It is undesirable to test vma->vm_file as it should be non-null
  358. * for valid hugetlb area. However, vm_file will be NULL in the error
  359. * cleanup path of do_mmap_pgoff. When hugetlbfs ->mmap method fails,
  360. * do_mmap_pgoff() nullifies vma->vm_file before calling this function
  361. * to clean up. Since no pte has actually been setup, it is safe to
  362. * do nothing in this case.
  363. */
  364. if (vma->vm_file) {
  365. spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
  366. __unmap_hugepage_range(vma, start, end);
  367. spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
  368. }
  369. }
  370. static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma,
  371. unsigned long address, pte_t *ptep, pte_t pte)
  372. {
  373. struct page *old_page, *new_page;
  374. int avoidcopy;
  375. old_page = pte_page(pte);
  376. /* If no-one else is actually using this page, avoid the copy
  377. * and just make the page writable */
  378. avoidcopy = (page_count(old_page) == 1);
  379. if (avoidcopy) {
  380. set_huge_ptep_writable(vma, address, ptep);
  381. return VM_FAULT_MINOR;
  382. }
  383. page_cache_get(old_page);
  384. new_page = alloc_huge_page(vma, address);
  385. if (!new_page) {
  386. page_cache_release(old_page);
  387. return VM_FAULT_OOM;
  388. }
  389. spin_unlock(&mm->page_table_lock);
  390. copy_huge_page(new_page, old_page, address);
  391. spin_lock(&mm->page_table_lock);
  392. ptep = huge_pte_offset(mm, address & HPAGE_MASK);
  393. if (likely(pte_same(*ptep, pte))) {
  394. /* Break COW */
  395. set_huge_pte_at(mm, address, ptep,
  396. make_huge_pte(vma, new_page, 1));
  397. /* Make the old page be freed below */
  398. new_page = old_page;
  399. }
  400. page_cache_release(new_page);
  401. page_cache_release(old_page);
  402. return VM_FAULT_MINOR;
  403. }
  404. int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
  405. unsigned long address, pte_t *ptep, int write_access)
  406. {
  407. int ret = VM_FAULT_SIGBUS;
  408. unsigned long idx;
  409. unsigned long size;
  410. struct page *page;
  411. struct address_space *mapping;
  412. pte_t new_pte;
  413. mapping = vma->vm_file->f_mapping;
  414. idx = ((address - vma->vm_start) >> HPAGE_SHIFT)
  415. + (vma->vm_pgoff >> (HPAGE_SHIFT - PAGE_SHIFT));
  416. /*
  417. * Use page lock to guard against racing truncation
  418. * before we get page_table_lock.
  419. */
  420. retry:
  421. page = find_lock_page(mapping, idx);
  422. if (!page) {
  423. size = i_size_read(mapping->host) >> HPAGE_SHIFT;
  424. if (idx >= size)
  425. goto out;
  426. if (hugetlb_get_quota(mapping))
  427. goto out;
  428. page = alloc_huge_page(vma, address);
  429. if (!page) {
  430. hugetlb_put_quota(mapping);
  431. ret = VM_FAULT_OOM;
  432. goto out;
  433. }
  434. clear_huge_page(page, address);
  435. if (vma->vm_flags & VM_SHARED) {
  436. int err;
  437. err = add_to_page_cache(page, mapping, idx, GFP_KERNEL);
  438. if (err) {
  439. put_page(page);
  440. hugetlb_put_quota(mapping);
  441. if (err == -EEXIST)
  442. goto retry;
  443. goto out;
  444. }
  445. } else
  446. lock_page(page);
  447. }
  448. spin_lock(&mm->page_table_lock);
  449. size = i_size_read(mapping->host) >> HPAGE_SHIFT;
  450. if (idx >= size)
  451. goto backout;
  452. ret = VM_FAULT_MINOR;
  453. if (!pte_none(*ptep))
  454. goto backout;
  455. add_mm_counter(mm, file_rss, HPAGE_SIZE / PAGE_SIZE);
  456. new_pte = make_huge_pte(vma, page, ((vma->vm_flags & VM_WRITE)
  457. && (vma->vm_flags & VM_SHARED)));
  458. set_huge_pte_at(mm, address, ptep, new_pte);
  459. if (write_access && !(vma->vm_flags & VM_SHARED)) {
  460. /* Optimization, do the COW without a second fault */
  461. ret = hugetlb_cow(mm, vma, address, ptep, new_pte);
  462. }
  463. spin_unlock(&mm->page_table_lock);
  464. unlock_page(page);
  465. out:
  466. return ret;
  467. backout:
  468. spin_unlock(&mm->page_table_lock);
  469. hugetlb_put_quota(mapping);
  470. unlock_page(page);
  471. put_page(page);
  472. goto out;
  473. }
  474. int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  475. unsigned long address, int write_access)
  476. {
  477. pte_t *ptep;
  478. pte_t entry;
  479. int ret;
  480. static DEFINE_MUTEX(hugetlb_instantiation_mutex);
  481. ptep = huge_pte_alloc(mm, address);
  482. if (!ptep)
  483. return VM_FAULT_OOM;
  484. /*
  485. * Serialize hugepage allocation and instantiation, so that we don't
  486. * get spurious allocation failures if two CPUs race to instantiate
  487. * the same page in the page cache.
  488. */
  489. mutex_lock(&hugetlb_instantiation_mutex);
  490. entry = *ptep;
  491. if (pte_none(entry)) {
  492. ret = hugetlb_no_page(mm, vma, address, ptep, write_access);
  493. mutex_unlock(&hugetlb_instantiation_mutex);
  494. return ret;
  495. }
  496. ret = VM_FAULT_MINOR;
  497. spin_lock(&mm->page_table_lock);
  498. /* Check for a racing update before calling hugetlb_cow */
  499. if (likely(pte_same(entry, *ptep)))
  500. if (write_access && !pte_write(entry))
  501. ret = hugetlb_cow(mm, vma, address, ptep, entry);
  502. spin_unlock(&mm->page_table_lock);
  503. mutex_unlock(&hugetlb_instantiation_mutex);
  504. return ret;
  505. }
  506. int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
  507. struct page **pages, struct vm_area_struct **vmas,
  508. unsigned long *position, int *length, int i)
  509. {
  510. unsigned long pfn_offset;
  511. unsigned long vaddr = *position;
  512. int remainder = *length;
  513. spin_lock(&mm->page_table_lock);
  514. while (vaddr < vma->vm_end && remainder) {
  515. pte_t *pte;
  516. struct page *page;
  517. /*
  518. * Some archs (sparc64, sh*) have multiple pte_ts to
  519. * each hugepage. We have to make * sure we get the
  520. * first, for the page indexing below to work.
  521. */
  522. pte = huge_pte_offset(mm, vaddr & HPAGE_MASK);
  523. if (!pte || pte_none(*pte)) {
  524. int ret;
  525. spin_unlock(&mm->page_table_lock);
  526. ret = hugetlb_fault(mm, vma, vaddr, 0);
  527. spin_lock(&mm->page_table_lock);
  528. if (ret == VM_FAULT_MINOR)
  529. continue;
  530. remainder = 0;
  531. if (!i)
  532. i = -EFAULT;
  533. break;
  534. }
  535. pfn_offset = (vaddr & ~HPAGE_MASK) >> PAGE_SHIFT;
  536. page = pte_page(*pte);
  537. same_page:
  538. if (pages) {
  539. get_page(page);
  540. pages[i] = page + pfn_offset;
  541. }
  542. if (vmas)
  543. vmas[i] = vma;
  544. vaddr += PAGE_SIZE;
  545. ++pfn_offset;
  546. --remainder;
  547. ++i;
  548. if (vaddr < vma->vm_end && remainder &&
  549. pfn_offset < HPAGE_SIZE/PAGE_SIZE) {
  550. /*
  551. * We use pfn_offset to avoid touching the pageframes
  552. * of this compound page.
  553. */
  554. goto same_page;
  555. }
  556. }
  557. spin_unlock(&mm->page_table_lock);
  558. *length = remainder;
  559. *position = vaddr;
  560. return i;
  561. }
  562. void hugetlb_change_protection(struct vm_area_struct *vma,
  563. unsigned long address, unsigned long end, pgprot_t newprot)
  564. {
  565. struct mm_struct *mm = vma->vm_mm;
  566. unsigned long start = address;
  567. pte_t *ptep;
  568. pte_t pte;
  569. BUG_ON(address >= end);
  570. flush_cache_range(vma, address, end);
  571. spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
  572. spin_lock(&mm->page_table_lock);
  573. for (; address < end; address += HPAGE_SIZE) {
  574. ptep = huge_pte_offset(mm, address);
  575. if (!ptep)
  576. continue;
  577. if (huge_pmd_unshare(mm, &address, ptep))
  578. continue;
  579. if (!pte_none(*ptep)) {
  580. pte = huge_ptep_get_and_clear(mm, address, ptep);
  581. pte = pte_mkhuge(pte_modify(pte, newprot));
  582. set_huge_pte_at(mm, address, ptep, pte);
  583. lazy_mmu_prot_update(pte);
  584. }
  585. }
  586. spin_unlock(&mm->page_table_lock);
  587. spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
  588. flush_tlb_range(vma, start, end);
  589. }
  590. struct file_region {
  591. struct list_head link;
  592. long from;
  593. long to;
  594. };
  595. static long region_add(struct list_head *head, long f, long t)
  596. {
  597. struct file_region *rg, *nrg, *trg;
  598. /* Locate the region we are either in or before. */
  599. list_for_each_entry(rg, head, link)
  600. if (f <= rg->to)
  601. break;
  602. /* Round our left edge to the current segment if it encloses us. */
  603. if (f > rg->from)
  604. f = rg->from;
  605. /* Check for and consume any regions we now overlap with. */
  606. nrg = rg;
  607. list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  608. if (&rg->link == head)
  609. break;
  610. if (rg->from > t)
  611. break;
  612. /* If this area reaches higher then extend our area to
  613. * include it completely. If this is not the first area
  614. * which we intend to reuse, free it. */
  615. if (rg->to > t)
  616. t = rg->to;
  617. if (rg != nrg) {
  618. list_del(&rg->link);
  619. kfree(rg);
  620. }
  621. }
  622. nrg->from = f;
  623. nrg->to = t;
  624. return 0;
  625. }
  626. static long region_chg(struct list_head *head, long f, long t)
  627. {
  628. struct file_region *rg, *nrg;
  629. long chg = 0;
  630. /* Locate the region we are before or in. */
  631. list_for_each_entry(rg, head, link)
  632. if (f <= rg->to)
  633. break;
  634. /* If we are below the current region then a new region is required.
  635. * Subtle, allocate a new region at the position but make it zero
  636. * size such that we can guarentee to record the reservation. */
  637. if (&rg->link == head || t < rg->from) {
  638. nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
  639. if (nrg == 0)
  640. return -ENOMEM;
  641. nrg->from = f;
  642. nrg->to = f;
  643. INIT_LIST_HEAD(&nrg->link);
  644. list_add(&nrg->link, rg->link.prev);
  645. return t - f;
  646. }
  647. /* Round our left edge to the current segment if it encloses us. */
  648. if (f > rg->from)
  649. f = rg->from;
  650. chg = t - f;
  651. /* Check for and consume any regions we now overlap with. */
  652. list_for_each_entry(rg, rg->link.prev, link) {
  653. if (&rg->link == head)
  654. break;
  655. if (rg->from > t)
  656. return chg;
  657. /* We overlap with this area, if it extends futher than
  658. * us then we must extend ourselves. Account for its
  659. * existing reservation. */
  660. if (rg->to > t) {
  661. chg += rg->to - t;
  662. t = rg->to;
  663. }
  664. chg -= rg->to - rg->from;
  665. }
  666. return chg;
  667. }
  668. static long region_truncate(struct list_head *head, long end)
  669. {
  670. struct file_region *rg, *trg;
  671. long chg = 0;
  672. /* Locate the region we are either in or before. */
  673. list_for_each_entry(rg, head, link)
  674. if (end <= rg->to)
  675. break;
  676. if (&rg->link == head)
  677. return 0;
  678. /* If we are in the middle of a region then adjust it. */
  679. if (end > rg->from) {
  680. chg = rg->to - end;
  681. rg->to = end;
  682. rg = list_entry(rg->link.next, typeof(*rg), link);
  683. }
  684. /* Drop any remaining regions. */
  685. list_for_each_entry_safe(rg, trg, rg->link.prev, link) {
  686. if (&rg->link == head)
  687. break;
  688. chg += rg->to - rg->from;
  689. list_del(&rg->link);
  690. kfree(rg);
  691. }
  692. return chg;
  693. }
  694. static int hugetlb_acct_memory(long delta)
  695. {
  696. int ret = -ENOMEM;
  697. spin_lock(&hugetlb_lock);
  698. if ((delta + resv_huge_pages) <= free_huge_pages) {
  699. resv_huge_pages += delta;
  700. ret = 0;
  701. }
  702. spin_unlock(&hugetlb_lock);
  703. return ret;
  704. }
  705. int hugetlb_reserve_pages(struct inode *inode, long from, long to)
  706. {
  707. long ret, chg;
  708. chg = region_chg(&inode->i_mapping->private_list, from, to);
  709. if (chg < 0)
  710. return chg;
  711. ret = hugetlb_acct_memory(chg);
  712. if (ret < 0)
  713. return ret;
  714. region_add(&inode->i_mapping->private_list, from, to);
  715. return 0;
  716. }
  717. void hugetlb_unreserve_pages(struct inode *inode, long offset, long freed)
  718. {
  719. long chg = region_truncate(&inode->i_mapping->private_list, offset);
  720. hugetlb_acct_memory(freed - chg);
  721. }