mmap.c 64 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489
  1. /*
  2. * mm/mmap.c
  3. *
  4. * Written by obz.
  5. *
  6. * Address space accounting code <alan@lxorguk.ukuu.org.uk>
  7. */
  8. #include <linux/slab.h>
  9. #include <linux/backing-dev.h>
  10. #include <linux/mm.h>
  11. #include <linux/shm.h>
  12. #include <linux/mman.h>
  13. #include <linux/pagemap.h>
  14. #include <linux/swap.h>
  15. #include <linux/syscalls.h>
  16. #include <linux/capability.h>
  17. #include <linux/init.h>
  18. #include <linux/file.h>
  19. #include <linux/fs.h>
  20. #include <linux/personality.h>
  21. #include <linux/security.h>
  22. #include <linux/ima.h>
  23. #include <linux/hugetlb.h>
  24. #include <linux/profile.h>
  25. #include <linux/module.h>
  26. #include <linux/mount.h>
  27. #include <linux/mempolicy.h>
  28. #include <linux/rmap.h>
  29. #include <linux/mmu_notifier.h>
  30. #include <linux/perf_counter.h>
  31. #include <asm/uaccess.h>
  32. #include <asm/cacheflush.h>
  33. #include <asm/tlb.h>
  34. #include <asm/mmu_context.h>
  35. #include "internal.h"
  36. #ifndef arch_mmap_check
  37. #define arch_mmap_check(addr, len, flags) (0)
  38. #endif
  39. #ifndef arch_rebalance_pgtables
  40. #define arch_rebalance_pgtables(addr, len) (addr)
  41. #endif
  42. static void unmap_region(struct mm_struct *mm,
  43. struct vm_area_struct *vma, struct vm_area_struct *prev,
  44. unsigned long start, unsigned long end);
  45. /*
  46. * WARNING: the debugging will use recursive algorithms so never enable this
  47. * unless you know what you are doing.
  48. */
  49. #undef DEBUG_MM_RB
  50. /* description of effects of mapping type and prot in current implementation.
  51. * this is due to the limited x86 page protection hardware. The expected
  52. * behavior is in parens:
  53. *
  54. * map_type prot
  55. * PROT_NONE PROT_READ PROT_WRITE PROT_EXEC
  56. * MAP_SHARED r: (no) no r: (yes) yes r: (no) yes r: (no) yes
  57. * w: (no) no w: (no) no w: (yes) yes w: (no) no
  58. * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
  59. *
  60. * MAP_PRIVATE r: (no) no r: (yes) yes r: (no) yes r: (no) yes
  61. * w: (no) no w: (no) no w: (copy) copy w: (no) no
  62. * x: (no) no x: (no) yes x: (no) yes x: (yes) yes
  63. *
  64. */
  65. pgprot_t protection_map[16] = {
  66. __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
  67. __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
  68. };
  69. pgprot_t vm_get_page_prot(unsigned long vm_flags)
  70. {
  71. return __pgprot(pgprot_val(protection_map[vm_flags &
  72. (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
  73. pgprot_val(arch_vm_get_page_prot(vm_flags)));
  74. }
  75. EXPORT_SYMBOL(vm_get_page_prot);
  76. int sysctl_overcommit_memory = OVERCOMMIT_GUESS; /* heuristic overcommit */
  77. int sysctl_overcommit_ratio = 50; /* default is 50% */
  78. int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
  79. struct percpu_counter vm_committed_as;
  80. /*
  81. * Check that a process has enough memory to allocate a new virtual
  82. * mapping. 0 means there is enough memory for the allocation to
  83. * succeed and -ENOMEM implies there is not.
  84. *
  85. * We currently support three overcommit policies, which are set via the
  86. * vm.overcommit_memory sysctl. See Documentation/vm/overcommit-accounting
  87. *
  88. * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
  89. * Additional code 2002 Jul 20 by Robert Love.
  90. *
  91. * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
  92. *
  93. * Note this is a helper function intended to be used by LSMs which
  94. * wish to use this logic.
  95. */
  96. int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
  97. {
  98. unsigned long free, allowed;
  99. vm_acct_memory(pages);
  100. /*
  101. * Sometimes we want to use more memory than we have
  102. */
  103. if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
  104. return 0;
  105. if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
  106. unsigned long n;
  107. free = global_page_state(NR_FILE_PAGES);
  108. free += nr_swap_pages;
  109. /*
  110. * Any slabs which are created with the
  111. * SLAB_RECLAIM_ACCOUNT flag claim to have contents
  112. * which are reclaimable, under pressure. The dentry
  113. * cache and most inode caches should fall into this
  114. */
  115. free += global_page_state(NR_SLAB_RECLAIMABLE);
  116. /*
  117. * Leave the last 3% for root
  118. */
  119. if (!cap_sys_admin)
  120. free -= free / 32;
  121. if (free > pages)
  122. return 0;
  123. /*
  124. * nr_free_pages() is very expensive on large systems,
  125. * only call if we're about to fail.
  126. */
  127. n = nr_free_pages();
  128. /*
  129. * Leave reserved pages. The pages are not for anonymous pages.
  130. */
  131. if (n <= totalreserve_pages)
  132. goto error;
  133. else
  134. n -= totalreserve_pages;
  135. /*
  136. * Leave the last 3% for root
  137. */
  138. if (!cap_sys_admin)
  139. n -= n / 32;
  140. free += n;
  141. if (free > pages)
  142. return 0;
  143. goto error;
  144. }
  145. allowed = (totalram_pages - hugetlb_total_pages())
  146. * sysctl_overcommit_ratio / 100;
  147. /*
  148. * Leave the last 3% for root
  149. */
  150. if (!cap_sys_admin)
  151. allowed -= allowed / 32;
  152. allowed += total_swap_pages;
  153. /* Don't let a single process grow too big:
  154. leave 3% of the size of this process for other processes */
  155. if (mm)
  156. allowed -= mm->total_vm / 32;
  157. if (percpu_counter_read_positive(&vm_committed_as) < allowed)
  158. return 0;
  159. error:
  160. vm_unacct_memory(pages);
  161. return -ENOMEM;
  162. }
  163. /*
  164. * Requires inode->i_mapping->i_mmap_lock
  165. */
  166. static void __remove_shared_vm_struct(struct vm_area_struct *vma,
  167. struct file *file, struct address_space *mapping)
  168. {
  169. if (vma->vm_flags & VM_DENYWRITE)
  170. atomic_inc(&file->f_path.dentry->d_inode->i_writecount);
  171. if (vma->vm_flags & VM_SHARED)
  172. mapping->i_mmap_writable--;
  173. flush_dcache_mmap_lock(mapping);
  174. if (unlikely(vma->vm_flags & VM_NONLINEAR))
  175. list_del_init(&vma->shared.vm_set.list);
  176. else
  177. vma_prio_tree_remove(vma, &mapping->i_mmap);
  178. flush_dcache_mmap_unlock(mapping);
  179. }
  180. /*
  181. * Unlink a file-based vm structure from its prio_tree, to hide
  182. * vma from rmap and vmtruncate before freeing its page tables.
  183. */
  184. void unlink_file_vma(struct vm_area_struct *vma)
  185. {
  186. struct file *file = vma->vm_file;
  187. if (file) {
  188. struct address_space *mapping = file->f_mapping;
  189. spin_lock(&mapping->i_mmap_lock);
  190. __remove_shared_vm_struct(vma, file, mapping);
  191. spin_unlock(&mapping->i_mmap_lock);
  192. }
  193. }
  194. /*
  195. * Close a vm structure and free it, returning the next.
  196. */
  197. static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
  198. {
  199. struct vm_area_struct *next = vma->vm_next;
  200. might_sleep();
  201. if (vma->vm_ops && vma->vm_ops->close)
  202. vma->vm_ops->close(vma);
  203. if (vma->vm_file) {
  204. fput(vma->vm_file);
  205. if (vma->vm_flags & VM_EXECUTABLE)
  206. removed_exe_file_vma(vma->vm_mm);
  207. }
  208. mpol_put(vma_policy(vma));
  209. kmem_cache_free(vm_area_cachep, vma);
  210. return next;
  211. }
  212. SYSCALL_DEFINE1(brk, unsigned long, brk)
  213. {
  214. unsigned long rlim, retval;
  215. unsigned long newbrk, oldbrk;
  216. struct mm_struct *mm = current->mm;
  217. unsigned long min_brk;
  218. down_write(&mm->mmap_sem);
  219. #ifdef CONFIG_COMPAT_BRK
  220. min_brk = mm->end_code;
  221. #else
  222. min_brk = mm->start_brk;
  223. #endif
  224. if (brk < min_brk)
  225. goto out;
  226. /*
  227. * Check against rlimit here. If this check is done later after the test
  228. * of oldbrk with newbrk then it can escape the test and let the data
  229. * segment grow beyond its set limit the in case where the limit is
  230. * not page aligned -Ram Gupta
  231. */
  232. rlim = current->signal->rlim[RLIMIT_DATA].rlim_cur;
  233. if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
  234. (mm->end_data - mm->start_data) > rlim)
  235. goto out;
  236. newbrk = PAGE_ALIGN(brk);
  237. oldbrk = PAGE_ALIGN(mm->brk);
  238. if (oldbrk == newbrk)
  239. goto set_brk;
  240. /* Always allow shrinking brk. */
  241. if (brk <= mm->brk) {
  242. if (!do_munmap(mm, newbrk, oldbrk-newbrk))
  243. goto set_brk;
  244. goto out;
  245. }
  246. /* Check against existing mmap mappings. */
  247. if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
  248. goto out;
  249. /* Ok, looks good - let it rip. */
  250. if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
  251. goto out;
  252. set_brk:
  253. mm->brk = brk;
  254. out:
  255. retval = mm->brk;
  256. up_write(&mm->mmap_sem);
  257. return retval;
  258. }
  259. #ifdef DEBUG_MM_RB
  260. static int browse_rb(struct rb_root *root)
  261. {
  262. int i = 0, j;
  263. struct rb_node *nd, *pn = NULL;
  264. unsigned long prev = 0, pend = 0;
  265. for (nd = rb_first(root); nd; nd = rb_next(nd)) {
  266. struct vm_area_struct *vma;
  267. vma = rb_entry(nd, struct vm_area_struct, vm_rb);
  268. if (vma->vm_start < prev)
  269. printk("vm_start %lx prev %lx\n", vma->vm_start, prev), i = -1;
  270. if (vma->vm_start < pend)
  271. printk("vm_start %lx pend %lx\n", vma->vm_start, pend);
  272. if (vma->vm_start > vma->vm_end)
  273. printk("vm_end %lx < vm_start %lx\n", vma->vm_end, vma->vm_start);
  274. i++;
  275. pn = nd;
  276. prev = vma->vm_start;
  277. pend = vma->vm_end;
  278. }
  279. j = 0;
  280. for (nd = pn; nd; nd = rb_prev(nd)) {
  281. j++;
  282. }
  283. if (i != j)
  284. printk("backwards %d, forwards %d\n", j, i), i = 0;
  285. return i;
  286. }
  287. void validate_mm(struct mm_struct *mm)
  288. {
  289. int bug = 0;
  290. int i = 0;
  291. struct vm_area_struct *tmp = mm->mmap;
  292. while (tmp) {
  293. tmp = tmp->vm_next;
  294. i++;
  295. }
  296. if (i != mm->map_count)
  297. printk("map_count %d vm_next %d\n", mm->map_count, i), bug = 1;
  298. i = browse_rb(&mm->mm_rb);
  299. if (i != mm->map_count)
  300. printk("map_count %d rb %d\n", mm->map_count, i), bug = 1;
  301. BUG_ON(bug);
  302. }
  303. #else
  304. #define validate_mm(mm) do { } while (0)
  305. #endif
  306. static struct vm_area_struct *
  307. find_vma_prepare(struct mm_struct *mm, unsigned long addr,
  308. struct vm_area_struct **pprev, struct rb_node ***rb_link,
  309. struct rb_node ** rb_parent)
  310. {
  311. struct vm_area_struct * vma;
  312. struct rb_node ** __rb_link, * __rb_parent, * rb_prev;
  313. __rb_link = &mm->mm_rb.rb_node;
  314. rb_prev = __rb_parent = NULL;
  315. vma = NULL;
  316. while (*__rb_link) {
  317. struct vm_area_struct *vma_tmp;
  318. __rb_parent = *__rb_link;
  319. vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
  320. if (vma_tmp->vm_end > addr) {
  321. vma = vma_tmp;
  322. if (vma_tmp->vm_start <= addr)
  323. break;
  324. __rb_link = &__rb_parent->rb_left;
  325. } else {
  326. rb_prev = __rb_parent;
  327. __rb_link = &__rb_parent->rb_right;
  328. }
  329. }
  330. *pprev = NULL;
  331. if (rb_prev)
  332. *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
  333. *rb_link = __rb_link;
  334. *rb_parent = __rb_parent;
  335. return vma;
  336. }
  337. static inline void
  338. __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
  339. struct vm_area_struct *prev, struct rb_node *rb_parent)
  340. {
  341. if (prev) {
  342. vma->vm_next = prev->vm_next;
  343. prev->vm_next = vma;
  344. } else {
  345. mm->mmap = vma;
  346. if (rb_parent)
  347. vma->vm_next = rb_entry(rb_parent,
  348. struct vm_area_struct, vm_rb);
  349. else
  350. vma->vm_next = NULL;
  351. }
  352. }
  353. void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
  354. struct rb_node **rb_link, struct rb_node *rb_parent)
  355. {
  356. rb_link_node(&vma->vm_rb, rb_parent, rb_link);
  357. rb_insert_color(&vma->vm_rb, &mm->mm_rb);
  358. }
  359. static void __vma_link_file(struct vm_area_struct *vma)
  360. {
  361. struct file *file;
  362. file = vma->vm_file;
  363. if (file) {
  364. struct address_space *mapping = file->f_mapping;
  365. if (vma->vm_flags & VM_DENYWRITE)
  366. atomic_dec(&file->f_path.dentry->d_inode->i_writecount);
  367. if (vma->vm_flags & VM_SHARED)
  368. mapping->i_mmap_writable++;
  369. flush_dcache_mmap_lock(mapping);
  370. if (unlikely(vma->vm_flags & VM_NONLINEAR))
  371. vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
  372. else
  373. vma_prio_tree_insert(vma, &mapping->i_mmap);
  374. flush_dcache_mmap_unlock(mapping);
  375. }
  376. }
  377. static void
  378. __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
  379. struct vm_area_struct *prev, struct rb_node **rb_link,
  380. struct rb_node *rb_parent)
  381. {
  382. __vma_link_list(mm, vma, prev, rb_parent);
  383. __vma_link_rb(mm, vma, rb_link, rb_parent);
  384. __anon_vma_link(vma);
  385. }
  386. static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
  387. struct vm_area_struct *prev, struct rb_node **rb_link,
  388. struct rb_node *rb_parent)
  389. {
  390. struct address_space *mapping = NULL;
  391. if (vma->vm_file)
  392. mapping = vma->vm_file->f_mapping;
  393. if (mapping) {
  394. spin_lock(&mapping->i_mmap_lock);
  395. vma->vm_truncate_count = mapping->truncate_count;
  396. }
  397. anon_vma_lock(vma);
  398. __vma_link(mm, vma, prev, rb_link, rb_parent);
  399. __vma_link_file(vma);
  400. anon_vma_unlock(vma);
  401. if (mapping)
  402. spin_unlock(&mapping->i_mmap_lock);
  403. mm->map_count++;
  404. validate_mm(mm);
  405. }
  406. /*
  407. * Helper for vma_adjust in the split_vma insert case:
  408. * insert vm structure into list and rbtree and anon_vma,
  409. * but it has already been inserted into prio_tree earlier.
  410. */
  411. static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
  412. {
  413. struct vm_area_struct *__vma, *prev;
  414. struct rb_node **rb_link, *rb_parent;
  415. __vma = find_vma_prepare(mm, vma->vm_start,&prev, &rb_link, &rb_parent);
  416. BUG_ON(__vma && __vma->vm_start < vma->vm_end);
  417. __vma_link(mm, vma, prev, rb_link, rb_parent);
  418. mm->map_count++;
  419. }
  420. static inline void
  421. __vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
  422. struct vm_area_struct *prev)
  423. {
  424. prev->vm_next = vma->vm_next;
  425. rb_erase(&vma->vm_rb, &mm->mm_rb);
  426. if (mm->mmap_cache == vma)
  427. mm->mmap_cache = prev;
  428. }
  429. /*
  430. * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
  431. * is already present in an i_mmap tree without adjusting the tree.
  432. * The following helper function should be used when such adjustments
  433. * are necessary. The "insert" vma (if any) is to be inserted
  434. * before we drop the necessary locks.
  435. */
  436. void vma_adjust(struct vm_area_struct *vma, unsigned long start,
  437. unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
  438. {
  439. struct mm_struct *mm = vma->vm_mm;
  440. struct vm_area_struct *next = vma->vm_next;
  441. struct vm_area_struct *importer = NULL;
  442. struct address_space *mapping = NULL;
  443. struct prio_tree_root *root = NULL;
  444. struct file *file = vma->vm_file;
  445. struct anon_vma *anon_vma = NULL;
  446. long adjust_next = 0;
  447. int remove_next = 0;
  448. if (next && !insert) {
  449. if (end >= next->vm_end) {
  450. /*
  451. * vma expands, overlapping all the next, and
  452. * perhaps the one after too (mprotect case 6).
  453. */
  454. again: remove_next = 1 + (end > next->vm_end);
  455. end = next->vm_end;
  456. anon_vma = next->anon_vma;
  457. importer = vma;
  458. } else if (end > next->vm_start) {
  459. /*
  460. * vma expands, overlapping part of the next:
  461. * mprotect case 5 shifting the boundary up.
  462. */
  463. adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
  464. anon_vma = next->anon_vma;
  465. importer = vma;
  466. } else if (end < vma->vm_end) {
  467. /*
  468. * vma shrinks, and !insert tells it's not
  469. * split_vma inserting another: so it must be
  470. * mprotect case 4 shifting the boundary down.
  471. */
  472. adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
  473. anon_vma = next->anon_vma;
  474. importer = next;
  475. }
  476. }
  477. if (file) {
  478. mapping = file->f_mapping;
  479. if (!(vma->vm_flags & VM_NONLINEAR))
  480. root = &mapping->i_mmap;
  481. spin_lock(&mapping->i_mmap_lock);
  482. if (importer &&
  483. vma->vm_truncate_count != next->vm_truncate_count) {
  484. /*
  485. * unmap_mapping_range might be in progress:
  486. * ensure that the expanding vma is rescanned.
  487. */
  488. importer->vm_truncate_count = 0;
  489. }
  490. if (insert) {
  491. insert->vm_truncate_count = vma->vm_truncate_count;
  492. /*
  493. * Put into prio_tree now, so instantiated pages
  494. * are visible to arm/parisc __flush_dcache_page
  495. * throughout; but we cannot insert into address
  496. * space until vma start or end is updated.
  497. */
  498. __vma_link_file(insert);
  499. }
  500. }
  501. /*
  502. * When changing only vma->vm_end, we don't really need
  503. * anon_vma lock: but is that case worth optimizing out?
  504. */
  505. if (vma->anon_vma)
  506. anon_vma = vma->anon_vma;
  507. if (anon_vma) {
  508. spin_lock(&anon_vma->lock);
  509. /*
  510. * Easily overlooked: when mprotect shifts the boundary,
  511. * make sure the expanding vma has anon_vma set if the
  512. * shrinking vma had, to cover any anon pages imported.
  513. */
  514. if (importer && !importer->anon_vma) {
  515. importer->anon_vma = anon_vma;
  516. __anon_vma_link(importer);
  517. }
  518. }
  519. if (root) {
  520. flush_dcache_mmap_lock(mapping);
  521. vma_prio_tree_remove(vma, root);
  522. if (adjust_next)
  523. vma_prio_tree_remove(next, root);
  524. }
  525. vma->vm_start = start;
  526. vma->vm_end = end;
  527. vma->vm_pgoff = pgoff;
  528. if (adjust_next) {
  529. next->vm_start += adjust_next << PAGE_SHIFT;
  530. next->vm_pgoff += adjust_next;
  531. }
  532. if (root) {
  533. if (adjust_next)
  534. vma_prio_tree_insert(next, root);
  535. vma_prio_tree_insert(vma, root);
  536. flush_dcache_mmap_unlock(mapping);
  537. }
  538. if (remove_next) {
  539. /*
  540. * vma_merge has merged next into vma, and needs
  541. * us to remove next before dropping the locks.
  542. */
  543. __vma_unlink(mm, next, vma);
  544. if (file)
  545. __remove_shared_vm_struct(next, file, mapping);
  546. if (next->anon_vma)
  547. __anon_vma_merge(vma, next);
  548. } else if (insert) {
  549. /*
  550. * split_vma has split insert from vma, and needs
  551. * us to insert it before dropping the locks
  552. * (it may either follow vma or precede it).
  553. */
  554. __insert_vm_struct(mm, insert);
  555. }
  556. if (anon_vma)
  557. spin_unlock(&anon_vma->lock);
  558. if (mapping)
  559. spin_unlock(&mapping->i_mmap_lock);
  560. if (remove_next) {
  561. if (file) {
  562. fput(file);
  563. if (next->vm_flags & VM_EXECUTABLE)
  564. removed_exe_file_vma(mm);
  565. }
  566. mm->map_count--;
  567. mpol_put(vma_policy(next));
  568. kmem_cache_free(vm_area_cachep, next);
  569. /*
  570. * In mprotect's case 6 (see comments on vma_merge),
  571. * we must remove another next too. It would clutter
  572. * up the code too much to do both in one go.
  573. */
  574. if (remove_next == 2) {
  575. next = vma->vm_next;
  576. goto again;
  577. }
  578. }
  579. validate_mm(mm);
  580. }
  581. /* Flags that can be inherited from an existing mapping when merging */
  582. #define VM_MERGEABLE_FLAGS (VM_CAN_NONLINEAR)
  583. /*
  584. * If the vma has a ->close operation then the driver probably needs to release
  585. * per-vma resources, so we don't attempt to merge those.
  586. */
  587. static inline int is_mergeable_vma(struct vm_area_struct *vma,
  588. struct file *file, unsigned long vm_flags)
  589. {
  590. if ((vma->vm_flags ^ vm_flags) & ~VM_MERGEABLE_FLAGS)
  591. return 0;
  592. if (vma->vm_file != file)
  593. return 0;
  594. if (vma->vm_ops && vma->vm_ops->close)
  595. return 0;
  596. return 1;
  597. }
  598. static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
  599. struct anon_vma *anon_vma2)
  600. {
  601. return !anon_vma1 || !anon_vma2 || (anon_vma1 == anon_vma2);
  602. }
  603. /*
  604. * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
  605. * in front of (at a lower virtual address and file offset than) the vma.
  606. *
  607. * We cannot merge two vmas if they have differently assigned (non-NULL)
  608. * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
  609. *
  610. * We don't check here for the merged mmap wrapping around the end of pagecache
  611. * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
  612. * wrap, nor mmaps which cover the final page at index -1UL.
  613. */
  614. static int
  615. can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
  616. struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
  617. {
  618. if (is_mergeable_vma(vma, file, vm_flags) &&
  619. is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
  620. if (vma->vm_pgoff == vm_pgoff)
  621. return 1;
  622. }
  623. return 0;
  624. }
  625. /*
  626. * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
  627. * beyond (at a higher virtual address and file offset than) the vma.
  628. *
  629. * We cannot merge two vmas if they have differently assigned (non-NULL)
  630. * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
  631. */
  632. static int
  633. can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
  634. struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
  635. {
  636. if (is_mergeable_vma(vma, file, vm_flags) &&
  637. is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
  638. pgoff_t vm_pglen;
  639. vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
  640. if (vma->vm_pgoff + vm_pglen == vm_pgoff)
  641. return 1;
  642. }
  643. return 0;
  644. }
  645. /*
  646. * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
  647. * whether that can be merged with its predecessor or its successor.
  648. * Or both (it neatly fills a hole).
  649. *
  650. * In most cases - when called for mmap, brk or mremap - [addr,end) is
  651. * certain not to be mapped by the time vma_merge is called; but when
  652. * called for mprotect, it is certain to be already mapped (either at
  653. * an offset within prev, or at the start of next), and the flags of
  654. * this area are about to be changed to vm_flags - and the no-change
  655. * case has already been eliminated.
  656. *
  657. * The following mprotect cases have to be considered, where AAAA is
  658. * the area passed down from mprotect_fixup, never extending beyond one
  659. * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
  660. *
  661. * AAAA AAAA AAAA AAAA
  662. * PPPPPPNNNNNN PPPPPPNNNNNN PPPPPPNNNNNN PPPPNNNNXXXX
  663. * cannot merge might become might become might become
  664. * PPNNNNNNNNNN PPPPPPPPPPNN PPPPPPPPPPPP 6 or
  665. * mmap, brk or case 4 below case 5 below PPPPPPPPXXXX 7 or
  666. * mremap move: PPPPNNNNNNNN 8
  667. * AAAA
  668. * PPPP NNNN PPPPPPPPPPPP PPPPPPPPNNNN PPPPNNNNNNNN
  669. * might become case 1 below case 2 below case 3 below
  670. *
  671. * Odd one out? Case 8, because it extends NNNN but needs flags of XXXX:
  672. * mprotect_fixup updates vm_flags & vm_page_prot on successful return.
  673. */
  674. struct vm_area_struct *vma_merge(struct mm_struct *mm,
  675. struct vm_area_struct *prev, unsigned long addr,
  676. unsigned long end, unsigned long vm_flags,
  677. struct anon_vma *anon_vma, struct file *file,
  678. pgoff_t pgoff, struct mempolicy *policy)
  679. {
  680. pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
  681. struct vm_area_struct *area, *next;
  682. /*
  683. * We later require that vma->vm_flags == vm_flags,
  684. * so this tests vma->vm_flags & VM_SPECIAL, too.
  685. */
  686. if (vm_flags & VM_SPECIAL)
  687. return NULL;
  688. if (prev)
  689. next = prev->vm_next;
  690. else
  691. next = mm->mmap;
  692. area = next;
  693. if (next && next->vm_end == end) /* cases 6, 7, 8 */
  694. next = next->vm_next;
  695. /*
  696. * Can it merge with the predecessor?
  697. */
  698. if (prev && prev->vm_end == addr &&
  699. mpol_equal(vma_policy(prev), policy) &&
  700. can_vma_merge_after(prev, vm_flags,
  701. anon_vma, file, pgoff)) {
  702. /*
  703. * OK, it can. Can we now merge in the successor as well?
  704. */
  705. if (next && end == next->vm_start &&
  706. mpol_equal(policy, vma_policy(next)) &&
  707. can_vma_merge_before(next, vm_flags,
  708. anon_vma, file, pgoff+pglen) &&
  709. is_mergeable_anon_vma(prev->anon_vma,
  710. next->anon_vma)) {
  711. /* cases 1, 6 */
  712. vma_adjust(prev, prev->vm_start,
  713. next->vm_end, prev->vm_pgoff, NULL);
  714. } else /* cases 2, 5, 7 */
  715. vma_adjust(prev, prev->vm_start,
  716. end, prev->vm_pgoff, NULL);
  717. return prev;
  718. }
  719. /*
  720. * Can this new request be merged in front of next?
  721. */
  722. if (next && end == next->vm_start &&
  723. mpol_equal(policy, vma_policy(next)) &&
  724. can_vma_merge_before(next, vm_flags,
  725. anon_vma, file, pgoff+pglen)) {
  726. if (prev && addr < prev->vm_end) /* case 4 */
  727. vma_adjust(prev, prev->vm_start,
  728. addr, prev->vm_pgoff, NULL);
  729. else /* cases 3, 8 */
  730. vma_adjust(area, addr, next->vm_end,
  731. next->vm_pgoff - pglen, NULL);
  732. return area;
  733. }
  734. return NULL;
  735. }
  736. /*
  737. * find_mergeable_anon_vma is used by anon_vma_prepare, to check
  738. * neighbouring vmas for a suitable anon_vma, before it goes off
  739. * to allocate a new anon_vma. It checks because a repetitive
  740. * sequence of mprotects and faults may otherwise lead to distinct
  741. * anon_vmas being allocated, preventing vma merge in subsequent
  742. * mprotect.
  743. */
  744. struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
  745. {
  746. struct vm_area_struct *near;
  747. unsigned long vm_flags;
  748. near = vma->vm_next;
  749. if (!near)
  750. goto try_prev;
  751. /*
  752. * Since only mprotect tries to remerge vmas, match flags
  753. * which might be mprotected into each other later on.
  754. * Neither mlock nor madvise tries to remerge at present,
  755. * so leave their flags as obstructing a merge.
  756. */
  757. vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
  758. vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
  759. if (near->anon_vma && vma->vm_end == near->vm_start &&
  760. mpol_equal(vma_policy(vma), vma_policy(near)) &&
  761. can_vma_merge_before(near, vm_flags,
  762. NULL, vma->vm_file, vma->vm_pgoff +
  763. ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT)))
  764. return near->anon_vma;
  765. try_prev:
  766. /*
  767. * It is potentially slow to have to call find_vma_prev here.
  768. * But it's only on the first write fault on the vma, not
  769. * every time, and we could devise a way to avoid it later
  770. * (e.g. stash info in next's anon_vma_node when assigning
  771. * an anon_vma, or when trying vma_merge). Another time.
  772. */
  773. BUG_ON(find_vma_prev(vma->vm_mm, vma->vm_start, &near) != vma);
  774. if (!near)
  775. goto none;
  776. vm_flags = vma->vm_flags & ~(VM_READ|VM_WRITE|VM_EXEC);
  777. vm_flags |= near->vm_flags & (VM_READ|VM_WRITE|VM_EXEC);
  778. if (near->anon_vma && near->vm_end == vma->vm_start &&
  779. mpol_equal(vma_policy(near), vma_policy(vma)) &&
  780. can_vma_merge_after(near, vm_flags,
  781. NULL, vma->vm_file, vma->vm_pgoff))
  782. return near->anon_vma;
  783. none:
  784. /*
  785. * There's no absolute need to look only at touching neighbours:
  786. * we could search further afield for "compatible" anon_vmas.
  787. * But it would probably just be a waste of time searching,
  788. * or lead to too many vmas hanging off the same anon_vma.
  789. * We're trying to allow mprotect remerging later on,
  790. * not trying to minimize memory used for anon_vmas.
  791. */
  792. return NULL;
  793. }
  794. #ifdef CONFIG_PROC_FS
  795. void vm_stat_account(struct mm_struct *mm, unsigned long flags,
  796. struct file *file, long pages)
  797. {
  798. const unsigned long stack_flags
  799. = VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);
  800. if (file) {
  801. mm->shared_vm += pages;
  802. if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
  803. mm->exec_vm += pages;
  804. } else if (flags & stack_flags)
  805. mm->stack_vm += pages;
  806. if (flags & (VM_RESERVED|VM_IO))
  807. mm->reserved_vm += pages;
  808. }
  809. #endif /* CONFIG_PROC_FS */
  810. /*
  811. * The caller must hold down_write(current->mm->mmap_sem).
  812. */
  813. unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
  814. unsigned long len, unsigned long prot,
  815. unsigned long flags, unsigned long pgoff)
  816. {
  817. struct mm_struct * mm = current->mm;
  818. struct inode *inode;
  819. unsigned int vm_flags;
  820. int error;
  821. unsigned long reqprot = prot;
  822. /*
  823. * Does the application expect PROT_READ to imply PROT_EXEC?
  824. *
  825. * (the exception is when the underlying filesystem is noexec
  826. * mounted, in which case we dont add PROT_EXEC.)
  827. */
  828. if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
  829. if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))
  830. prot |= PROT_EXEC;
  831. if (!len)
  832. return -EINVAL;
  833. if (!(flags & MAP_FIXED))
  834. addr = round_hint_to_min(addr);
  835. error = arch_mmap_check(addr, len, flags);
  836. if (error)
  837. return error;
  838. /* Careful about overflows.. */
  839. len = PAGE_ALIGN(len);
  840. if (!len || len > TASK_SIZE)
  841. return -ENOMEM;
  842. /* offset overflow? */
  843. if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
  844. return -EOVERFLOW;
  845. /* Too many mappings? */
  846. if (mm->map_count > sysctl_max_map_count)
  847. return -ENOMEM;
  848. /* Obtain the address to map to. we verify (or select) it and ensure
  849. * that it represents a valid section of the address space.
  850. */
  851. addr = get_unmapped_area(file, addr, len, pgoff, flags);
  852. if (addr & ~PAGE_MASK)
  853. return addr;
  854. /* Do simple checking here so the lower-level routines won't have
  855. * to. we assume access permissions have been handled by the open
  856. * of the memory object, so we don't do any here.
  857. */
  858. vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
  859. mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
  860. if (flags & MAP_LOCKED) {
  861. if (!can_do_mlock())
  862. return -EPERM;
  863. vm_flags |= VM_LOCKED;
  864. }
  865. /* mlock MCL_FUTURE? */
  866. if (vm_flags & VM_LOCKED) {
  867. unsigned long locked, lock_limit;
  868. locked = len >> PAGE_SHIFT;
  869. locked += mm->locked_vm;
  870. lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
  871. lock_limit >>= PAGE_SHIFT;
  872. if (locked > lock_limit && !capable(CAP_IPC_LOCK))
  873. return -EAGAIN;
  874. }
  875. inode = file ? file->f_path.dentry->d_inode : NULL;
  876. if (file) {
  877. switch (flags & MAP_TYPE) {
  878. case MAP_SHARED:
  879. if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
  880. return -EACCES;
  881. /*
  882. * Make sure we don't allow writing to an append-only
  883. * file..
  884. */
  885. if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
  886. return -EACCES;
  887. /*
  888. * Make sure there are no mandatory locks on the file.
  889. */
  890. if (locks_verify_locked(inode))
  891. return -EAGAIN;
  892. vm_flags |= VM_SHARED | VM_MAYSHARE;
  893. if (!(file->f_mode & FMODE_WRITE))
  894. vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
  895. /* fall through */
  896. case MAP_PRIVATE:
  897. if (!(file->f_mode & FMODE_READ))
  898. return -EACCES;
  899. if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
  900. if (vm_flags & VM_EXEC)
  901. return -EPERM;
  902. vm_flags &= ~VM_MAYEXEC;
  903. }
  904. if (!file->f_op || !file->f_op->mmap)
  905. return -ENODEV;
  906. break;
  907. default:
  908. return -EINVAL;
  909. }
  910. } else {
  911. switch (flags & MAP_TYPE) {
  912. case MAP_SHARED:
  913. /*
  914. * Ignore pgoff.
  915. */
  916. pgoff = 0;
  917. vm_flags |= VM_SHARED | VM_MAYSHARE;
  918. break;
  919. case MAP_PRIVATE:
  920. /*
  921. * Set pgoff according to addr for anon_vma.
  922. */
  923. pgoff = addr >> PAGE_SHIFT;
  924. break;
  925. default:
  926. return -EINVAL;
  927. }
  928. }
  929. error = security_file_mmap(file, reqprot, prot, flags, addr, 0);
  930. if (error)
  931. return error;
  932. error = ima_file_mmap(file, prot);
  933. if (error)
  934. return error;
  935. return mmap_region(file, addr, len, flags, vm_flags, pgoff);
  936. }
  937. EXPORT_SYMBOL(do_mmap_pgoff);
  938. /*
  939. * Some shared mappigns will want the pages marked read-only
  940. * to track write events. If so, we'll downgrade vm_page_prot
  941. * to the private version (using protection_map[] without the
  942. * VM_SHARED bit).
  943. */
  944. int vma_wants_writenotify(struct vm_area_struct *vma)
  945. {
  946. unsigned int vm_flags = vma->vm_flags;
  947. /* If it was private or non-writable, the write bit is already clear */
  948. if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
  949. return 0;
  950. /* The backer wishes to know when pages are first written to? */
  951. if (vma->vm_ops && vma->vm_ops->page_mkwrite)
  952. return 1;
  953. /* The open routine did something to the protections already? */
  954. if (pgprot_val(vma->vm_page_prot) !=
  955. pgprot_val(vm_get_page_prot(vm_flags)))
  956. return 0;
  957. /* Specialty mapping? */
  958. if (vm_flags & (VM_PFNMAP|VM_INSERTPAGE))
  959. return 0;
  960. /* Can the mapping track the dirty pages? */
  961. return vma->vm_file && vma->vm_file->f_mapping &&
  962. mapping_cap_account_dirty(vma->vm_file->f_mapping);
  963. }
  964. /*
  965. * We account for memory if it's a private writeable mapping,
  966. * not hugepages and VM_NORESERVE wasn't set.
  967. */
  968. static inline int accountable_mapping(struct file *file, unsigned int vm_flags)
  969. {
  970. /*
  971. * hugetlb has its own accounting separate from the core VM
  972. * VM_HUGETLB may not be set yet so we cannot check for that flag.
  973. */
  974. if (file && is_file_hugepages(file))
  975. return 0;
  976. return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
  977. }
  978. unsigned long mmap_region(struct file *file, unsigned long addr,
  979. unsigned long len, unsigned long flags,
  980. unsigned int vm_flags, unsigned long pgoff)
  981. {
  982. struct mm_struct *mm = current->mm;
  983. struct vm_area_struct *vma, *prev;
  984. int correct_wcount = 0;
  985. int error;
  986. struct rb_node **rb_link, *rb_parent;
  987. unsigned long charged = 0;
  988. struct inode *inode = file ? file->f_path.dentry->d_inode : NULL;
  989. /* Clear old maps */
  990. error = -ENOMEM;
  991. munmap_back:
  992. vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
  993. if (vma && vma->vm_start < addr + len) {
  994. if (do_munmap(mm, addr, len))
  995. return -ENOMEM;
  996. goto munmap_back;
  997. }
  998. /* Check against address space limit. */
  999. if (!may_expand_vm(mm, len >> PAGE_SHIFT))
  1000. return -ENOMEM;
  1001. /*
  1002. * Set 'VM_NORESERVE' if we should not account for the
  1003. * memory use of this mapping.
  1004. */
  1005. if ((flags & MAP_NORESERVE)) {
  1006. /* We honor MAP_NORESERVE if allowed to overcommit */
  1007. if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
  1008. vm_flags |= VM_NORESERVE;
  1009. /* hugetlb applies strict overcommit unless MAP_NORESERVE */
  1010. if (file && is_file_hugepages(file))
  1011. vm_flags |= VM_NORESERVE;
  1012. }
  1013. /*
  1014. * Private writable mapping: check memory availability
  1015. */
  1016. if (accountable_mapping(file, vm_flags)) {
  1017. charged = len >> PAGE_SHIFT;
  1018. if (security_vm_enough_memory(charged))
  1019. return -ENOMEM;
  1020. vm_flags |= VM_ACCOUNT;
  1021. }
  1022. /*
  1023. * Can we just expand an old mapping?
  1024. */
  1025. vma = vma_merge(mm, prev, addr, addr + len, vm_flags, NULL, file, pgoff, NULL);
  1026. if (vma)
  1027. goto out;
  1028. /*
  1029. * Determine the object being mapped and call the appropriate
  1030. * specific mapper. the address has already been validated, but
  1031. * not unmapped, but the maps are removed from the list.
  1032. */
  1033. vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
  1034. if (!vma) {
  1035. error = -ENOMEM;
  1036. goto unacct_error;
  1037. }
  1038. vma->vm_mm = mm;
  1039. vma->vm_start = addr;
  1040. vma->vm_end = addr + len;
  1041. vma->vm_flags = vm_flags;
  1042. vma->vm_page_prot = vm_get_page_prot(vm_flags);
  1043. vma->vm_pgoff = pgoff;
  1044. if (file) {
  1045. error = -EINVAL;
  1046. if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
  1047. goto free_vma;
  1048. if (vm_flags & VM_DENYWRITE) {
  1049. error = deny_write_access(file);
  1050. if (error)
  1051. goto free_vma;
  1052. correct_wcount = 1;
  1053. }
  1054. vma->vm_file = file;
  1055. get_file(file);
  1056. error = file->f_op->mmap(file, vma);
  1057. if (error)
  1058. goto unmap_and_free_vma;
  1059. if (vm_flags & VM_EXECUTABLE)
  1060. added_exe_file_vma(mm);
  1061. } else if (vm_flags & VM_SHARED) {
  1062. error = shmem_zero_setup(vma);
  1063. if (error)
  1064. goto free_vma;
  1065. }
  1066. /* Can addr have changed??
  1067. *
  1068. * Answer: Yes, several device drivers can do it in their
  1069. * f_op->mmap method. -DaveM
  1070. */
  1071. addr = vma->vm_start;
  1072. pgoff = vma->vm_pgoff;
  1073. vm_flags = vma->vm_flags;
  1074. if (vma_wants_writenotify(vma))
  1075. vma->vm_page_prot = vm_get_page_prot(vm_flags & ~VM_SHARED);
  1076. vma_link(mm, vma, prev, rb_link, rb_parent);
  1077. file = vma->vm_file;
  1078. /* Once vma denies write, undo our temporary denial count */
  1079. if (correct_wcount)
  1080. atomic_inc(&inode->i_writecount);
  1081. out:
  1082. perf_counter_mmap(vma);
  1083. mm->total_vm += len >> PAGE_SHIFT;
  1084. vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
  1085. if (vm_flags & VM_LOCKED) {
  1086. /*
  1087. * makes pages present; downgrades, drops, reacquires mmap_sem
  1088. */
  1089. long nr_pages = mlock_vma_pages_range(vma, addr, addr + len);
  1090. if (nr_pages < 0)
  1091. return nr_pages; /* vma gone! */
  1092. mm->locked_vm += (len >> PAGE_SHIFT) - nr_pages;
  1093. } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
  1094. make_pages_present(addr, addr + len);
  1095. return addr;
  1096. unmap_and_free_vma:
  1097. if (correct_wcount)
  1098. atomic_inc(&inode->i_writecount);
  1099. vma->vm_file = NULL;
  1100. fput(file);
  1101. /* Undo any partial mapping done by a device driver. */
  1102. unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
  1103. charged = 0;
  1104. free_vma:
  1105. kmem_cache_free(vm_area_cachep, vma);
  1106. unacct_error:
  1107. if (charged)
  1108. vm_unacct_memory(charged);
  1109. return error;
  1110. }
  1111. /* Get an address range which is currently unmapped.
  1112. * For shmat() with addr=0.
  1113. *
  1114. * Ugly calling convention alert:
  1115. * Return value with the low bits set means error value,
  1116. * ie
  1117. * if (ret & ~PAGE_MASK)
  1118. * error = ret;
  1119. *
  1120. * This function "knows" that -ENOMEM has the bits set.
  1121. */
  1122. #ifndef HAVE_ARCH_UNMAPPED_AREA
  1123. unsigned long
  1124. arch_get_unmapped_area(struct file *filp, unsigned long addr,
  1125. unsigned long len, unsigned long pgoff, unsigned long flags)
  1126. {
  1127. struct mm_struct *mm = current->mm;
  1128. struct vm_area_struct *vma;
  1129. unsigned long start_addr;
  1130. if (len > TASK_SIZE)
  1131. return -ENOMEM;
  1132. if (flags & MAP_FIXED)
  1133. return addr;
  1134. if (addr) {
  1135. addr = PAGE_ALIGN(addr);
  1136. vma = find_vma(mm, addr);
  1137. if (TASK_SIZE - len >= addr &&
  1138. (!vma || addr + len <= vma->vm_start))
  1139. return addr;
  1140. }
  1141. if (len > mm->cached_hole_size) {
  1142. start_addr = addr = mm->free_area_cache;
  1143. } else {
  1144. start_addr = addr = TASK_UNMAPPED_BASE;
  1145. mm->cached_hole_size = 0;
  1146. }
  1147. full_search:
  1148. for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
  1149. /* At this point: (!vma || addr < vma->vm_end). */
  1150. if (TASK_SIZE - len < addr) {
  1151. /*
  1152. * Start a new search - just in case we missed
  1153. * some holes.
  1154. */
  1155. if (start_addr != TASK_UNMAPPED_BASE) {
  1156. addr = TASK_UNMAPPED_BASE;
  1157. start_addr = addr;
  1158. mm->cached_hole_size = 0;
  1159. goto full_search;
  1160. }
  1161. return -ENOMEM;
  1162. }
  1163. if (!vma || addr + len <= vma->vm_start) {
  1164. /*
  1165. * Remember the place where we stopped the search:
  1166. */
  1167. mm->free_area_cache = addr + len;
  1168. return addr;
  1169. }
  1170. if (addr + mm->cached_hole_size < vma->vm_start)
  1171. mm->cached_hole_size = vma->vm_start - addr;
  1172. addr = vma->vm_end;
  1173. }
  1174. }
  1175. #endif
  1176. void arch_unmap_area(struct mm_struct *mm, unsigned long addr)
  1177. {
  1178. /*
  1179. * Is this a new hole at the lowest possible address?
  1180. */
  1181. if (addr >= TASK_UNMAPPED_BASE && addr < mm->free_area_cache) {
  1182. mm->free_area_cache = addr;
  1183. mm->cached_hole_size = ~0UL;
  1184. }
  1185. }
  1186. /*
  1187. * This mmap-allocator allocates new areas top-down from below the
  1188. * stack's low limit (the base):
  1189. */
  1190. #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
  1191. unsigned long
  1192. arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
  1193. const unsigned long len, const unsigned long pgoff,
  1194. const unsigned long flags)
  1195. {
  1196. struct vm_area_struct *vma;
  1197. struct mm_struct *mm = current->mm;
  1198. unsigned long addr = addr0;
  1199. /* requested length too big for entire address space */
  1200. if (len > TASK_SIZE)
  1201. return -ENOMEM;
  1202. if (flags & MAP_FIXED)
  1203. return addr;
  1204. /* requesting a specific address */
  1205. if (addr) {
  1206. addr = PAGE_ALIGN(addr);
  1207. vma = find_vma(mm, addr);
  1208. if (TASK_SIZE - len >= addr &&
  1209. (!vma || addr + len <= vma->vm_start))
  1210. return addr;
  1211. }
  1212. /* check if free_area_cache is useful for us */
  1213. if (len <= mm->cached_hole_size) {
  1214. mm->cached_hole_size = 0;
  1215. mm->free_area_cache = mm->mmap_base;
  1216. }
  1217. /* either no address requested or can't fit in requested address hole */
  1218. addr = mm->free_area_cache;
  1219. /* make sure it can fit in the remaining address space */
  1220. if (addr > len) {
  1221. vma = find_vma(mm, addr-len);
  1222. if (!vma || addr <= vma->vm_start)
  1223. /* remember the address as a hint for next time */
  1224. return (mm->free_area_cache = addr-len);
  1225. }
  1226. if (mm->mmap_base < len)
  1227. goto bottomup;
  1228. addr = mm->mmap_base-len;
  1229. do {
  1230. /*
  1231. * Lookup failure means no vma is above this address,
  1232. * else if new region fits below vma->vm_start,
  1233. * return with success:
  1234. */
  1235. vma = find_vma(mm, addr);
  1236. if (!vma || addr+len <= vma->vm_start)
  1237. /* remember the address as a hint for next time */
  1238. return (mm->free_area_cache = addr);
  1239. /* remember the largest hole we saw so far */
  1240. if (addr + mm->cached_hole_size < vma->vm_start)
  1241. mm->cached_hole_size = vma->vm_start - addr;
  1242. /* try just below the current vma->vm_start */
  1243. addr = vma->vm_start-len;
  1244. } while (len < vma->vm_start);
  1245. bottomup:
  1246. /*
  1247. * A failed mmap() very likely causes application failure,
  1248. * so fall back to the bottom-up function here. This scenario
  1249. * can happen with large stack limits and large mmap()
  1250. * allocations.
  1251. */
  1252. mm->cached_hole_size = ~0UL;
  1253. mm->free_area_cache = TASK_UNMAPPED_BASE;
  1254. addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
  1255. /*
  1256. * Restore the topdown base:
  1257. */
  1258. mm->free_area_cache = mm->mmap_base;
  1259. mm->cached_hole_size = ~0UL;
  1260. return addr;
  1261. }
  1262. #endif
  1263. void arch_unmap_area_topdown(struct mm_struct *mm, unsigned long addr)
  1264. {
  1265. /*
  1266. * Is this a new hole at the highest possible address?
  1267. */
  1268. if (addr > mm->free_area_cache)
  1269. mm->free_area_cache = addr;
  1270. /* dont allow allocations above current base */
  1271. if (mm->free_area_cache > mm->mmap_base)
  1272. mm->free_area_cache = mm->mmap_base;
  1273. }
  1274. unsigned long
  1275. get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
  1276. unsigned long pgoff, unsigned long flags)
  1277. {
  1278. unsigned long (*get_area)(struct file *, unsigned long,
  1279. unsigned long, unsigned long, unsigned long);
  1280. get_area = current->mm->get_unmapped_area;
  1281. if (file && file->f_op && file->f_op->get_unmapped_area)
  1282. get_area = file->f_op->get_unmapped_area;
  1283. addr = get_area(file, addr, len, pgoff, flags);
  1284. if (IS_ERR_VALUE(addr))
  1285. return addr;
  1286. if (addr > TASK_SIZE - len)
  1287. return -ENOMEM;
  1288. if (addr & ~PAGE_MASK)
  1289. return -EINVAL;
  1290. return arch_rebalance_pgtables(addr, len);
  1291. }
  1292. EXPORT_SYMBOL(get_unmapped_area);
  1293. /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
  1294. struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
  1295. {
  1296. struct vm_area_struct *vma = NULL;
  1297. if (mm) {
  1298. /* Check the cache first. */
  1299. /* (Cache hit rate is typically around 35%.) */
  1300. vma = mm->mmap_cache;
  1301. if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) {
  1302. struct rb_node * rb_node;
  1303. rb_node = mm->mm_rb.rb_node;
  1304. vma = NULL;
  1305. while (rb_node) {
  1306. struct vm_area_struct * vma_tmp;
  1307. vma_tmp = rb_entry(rb_node,
  1308. struct vm_area_struct, vm_rb);
  1309. if (vma_tmp->vm_end > addr) {
  1310. vma = vma_tmp;
  1311. if (vma_tmp->vm_start <= addr)
  1312. break;
  1313. rb_node = rb_node->rb_left;
  1314. } else
  1315. rb_node = rb_node->rb_right;
  1316. }
  1317. if (vma)
  1318. mm->mmap_cache = vma;
  1319. }
  1320. }
  1321. return vma;
  1322. }
  1323. EXPORT_SYMBOL(find_vma);
  1324. /* Same as find_vma, but also return a pointer to the previous VMA in *pprev. */
  1325. struct vm_area_struct *
  1326. find_vma_prev(struct mm_struct *mm, unsigned long addr,
  1327. struct vm_area_struct **pprev)
  1328. {
  1329. struct vm_area_struct *vma = NULL, *prev = NULL;
  1330. struct rb_node *rb_node;
  1331. if (!mm)
  1332. goto out;
  1333. /* Guard against addr being lower than the first VMA */
  1334. vma = mm->mmap;
  1335. /* Go through the RB tree quickly. */
  1336. rb_node = mm->mm_rb.rb_node;
  1337. while (rb_node) {
  1338. struct vm_area_struct *vma_tmp;
  1339. vma_tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
  1340. if (addr < vma_tmp->vm_end) {
  1341. rb_node = rb_node->rb_left;
  1342. } else {
  1343. prev = vma_tmp;
  1344. if (!prev->vm_next || (addr < prev->vm_next->vm_end))
  1345. break;
  1346. rb_node = rb_node->rb_right;
  1347. }
  1348. }
  1349. out:
  1350. *pprev = prev;
  1351. return prev ? prev->vm_next : vma;
  1352. }
  1353. /*
  1354. * Verify that the stack growth is acceptable and
  1355. * update accounting. This is shared with both the
  1356. * grow-up and grow-down cases.
  1357. */
  1358. static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
  1359. {
  1360. struct mm_struct *mm = vma->vm_mm;
  1361. struct rlimit *rlim = current->signal->rlim;
  1362. unsigned long new_start;
  1363. /* address space limit tests */
  1364. if (!may_expand_vm(mm, grow))
  1365. return -ENOMEM;
  1366. /* Stack limit test */
  1367. if (size > rlim[RLIMIT_STACK].rlim_cur)
  1368. return -ENOMEM;
  1369. /* mlock limit tests */
  1370. if (vma->vm_flags & VM_LOCKED) {
  1371. unsigned long locked;
  1372. unsigned long limit;
  1373. locked = mm->locked_vm + grow;
  1374. limit = rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
  1375. if (locked > limit && !capable(CAP_IPC_LOCK))
  1376. return -ENOMEM;
  1377. }
  1378. /* Check to ensure the stack will not grow into a hugetlb-only region */
  1379. new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
  1380. vma->vm_end - size;
  1381. if (is_hugepage_only_range(vma->vm_mm, new_start, size))
  1382. return -EFAULT;
  1383. /*
  1384. * Overcommit.. This must be the final test, as it will
  1385. * update security statistics.
  1386. */
  1387. if (security_vm_enough_memory_mm(mm, grow))
  1388. return -ENOMEM;
  1389. /* Ok, everything looks good - let it rip */
  1390. mm->total_vm += grow;
  1391. if (vma->vm_flags & VM_LOCKED)
  1392. mm->locked_vm += grow;
  1393. vm_stat_account(mm, vma->vm_flags, vma->vm_file, grow);
  1394. return 0;
  1395. }
  1396. #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
  1397. /*
  1398. * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
  1399. * vma is the last one with address > vma->vm_end. Have to extend vma.
  1400. */
  1401. #ifndef CONFIG_IA64
  1402. static
  1403. #endif
  1404. int expand_upwards(struct vm_area_struct *vma, unsigned long address)
  1405. {
  1406. int error;
  1407. if (!(vma->vm_flags & VM_GROWSUP))
  1408. return -EFAULT;
  1409. /*
  1410. * We must make sure the anon_vma is allocated
  1411. * so that the anon_vma locking is not a noop.
  1412. */
  1413. if (unlikely(anon_vma_prepare(vma)))
  1414. return -ENOMEM;
  1415. anon_vma_lock(vma);
  1416. /*
  1417. * vma->vm_start/vm_end cannot change under us because the caller
  1418. * is required to hold the mmap_sem in read mode. We need the
  1419. * anon_vma lock to serialize against concurrent expand_stacks.
  1420. * Also guard against wrapping around to address 0.
  1421. */
  1422. if (address < PAGE_ALIGN(address+4))
  1423. address = PAGE_ALIGN(address+4);
  1424. else {
  1425. anon_vma_unlock(vma);
  1426. return -ENOMEM;
  1427. }
  1428. error = 0;
  1429. /* Somebody else might have raced and expanded it already */
  1430. if (address > vma->vm_end) {
  1431. unsigned long size, grow;
  1432. size = address - vma->vm_start;
  1433. grow = (address - vma->vm_end) >> PAGE_SHIFT;
  1434. error = acct_stack_growth(vma, size, grow);
  1435. if (!error)
  1436. vma->vm_end = address;
  1437. }
  1438. anon_vma_unlock(vma);
  1439. return error;
  1440. }
  1441. #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
  1442. /*
  1443. * vma is the first one with address < vma->vm_start. Have to extend vma.
  1444. */
  1445. static int expand_downwards(struct vm_area_struct *vma,
  1446. unsigned long address)
  1447. {
  1448. int error;
  1449. /*
  1450. * We must make sure the anon_vma is allocated
  1451. * so that the anon_vma locking is not a noop.
  1452. */
  1453. if (unlikely(anon_vma_prepare(vma)))
  1454. return -ENOMEM;
  1455. address &= PAGE_MASK;
  1456. error = security_file_mmap(NULL, 0, 0, 0, address, 1);
  1457. if (error)
  1458. return error;
  1459. anon_vma_lock(vma);
  1460. /*
  1461. * vma->vm_start/vm_end cannot change under us because the caller
  1462. * is required to hold the mmap_sem in read mode. We need the
  1463. * anon_vma lock to serialize against concurrent expand_stacks.
  1464. */
  1465. /* Somebody else might have raced and expanded it already */
  1466. if (address < vma->vm_start) {
  1467. unsigned long size, grow;
  1468. size = vma->vm_end - address;
  1469. grow = (vma->vm_start - address) >> PAGE_SHIFT;
  1470. error = acct_stack_growth(vma, size, grow);
  1471. if (!error) {
  1472. vma->vm_start = address;
  1473. vma->vm_pgoff -= grow;
  1474. }
  1475. }
  1476. anon_vma_unlock(vma);
  1477. return error;
  1478. }
  1479. int expand_stack_downwards(struct vm_area_struct *vma, unsigned long address)
  1480. {
  1481. return expand_downwards(vma, address);
  1482. }
  1483. #ifdef CONFIG_STACK_GROWSUP
  1484. int expand_stack(struct vm_area_struct *vma, unsigned long address)
  1485. {
  1486. return expand_upwards(vma, address);
  1487. }
  1488. struct vm_area_struct *
  1489. find_extend_vma(struct mm_struct *mm, unsigned long addr)
  1490. {
  1491. struct vm_area_struct *vma, *prev;
  1492. addr &= PAGE_MASK;
  1493. vma = find_vma_prev(mm, addr, &prev);
  1494. if (vma && (vma->vm_start <= addr))
  1495. return vma;
  1496. if (!prev || expand_stack(prev, addr))
  1497. return NULL;
  1498. if (prev->vm_flags & VM_LOCKED) {
  1499. if (mlock_vma_pages_range(prev, addr, prev->vm_end) < 0)
  1500. return NULL; /* vma gone! */
  1501. }
  1502. return prev;
  1503. }
  1504. #else
  1505. int expand_stack(struct vm_area_struct *vma, unsigned long address)
  1506. {
  1507. return expand_downwards(vma, address);
  1508. }
  1509. struct vm_area_struct *
  1510. find_extend_vma(struct mm_struct * mm, unsigned long addr)
  1511. {
  1512. struct vm_area_struct * vma;
  1513. unsigned long start;
  1514. addr &= PAGE_MASK;
  1515. vma = find_vma(mm,addr);
  1516. if (!vma)
  1517. return NULL;
  1518. if (vma->vm_start <= addr)
  1519. return vma;
  1520. if (!(vma->vm_flags & VM_GROWSDOWN))
  1521. return NULL;
  1522. start = vma->vm_start;
  1523. if (expand_stack(vma, addr))
  1524. return NULL;
  1525. if (vma->vm_flags & VM_LOCKED) {
  1526. if (mlock_vma_pages_range(vma, addr, start) < 0)
  1527. return NULL; /* vma gone! */
  1528. }
  1529. return vma;
  1530. }
  1531. #endif
  1532. /*
  1533. * Ok - we have the memory areas we should free on the vma list,
  1534. * so release them, and do the vma updates.
  1535. *
  1536. * Called with the mm semaphore held.
  1537. */
  1538. static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
  1539. {
  1540. /* Update high watermark before we lower total_vm */
  1541. update_hiwater_vm(mm);
  1542. do {
  1543. long nrpages = vma_pages(vma);
  1544. mm->total_vm -= nrpages;
  1545. vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
  1546. vma = remove_vma(vma);
  1547. } while (vma);
  1548. validate_mm(mm);
  1549. }
  1550. /*
  1551. * Get rid of page table information in the indicated region.
  1552. *
  1553. * Called with the mm semaphore held.
  1554. */
  1555. static void unmap_region(struct mm_struct *mm,
  1556. struct vm_area_struct *vma, struct vm_area_struct *prev,
  1557. unsigned long start, unsigned long end)
  1558. {
  1559. struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
  1560. struct mmu_gather *tlb;
  1561. unsigned long nr_accounted = 0;
  1562. lru_add_drain();
  1563. tlb = tlb_gather_mmu(mm, 0);
  1564. update_hiwater_rss(mm);
  1565. unmap_vmas(&tlb, vma, start, end, &nr_accounted, NULL);
  1566. vm_unacct_memory(nr_accounted);
  1567. free_pgtables(tlb, vma, prev? prev->vm_end: FIRST_USER_ADDRESS,
  1568. next? next->vm_start: 0);
  1569. tlb_finish_mmu(tlb, start, end);
  1570. }
  1571. /*
  1572. * Create a list of vma's touched by the unmap, removing them from the mm's
  1573. * vma list as we go..
  1574. */
  1575. static void
  1576. detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
  1577. struct vm_area_struct *prev, unsigned long end)
  1578. {
  1579. struct vm_area_struct **insertion_point;
  1580. struct vm_area_struct *tail_vma = NULL;
  1581. unsigned long addr;
  1582. insertion_point = (prev ? &prev->vm_next : &mm->mmap);
  1583. do {
  1584. rb_erase(&vma->vm_rb, &mm->mm_rb);
  1585. mm->map_count--;
  1586. tail_vma = vma;
  1587. vma = vma->vm_next;
  1588. } while (vma && vma->vm_start < end);
  1589. *insertion_point = vma;
  1590. tail_vma->vm_next = NULL;
  1591. if (mm->unmap_area == arch_unmap_area)
  1592. addr = prev ? prev->vm_end : mm->mmap_base;
  1593. else
  1594. addr = vma ? vma->vm_start : mm->mmap_base;
  1595. mm->unmap_area(mm, addr);
  1596. mm->mmap_cache = NULL; /* Kill the cache. */
  1597. }
  1598. /*
  1599. * Split a vma into two pieces at address 'addr', a new vma is allocated
  1600. * either for the first part or the tail.
  1601. */
  1602. int split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
  1603. unsigned long addr, int new_below)
  1604. {
  1605. struct mempolicy *pol;
  1606. struct vm_area_struct *new;
  1607. if (is_vm_hugetlb_page(vma) && (addr &
  1608. ~(huge_page_mask(hstate_vma(vma)))))
  1609. return -EINVAL;
  1610. if (mm->map_count >= sysctl_max_map_count)
  1611. return -ENOMEM;
  1612. new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
  1613. if (!new)
  1614. return -ENOMEM;
  1615. /* most fields are the same, copy all, and then fixup */
  1616. *new = *vma;
  1617. if (new_below)
  1618. new->vm_end = addr;
  1619. else {
  1620. new->vm_start = addr;
  1621. new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
  1622. }
  1623. pol = mpol_dup(vma_policy(vma));
  1624. if (IS_ERR(pol)) {
  1625. kmem_cache_free(vm_area_cachep, new);
  1626. return PTR_ERR(pol);
  1627. }
  1628. vma_set_policy(new, pol);
  1629. if (new->vm_file) {
  1630. get_file(new->vm_file);
  1631. if (vma->vm_flags & VM_EXECUTABLE)
  1632. added_exe_file_vma(mm);
  1633. }
  1634. if (new->vm_ops && new->vm_ops->open)
  1635. new->vm_ops->open(new);
  1636. if (new_below)
  1637. vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
  1638. ((addr - new->vm_start) >> PAGE_SHIFT), new);
  1639. else
  1640. vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
  1641. return 0;
  1642. }
  1643. /* Munmap is split into 2 main parts -- this part which finds
  1644. * what needs doing, and the areas themselves, which do the
  1645. * work. This now handles partial unmappings.
  1646. * Jeremy Fitzhardinge <jeremy@goop.org>
  1647. */
  1648. int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
  1649. {
  1650. unsigned long end;
  1651. struct vm_area_struct *vma, *prev, *last;
  1652. if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start)
  1653. return -EINVAL;
  1654. if ((len = PAGE_ALIGN(len)) == 0)
  1655. return -EINVAL;
  1656. /* Find the first overlapping VMA */
  1657. vma = find_vma_prev(mm, start, &prev);
  1658. if (!vma)
  1659. return 0;
  1660. /* we have start < vma->vm_end */
  1661. /* if it doesn't overlap, we have nothing.. */
  1662. end = start + len;
  1663. if (vma->vm_start >= end)
  1664. return 0;
  1665. /*
  1666. * If we need to split any vma, do it now to save pain later.
  1667. *
  1668. * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
  1669. * unmapped vm_area_struct will remain in use: so lower split_vma
  1670. * places tmp vma above, and higher split_vma places tmp vma below.
  1671. */
  1672. if (start > vma->vm_start) {
  1673. int error = split_vma(mm, vma, start, 0);
  1674. if (error)
  1675. return error;
  1676. prev = vma;
  1677. }
  1678. /* Does it split the last one? */
  1679. last = find_vma(mm, end);
  1680. if (last && end > last->vm_start) {
  1681. int error = split_vma(mm, last, end, 1);
  1682. if (error)
  1683. return error;
  1684. }
  1685. vma = prev? prev->vm_next: mm->mmap;
  1686. /*
  1687. * unlock any mlock()ed ranges before detaching vmas
  1688. */
  1689. if (mm->locked_vm) {
  1690. struct vm_area_struct *tmp = vma;
  1691. while (tmp && tmp->vm_start < end) {
  1692. if (tmp->vm_flags & VM_LOCKED) {
  1693. mm->locked_vm -= vma_pages(tmp);
  1694. munlock_vma_pages_all(tmp);
  1695. }
  1696. tmp = tmp->vm_next;
  1697. }
  1698. }
  1699. /*
  1700. * Remove the vma's, and unmap the actual pages
  1701. */
  1702. detach_vmas_to_be_unmapped(mm, vma, prev, end);
  1703. unmap_region(mm, vma, prev, start, end);
  1704. /* Fix up all other VM information */
  1705. remove_vma_list(mm, vma);
  1706. return 0;
  1707. }
  1708. EXPORT_SYMBOL(do_munmap);
  1709. SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
  1710. {
  1711. int ret;
  1712. struct mm_struct *mm = current->mm;
  1713. profile_munmap(addr);
  1714. down_write(&mm->mmap_sem);
  1715. ret = do_munmap(mm, addr, len);
  1716. up_write(&mm->mmap_sem);
  1717. return ret;
  1718. }
  1719. static inline void verify_mm_writelocked(struct mm_struct *mm)
  1720. {
  1721. #ifdef CONFIG_DEBUG_VM
  1722. if (unlikely(down_read_trylock(&mm->mmap_sem))) {
  1723. WARN_ON(1);
  1724. up_read(&mm->mmap_sem);
  1725. }
  1726. #endif
  1727. }
  1728. /*
  1729. * this is really a simplified "do_mmap". it only handles
  1730. * anonymous maps. eventually we may be able to do some
  1731. * brk-specific accounting here.
  1732. */
  1733. unsigned long do_brk(unsigned long addr, unsigned long len)
  1734. {
  1735. struct mm_struct * mm = current->mm;
  1736. struct vm_area_struct * vma, * prev;
  1737. unsigned long flags;
  1738. struct rb_node ** rb_link, * rb_parent;
  1739. pgoff_t pgoff = addr >> PAGE_SHIFT;
  1740. int error;
  1741. len = PAGE_ALIGN(len);
  1742. if (!len)
  1743. return addr;
  1744. if ((addr + len) > TASK_SIZE || (addr + len) < addr)
  1745. return -EINVAL;
  1746. if (is_hugepage_only_range(mm, addr, len))
  1747. return -EINVAL;
  1748. error = security_file_mmap(NULL, 0, 0, 0, addr, 1);
  1749. if (error)
  1750. return error;
  1751. flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
  1752. error = arch_mmap_check(addr, len, flags);
  1753. if (error)
  1754. return error;
  1755. /*
  1756. * mlock MCL_FUTURE?
  1757. */
  1758. if (mm->def_flags & VM_LOCKED) {
  1759. unsigned long locked, lock_limit;
  1760. locked = len >> PAGE_SHIFT;
  1761. locked += mm->locked_vm;
  1762. lock_limit = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur;
  1763. lock_limit >>= PAGE_SHIFT;
  1764. if (locked > lock_limit && !capable(CAP_IPC_LOCK))
  1765. return -EAGAIN;
  1766. }
  1767. /*
  1768. * mm->mmap_sem is required to protect against another thread
  1769. * changing the mappings in case we sleep.
  1770. */
  1771. verify_mm_writelocked(mm);
  1772. /*
  1773. * Clear old maps. this also does some error checking for us
  1774. */
  1775. munmap_back:
  1776. vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
  1777. if (vma && vma->vm_start < addr + len) {
  1778. if (do_munmap(mm, addr, len))
  1779. return -ENOMEM;
  1780. goto munmap_back;
  1781. }
  1782. /* Check against address space limits *after* clearing old maps... */
  1783. if (!may_expand_vm(mm, len >> PAGE_SHIFT))
  1784. return -ENOMEM;
  1785. if (mm->map_count > sysctl_max_map_count)
  1786. return -ENOMEM;
  1787. if (security_vm_enough_memory(len >> PAGE_SHIFT))
  1788. return -ENOMEM;
  1789. /* Can we just expand an old private anonymous mapping? */
  1790. vma = vma_merge(mm, prev, addr, addr + len, flags,
  1791. NULL, NULL, pgoff, NULL);
  1792. if (vma)
  1793. goto out;
  1794. /*
  1795. * create a vma struct for an anonymous mapping
  1796. */
  1797. vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
  1798. if (!vma) {
  1799. vm_unacct_memory(len >> PAGE_SHIFT);
  1800. return -ENOMEM;
  1801. }
  1802. vma->vm_mm = mm;
  1803. vma->vm_start = addr;
  1804. vma->vm_end = addr + len;
  1805. vma->vm_pgoff = pgoff;
  1806. vma->vm_flags = flags;
  1807. vma->vm_page_prot = vm_get_page_prot(flags);
  1808. vma_link(mm, vma, prev, rb_link, rb_parent);
  1809. out:
  1810. mm->total_vm += len >> PAGE_SHIFT;
  1811. if (flags & VM_LOCKED) {
  1812. if (!mlock_vma_pages_range(vma, addr, addr + len))
  1813. mm->locked_vm += (len >> PAGE_SHIFT);
  1814. }
  1815. return addr;
  1816. }
  1817. EXPORT_SYMBOL(do_brk);
  1818. /* Release all mmaps. */
  1819. void exit_mmap(struct mm_struct *mm)
  1820. {
  1821. struct mmu_gather *tlb;
  1822. struct vm_area_struct *vma;
  1823. unsigned long nr_accounted = 0;
  1824. unsigned long end;
  1825. /* mm's last user has gone, and its about to be pulled down */
  1826. mmu_notifier_release(mm);
  1827. if (mm->locked_vm) {
  1828. vma = mm->mmap;
  1829. while (vma) {
  1830. if (vma->vm_flags & VM_LOCKED)
  1831. munlock_vma_pages_all(vma);
  1832. vma = vma->vm_next;
  1833. }
  1834. }
  1835. arch_exit_mmap(mm);
  1836. vma = mm->mmap;
  1837. if (!vma) /* Can happen if dup_mmap() received an OOM */
  1838. return;
  1839. lru_add_drain();
  1840. flush_cache_mm(mm);
  1841. tlb = tlb_gather_mmu(mm, 1);
  1842. /* update_hiwater_rss(mm) here? but nobody should be looking */
  1843. /* Use -1 here to ensure all VMAs in the mm are unmapped */
  1844. end = unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL);
  1845. vm_unacct_memory(nr_accounted);
  1846. free_pgtables(tlb, vma, FIRST_USER_ADDRESS, 0);
  1847. tlb_finish_mmu(tlb, 0, end);
  1848. /*
  1849. * Walk the list again, actually closing and freeing it,
  1850. * with preemption enabled, without holding any MM locks.
  1851. */
  1852. while (vma)
  1853. vma = remove_vma(vma);
  1854. BUG_ON(mm->nr_ptes > (FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
  1855. }
  1856. /* Insert vm structure into process list sorted by address
  1857. * and into the inode's i_mmap tree. If vm_file is non-NULL
  1858. * then i_mmap_lock is taken here.
  1859. */
  1860. int insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
  1861. {
  1862. struct vm_area_struct * __vma, * prev;
  1863. struct rb_node ** rb_link, * rb_parent;
  1864. /*
  1865. * The vm_pgoff of a purely anonymous vma should be irrelevant
  1866. * until its first write fault, when page's anon_vma and index
  1867. * are set. But now set the vm_pgoff it will almost certainly
  1868. * end up with (unless mremap moves it elsewhere before that
  1869. * first wfault), so /proc/pid/maps tells a consistent story.
  1870. *
  1871. * By setting it to reflect the virtual start address of the
  1872. * vma, merges and splits can happen in a seamless way, just
  1873. * using the existing file pgoff checks and manipulations.
  1874. * Similarly in do_mmap_pgoff and in do_brk.
  1875. */
  1876. if (!vma->vm_file) {
  1877. BUG_ON(vma->anon_vma);
  1878. vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
  1879. }
  1880. __vma = find_vma_prepare(mm,vma->vm_start,&prev,&rb_link,&rb_parent);
  1881. if (__vma && __vma->vm_start < vma->vm_end)
  1882. return -ENOMEM;
  1883. if ((vma->vm_flags & VM_ACCOUNT) &&
  1884. security_vm_enough_memory_mm(mm, vma_pages(vma)))
  1885. return -ENOMEM;
  1886. vma_link(mm, vma, prev, rb_link, rb_parent);
  1887. return 0;
  1888. }
  1889. /*
  1890. * Copy the vma structure to a new location in the same mm,
  1891. * prior to moving page table entries, to effect an mremap move.
  1892. */
  1893. struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
  1894. unsigned long addr, unsigned long len, pgoff_t pgoff)
  1895. {
  1896. struct vm_area_struct *vma = *vmap;
  1897. unsigned long vma_start = vma->vm_start;
  1898. struct mm_struct *mm = vma->vm_mm;
  1899. struct vm_area_struct *new_vma, *prev;
  1900. struct rb_node **rb_link, *rb_parent;
  1901. struct mempolicy *pol;
  1902. /*
  1903. * If anonymous vma has not yet been faulted, update new pgoff
  1904. * to match new location, to increase its chance of merging.
  1905. */
  1906. if (!vma->vm_file && !vma->anon_vma)
  1907. pgoff = addr >> PAGE_SHIFT;
  1908. find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
  1909. new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
  1910. vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
  1911. if (new_vma) {
  1912. /*
  1913. * Source vma may have been merged into new_vma
  1914. */
  1915. if (vma_start >= new_vma->vm_start &&
  1916. vma_start < new_vma->vm_end)
  1917. *vmap = new_vma;
  1918. } else {
  1919. new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
  1920. if (new_vma) {
  1921. *new_vma = *vma;
  1922. pol = mpol_dup(vma_policy(vma));
  1923. if (IS_ERR(pol)) {
  1924. kmem_cache_free(vm_area_cachep, new_vma);
  1925. return NULL;
  1926. }
  1927. vma_set_policy(new_vma, pol);
  1928. new_vma->vm_start = addr;
  1929. new_vma->vm_end = addr + len;
  1930. new_vma->vm_pgoff = pgoff;
  1931. if (new_vma->vm_file) {
  1932. get_file(new_vma->vm_file);
  1933. if (vma->vm_flags & VM_EXECUTABLE)
  1934. added_exe_file_vma(mm);
  1935. }
  1936. if (new_vma->vm_ops && new_vma->vm_ops->open)
  1937. new_vma->vm_ops->open(new_vma);
  1938. vma_link(mm, new_vma, prev, rb_link, rb_parent);
  1939. }
  1940. }
  1941. return new_vma;
  1942. }
  1943. /*
  1944. * Return true if the calling process may expand its vm space by the passed
  1945. * number of pages
  1946. */
  1947. int may_expand_vm(struct mm_struct *mm, unsigned long npages)
  1948. {
  1949. unsigned long cur = mm->total_vm; /* pages */
  1950. unsigned long lim;
  1951. lim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
  1952. if (cur + npages > lim)
  1953. return 0;
  1954. return 1;
  1955. }
  1956. static int special_mapping_fault(struct vm_area_struct *vma,
  1957. struct vm_fault *vmf)
  1958. {
  1959. pgoff_t pgoff;
  1960. struct page **pages;
  1961. /*
  1962. * special mappings have no vm_file, and in that case, the mm
  1963. * uses vm_pgoff internally. So we have to subtract it from here.
  1964. * We are allowed to do this because we are the mm; do not copy
  1965. * this code into drivers!
  1966. */
  1967. pgoff = vmf->pgoff - vma->vm_pgoff;
  1968. for (pages = vma->vm_private_data; pgoff && *pages; ++pages)
  1969. pgoff--;
  1970. if (*pages) {
  1971. struct page *page = *pages;
  1972. get_page(page);
  1973. vmf->page = page;
  1974. return 0;
  1975. }
  1976. return VM_FAULT_SIGBUS;
  1977. }
  1978. /*
  1979. * Having a close hook prevents vma merging regardless of flags.
  1980. */
  1981. static void special_mapping_close(struct vm_area_struct *vma)
  1982. {
  1983. }
  1984. static struct vm_operations_struct special_mapping_vmops = {
  1985. .close = special_mapping_close,
  1986. .fault = special_mapping_fault,
  1987. };
  1988. /*
  1989. * Called with mm->mmap_sem held for writing.
  1990. * Insert a new vma covering the given region, with the given flags.
  1991. * Its pages are supplied by the given array of struct page *.
  1992. * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
  1993. * The region past the last page supplied will always produce SIGBUS.
  1994. * The array pointer and the pages it points to are assumed to stay alive
  1995. * for as long as this mapping might exist.
  1996. */
  1997. int install_special_mapping(struct mm_struct *mm,
  1998. unsigned long addr, unsigned long len,
  1999. unsigned long vm_flags, struct page **pages)
  2000. {
  2001. struct vm_area_struct *vma;
  2002. vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
  2003. if (unlikely(vma == NULL))
  2004. return -ENOMEM;
  2005. vma->vm_mm = mm;
  2006. vma->vm_start = addr;
  2007. vma->vm_end = addr + len;
  2008. vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND;
  2009. vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
  2010. vma->vm_ops = &special_mapping_vmops;
  2011. vma->vm_private_data = pages;
  2012. if (unlikely(insert_vm_struct(mm, vma))) {
  2013. kmem_cache_free(vm_area_cachep, vma);
  2014. return -ENOMEM;
  2015. }
  2016. mm->total_vm += len >> PAGE_SHIFT;
  2017. perf_counter_mmap(vma);
  2018. return 0;
  2019. }
  2020. static DEFINE_MUTEX(mm_all_locks_mutex);
  2021. static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
  2022. {
  2023. if (!test_bit(0, (unsigned long *) &anon_vma->head.next)) {
  2024. /*
  2025. * The LSB of head.next can't change from under us
  2026. * because we hold the mm_all_locks_mutex.
  2027. */
  2028. spin_lock_nest_lock(&anon_vma->lock, &mm->mmap_sem);
  2029. /*
  2030. * We can safely modify head.next after taking the
  2031. * anon_vma->lock. If some other vma in this mm shares
  2032. * the same anon_vma we won't take it again.
  2033. *
  2034. * No need of atomic instructions here, head.next
  2035. * can't change from under us thanks to the
  2036. * anon_vma->lock.
  2037. */
  2038. if (__test_and_set_bit(0, (unsigned long *)
  2039. &anon_vma->head.next))
  2040. BUG();
  2041. }
  2042. }
  2043. static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
  2044. {
  2045. if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
  2046. /*
  2047. * AS_MM_ALL_LOCKS can't change from under us because
  2048. * we hold the mm_all_locks_mutex.
  2049. *
  2050. * Operations on ->flags have to be atomic because
  2051. * even if AS_MM_ALL_LOCKS is stable thanks to the
  2052. * mm_all_locks_mutex, there may be other cpus
  2053. * changing other bitflags in parallel to us.
  2054. */
  2055. if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
  2056. BUG();
  2057. spin_lock_nest_lock(&mapping->i_mmap_lock, &mm->mmap_sem);
  2058. }
  2059. }
  2060. /*
  2061. * This operation locks against the VM for all pte/vma/mm related
  2062. * operations that could ever happen on a certain mm. This includes
  2063. * vmtruncate, try_to_unmap, and all page faults.
  2064. *
  2065. * The caller must take the mmap_sem in write mode before calling
  2066. * mm_take_all_locks(). The caller isn't allowed to release the
  2067. * mmap_sem until mm_drop_all_locks() returns.
  2068. *
  2069. * mmap_sem in write mode is required in order to block all operations
  2070. * that could modify pagetables and free pages without need of
  2071. * altering the vma layout (for example populate_range() with
  2072. * nonlinear vmas). It's also needed in write mode to avoid new
  2073. * anon_vmas to be associated with existing vmas.
  2074. *
  2075. * A single task can't take more than one mm_take_all_locks() in a row
  2076. * or it would deadlock.
  2077. *
  2078. * The LSB in anon_vma->head.next and the AS_MM_ALL_LOCKS bitflag in
  2079. * mapping->flags avoid to take the same lock twice, if more than one
  2080. * vma in this mm is backed by the same anon_vma or address_space.
  2081. *
  2082. * We can take all the locks in random order because the VM code
  2083. * taking i_mmap_lock or anon_vma->lock outside the mmap_sem never
  2084. * takes more than one of them in a row. Secondly we're protected
  2085. * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
  2086. *
  2087. * mm_take_all_locks() and mm_drop_all_locks are expensive operations
  2088. * that may have to take thousand of locks.
  2089. *
  2090. * mm_take_all_locks() can fail if it's interrupted by signals.
  2091. */
  2092. int mm_take_all_locks(struct mm_struct *mm)
  2093. {
  2094. struct vm_area_struct *vma;
  2095. int ret = -EINTR;
  2096. BUG_ON(down_read_trylock(&mm->mmap_sem));
  2097. mutex_lock(&mm_all_locks_mutex);
  2098. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  2099. if (signal_pending(current))
  2100. goto out_unlock;
  2101. if (vma->vm_file && vma->vm_file->f_mapping)
  2102. vm_lock_mapping(mm, vma->vm_file->f_mapping);
  2103. }
  2104. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  2105. if (signal_pending(current))
  2106. goto out_unlock;
  2107. if (vma->anon_vma)
  2108. vm_lock_anon_vma(mm, vma->anon_vma);
  2109. }
  2110. ret = 0;
  2111. out_unlock:
  2112. if (ret)
  2113. mm_drop_all_locks(mm);
  2114. return ret;
  2115. }
  2116. static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
  2117. {
  2118. if (test_bit(0, (unsigned long *) &anon_vma->head.next)) {
  2119. /*
  2120. * The LSB of head.next can't change to 0 from under
  2121. * us because we hold the mm_all_locks_mutex.
  2122. *
  2123. * We must however clear the bitflag before unlocking
  2124. * the vma so the users using the anon_vma->head will
  2125. * never see our bitflag.
  2126. *
  2127. * No need of atomic instructions here, head.next
  2128. * can't change from under us until we release the
  2129. * anon_vma->lock.
  2130. */
  2131. if (!__test_and_clear_bit(0, (unsigned long *)
  2132. &anon_vma->head.next))
  2133. BUG();
  2134. spin_unlock(&anon_vma->lock);
  2135. }
  2136. }
  2137. static void vm_unlock_mapping(struct address_space *mapping)
  2138. {
  2139. if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
  2140. /*
  2141. * AS_MM_ALL_LOCKS can't change to 0 from under us
  2142. * because we hold the mm_all_locks_mutex.
  2143. */
  2144. spin_unlock(&mapping->i_mmap_lock);
  2145. if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
  2146. &mapping->flags))
  2147. BUG();
  2148. }
  2149. }
  2150. /*
  2151. * The mmap_sem cannot be released by the caller until
  2152. * mm_drop_all_locks() returns.
  2153. */
  2154. void mm_drop_all_locks(struct mm_struct *mm)
  2155. {
  2156. struct vm_area_struct *vma;
  2157. BUG_ON(down_read_trylock(&mm->mmap_sem));
  2158. BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
  2159. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  2160. if (vma->anon_vma)
  2161. vm_unlock_anon_vma(vma->anon_vma);
  2162. if (vma->vm_file && vma->vm_file->f_mapping)
  2163. vm_unlock_mapping(vma->vm_file->f_mapping);
  2164. }
  2165. mutex_unlock(&mm_all_locks_mutex);
  2166. }
  2167. /*
  2168. * initialise the VMA slab
  2169. */
  2170. void __init mmap_init(void)
  2171. {
  2172. int ret;
  2173. ret = percpu_counter_init(&vm_committed_as, 0);
  2174. VM_BUG_ON(ret);
  2175. }