memory.c 81 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010
  1. /*
  2. * linux/mm/memory.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. */
  6. /*
  7. * demand-loading started 01.12.91 - seems it is high on the list of
  8. * things wanted, and it should be easy to implement. - Linus
  9. */
  10. /*
  11. * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
  12. * pages started 02.12.91, seems to work. - Linus.
  13. *
  14. * Tested sharing by executing about 30 /bin/sh: under the old kernel it
  15. * would have taken more than the 6M I have free, but it worked well as
  16. * far as I could see.
  17. *
  18. * Also corrected some "invalidate()"s - I wasn't doing enough of them.
  19. */
  20. /*
  21. * Real VM (paging to/from disk) started 18.12.91. Much more work and
  22. * thought has to go into this. Oh, well..
  23. * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
  24. * Found it. Everything seems to work now.
  25. * 20.12.91 - Ok, making the swap-device changeable like the root.
  26. */
  27. /*
  28. * 05.04.94 - Multi-page memory management added for v1.1.
  29. * Idea by Alex Bligh (alex@cconcepts.co.uk)
  30. *
  31. * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
  32. * (Gerhard.Wichert@pdb.siemens.de)
  33. *
  34. * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
  35. */
  36. #include <linux/kernel_stat.h>
  37. #include <linux/mm.h>
  38. #include <linux/hugetlb.h>
  39. #include <linux/mman.h>
  40. #include <linux/swap.h>
  41. #include <linux/highmem.h>
  42. #include <linux/pagemap.h>
  43. #include <linux/rmap.h>
  44. #include <linux/module.h>
  45. #include <linux/delayacct.h>
  46. #include <linux/init.h>
  47. #include <linux/writeback.h>
  48. #include <linux/memcontrol.h>
  49. #include <linux/mmu_notifier.h>
  50. #include <asm/pgalloc.h>
  51. #include <asm/uaccess.h>
  52. #include <asm/tlb.h>
  53. #include <asm/tlbflush.h>
  54. #include <asm/pgtable.h>
  55. #include <linux/swapops.h>
  56. #include <linux/elf.h>
  57. #include "internal.h"
  58. #ifndef CONFIG_NEED_MULTIPLE_NODES
  59. /* use the per-pgdat data instead for discontigmem - mbligh */
  60. unsigned long max_mapnr;
  61. struct page *mem_map;
  62. EXPORT_SYMBOL(max_mapnr);
  63. EXPORT_SYMBOL(mem_map);
  64. #endif
  65. unsigned long num_physpages;
  66. /*
  67. * A number of key systems in x86 including ioremap() rely on the assumption
  68. * that high_memory defines the upper bound on direct map memory, then end
  69. * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
  70. * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
  71. * and ZONE_HIGHMEM.
  72. */
  73. void * high_memory;
  74. EXPORT_SYMBOL(num_physpages);
  75. EXPORT_SYMBOL(high_memory);
  76. /*
  77. * Randomize the address space (stacks, mmaps, brk, etc.).
  78. *
  79. * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
  80. * as ancient (libc5 based) binaries can segfault. )
  81. */
  82. int randomize_va_space __read_mostly =
  83. #ifdef CONFIG_COMPAT_BRK
  84. 1;
  85. #else
  86. 2;
  87. #endif
  88. static int __init disable_randmaps(char *s)
  89. {
  90. randomize_va_space = 0;
  91. return 1;
  92. }
  93. __setup("norandmaps", disable_randmaps);
  94. /*
  95. * If a p?d_bad entry is found while walking page tables, report
  96. * the error, before resetting entry to p?d_none. Usually (but
  97. * very seldom) called out from the p?d_none_or_clear_bad macros.
  98. */
  99. void pgd_clear_bad(pgd_t *pgd)
  100. {
  101. pgd_ERROR(*pgd);
  102. pgd_clear(pgd);
  103. }
  104. void pud_clear_bad(pud_t *pud)
  105. {
  106. pud_ERROR(*pud);
  107. pud_clear(pud);
  108. }
  109. void pmd_clear_bad(pmd_t *pmd)
  110. {
  111. pmd_ERROR(*pmd);
  112. pmd_clear(pmd);
  113. }
  114. /*
  115. * Note: this doesn't free the actual pages themselves. That
  116. * has been handled earlier when unmapping all the memory regions.
  117. */
  118. static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd)
  119. {
  120. pgtable_t token = pmd_pgtable(*pmd);
  121. pmd_clear(pmd);
  122. pte_free_tlb(tlb, token);
  123. tlb->mm->nr_ptes--;
  124. }
  125. static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
  126. unsigned long addr, unsigned long end,
  127. unsigned long floor, unsigned long ceiling)
  128. {
  129. pmd_t *pmd;
  130. unsigned long next;
  131. unsigned long start;
  132. start = addr;
  133. pmd = pmd_offset(pud, addr);
  134. do {
  135. next = pmd_addr_end(addr, end);
  136. if (pmd_none_or_clear_bad(pmd))
  137. continue;
  138. free_pte_range(tlb, pmd);
  139. } while (pmd++, addr = next, addr != end);
  140. start &= PUD_MASK;
  141. if (start < floor)
  142. return;
  143. if (ceiling) {
  144. ceiling &= PUD_MASK;
  145. if (!ceiling)
  146. return;
  147. }
  148. if (end - 1 > ceiling - 1)
  149. return;
  150. pmd = pmd_offset(pud, start);
  151. pud_clear(pud);
  152. pmd_free_tlb(tlb, pmd);
  153. }
  154. static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
  155. unsigned long addr, unsigned long end,
  156. unsigned long floor, unsigned long ceiling)
  157. {
  158. pud_t *pud;
  159. unsigned long next;
  160. unsigned long start;
  161. start = addr;
  162. pud = pud_offset(pgd, addr);
  163. do {
  164. next = pud_addr_end(addr, end);
  165. if (pud_none_or_clear_bad(pud))
  166. continue;
  167. free_pmd_range(tlb, pud, addr, next, floor, ceiling);
  168. } while (pud++, addr = next, addr != end);
  169. start &= PGDIR_MASK;
  170. if (start < floor)
  171. return;
  172. if (ceiling) {
  173. ceiling &= PGDIR_MASK;
  174. if (!ceiling)
  175. return;
  176. }
  177. if (end - 1 > ceiling - 1)
  178. return;
  179. pud = pud_offset(pgd, start);
  180. pgd_clear(pgd);
  181. pud_free_tlb(tlb, pud);
  182. }
  183. /*
  184. * This function frees user-level page tables of a process.
  185. *
  186. * Must be called with pagetable lock held.
  187. */
  188. void free_pgd_range(struct mmu_gather *tlb,
  189. unsigned long addr, unsigned long end,
  190. unsigned long floor, unsigned long ceiling)
  191. {
  192. pgd_t *pgd;
  193. unsigned long next;
  194. unsigned long start;
  195. /*
  196. * The next few lines have given us lots of grief...
  197. *
  198. * Why are we testing PMD* at this top level? Because often
  199. * there will be no work to do at all, and we'd prefer not to
  200. * go all the way down to the bottom just to discover that.
  201. *
  202. * Why all these "- 1"s? Because 0 represents both the bottom
  203. * of the address space and the top of it (using -1 for the
  204. * top wouldn't help much: the masks would do the wrong thing).
  205. * The rule is that addr 0 and floor 0 refer to the bottom of
  206. * the address space, but end 0 and ceiling 0 refer to the top
  207. * Comparisons need to use "end - 1" and "ceiling - 1" (though
  208. * that end 0 case should be mythical).
  209. *
  210. * Wherever addr is brought up or ceiling brought down, we must
  211. * be careful to reject "the opposite 0" before it confuses the
  212. * subsequent tests. But what about where end is brought down
  213. * by PMD_SIZE below? no, end can't go down to 0 there.
  214. *
  215. * Whereas we round start (addr) and ceiling down, by different
  216. * masks at different levels, in order to test whether a table
  217. * now has no other vmas using it, so can be freed, we don't
  218. * bother to round floor or end up - the tests don't need that.
  219. */
  220. addr &= PMD_MASK;
  221. if (addr < floor) {
  222. addr += PMD_SIZE;
  223. if (!addr)
  224. return;
  225. }
  226. if (ceiling) {
  227. ceiling &= PMD_MASK;
  228. if (!ceiling)
  229. return;
  230. }
  231. if (end - 1 > ceiling - 1)
  232. end -= PMD_SIZE;
  233. if (addr > end - 1)
  234. return;
  235. start = addr;
  236. pgd = pgd_offset(tlb->mm, addr);
  237. do {
  238. next = pgd_addr_end(addr, end);
  239. if (pgd_none_or_clear_bad(pgd))
  240. continue;
  241. free_pud_range(tlb, pgd, addr, next, floor, ceiling);
  242. } while (pgd++, addr = next, addr != end);
  243. }
  244. void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
  245. unsigned long floor, unsigned long ceiling)
  246. {
  247. while (vma) {
  248. struct vm_area_struct *next = vma->vm_next;
  249. unsigned long addr = vma->vm_start;
  250. /*
  251. * Hide vma from rmap and vmtruncate before freeing pgtables
  252. */
  253. anon_vma_unlink(vma);
  254. unlink_file_vma(vma);
  255. if (is_vm_hugetlb_page(vma)) {
  256. hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
  257. floor, next? next->vm_start: ceiling);
  258. } else {
  259. /*
  260. * Optimization: gather nearby vmas into one call down
  261. */
  262. while (next && next->vm_start <= vma->vm_end + PMD_SIZE
  263. && !is_vm_hugetlb_page(next)) {
  264. vma = next;
  265. next = vma->vm_next;
  266. anon_vma_unlink(vma);
  267. unlink_file_vma(vma);
  268. }
  269. free_pgd_range(tlb, addr, vma->vm_end,
  270. floor, next? next->vm_start: ceiling);
  271. }
  272. vma = next;
  273. }
  274. }
  275. int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
  276. {
  277. pgtable_t new = pte_alloc_one(mm, address);
  278. if (!new)
  279. return -ENOMEM;
  280. /*
  281. * Ensure all pte setup (eg. pte page lock and page clearing) are
  282. * visible before the pte is made visible to other CPUs by being
  283. * put into page tables.
  284. *
  285. * The other side of the story is the pointer chasing in the page
  286. * table walking code (when walking the page table without locking;
  287. * ie. most of the time). Fortunately, these data accesses consist
  288. * of a chain of data-dependent loads, meaning most CPUs (alpha
  289. * being the notable exception) will already guarantee loads are
  290. * seen in-order. See the alpha page table accessors for the
  291. * smp_read_barrier_depends() barriers in page table walking code.
  292. */
  293. smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
  294. spin_lock(&mm->page_table_lock);
  295. if (!pmd_present(*pmd)) { /* Has another populated it ? */
  296. mm->nr_ptes++;
  297. pmd_populate(mm, pmd, new);
  298. new = NULL;
  299. }
  300. spin_unlock(&mm->page_table_lock);
  301. if (new)
  302. pte_free(mm, new);
  303. return 0;
  304. }
  305. int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
  306. {
  307. pte_t *new = pte_alloc_one_kernel(&init_mm, address);
  308. if (!new)
  309. return -ENOMEM;
  310. smp_wmb(); /* See comment in __pte_alloc */
  311. spin_lock(&init_mm.page_table_lock);
  312. if (!pmd_present(*pmd)) { /* Has another populated it ? */
  313. pmd_populate_kernel(&init_mm, pmd, new);
  314. new = NULL;
  315. }
  316. spin_unlock(&init_mm.page_table_lock);
  317. if (new)
  318. pte_free_kernel(&init_mm, new);
  319. return 0;
  320. }
  321. static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss)
  322. {
  323. if (file_rss)
  324. add_mm_counter(mm, file_rss, file_rss);
  325. if (anon_rss)
  326. add_mm_counter(mm, anon_rss, anon_rss);
  327. }
  328. /*
  329. * This function is called to print an error when a bad pte
  330. * is found. For example, we might have a PFN-mapped pte in
  331. * a region that doesn't allow it.
  332. *
  333. * The calling function must still handle the error.
  334. */
  335. static void print_bad_pte(struct vm_area_struct *vma, pte_t pte,
  336. unsigned long vaddr)
  337. {
  338. printk(KERN_ERR "Bad pte = %08llx, process = %s, "
  339. "vm_flags = %lx, vaddr = %lx\n",
  340. (long long)pte_val(pte),
  341. (vma->vm_mm == current->mm ? current->comm : "???"),
  342. vma->vm_flags, vaddr);
  343. dump_stack();
  344. }
  345. static inline int is_cow_mapping(unsigned int flags)
  346. {
  347. return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
  348. }
  349. /*
  350. * vm_normal_page -- This function gets the "struct page" associated with a pte.
  351. *
  352. * "Special" mappings do not wish to be associated with a "struct page" (either
  353. * it doesn't exist, or it exists but they don't want to touch it). In this
  354. * case, NULL is returned here. "Normal" mappings do have a struct page.
  355. *
  356. * There are 2 broad cases. Firstly, an architecture may define a pte_special()
  357. * pte bit, in which case this function is trivial. Secondly, an architecture
  358. * may not have a spare pte bit, which requires a more complicated scheme,
  359. * described below.
  360. *
  361. * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
  362. * special mapping (even if there are underlying and valid "struct pages").
  363. * COWed pages of a VM_PFNMAP are always normal.
  364. *
  365. * The way we recognize COWed pages within VM_PFNMAP mappings is through the
  366. * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
  367. * set, and the vm_pgoff will point to the first PFN mapped: thus every special
  368. * mapping will always honor the rule
  369. *
  370. * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
  371. *
  372. * And for normal mappings this is false.
  373. *
  374. * This restricts such mappings to be a linear translation from virtual address
  375. * to pfn. To get around this restriction, we allow arbitrary mappings so long
  376. * as the vma is not a COW mapping; in that case, we know that all ptes are
  377. * special (because none can have been COWed).
  378. *
  379. *
  380. * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
  381. *
  382. * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
  383. * page" backing, however the difference is that _all_ pages with a struct
  384. * page (that is, those where pfn_valid is true) are refcounted and considered
  385. * normal pages by the VM. The disadvantage is that pages are refcounted
  386. * (which can be slower and simply not an option for some PFNMAP users). The
  387. * advantage is that we don't have to follow the strict linearity rule of
  388. * PFNMAP mappings in order to support COWable mappings.
  389. *
  390. */
  391. #ifdef __HAVE_ARCH_PTE_SPECIAL
  392. # define HAVE_PTE_SPECIAL 1
  393. #else
  394. # define HAVE_PTE_SPECIAL 0
  395. #endif
  396. struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
  397. pte_t pte)
  398. {
  399. unsigned long pfn;
  400. if (HAVE_PTE_SPECIAL) {
  401. if (likely(!pte_special(pte))) {
  402. VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
  403. return pte_page(pte);
  404. }
  405. VM_BUG_ON(!(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
  406. return NULL;
  407. }
  408. /* !HAVE_PTE_SPECIAL case follows: */
  409. pfn = pte_pfn(pte);
  410. if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
  411. if (vma->vm_flags & VM_MIXEDMAP) {
  412. if (!pfn_valid(pfn))
  413. return NULL;
  414. goto out;
  415. } else {
  416. unsigned long off;
  417. off = (addr - vma->vm_start) >> PAGE_SHIFT;
  418. if (pfn == vma->vm_pgoff + off)
  419. return NULL;
  420. if (!is_cow_mapping(vma->vm_flags))
  421. return NULL;
  422. }
  423. }
  424. VM_BUG_ON(!pfn_valid(pfn));
  425. /*
  426. * NOTE! We still have PageReserved() pages in the page tables.
  427. *
  428. * eg. VDSO mappings can cause them to exist.
  429. */
  430. out:
  431. return pfn_to_page(pfn);
  432. }
  433. /*
  434. * copy one vm_area from one task to the other. Assumes the page tables
  435. * already present in the new task to be cleared in the whole range
  436. * covered by this vma.
  437. */
  438. static inline void
  439. copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  440. pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
  441. unsigned long addr, int *rss)
  442. {
  443. unsigned long vm_flags = vma->vm_flags;
  444. pte_t pte = *src_pte;
  445. struct page *page;
  446. /* pte contains position in swap or file, so copy. */
  447. if (unlikely(!pte_present(pte))) {
  448. if (!pte_file(pte)) {
  449. swp_entry_t entry = pte_to_swp_entry(pte);
  450. swap_duplicate(entry);
  451. /* make sure dst_mm is on swapoff's mmlist. */
  452. if (unlikely(list_empty(&dst_mm->mmlist))) {
  453. spin_lock(&mmlist_lock);
  454. if (list_empty(&dst_mm->mmlist))
  455. list_add(&dst_mm->mmlist,
  456. &src_mm->mmlist);
  457. spin_unlock(&mmlist_lock);
  458. }
  459. if (is_write_migration_entry(entry) &&
  460. is_cow_mapping(vm_flags)) {
  461. /*
  462. * COW mappings require pages in both parent
  463. * and child to be set to read.
  464. */
  465. make_migration_entry_read(&entry);
  466. pte = swp_entry_to_pte(entry);
  467. set_pte_at(src_mm, addr, src_pte, pte);
  468. }
  469. }
  470. goto out_set_pte;
  471. }
  472. /*
  473. * If it's a COW mapping, write protect it both
  474. * in the parent and the child
  475. */
  476. if (is_cow_mapping(vm_flags)) {
  477. ptep_set_wrprotect(src_mm, addr, src_pte);
  478. pte = pte_wrprotect(pte);
  479. }
  480. /*
  481. * If it's a shared mapping, mark it clean in
  482. * the child
  483. */
  484. if (vm_flags & VM_SHARED)
  485. pte = pte_mkclean(pte);
  486. pte = pte_mkold(pte);
  487. page = vm_normal_page(vma, addr, pte);
  488. if (page) {
  489. get_page(page);
  490. page_dup_rmap(page, vma, addr);
  491. rss[!!PageAnon(page)]++;
  492. }
  493. out_set_pte:
  494. set_pte_at(dst_mm, addr, dst_pte, pte);
  495. }
  496. static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  497. pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
  498. unsigned long addr, unsigned long end)
  499. {
  500. pte_t *src_pte, *dst_pte;
  501. spinlock_t *src_ptl, *dst_ptl;
  502. int progress = 0;
  503. int rss[2];
  504. again:
  505. rss[1] = rss[0] = 0;
  506. dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
  507. if (!dst_pte)
  508. return -ENOMEM;
  509. src_pte = pte_offset_map_nested(src_pmd, addr);
  510. src_ptl = pte_lockptr(src_mm, src_pmd);
  511. spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
  512. arch_enter_lazy_mmu_mode();
  513. do {
  514. /*
  515. * We are holding two locks at this point - either of them
  516. * could generate latencies in another task on another CPU.
  517. */
  518. if (progress >= 32) {
  519. progress = 0;
  520. if (need_resched() ||
  521. spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
  522. break;
  523. }
  524. if (pte_none(*src_pte)) {
  525. progress++;
  526. continue;
  527. }
  528. copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss);
  529. progress += 8;
  530. } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
  531. arch_leave_lazy_mmu_mode();
  532. spin_unlock(src_ptl);
  533. pte_unmap_nested(src_pte - 1);
  534. add_mm_rss(dst_mm, rss[0], rss[1]);
  535. pte_unmap_unlock(dst_pte - 1, dst_ptl);
  536. cond_resched();
  537. if (addr != end)
  538. goto again;
  539. return 0;
  540. }
  541. static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  542. pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
  543. unsigned long addr, unsigned long end)
  544. {
  545. pmd_t *src_pmd, *dst_pmd;
  546. unsigned long next;
  547. dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
  548. if (!dst_pmd)
  549. return -ENOMEM;
  550. src_pmd = pmd_offset(src_pud, addr);
  551. do {
  552. next = pmd_addr_end(addr, end);
  553. if (pmd_none_or_clear_bad(src_pmd))
  554. continue;
  555. if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
  556. vma, addr, next))
  557. return -ENOMEM;
  558. } while (dst_pmd++, src_pmd++, addr = next, addr != end);
  559. return 0;
  560. }
  561. static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  562. pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
  563. unsigned long addr, unsigned long end)
  564. {
  565. pud_t *src_pud, *dst_pud;
  566. unsigned long next;
  567. dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
  568. if (!dst_pud)
  569. return -ENOMEM;
  570. src_pud = pud_offset(src_pgd, addr);
  571. do {
  572. next = pud_addr_end(addr, end);
  573. if (pud_none_or_clear_bad(src_pud))
  574. continue;
  575. if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
  576. vma, addr, next))
  577. return -ENOMEM;
  578. } while (dst_pud++, src_pud++, addr = next, addr != end);
  579. return 0;
  580. }
  581. int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  582. struct vm_area_struct *vma)
  583. {
  584. pgd_t *src_pgd, *dst_pgd;
  585. unsigned long next;
  586. unsigned long addr = vma->vm_start;
  587. unsigned long end = vma->vm_end;
  588. int ret;
  589. /*
  590. * Don't copy ptes where a page fault will fill them correctly.
  591. * Fork becomes much lighter when there are big shared or private
  592. * readonly mappings. The tradeoff is that copy_page_range is more
  593. * efficient than faulting.
  594. */
  595. if (!(vma->vm_flags & (VM_HUGETLB|VM_NONLINEAR|VM_PFNMAP|VM_INSERTPAGE))) {
  596. if (!vma->anon_vma)
  597. return 0;
  598. }
  599. if (is_vm_hugetlb_page(vma))
  600. return copy_hugetlb_page_range(dst_mm, src_mm, vma);
  601. /*
  602. * We need to invalidate the secondary MMU mappings only when
  603. * there could be a permission downgrade on the ptes of the
  604. * parent mm. And a permission downgrade will only happen if
  605. * is_cow_mapping() returns true.
  606. */
  607. if (is_cow_mapping(vma->vm_flags))
  608. mmu_notifier_invalidate_range_start(src_mm, addr, end);
  609. ret = 0;
  610. dst_pgd = pgd_offset(dst_mm, addr);
  611. src_pgd = pgd_offset(src_mm, addr);
  612. do {
  613. next = pgd_addr_end(addr, end);
  614. if (pgd_none_or_clear_bad(src_pgd))
  615. continue;
  616. if (unlikely(copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
  617. vma, addr, next))) {
  618. ret = -ENOMEM;
  619. break;
  620. }
  621. } while (dst_pgd++, src_pgd++, addr = next, addr != end);
  622. if (is_cow_mapping(vma->vm_flags))
  623. mmu_notifier_invalidate_range_end(src_mm,
  624. vma->vm_start, end);
  625. return ret;
  626. }
  627. static unsigned long zap_pte_range(struct mmu_gather *tlb,
  628. struct vm_area_struct *vma, pmd_t *pmd,
  629. unsigned long addr, unsigned long end,
  630. long *zap_work, struct zap_details *details)
  631. {
  632. struct mm_struct *mm = tlb->mm;
  633. pte_t *pte;
  634. spinlock_t *ptl;
  635. int file_rss = 0;
  636. int anon_rss = 0;
  637. pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  638. arch_enter_lazy_mmu_mode();
  639. do {
  640. pte_t ptent = *pte;
  641. if (pte_none(ptent)) {
  642. (*zap_work)--;
  643. continue;
  644. }
  645. (*zap_work) -= PAGE_SIZE;
  646. if (pte_present(ptent)) {
  647. struct page *page;
  648. page = vm_normal_page(vma, addr, ptent);
  649. if (unlikely(details) && page) {
  650. /*
  651. * unmap_shared_mapping_pages() wants to
  652. * invalidate cache without truncating:
  653. * unmap shared but keep private pages.
  654. */
  655. if (details->check_mapping &&
  656. details->check_mapping != page->mapping)
  657. continue;
  658. /*
  659. * Each page->index must be checked when
  660. * invalidating or truncating nonlinear.
  661. */
  662. if (details->nonlinear_vma &&
  663. (page->index < details->first_index ||
  664. page->index > details->last_index))
  665. continue;
  666. }
  667. ptent = ptep_get_and_clear_full(mm, addr, pte,
  668. tlb->fullmm);
  669. tlb_remove_tlb_entry(tlb, pte, addr);
  670. if (unlikely(!page))
  671. continue;
  672. if (unlikely(details) && details->nonlinear_vma
  673. && linear_page_index(details->nonlinear_vma,
  674. addr) != page->index)
  675. set_pte_at(mm, addr, pte,
  676. pgoff_to_pte(page->index));
  677. if (PageAnon(page))
  678. anon_rss--;
  679. else {
  680. if (pte_dirty(ptent))
  681. set_page_dirty(page);
  682. if (pte_young(ptent))
  683. SetPageReferenced(page);
  684. file_rss--;
  685. }
  686. page_remove_rmap(page, vma);
  687. tlb_remove_page(tlb, page);
  688. continue;
  689. }
  690. /*
  691. * If details->check_mapping, we leave swap entries;
  692. * if details->nonlinear_vma, we leave file entries.
  693. */
  694. if (unlikely(details))
  695. continue;
  696. if (!pte_file(ptent))
  697. free_swap_and_cache(pte_to_swp_entry(ptent));
  698. pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
  699. } while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0));
  700. add_mm_rss(mm, file_rss, anon_rss);
  701. arch_leave_lazy_mmu_mode();
  702. pte_unmap_unlock(pte - 1, ptl);
  703. return addr;
  704. }
  705. static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
  706. struct vm_area_struct *vma, pud_t *pud,
  707. unsigned long addr, unsigned long end,
  708. long *zap_work, struct zap_details *details)
  709. {
  710. pmd_t *pmd;
  711. unsigned long next;
  712. pmd = pmd_offset(pud, addr);
  713. do {
  714. next = pmd_addr_end(addr, end);
  715. if (pmd_none_or_clear_bad(pmd)) {
  716. (*zap_work)--;
  717. continue;
  718. }
  719. next = zap_pte_range(tlb, vma, pmd, addr, next,
  720. zap_work, details);
  721. } while (pmd++, addr = next, (addr != end && *zap_work > 0));
  722. return addr;
  723. }
  724. static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
  725. struct vm_area_struct *vma, pgd_t *pgd,
  726. unsigned long addr, unsigned long end,
  727. long *zap_work, struct zap_details *details)
  728. {
  729. pud_t *pud;
  730. unsigned long next;
  731. pud = pud_offset(pgd, addr);
  732. do {
  733. next = pud_addr_end(addr, end);
  734. if (pud_none_or_clear_bad(pud)) {
  735. (*zap_work)--;
  736. continue;
  737. }
  738. next = zap_pmd_range(tlb, vma, pud, addr, next,
  739. zap_work, details);
  740. } while (pud++, addr = next, (addr != end && *zap_work > 0));
  741. return addr;
  742. }
  743. static unsigned long unmap_page_range(struct mmu_gather *tlb,
  744. struct vm_area_struct *vma,
  745. unsigned long addr, unsigned long end,
  746. long *zap_work, struct zap_details *details)
  747. {
  748. pgd_t *pgd;
  749. unsigned long next;
  750. if (details && !details->check_mapping && !details->nonlinear_vma)
  751. details = NULL;
  752. BUG_ON(addr >= end);
  753. tlb_start_vma(tlb, vma);
  754. pgd = pgd_offset(vma->vm_mm, addr);
  755. do {
  756. next = pgd_addr_end(addr, end);
  757. if (pgd_none_or_clear_bad(pgd)) {
  758. (*zap_work)--;
  759. continue;
  760. }
  761. next = zap_pud_range(tlb, vma, pgd, addr, next,
  762. zap_work, details);
  763. } while (pgd++, addr = next, (addr != end && *zap_work > 0));
  764. tlb_end_vma(tlb, vma);
  765. return addr;
  766. }
  767. #ifdef CONFIG_PREEMPT
  768. # define ZAP_BLOCK_SIZE (8 * PAGE_SIZE)
  769. #else
  770. /* No preempt: go for improved straight-line efficiency */
  771. # define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE)
  772. #endif
  773. /**
  774. * unmap_vmas - unmap a range of memory covered by a list of vma's
  775. * @tlbp: address of the caller's struct mmu_gather
  776. * @vma: the starting vma
  777. * @start_addr: virtual address at which to start unmapping
  778. * @end_addr: virtual address at which to end unmapping
  779. * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here
  780. * @details: details of nonlinear truncation or shared cache invalidation
  781. *
  782. * Returns the end address of the unmapping (restart addr if interrupted).
  783. *
  784. * Unmap all pages in the vma list.
  785. *
  786. * We aim to not hold locks for too long (for scheduling latency reasons).
  787. * So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to
  788. * return the ending mmu_gather to the caller.
  789. *
  790. * Only addresses between `start' and `end' will be unmapped.
  791. *
  792. * The VMA list must be sorted in ascending virtual address order.
  793. *
  794. * unmap_vmas() assumes that the caller will flush the whole unmapped address
  795. * range after unmap_vmas() returns. So the only responsibility here is to
  796. * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
  797. * drops the lock and schedules.
  798. */
  799. unsigned long unmap_vmas(struct mmu_gather **tlbp,
  800. struct vm_area_struct *vma, unsigned long start_addr,
  801. unsigned long end_addr, unsigned long *nr_accounted,
  802. struct zap_details *details)
  803. {
  804. long zap_work = ZAP_BLOCK_SIZE;
  805. unsigned long tlb_start = 0; /* For tlb_finish_mmu */
  806. int tlb_start_valid = 0;
  807. unsigned long start = start_addr;
  808. spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
  809. int fullmm = (*tlbp)->fullmm;
  810. struct mm_struct *mm = vma->vm_mm;
  811. mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
  812. for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
  813. unsigned long end;
  814. start = max(vma->vm_start, start_addr);
  815. if (start >= vma->vm_end)
  816. continue;
  817. end = min(vma->vm_end, end_addr);
  818. if (end <= vma->vm_start)
  819. continue;
  820. if (vma->vm_flags & VM_ACCOUNT)
  821. *nr_accounted += (end - start) >> PAGE_SHIFT;
  822. while (start != end) {
  823. if (!tlb_start_valid) {
  824. tlb_start = start;
  825. tlb_start_valid = 1;
  826. }
  827. if (unlikely(is_vm_hugetlb_page(vma))) {
  828. /*
  829. * It is undesirable to test vma->vm_file as it
  830. * should be non-null for valid hugetlb area.
  831. * However, vm_file will be NULL in the error
  832. * cleanup path of do_mmap_pgoff. When
  833. * hugetlbfs ->mmap method fails,
  834. * do_mmap_pgoff() nullifies vma->vm_file
  835. * before calling this function to clean up.
  836. * Since no pte has actually been setup, it is
  837. * safe to do nothing in this case.
  838. */
  839. if (vma->vm_file) {
  840. unmap_hugepage_range(vma, start, end, NULL);
  841. zap_work -= (end - start) /
  842. pages_per_huge_page(hstate_vma(vma));
  843. }
  844. start = end;
  845. } else
  846. start = unmap_page_range(*tlbp, vma,
  847. start, end, &zap_work, details);
  848. if (zap_work > 0) {
  849. BUG_ON(start != end);
  850. break;
  851. }
  852. tlb_finish_mmu(*tlbp, tlb_start, start);
  853. if (need_resched() ||
  854. (i_mmap_lock && spin_needbreak(i_mmap_lock))) {
  855. if (i_mmap_lock) {
  856. *tlbp = NULL;
  857. goto out;
  858. }
  859. cond_resched();
  860. }
  861. *tlbp = tlb_gather_mmu(vma->vm_mm, fullmm);
  862. tlb_start_valid = 0;
  863. zap_work = ZAP_BLOCK_SIZE;
  864. }
  865. }
  866. out:
  867. mmu_notifier_invalidate_range_end(mm, start_addr, end_addr);
  868. return start; /* which is now the end (or restart) address */
  869. }
  870. /**
  871. * zap_page_range - remove user pages in a given range
  872. * @vma: vm_area_struct holding the applicable pages
  873. * @address: starting address of pages to zap
  874. * @size: number of bytes to zap
  875. * @details: details of nonlinear truncation or shared cache invalidation
  876. */
  877. unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
  878. unsigned long size, struct zap_details *details)
  879. {
  880. struct mm_struct *mm = vma->vm_mm;
  881. struct mmu_gather *tlb;
  882. unsigned long end = address + size;
  883. unsigned long nr_accounted = 0;
  884. lru_add_drain();
  885. tlb = tlb_gather_mmu(mm, 0);
  886. update_hiwater_rss(mm);
  887. end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
  888. if (tlb)
  889. tlb_finish_mmu(tlb, address, end);
  890. return end;
  891. }
  892. EXPORT_SYMBOL_GPL(zap_page_range);
  893. /**
  894. * zap_vma_ptes - remove ptes mapping the vma
  895. * @vma: vm_area_struct holding ptes to be zapped
  896. * @address: starting address of pages to zap
  897. * @size: number of bytes to zap
  898. *
  899. * This function only unmaps ptes assigned to VM_PFNMAP vmas.
  900. *
  901. * The entire address range must be fully contained within the vma.
  902. *
  903. * Returns 0 if successful.
  904. */
  905. int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
  906. unsigned long size)
  907. {
  908. if (address < vma->vm_start || address + size > vma->vm_end ||
  909. !(vma->vm_flags & VM_PFNMAP))
  910. return -1;
  911. zap_page_range(vma, address, size, NULL);
  912. return 0;
  913. }
  914. EXPORT_SYMBOL_GPL(zap_vma_ptes);
  915. /*
  916. * Do a quick page-table lookup for a single page.
  917. */
  918. struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
  919. unsigned int flags)
  920. {
  921. pgd_t *pgd;
  922. pud_t *pud;
  923. pmd_t *pmd;
  924. pte_t *ptep, pte;
  925. spinlock_t *ptl;
  926. struct page *page;
  927. struct mm_struct *mm = vma->vm_mm;
  928. page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
  929. if (!IS_ERR(page)) {
  930. BUG_ON(flags & FOLL_GET);
  931. goto out;
  932. }
  933. page = NULL;
  934. pgd = pgd_offset(mm, address);
  935. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  936. goto no_page_table;
  937. pud = pud_offset(pgd, address);
  938. if (pud_none(*pud))
  939. goto no_page_table;
  940. if (pud_huge(*pud)) {
  941. BUG_ON(flags & FOLL_GET);
  942. page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE);
  943. goto out;
  944. }
  945. if (unlikely(pud_bad(*pud)))
  946. goto no_page_table;
  947. pmd = pmd_offset(pud, address);
  948. if (pmd_none(*pmd))
  949. goto no_page_table;
  950. if (pmd_huge(*pmd)) {
  951. BUG_ON(flags & FOLL_GET);
  952. page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
  953. goto out;
  954. }
  955. if (unlikely(pmd_bad(*pmd)))
  956. goto no_page_table;
  957. ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
  958. pte = *ptep;
  959. if (!pte_present(pte))
  960. goto no_page;
  961. if ((flags & FOLL_WRITE) && !pte_write(pte))
  962. goto unlock;
  963. page = vm_normal_page(vma, address, pte);
  964. if (unlikely(!page))
  965. goto bad_page;
  966. if (flags & FOLL_GET)
  967. get_page(page);
  968. if (flags & FOLL_TOUCH) {
  969. if ((flags & FOLL_WRITE) &&
  970. !pte_dirty(pte) && !PageDirty(page))
  971. set_page_dirty(page);
  972. mark_page_accessed(page);
  973. }
  974. unlock:
  975. pte_unmap_unlock(ptep, ptl);
  976. out:
  977. return page;
  978. bad_page:
  979. pte_unmap_unlock(ptep, ptl);
  980. return ERR_PTR(-EFAULT);
  981. no_page:
  982. pte_unmap_unlock(ptep, ptl);
  983. if (!pte_none(pte))
  984. return page;
  985. /* Fall through to ZERO_PAGE handling */
  986. no_page_table:
  987. /*
  988. * When core dumping an enormous anonymous area that nobody
  989. * has touched so far, we don't want to allocate page tables.
  990. */
  991. if (flags & FOLL_ANON) {
  992. page = ZERO_PAGE(0);
  993. if (flags & FOLL_GET)
  994. get_page(page);
  995. BUG_ON(flags & FOLL_WRITE);
  996. }
  997. return page;
  998. }
  999. EXPORT_SYMBOL_GPL(follow_page);
  1000. /* Can we do the FOLL_ANON optimization? */
  1001. static inline int use_zero_page(struct vm_area_struct *vma)
  1002. {
  1003. /*
  1004. * We don't want to optimize FOLL_ANON for make_pages_present()
  1005. * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
  1006. * we want to get the page from the page tables to make sure
  1007. * that we serialize and update with any other user of that
  1008. * mapping.
  1009. */
  1010. if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
  1011. return 0;
  1012. /*
  1013. * And if we have a fault routine, it's not an anonymous region.
  1014. */
  1015. return !vma->vm_ops || !vma->vm_ops->fault;
  1016. }
  1017. int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  1018. unsigned long start, int len, int write, int force,
  1019. struct page **pages, struct vm_area_struct **vmas)
  1020. {
  1021. int i;
  1022. unsigned int vm_flags;
  1023. if (len <= 0)
  1024. return 0;
  1025. /*
  1026. * Require read or write permissions.
  1027. * If 'force' is set, we only require the "MAY" flags.
  1028. */
  1029. vm_flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
  1030. vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
  1031. i = 0;
  1032. do {
  1033. struct vm_area_struct *vma;
  1034. unsigned int foll_flags;
  1035. vma = find_extend_vma(mm, start);
  1036. if (!vma && in_gate_area(tsk, start)) {
  1037. unsigned long pg = start & PAGE_MASK;
  1038. struct vm_area_struct *gate_vma = get_gate_vma(tsk);
  1039. pgd_t *pgd;
  1040. pud_t *pud;
  1041. pmd_t *pmd;
  1042. pte_t *pte;
  1043. if (write) /* user gate pages are read-only */
  1044. return i ? : -EFAULT;
  1045. if (pg > TASK_SIZE)
  1046. pgd = pgd_offset_k(pg);
  1047. else
  1048. pgd = pgd_offset_gate(mm, pg);
  1049. BUG_ON(pgd_none(*pgd));
  1050. pud = pud_offset(pgd, pg);
  1051. BUG_ON(pud_none(*pud));
  1052. pmd = pmd_offset(pud, pg);
  1053. if (pmd_none(*pmd))
  1054. return i ? : -EFAULT;
  1055. pte = pte_offset_map(pmd, pg);
  1056. if (pte_none(*pte)) {
  1057. pte_unmap(pte);
  1058. return i ? : -EFAULT;
  1059. }
  1060. if (pages) {
  1061. struct page *page = vm_normal_page(gate_vma, start, *pte);
  1062. pages[i] = page;
  1063. if (page)
  1064. get_page(page);
  1065. }
  1066. pte_unmap(pte);
  1067. if (vmas)
  1068. vmas[i] = gate_vma;
  1069. i++;
  1070. start += PAGE_SIZE;
  1071. len--;
  1072. continue;
  1073. }
  1074. if (!vma || (vma->vm_flags & (VM_IO | VM_PFNMAP))
  1075. || !(vm_flags & vma->vm_flags))
  1076. return i ? : -EFAULT;
  1077. if (is_vm_hugetlb_page(vma)) {
  1078. i = follow_hugetlb_page(mm, vma, pages, vmas,
  1079. &start, &len, i, write);
  1080. continue;
  1081. }
  1082. foll_flags = FOLL_TOUCH;
  1083. if (pages)
  1084. foll_flags |= FOLL_GET;
  1085. if (!write && use_zero_page(vma))
  1086. foll_flags |= FOLL_ANON;
  1087. do {
  1088. struct page *page;
  1089. /*
  1090. * If tsk is ooming, cut off its access to large memory
  1091. * allocations. It has a pending SIGKILL, but it can't
  1092. * be processed until returning to user space.
  1093. */
  1094. if (unlikely(test_tsk_thread_flag(tsk, TIF_MEMDIE)))
  1095. return i ? i : -ENOMEM;
  1096. if (write)
  1097. foll_flags |= FOLL_WRITE;
  1098. cond_resched();
  1099. while (!(page = follow_page(vma, start, foll_flags))) {
  1100. int ret;
  1101. ret = handle_mm_fault(mm, vma, start,
  1102. foll_flags & FOLL_WRITE);
  1103. if (ret & VM_FAULT_ERROR) {
  1104. if (ret & VM_FAULT_OOM)
  1105. return i ? i : -ENOMEM;
  1106. else if (ret & VM_FAULT_SIGBUS)
  1107. return i ? i : -EFAULT;
  1108. BUG();
  1109. }
  1110. if (ret & VM_FAULT_MAJOR)
  1111. tsk->maj_flt++;
  1112. else
  1113. tsk->min_flt++;
  1114. /*
  1115. * The VM_FAULT_WRITE bit tells us that
  1116. * do_wp_page has broken COW when necessary,
  1117. * even if maybe_mkwrite decided not to set
  1118. * pte_write. We can thus safely do subsequent
  1119. * page lookups as if they were reads.
  1120. */
  1121. if (ret & VM_FAULT_WRITE)
  1122. foll_flags &= ~FOLL_WRITE;
  1123. cond_resched();
  1124. }
  1125. if (IS_ERR(page))
  1126. return i ? i : PTR_ERR(page);
  1127. if (pages) {
  1128. pages[i] = page;
  1129. flush_anon_page(vma, page, start);
  1130. flush_dcache_page(page);
  1131. }
  1132. if (vmas)
  1133. vmas[i] = vma;
  1134. i++;
  1135. start += PAGE_SIZE;
  1136. len--;
  1137. } while (len && start < vma->vm_end);
  1138. } while (len);
  1139. return i;
  1140. }
  1141. EXPORT_SYMBOL(get_user_pages);
  1142. pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
  1143. spinlock_t **ptl)
  1144. {
  1145. pgd_t * pgd = pgd_offset(mm, addr);
  1146. pud_t * pud = pud_alloc(mm, pgd, addr);
  1147. if (pud) {
  1148. pmd_t * pmd = pmd_alloc(mm, pud, addr);
  1149. if (pmd)
  1150. return pte_alloc_map_lock(mm, pmd, addr, ptl);
  1151. }
  1152. return NULL;
  1153. }
  1154. /*
  1155. * This is the old fallback for page remapping.
  1156. *
  1157. * For historical reasons, it only allows reserved pages. Only
  1158. * old drivers should use this, and they needed to mark their
  1159. * pages reserved for the old functions anyway.
  1160. */
  1161. static int insert_page(struct vm_area_struct *vma, unsigned long addr,
  1162. struct page *page, pgprot_t prot)
  1163. {
  1164. struct mm_struct *mm = vma->vm_mm;
  1165. int retval;
  1166. pte_t *pte;
  1167. spinlock_t *ptl;
  1168. retval = mem_cgroup_charge(page, mm, GFP_KERNEL);
  1169. if (retval)
  1170. goto out;
  1171. retval = -EINVAL;
  1172. if (PageAnon(page))
  1173. goto out_uncharge;
  1174. retval = -ENOMEM;
  1175. flush_dcache_page(page);
  1176. pte = get_locked_pte(mm, addr, &ptl);
  1177. if (!pte)
  1178. goto out_uncharge;
  1179. retval = -EBUSY;
  1180. if (!pte_none(*pte))
  1181. goto out_unlock;
  1182. /* Ok, finally just insert the thing.. */
  1183. get_page(page);
  1184. inc_mm_counter(mm, file_rss);
  1185. page_add_file_rmap(page);
  1186. set_pte_at(mm, addr, pte, mk_pte(page, prot));
  1187. retval = 0;
  1188. pte_unmap_unlock(pte, ptl);
  1189. return retval;
  1190. out_unlock:
  1191. pte_unmap_unlock(pte, ptl);
  1192. out_uncharge:
  1193. mem_cgroup_uncharge_page(page);
  1194. out:
  1195. return retval;
  1196. }
  1197. /**
  1198. * vm_insert_page - insert single page into user vma
  1199. * @vma: user vma to map to
  1200. * @addr: target user address of this page
  1201. * @page: source kernel page
  1202. *
  1203. * This allows drivers to insert individual pages they've allocated
  1204. * into a user vma.
  1205. *
  1206. * The page has to be a nice clean _individual_ kernel allocation.
  1207. * If you allocate a compound page, you need to have marked it as
  1208. * such (__GFP_COMP), or manually just split the page up yourself
  1209. * (see split_page()).
  1210. *
  1211. * NOTE! Traditionally this was done with "remap_pfn_range()" which
  1212. * took an arbitrary page protection parameter. This doesn't allow
  1213. * that. Your vma protection will have to be set up correctly, which
  1214. * means that if you want a shared writable mapping, you'd better
  1215. * ask for a shared writable mapping!
  1216. *
  1217. * The page does not need to be reserved.
  1218. */
  1219. int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
  1220. struct page *page)
  1221. {
  1222. if (addr < vma->vm_start || addr >= vma->vm_end)
  1223. return -EFAULT;
  1224. if (!page_count(page))
  1225. return -EINVAL;
  1226. vma->vm_flags |= VM_INSERTPAGE;
  1227. return insert_page(vma, addr, page, vma->vm_page_prot);
  1228. }
  1229. EXPORT_SYMBOL(vm_insert_page);
  1230. static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1231. unsigned long pfn, pgprot_t prot)
  1232. {
  1233. struct mm_struct *mm = vma->vm_mm;
  1234. int retval;
  1235. pte_t *pte, entry;
  1236. spinlock_t *ptl;
  1237. retval = -ENOMEM;
  1238. pte = get_locked_pte(mm, addr, &ptl);
  1239. if (!pte)
  1240. goto out;
  1241. retval = -EBUSY;
  1242. if (!pte_none(*pte))
  1243. goto out_unlock;
  1244. /* Ok, finally just insert the thing.. */
  1245. entry = pte_mkspecial(pfn_pte(pfn, prot));
  1246. set_pte_at(mm, addr, pte, entry);
  1247. update_mmu_cache(vma, addr, entry); /* XXX: why not for insert_page? */
  1248. retval = 0;
  1249. out_unlock:
  1250. pte_unmap_unlock(pte, ptl);
  1251. out:
  1252. return retval;
  1253. }
  1254. /**
  1255. * vm_insert_pfn - insert single pfn into user vma
  1256. * @vma: user vma to map to
  1257. * @addr: target user address of this page
  1258. * @pfn: source kernel pfn
  1259. *
  1260. * Similar to vm_inert_page, this allows drivers to insert individual pages
  1261. * they've allocated into a user vma. Same comments apply.
  1262. *
  1263. * This function should only be called from a vm_ops->fault handler, and
  1264. * in that case the handler should return NULL.
  1265. *
  1266. * vma cannot be a COW mapping.
  1267. *
  1268. * As this is called only for pages that do not currently exist, we
  1269. * do not need to flush old virtual caches or the TLB.
  1270. */
  1271. int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1272. unsigned long pfn)
  1273. {
  1274. /*
  1275. * Technically, architectures with pte_special can avoid all these
  1276. * restrictions (same for remap_pfn_range). However we would like
  1277. * consistency in testing and feature parity among all, so we should
  1278. * try to keep these invariants in place for everybody.
  1279. */
  1280. BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
  1281. BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
  1282. (VM_PFNMAP|VM_MIXEDMAP));
  1283. BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
  1284. BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
  1285. if (addr < vma->vm_start || addr >= vma->vm_end)
  1286. return -EFAULT;
  1287. return insert_pfn(vma, addr, pfn, vma->vm_page_prot);
  1288. }
  1289. EXPORT_SYMBOL(vm_insert_pfn);
  1290. int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
  1291. unsigned long pfn)
  1292. {
  1293. BUG_ON(!(vma->vm_flags & VM_MIXEDMAP));
  1294. if (addr < vma->vm_start || addr >= vma->vm_end)
  1295. return -EFAULT;
  1296. /*
  1297. * If we don't have pte special, then we have to use the pfn_valid()
  1298. * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
  1299. * refcount the page if pfn_valid is true (hence insert_page rather
  1300. * than insert_pfn).
  1301. */
  1302. if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) {
  1303. struct page *page;
  1304. page = pfn_to_page(pfn);
  1305. return insert_page(vma, addr, page, vma->vm_page_prot);
  1306. }
  1307. return insert_pfn(vma, addr, pfn, vma->vm_page_prot);
  1308. }
  1309. EXPORT_SYMBOL(vm_insert_mixed);
  1310. /*
  1311. * maps a range of physical memory into the requested pages. the old
  1312. * mappings are removed. any references to nonexistent pages results
  1313. * in null mappings (currently treated as "copy-on-access")
  1314. */
  1315. static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
  1316. unsigned long addr, unsigned long end,
  1317. unsigned long pfn, pgprot_t prot)
  1318. {
  1319. pte_t *pte;
  1320. spinlock_t *ptl;
  1321. pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
  1322. if (!pte)
  1323. return -ENOMEM;
  1324. arch_enter_lazy_mmu_mode();
  1325. do {
  1326. BUG_ON(!pte_none(*pte));
  1327. set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
  1328. pfn++;
  1329. } while (pte++, addr += PAGE_SIZE, addr != end);
  1330. arch_leave_lazy_mmu_mode();
  1331. pte_unmap_unlock(pte - 1, ptl);
  1332. return 0;
  1333. }
  1334. static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
  1335. unsigned long addr, unsigned long end,
  1336. unsigned long pfn, pgprot_t prot)
  1337. {
  1338. pmd_t *pmd;
  1339. unsigned long next;
  1340. pfn -= addr >> PAGE_SHIFT;
  1341. pmd = pmd_alloc(mm, pud, addr);
  1342. if (!pmd)
  1343. return -ENOMEM;
  1344. do {
  1345. next = pmd_addr_end(addr, end);
  1346. if (remap_pte_range(mm, pmd, addr, next,
  1347. pfn + (addr >> PAGE_SHIFT), prot))
  1348. return -ENOMEM;
  1349. } while (pmd++, addr = next, addr != end);
  1350. return 0;
  1351. }
  1352. static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
  1353. unsigned long addr, unsigned long end,
  1354. unsigned long pfn, pgprot_t prot)
  1355. {
  1356. pud_t *pud;
  1357. unsigned long next;
  1358. pfn -= addr >> PAGE_SHIFT;
  1359. pud = pud_alloc(mm, pgd, addr);
  1360. if (!pud)
  1361. return -ENOMEM;
  1362. do {
  1363. next = pud_addr_end(addr, end);
  1364. if (remap_pmd_range(mm, pud, addr, next,
  1365. pfn + (addr >> PAGE_SHIFT), prot))
  1366. return -ENOMEM;
  1367. } while (pud++, addr = next, addr != end);
  1368. return 0;
  1369. }
  1370. /**
  1371. * remap_pfn_range - remap kernel memory to userspace
  1372. * @vma: user vma to map to
  1373. * @addr: target user address to start at
  1374. * @pfn: physical address of kernel memory
  1375. * @size: size of map area
  1376. * @prot: page protection flags for this mapping
  1377. *
  1378. * Note: this is only safe if the mm semaphore is held when called.
  1379. */
  1380. int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
  1381. unsigned long pfn, unsigned long size, pgprot_t prot)
  1382. {
  1383. pgd_t *pgd;
  1384. unsigned long next;
  1385. unsigned long end = addr + PAGE_ALIGN(size);
  1386. struct mm_struct *mm = vma->vm_mm;
  1387. int err;
  1388. /*
  1389. * Physically remapped pages are special. Tell the
  1390. * rest of the world about it:
  1391. * VM_IO tells people not to look at these pages
  1392. * (accesses can have side effects).
  1393. * VM_RESERVED is specified all over the place, because
  1394. * in 2.4 it kept swapout's vma scan off this vma; but
  1395. * in 2.6 the LRU scan won't even find its pages, so this
  1396. * flag means no more than count its pages in reserved_vm,
  1397. * and omit it from core dump, even when VM_IO turned off.
  1398. * VM_PFNMAP tells the core MM that the base pages are just
  1399. * raw PFN mappings, and do not have a "struct page" associated
  1400. * with them.
  1401. *
  1402. * There's a horrible special case to handle copy-on-write
  1403. * behaviour that some programs depend on. We mark the "original"
  1404. * un-COW'ed pages by matching them up with "vma->vm_pgoff".
  1405. */
  1406. if (is_cow_mapping(vma->vm_flags)) {
  1407. if (addr != vma->vm_start || end != vma->vm_end)
  1408. return -EINVAL;
  1409. vma->vm_pgoff = pfn;
  1410. }
  1411. vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP;
  1412. BUG_ON(addr >= end);
  1413. pfn -= addr >> PAGE_SHIFT;
  1414. pgd = pgd_offset(mm, addr);
  1415. flush_cache_range(vma, addr, end);
  1416. do {
  1417. next = pgd_addr_end(addr, end);
  1418. err = remap_pud_range(mm, pgd, addr, next,
  1419. pfn + (addr >> PAGE_SHIFT), prot);
  1420. if (err)
  1421. break;
  1422. } while (pgd++, addr = next, addr != end);
  1423. return err;
  1424. }
  1425. EXPORT_SYMBOL(remap_pfn_range);
  1426. static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
  1427. unsigned long addr, unsigned long end,
  1428. pte_fn_t fn, void *data)
  1429. {
  1430. pte_t *pte;
  1431. int err;
  1432. pgtable_t token;
  1433. spinlock_t *uninitialized_var(ptl);
  1434. pte = (mm == &init_mm) ?
  1435. pte_alloc_kernel(pmd, addr) :
  1436. pte_alloc_map_lock(mm, pmd, addr, &ptl);
  1437. if (!pte)
  1438. return -ENOMEM;
  1439. BUG_ON(pmd_huge(*pmd));
  1440. token = pmd_pgtable(*pmd);
  1441. do {
  1442. err = fn(pte, token, addr, data);
  1443. if (err)
  1444. break;
  1445. } while (pte++, addr += PAGE_SIZE, addr != end);
  1446. if (mm != &init_mm)
  1447. pte_unmap_unlock(pte-1, ptl);
  1448. return err;
  1449. }
  1450. static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
  1451. unsigned long addr, unsigned long end,
  1452. pte_fn_t fn, void *data)
  1453. {
  1454. pmd_t *pmd;
  1455. unsigned long next;
  1456. int err;
  1457. BUG_ON(pud_huge(*pud));
  1458. pmd = pmd_alloc(mm, pud, addr);
  1459. if (!pmd)
  1460. return -ENOMEM;
  1461. do {
  1462. next = pmd_addr_end(addr, end);
  1463. err = apply_to_pte_range(mm, pmd, addr, next, fn, data);
  1464. if (err)
  1465. break;
  1466. } while (pmd++, addr = next, addr != end);
  1467. return err;
  1468. }
  1469. static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
  1470. unsigned long addr, unsigned long end,
  1471. pte_fn_t fn, void *data)
  1472. {
  1473. pud_t *pud;
  1474. unsigned long next;
  1475. int err;
  1476. pud = pud_alloc(mm, pgd, addr);
  1477. if (!pud)
  1478. return -ENOMEM;
  1479. do {
  1480. next = pud_addr_end(addr, end);
  1481. err = apply_to_pmd_range(mm, pud, addr, next, fn, data);
  1482. if (err)
  1483. break;
  1484. } while (pud++, addr = next, addr != end);
  1485. return err;
  1486. }
  1487. /*
  1488. * Scan a region of virtual memory, filling in page tables as necessary
  1489. * and calling a provided function on each leaf page table.
  1490. */
  1491. int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
  1492. unsigned long size, pte_fn_t fn, void *data)
  1493. {
  1494. pgd_t *pgd;
  1495. unsigned long next;
  1496. unsigned long start = addr, end = addr + size;
  1497. int err;
  1498. BUG_ON(addr >= end);
  1499. mmu_notifier_invalidate_range_start(mm, start, end);
  1500. pgd = pgd_offset(mm, addr);
  1501. do {
  1502. next = pgd_addr_end(addr, end);
  1503. err = apply_to_pud_range(mm, pgd, addr, next, fn, data);
  1504. if (err)
  1505. break;
  1506. } while (pgd++, addr = next, addr != end);
  1507. mmu_notifier_invalidate_range_end(mm, start, end);
  1508. return err;
  1509. }
  1510. EXPORT_SYMBOL_GPL(apply_to_page_range);
  1511. /*
  1512. * handle_pte_fault chooses page fault handler according to an entry
  1513. * which was read non-atomically. Before making any commitment, on
  1514. * those architectures or configurations (e.g. i386 with PAE) which
  1515. * might give a mix of unmatched parts, do_swap_page and do_file_page
  1516. * must check under lock before unmapping the pte and proceeding
  1517. * (but do_wp_page is only called after already making such a check;
  1518. * and do_anonymous_page and do_no_page can safely check later on).
  1519. */
  1520. static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
  1521. pte_t *page_table, pte_t orig_pte)
  1522. {
  1523. int same = 1;
  1524. #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT)
  1525. if (sizeof(pte_t) > sizeof(unsigned long)) {
  1526. spinlock_t *ptl = pte_lockptr(mm, pmd);
  1527. spin_lock(ptl);
  1528. same = pte_same(*page_table, orig_pte);
  1529. spin_unlock(ptl);
  1530. }
  1531. #endif
  1532. pte_unmap(page_table);
  1533. return same;
  1534. }
  1535. /*
  1536. * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
  1537. * servicing faults for write access. In the normal case, do always want
  1538. * pte_mkwrite. But get_user_pages can cause write faults for mappings
  1539. * that do not have writing enabled, when used by access_process_vm.
  1540. */
  1541. static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
  1542. {
  1543. if (likely(vma->vm_flags & VM_WRITE))
  1544. pte = pte_mkwrite(pte);
  1545. return pte;
  1546. }
  1547. static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
  1548. {
  1549. /*
  1550. * If the source page was a PFN mapping, we don't have
  1551. * a "struct page" for it. We do a best-effort copy by
  1552. * just copying from the original user address. If that
  1553. * fails, we just zero-fill it. Live with it.
  1554. */
  1555. if (unlikely(!src)) {
  1556. void *kaddr = kmap_atomic(dst, KM_USER0);
  1557. void __user *uaddr = (void __user *)(va & PAGE_MASK);
  1558. /*
  1559. * This really shouldn't fail, because the page is there
  1560. * in the page tables. But it might just be unreadable,
  1561. * in which case we just give up and fill the result with
  1562. * zeroes.
  1563. */
  1564. if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
  1565. memset(kaddr, 0, PAGE_SIZE);
  1566. kunmap_atomic(kaddr, KM_USER0);
  1567. flush_dcache_page(dst);
  1568. } else
  1569. copy_user_highpage(dst, src, va, vma);
  1570. }
  1571. /*
  1572. * This routine handles present pages, when users try to write
  1573. * to a shared page. It is done by copying the page to a new address
  1574. * and decrementing the shared-page counter for the old page.
  1575. *
  1576. * Note that this routine assumes that the protection checks have been
  1577. * done by the caller (the low-level page fault routine in most cases).
  1578. * Thus we can safely just mark it writable once we've done any necessary
  1579. * COW.
  1580. *
  1581. * We also mark the page dirty at this point even though the page will
  1582. * change only once the write actually happens. This avoids a few races,
  1583. * and potentially makes it more efficient.
  1584. *
  1585. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  1586. * but allow concurrent faults), with pte both mapped and locked.
  1587. * We return with mmap_sem still held, but pte unmapped and unlocked.
  1588. */
  1589. static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
  1590. unsigned long address, pte_t *page_table, pmd_t *pmd,
  1591. spinlock_t *ptl, pte_t orig_pte)
  1592. {
  1593. struct page *old_page, *new_page;
  1594. pte_t entry;
  1595. int reuse = 0, ret = 0;
  1596. int page_mkwrite = 0;
  1597. struct page *dirty_page = NULL;
  1598. old_page = vm_normal_page(vma, address, orig_pte);
  1599. if (!old_page) {
  1600. /*
  1601. * VM_MIXEDMAP !pfn_valid() case
  1602. *
  1603. * We should not cow pages in a shared writeable mapping.
  1604. * Just mark the pages writable as we can't do any dirty
  1605. * accounting on raw pfn maps.
  1606. */
  1607. if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
  1608. (VM_WRITE|VM_SHARED))
  1609. goto reuse;
  1610. goto gotten;
  1611. }
  1612. /*
  1613. * Take out anonymous pages first, anonymous shared vmas are
  1614. * not dirty accountable.
  1615. */
  1616. if (PageAnon(old_page)) {
  1617. if (!TestSetPageLocked(old_page)) {
  1618. reuse = can_share_swap_page(old_page);
  1619. unlock_page(old_page);
  1620. }
  1621. } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
  1622. (VM_WRITE|VM_SHARED))) {
  1623. /*
  1624. * Only catch write-faults on shared writable pages,
  1625. * read-only shared pages can get COWed by
  1626. * get_user_pages(.write=1, .force=1).
  1627. */
  1628. if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
  1629. /*
  1630. * Notify the address space that the page is about to
  1631. * become writable so that it can prohibit this or wait
  1632. * for the page to get into an appropriate state.
  1633. *
  1634. * We do this without the lock held, so that it can
  1635. * sleep if it needs to.
  1636. */
  1637. page_cache_get(old_page);
  1638. pte_unmap_unlock(page_table, ptl);
  1639. if (vma->vm_ops->page_mkwrite(vma, old_page) < 0)
  1640. goto unwritable_page;
  1641. /*
  1642. * Since we dropped the lock we need to revalidate
  1643. * the PTE as someone else may have changed it. If
  1644. * they did, we just return, as we can count on the
  1645. * MMU to tell us if they didn't also make it writable.
  1646. */
  1647. page_table = pte_offset_map_lock(mm, pmd, address,
  1648. &ptl);
  1649. page_cache_release(old_page);
  1650. if (!pte_same(*page_table, orig_pte))
  1651. goto unlock;
  1652. page_mkwrite = 1;
  1653. }
  1654. dirty_page = old_page;
  1655. get_page(dirty_page);
  1656. reuse = 1;
  1657. }
  1658. if (reuse) {
  1659. reuse:
  1660. flush_cache_page(vma, address, pte_pfn(orig_pte));
  1661. entry = pte_mkyoung(orig_pte);
  1662. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1663. if (ptep_set_access_flags(vma, address, page_table, entry,1))
  1664. update_mmu_cache(vma, address, entry);
  1665. ret |= VM_FAULT_WRITE;
  1666. goto unlock;
  1667. }
  1668. /*
  1669. * Ok, we need to copy. Oh, well..
  1670. */
  1671. page_cache_get(old_page);
  1672. gotten:
  1673. pte_unmap_unlock(page_table, ptl);
  1674. if (unlikely(anon_vma_prepare(vma)))
  1675. goto oom;
  1676. VM_BUG_ON(old_page == ZERO_PAGE(0));
  1677. new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
  1678. if (!new_page)
  1679. goto oom;
  1680. cow_user_page(new_page, old_page, address, vma);
  1681. __SetPageUptodate(new_page);
  1682. if (mem_cgroup_charge(new_page, mm, GFP_KERNEL))
  1683. goto oom_free_new;
  1684. /*
  1685. * Re-check the pte - we dropped the lock
  1686. */
  1687. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  1688. if (likely(pte_same(*page_table, orig_pte))) {
  1689. if (old_page) {
  1690. if (!PageAnon(old_page)) {
  1691. dec_mm_counter(mm, file_rss);
  1692. inc_mm_counter(mm, anon_rss);
  1693. }
  1694. } else
  1695. inc_mm_counter(mm, anon_rss);
  1696. flush_cache_page(vma, address, pte_pfn(orig_pte));
  1697. entry = mk_pte(new_page, vma->vm_page_prot);
  1698. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1699. /*
  1700. * Clear the pte entry and flush it first, before updating the
  1701. * pte with the new entry. This will avoid a race condition
  1702. * seen in the presence of one thread doing SMC and another
  1703. * thread doing COW.
  1704. */
  1705. ptep_clear_flush_notify(vma, address, page_table);
  1706. set_pte_at(mm, address, page_table, entry);
  1707. update_mmu_cache(vma, address, entry);
  1708. lru_cache_add_active(new_page);
  1709. page_add_new_anon_rmap(new_page, vma, address);
  1710. if (old_page) {
  1711. /*
  1712. * Only after switching the pte to the new page may
  1713. * we remove the mapcount here. Otherwise another
  1714. * process may come and find the rmap count decremented
  1715. * before the pte is switched to the new page, and
  1716. * "reuse" the old page writing into it while our pte
  1717. * here still points into it and can be read by other
  1718. * threads.
  1719. *
  1720. * The critical issue is to order this
  1721. * page_remove_rmap with the ptp_clear_flush above.
  1722. * Those stores are ordered by (if nothing else,)
  1723. * the barrier present in the atomic_add_negative
  1724. * in page_remove_rmap.
  1725. *
  1726. * Then the TLB flush in ptep_clear_flush ensures that
  1727. * no process can access the old page before the
  1728. * decremented mapcount is visible. And the old page
  1729. * cannot be reused until after the decremented
  1730. * mapcount is visible. So transitively, TLBs to
  1731. * old page will be flushed before it can be reused.
  1732. */
  1733. page_remove_rmap(old_page, vma);
  1734. }
  1735. /* Free the old page.. */
  1736. new_page = old_page;
  1737. ret |= VM_FAULT_WRITE;
  1738. } else
  1739. mem_cgroup_uncharge_page(new_page);
  1740. if (new_page)
  1741. page_cache_release(new_page);
  1742. if (old_page)
  1743. page_cache_release(old_page);
  1744. unlock:
  1745. pte_unmap_unlock(page_table, ptl);
  1746. if (dirty_page) {
  1747. if (vma->vm_file)
  1748. file_update_time(vma->vm_file);
  1749. /*
  1750. * Yes, Virginia, this is actually required to prevent a race
  1751. * with clear_page_dirty_for_io() from clearing the page dirty
  1752. * bit after it clear all dirty ptes, but before a racing
  1753. * do_wp_page installs a dirty pte.
  1754. *
  1755. * do_no_page is protected similarly.
  1756. */
  1757. wait_on_page_locked(dirty_page);
  1758. set_page_dirty_balance(dirty_page, page_mkwrite);
  1759. put_page(dirty_page);
  1760. }
  1761. return ret;
  1762. oom_free_new:
  1763. page_cache_release(new_page);
  1764. oom:
  1765. if (old_page)
  1766. page_cache_release(old_page);
  1767. return VM_FAULT_OOM;
  1768. unwritable_page:
  1769. page_cache_release(old_page);
  1770. return VM_FAULT_SIGBUS;
  1771. }
  1772. /*
  1773. * Helper functions for unmap_mapping_range().
  1774. *
  1775. * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __
  1776. *
  1777. * We have to restart searching the prio_tree whenever we drop the lock,
  1778. * since the iterator is only valid while the lock is held, and anyway
  1779. * a later vma might be split and reinserted earlier while lock dropped.
  1780. *
  1781. * The list of nonlinear vmas could be handled more efficiently, using
  1782. * a placeholder, but handle it in the same way until a need is shown.
  1783. * It is important to search the prio_tree before nonlinear list: a vma
  1784. * may become nonlinear and be shifted from prio_tree to nonlinear list
  1785. * while the lock is dropped; but never shifted from list to prio_tree.
  1786. *
  1787. * In order to make forward progress despite restarting the search,
  1788. * vm_truncate_count is used to mark a vma as now dealt with, so we can
  1789. * quickly skip it next time around. Since the prio_tree search only
  1790. * shows us those vmas affected by unmapping the range in question, we
  1791. * can't efficiently keep all vmas in step with mapping->truncate_count:
  1792. * so instead reset them all whenever it wraps back to 0 (then go to 1).
  1793. * mapping->truncate_count and vma->vm_truncate_count are protected by
  1794. * i_mmap_lock.
  1795. *
  1796. * In order to make forward progress despite repeatedly restarting some
  1797. * large vma, note the restart_addr from unmap_vmas when it breaks out:
  1798. * and restart from that address when we reach that vma again. It might
  1799. * have been split or merged, shrunk or extended, but never shifted: so
  1800. * restart_addr remains valid so long as it remains in the vma's range.
  1801. * unmap_mapping_range forces truncate_count to leap over page-aligned
  1802. * values so we can save vma's restart_addr in its truncate_count field.
  1803. */
  1804. #define is_restart_addr(truncate_count) (!((truncate_count) & ~PAGE_MASK))
  1805. static void reset_vma_truncate_counts(struct address_space *mapping)
  1806. {
  1807. struct vm_area_struct *vma;
  1808. struct prio_tree_iter iter;
  1809. vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX)
  1810. vma->vm_truncate_count = 0;
  1811. list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list)
  1812. vma->vm_truncate_count = 0;
  1813. }
  1814. static int unmap_mapping_range_vma(struct vm_area_struct *vma,
  1815. unsigned long start_addr, unsigned long end_addr,
  1816. struct zap_details *details)
  1817. {
  1818. unsigned long restart_addr;
  1819. int need_break;
  1820. /*
  1821. * files that support invalidating or truncating portions of the
  1822. * file from under mmaped areas must have their ->fault function
  1823. * return a locked page (and set VM_FAULT_LOCKED in the return).
  1824. * This provides synchronisation against concurrent unmapping here.
  1825. */
  1826. again:
  1827. restart_addr = vma->vm_truncate_count;
  1828. if (is_restart_addr(restart_addr) && start_addr < restart_addr) {
  1829. start_addr = restart_addr;
  1830. if (start_addr >= end_addr) {
  1831. /* Top of vma has been split off since last time */
  1832. vma->vm_truncate_count = details->truncate_count;
  1833. return 0;
  1834. }
  1835. }
  1836. restart_addr = zap_page_range(vma, start_addr,
  1837. end_addr - start_addr, details);
  1838. need_break = need_resched() || spin_needbreak(details->i_mmap_lock);
  1839. if (restart_addr >= end_addr) {
  1840. /* We have now completed this vma: mark it so */
  1841. vma->vm_truncate_count = details->truncate_count;
  1842. if (!need_break)
  1843. return 0;
  1844. } else {
  1845. /* Note restart_addr in vma's truncate_count field */
  1846. vma->vm_truncate_count = restart_addr;
  1847. if (!need_break)
  1848. goto again;
  1849. }
  1850. spin_unlock(details->i_mmap_lock);
  1851. cond_resched();
  1852. spin_lock(details->i_mmap_lock);
  1853. return -EINTR;
  1854. }
  1855. static inline void unmap_mapping_range_tree(struct prio_tree_root *root,
  1856. struct zap_details *details)
  1857. {
  1858. struct vm_area_struct *vma;
  1859. struct prio_tree_iter iter;
  1860. pgoff_t vba, vea, zba, zea;
  1861. restart:
  1862. vma_prio_tree_foreach(vma, &iter, root,
  1863. details->first_index, details->last_index) {
  1864. /* Skip quickly over those we have already dealt with */
  1865. if (vma->vm_truncate_count == details->truncate_count)
  1866. continue;
  1867. vba = vma->vm_pgoff;
  1868. vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1;
  1869. /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */
  1870. zba = details->first_index;
  1871. if (zba < vba)
  1872. zba = vba;
  1873. zea = details->last_index;
  1874. if (zea > vea)
  1875. zea = vea;
  1876. if (unmap_mapping_range_vma(vma,
  1877. ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
  1878. ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
  1879. details) < 0)
  1880. goto restart;
  1881. }
  1882. }
  1883. static inline void unmap_mapping_range_list(struct list_head *head,
  1884. struct zap_details *details)
  1885. {
  1886. struct vm_area_struct *vma;
  1887. /*
  1888. * In nonlinear VMAs there is no correspondence between virtual address
  1889. * offset and file offset. So we must perform an exhaustive search
  1890. * across *all* the pages in each nonlinear VMA, not just the pages
  1891. * whose virtual address lies outside the file truncation point.
  1892. */
  1893. restart:
  1894. list_for_each_entry(vma, head, shared.vm_set.list) {
  1895. /* Skip quickly over those we have already dealt with */
  1896. if (vma->vm_truncate_count == details->truncate_count)
  1897. continue;
  1898. details->nonlinear_vma = vma;
  1899. if (unmap_mapping_range_vma(vma, vma->vm_start,
  1900. vma->vm_end, details) < 0)
  1901. goto restart;
  1902. }
  1903. }
  1904. /**
  1905. * unmap_mapping_range - unmap the portion of all mmaps in the specified address_space corresponding to the specified page range in the underlying file.
  1906. * @mapping: the address space containing mmaps to be unmapped.
  1907. * @holebegin: byte in first page to unmap, relative to the start of
  1908. * the underlying file. This will be rounded down to a PAGE_SIZE
  1909. * boundary. Note that this is different from vmtruncate(), which
  1910. * must keep the partial page. In contrast, we must get rid of
  1911. * partial pages.
  1912. * @holelen: size of prospective hole in bytes. This will be rounded
  1913. * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
  1914. * end of the file.
  1915. * @even_cows: 1 when truncating a file, unmap even private COWed pages;
  1916. * but 0 when invalidating pagecache, don't throw away private data.
  1917. */
  1918. void unmap_mapping_range(struct address_space *mapping,
  1919. loff_t const holebegin, loff_t const holelen, int even_cows)
  1920. {
  1921. struct zap_details details;
  1922. pgoff_t hba = holebegin >> PAGE_SHIFT;
  1923. pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1924. /* Check for overflow. */
  1925. if (sizeof(holelen) > sizeof(hlen)) {
  1926. long long holeend =
  1927. (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1928. if (holeend & ~(long long)ULONG_MAX)
  1929. hlen = ULONG_MAX - hba + 1;
  1930. }
  1931. details.check_mapping = even_cows? NULL: mapping;
  1932. details.nonlinear_vma = NULL;
  1933. details.first_index = hba;
  1934. details.last_index = hba + hlen - 1;
  1935. if (details.last_index < details.first_index)
  1936. details.last_index = ULONG_MAX;
  1937. details.i_mmap_lock = &mapping->i_mmap_lock;
  1938. spin_lock(&mapping->i_mmap_lock);
  1939. /* Protect against endless unmapping loops */
  1940. mapping->truncate_count++;
  1941. if (unlikely(is_restart_addr(mapping->truncate_count))) {
  1942. if (mapping->truncate_count == 0)
  1943. reset_vma_truncate_counts(mapping);
  1944. mapping->truncate_count++;
  1945. }
  1946. details.truncate_count = mapping->truncate_count;
  1947. if (unlikely(!prio_tree_empty(&mapping->i_mmap)))
  1948. unmap_mapping_range_tree(&mapping->i_mmap, &details);
  1949. if (unlikely(!list_empty(&mapping->i_mmap_nonlinear)))
  1950. unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details);
  1951. spin_unlock(&mapping->i_mmap_lock);
  1952. }
  1953. EXPORT_SYMBOL(unmap_mapping_range);
  1954. /**
  1955. * vmtruncate - unmap mappings "freed" by truncate() syscall
  1956. * @inode: inode of the file used
  1957. * @offset: file offset to start truncating
  1958. *
  1959. * NOTE! We have to be ready to update the memory sharing
  1960. * between the file and the memory map for a potential last
  1961. * incomplete page. Ugly, but necessary.
  1962. */
  1963. int vmtruncate(struct inode * inode, loff_t offset)
  1964. {
  1965. if (inode->i_size < offset) {
  1966. unsigned long limit;
  1967. limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
  1968. if (limit != RLIM_INFINITY && offset > limit)
  1969. goto out_sig;
  1970. if (offset > inode->i_sb->s_maxbytes)
  1971. goto out_big;
  1972. i_size_write(inode, offset);
  1973. } else {
  1974. struct address_space *mapping = inode->i_mapping;
  1975. /*
  1976. * truncation of in-use swapfiles is disallowed - it would
  1977. * cause subsequent swapout to scribble on the now-freed
  1978. * blocks.
  1979. */
  1980. if (IS_SWAPFILE(inode))
  1981. return -ETXTBSY;
  1982. i_size_write(inode, offset);
  1983. /*
  1984. * unmap_mapping_range is called twice, first simply for
  1985. * efficiency so that truncate_inode_pages does fewer
  1986. * single-page unmaps. However after this first call, and
  1987. * before truncate_inode_pages finishes, it is possible for
  1988. * private pages to be COWed, which remain after
  1989. * truncate_inode_pages finishes, hence the second
  1990. * unmap_mapping_range call must be made for correctness.
  1991. */
  1992. unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1);
  1993. truncate_inode_pages(mapping, offset);
  1994. unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1);
  1995. }
  1996. if (inode->i_op && inode->i_op->truncate)
  1997. inode->i_op->truncate(inode);
  1998. return 0;
  1999. out_sig:
  2000. send_sig(SIGXFSZ, current, 0);
  2001. out_big:
  2002. return -EFBIG;
  2003. }
  2004. EXPORT_SYMBOL(vmtruncate);
  2005. int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end)
  2006. {
  2007. struct address_space *mapping = inode->i_mapping;
  2008. /*
  2009. * If the underlying filesystem is not going to provide
  2010. * a way to truncate a range of blocks (punch a hole) -
  2011. * we should return failure right now.
  2012. */
  2013. if (!inode->i_op || !inode->i_op->truncate_range)
  2014. return -ENOSYS;
  2015. mutex_lock(&inode->i_mutex);
  2016. down_write(&inode->i_alloc_sem);
  2017. unmap_mapping_range(mapping, offset, (end - offset), 1);
  2018. truncate_inode_pages_range(mapping, offset, end);
  2019. unmap_mapping_range(mapping, offset, (end - offset), 1);
  2020. inode->i_op->truncate_range(inode, offset, end);
  2021. up_write(&inode->i_alloc_sem);
  2022. mutex_unlock(&inode->i_mutex);
  2023. return 0;
  2024. }
  2025. /*
  2026. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2027. * but allow concurrent faults), and pte mapped but not yet locked.
  2028. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2029. */
  2030. static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
  2031. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2032. int write_access, pte_t orig_pte)
  2033. {
  2034. spinlock_t *ptl;
  2035. struct page *page;
  2036. swp_entry_t entry;
  2037. pte_t pte;
  2038. int ret = 0;
  2039. if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
  2040. goto out;
  2041. entry = pte_to_swp_entry(orig_pte);
  2042. if (is_migration_entry(entry)) {
  2043. migration_entry_wait(mm, pmd, address);
  2044. goto out;
  2045. }
  2046. delayacct_set_flag(DELAYACCT_PF_SWAPIN);
  2047. page = lookup_swap_cache(entry);
  2048. if (!page) {
  2049. grab_swap_token(); /* Contend for token _before_ read-in */
  2050. page = swapin_readahead(entry,
  2051. GFP_HIGHUSER_MOVABLE, vma, address);
  2052. if (!page) {
  2053. /*
  2054. * Back out if somebody else faulted in this pte
  2055. * while we released the pte lock.
  2056. */
  2057. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2058. if (likely(pte_same(*page_table, orig_pte)))
  2059. ret = VM_FAULT_OOM;
  2060. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2061. goto unlock;
  2062. }
  2063. /* Had to read the page from swap area: Major fault */
  2064. ret = VM_FAULT_MAJOR;
  2065. count_vm_event(PGMAJFAULT);
  2066. }
  2067. if (mem_cgroup_charge(page, mm, GFP_KERNEL)) {
  2068. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2069. ret = VM_FAULT_OOM;
  2070. goto out;
  2071. }
  2072. mark_page_accessed(page);
  2073. lock_page(page);
  2074. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2075. /*
  2076. * Back out if somebody else already faulted in this pte.
  2077. */
  2078. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2079. if (unlikely(!pte_same(*page_table, orig_pte)))
  2080. goto out_nomap;
  2081. if (unlikely(!PageUptodate(page))) {
  2082. ret = VM_FAULT_SIGBUS;
  2083. goto out_nomap;
  2084. }
  2085. /* The page isn't present yet, go ahead with the fault. */
  2086. inc_mm_counter(mm, anon_rss);
  2087. pte = mk_pte(page, vma->vm_page_prot);
  2088. if (write_access && can_share_swap_page(page)) {
  2089. pte = maybe_mkwrite(pte_mkdirty(pte), vma);
  2090. write_access = 0;
  2091. }
  2092. flush_icache_page(vma, page);
  2093. set_pte_at(mm, address, page_table, pte);
  2094. page_add_anon_rmap(page, vma, address);
  2095. swap_free(entry);
  2096. if (vm_swap_full())
  2097. remove_exclusive_swap_page(page);
  2098. unlock_page(page);
  2099. if (write_access) {
  2100. ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte);
  2101. if (ret & VM_FAULT_ERROR)
  2102. ret &= VM_FAULT_ERROR;
  2103. goto out;
  2104. }
  2105. /* No need to invalidate - it was non-present before */
  2106. update_mmu_cache(vma, address, pte);
  2107. unlock:
  2108. pte_unmap_unlock(page_table, ptl);
  2109. out:
  2110. return ret;
  2111. out_nomap:
  2112. mem_cgroup_uncharge_page(page);
  2113. pte_unmap_unlock(page_table, ptl);
  2114. unlock_page(page);
  2115. page_cache_release(page);
  2116. return ret;
  2117. }
  2118. /*
  2119. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2120. * but allow concurrent faults), and pte mapped but not yet locked.
  2121. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2122. */
  2123. static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
  2124. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2125. int write_access)
  2126. {
  2127. struct page *page;
  2128. spinlock_t *ptl;
  2129. pte_t entry;
  2130. /* Allocate our own private page. */
  2131. pte_unmap(page_table);
  2132. if (unlikely(anon_vma_prepare(vma)))
  2133. goto oom;
  2134. page = alloc_zeroed_user_highpage_movable(vma, address);
  2135. if (!page)
  2136. goto oom;
  2137. __SetPageUptodate(page);
  2138. if (mem_cgroup_charge(page, mm, GFP_KERNEL))
  2139. goto oom_free_page;
  2140. entry = mk_pte(page, vma->vm_page_prot);
  2141. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  2142. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2143. if (!pte_none(*page_table))
  2144. goto release;
  2145. inc_mm_counter(mm, anon_rss);
  2146. lru_cache_add_active(page);
  2147. page_add_new_anon_rmap(page, vma, address);
  2148. set_pte_at(mm, address, page_table, entry);
  2149. /* No need to invalidate - it was non-present before */
  2150. update_mmu_cache(vma, address, entry);
  2151. unlock:
  2152. pte_unmap_unlock(page_table, ptl);
  2153. return 0;
  2154. release:
  2155. mem_cgroup_uncharge_page(page);
  2156. page_cache_release(page);
  2157. goto unlock;
  2158. oom_free_page:
  2159. page_cache_release(page);
  2160. oom:
  2161. return VM_FAULT_OOM;
  2162. }
  2163. /*
  2164. * __do_fault() tries to create a new page mapping. It aggressively
  2165. * tries to share with existing pages, but makes a separate copy if
  2166. * the FAULT_FLAG_WRITE is set in the flags parameter in order to avoid
  2167. * the next page fault.
  2168. *
  2169. * As this is called only for pages that do not currently exist, we
  2170. * do not need to flush old virtual caches or the TLB.
  2171. *
  2172. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2173. * but allow concurrent faults), and pte neither mapped nor locked.
  2174. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2175. */
  2176. static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2177. unsigned long address, pmd_t *pmd,
  2178. pgoff_t pgoff, unsigned int flags, pte_t orig_pte)
  2179. {
  2180. pte_t *page_table;
  2181. spinlock_t *ptl;
  2182. struct page *page;
  2183. pte_t entry;
  2184. int anon = 0;
  2185. struct page *dirty_page = NULL;
  2186. struct vm_fault vmf;
  2187. int ret;
  2188. int page_mkwrite = 0;
  2189. vmf.virtual_address = (void __user *)(address & PAGE_MASK);
  2190. vmf.pgoff = pgoff;
  2191. vmf.flags = flags;
  2192. vmf.page = NULL;
  2193. ret = vma->vm_ops->fault(vma, &vmf);
  2194. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
  2195. return ret;
  2196. /*
  2197. * For consistency in subsequent calls, make the faulted page always
  2198. * locked.
  2199. */
  2200. if (unlikely(!(ret & VM_FAULT_LOCKED)))
  2201. lock_page(vmf.page);
  2202. else
  2203. VM_BUG_ON(!PageLocked(vmf.page));
  2204. /*
  2205. * Should we do an early C-O-W break?
  2206. */
  2207. page = vmf.page;
  2208. if (flags & FAULT_FLAG_WRITE) {
  2209. if (!(vma->vm_flags & VM_SHARED)) {
  2210. anon = 1;
  2211. if (unlikely(anon_vma_prepare(vma))) {
  2212. ret = VM_FAULT_OOM;
  2213. goto out;
  2214. }
  2215. page = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
  2216. vma, address);
  2217. if (!page) {
  2218. ret = VM_FAULT_OOM;
  2219. goto out;
  2220. }
  2221. copy_user_highpage(page, vmf.page, address, vma);
  2222. __SetPageUptodate(page);
  2223. } else {
  2224. /*
  2225. * If the page will be shareable, see if the backing
  2226. * address space wants to know that the page is about
  2227. * to become writable
  2228. */
  2229. if (vma->vm_ops->page_mkwrite) {
  2230. unlock_page(page);
  2231. if (vma->vm_ops->page_mkwrite(vma, page) < 0) {
  2232. ret = VM_FAULT_SIGBUS;
  2233. anon = 1; /* no anon but release vmf.page */
  2234. goto out_unlocked;
  2235. }
  2236. lock_page(page);
  2237. /*
  2238. * XXX: this is not quite right (racy vs
  2239. * invalidate) to unlock and relock the page
  2240. * like this, however a better fix requires
  2241. * reworking page_mkwrite locking API, which
  2242. * is better done later.
  2243. */
  2244. if (!page->mapping) {
  2245. ret = 0;
  2246. anon = 1; /* no anon but release vmf.page */
  2247. goto out;
  2248. }
  2249. page_mkwrite = 1;
  2250. }
  2251. }
  2252. }
  2253. if (mem_cgroup_charge(page, mm, GFP_KERNEL)) {
  2254. ret = VM_FAULT_OOM;
  2255. goto out;
  2256. }
  2257. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2258. /*
  2259. * This silly early PAGE_DIRTY setting removes a race
  2260. * due to the bad i386 page protection. But it's valid
  2261. * for other architectures too.
  2262. *
  2263. * Note that if write_access is true, we either now have
  2264. * an exclusive copy of the page, or this is a shared mapping,
  2265. * so we can make it writable and dirty to avoid having to
  2266. * handle that later.
  2267. */
  2268. /* Only go through if we didn't race with anybody else... */
  2269. if (likely(pte_same(*page_table, orig_pte))) {
  2270. flush_icache_page(vma, page);
  2271. entry = mk_pte(page, vma->vm_page_prot);
  2272. if (flags & FAULT_FLAG_WRITE)
  2273. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  2274. set_pte_at(mm, address, page_table, entry);
  2275. if (anon) {
  2276. inc_mm_counter(mm, anon_rss);
  2277. lru_cache_add_active(page);
  2278. page_add_new_anon_rmap(page, vma, address);
  2279. } else {
  2280. inc_mm_counter(mm, file_rss);
  2281. page_add_file_rmap(page);
  2282. if (flags & FAULT_FLAG_WRITE) {
  2283. dirty_page = page;
  2284. get_page(dirty_page);
  2285. }
  2286. }
  2287. /* no need to invalidate: a not-present page won't be cached */
  2288. update_mmu_cache(vma, address, entry);
  2289. } else {
  2290. mem_cgroup_uncharge_page(page);
  2291. if (anon)
  2292. page_cache_release(page);
  2293. else
  2294. anon = 1; /* no anon but release faulted_page */
  2295. }
  2296. pte_unmap_unlock(page_table, ptl);
  2297. out:
  2298. unlock_page(vmf.page);
  2299. out_unlocked:
  2300. if (anon)
  2301. page_cache_release(vmf.page);
  2302. else if (dirty_page) {
  2303. if (vma->vm_file)
  2304. file_update_time(vma->vm_file);
  2305. set_page_dirty_balance(dirty_page, page_mkwrite);
  2306. put_page(dirty_page);
  2307. }
  2308. return ret;
  2309. }
  2310. static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2311. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2312. int write_access, pte_t orig_pte)
  2313. {
  2314. pgoff_t pgoff = (((address & PAGE_MASK)
  2315. - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  2316. unsigned int flags = (write_access ? FAULT_FLAG_WRITE : 0);
  2317. pte_unmap(page_table);
  2318. return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte);
  2319. }
  2320. /*
  2321. * Fault of a previously existing named mapping. Repopulate the pte
  2322. * from the encoded file_pte if possible. This enables swappable
  2323. * nonlinear vmas.
  2324. *
  2325. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2326. * but allow concurrent faults), and pte mapped but not yet locked.
  2327. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2328. */
  2329. static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2330. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2331. int write_access, pte_t orig_pte)
  2332. {
  2333. unsigned int flags = FAULT_FLAG_NONLINEAR |
  2334. (write_access ? FAULT_FLAG_WRITE : 0);
  2335. pgoff_t pgoff;
  2336. if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
  2337. return 0;
  2338. if (unlikely(!(vma->vm_flags & VM_NONLINEAR) ||
  2339. !(vma->vm_flags & VM_CAN_NONLINEAR))) {
  2340. /*
  2341. * Page table corrupted: show pte and kill process.
  2342. */
  2343. print_bad_pte(vma, orig_pte, address);
  2344. return VM_FAULT_OOM;
  2345. }
  2346. pgoff = pte_to_pgoff(orig_pte);
  2347. return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte);
  2348. }
  2349. /*
  2350. * These routines also need to handle stuff like marking pages dirty
  2351. * and/or accessed for architectures that don't do it in hardware (most
  2352. * RISC architectures). The early dirtying is also good on the i386.
  2353. *
  2354. * There is also a hook called "update_mmu_cache()" that architectures
  2355. * with external mmu caches can use to update those (ie the Sparc or
  2356. * PowerPC hashed page tables that act as extended TLBs).
  2357. *
  2358. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2359. * but allow concurrent faults), and pte mapped but not yet locked.
  2360. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2361. */
  2362. static inline int handle_pte_fault(struct mm_struct *mm,
  2363. struct vm_area_struct *vma, unsigned long address,
  2364. pte_t *pte, pmd_t *pmd, int write_access)
  2365. {
  2366. pte_t entry;
  2367. spinlock_t *ptl;
  2368. entry = *pte;
  2369. if (!pte_present(entry)) {
  2370. if (pte_none(entry)) {
  2371. if (vma->vm_ops) {
  2372. if (likely(vma->vm_ops->fault))
  2373. return do_linear_fault(mm, vma, address,
  2374. pte, pmd, write_access, entry);
  2375. }
  2376. return do_anonymous_page(mm, vma, address,
  2377. pte, pmd, write_access);
  2378. }
  2379. if (pte_file(entry))
  2380. return do_nonlinear_fault(mm, vma, address,
  2381. pte, pmd, write_access, entry);
  2382. return do_swap_page(mm, vma, address,
  2383. pte, pmd, write_access, entry);
  2384. }
  2385. ptl = pte_lockptr(mm, pmd);
  2386. spin_lock(ptl);
  2387. if (unlikely(!pte_same(*pte, entry)))
  2388. goto unlock;
  2389. if (write_access) {
  2390. if (!pte_write(entry))
  2391. return do_wp_page(mm, vma, address,
  2392. pte, pmd, ptl, entry);
  2393. entry = pte_mkdirty(entry);
  2394. }
  2395. entry = pte_mkyoung(entry);
  2396. if (ptep_set_access_flags(vma, address, pte, entry, write_access)) {
  2397. update_mmu_cache(vma, address, entry);
  2398. } else {
  2399. /*
  2400. * This is needed only for protection faults but the arch code
  2401. * is not yet telling us if this is a protection fault or not.
  2402. * This still avoids useless tlb flushes for .text page faults
  2403. * with threads.
  2404. */
  2405. if (write_access)
  2406. flush_tlb_page(vma, address);
  2407. }
  2408. unlock:
  2409. pte_unmap_unlock(pte, ptl);
  2410. return 0;
  2411. }
  2412. /*
  2413. * By the time we get here, we already hold the mm semaphore
  2414. */
  2415. int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2416. unsigned long address, int write_access)
  2417. {
  2418. pgd_t *pgd;
  2419. pud_t *pud;
  2420. pmd_t *pmd;
  2421. pte_t *pte;
  2422. __set_current_state(TASK_RUNNING);
  2423. count_vm_event(PGFAULT);
  2424. if (unlikely(is_vm_hugetlb_page(vma)))
  2425. return hugetlb_fault(mm, vma, address, write_access);
  2426. pgd = pgd_offset(mm, address);
  2427. pud = pud_alloc(mm, pgd, address);
  2428. if (!pud)
  2429. return VM_FAULT_OOM;
  2430. pmd = pmd_alloc(mm, pud, address);
  2431. if (!pmd)
  2432. return VM_FAULT_OOM;
  2433. pte = pte_alloc_map(mm, pmd, address);
  2434. if (!pte)
  2435. return VM_FAULT_OOM;
  2436. return handle_pte_fault(mm, vma, address, pte, pmd, write_access);
  2437. }
  2438. #ifndef __PAGETABLE_PUD_FOLDED
  2439. /*
  2440. * Allocate page upper directory.
  2441. * We've already handled the fast-path in-line.
  2442. */
  2443. int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  2444. {
  2445. pud_t *new = pud_alloc_one(mm, address);
  2446. if (!new)
  2447. return -ENOMEM;
  2448. smp_wmb(); /* See comment in __pte_alloc */
  2449. spin_lock(&mm->page_table_lock);
  2450. if (pgd_present(*pgd)) /* Another has populated it */
  2451. pud_free(mm, new);
  2452. else
  2453. pgd_populate(mm, pgd, new);
  2454. spin_unlock(&mm->page_table_lock);
  2455. return 0;
  2456. }
  2457. #endif /* __PAGETABLE_PUD_FOLDED */
  2458. #ifndef __PAGETABLE_PMD_FOLDED
  2459. /*
  2460. * Allocate page middle directory.
  2461. * We've already handled the fast-path in-line.
  2462. */
  2463. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  2464. {
  2465. pmd_t *new = pmd_alloc_one(mm, address);
  2466. if (!new)
  2467. return -ENOMEM;
  2468. smp_wmb(); /* See comment in __pte_alloc */
  2469. spin_lock(&mm->page_table_lock);
  2470. #ifndef __ARCH_HAS_4LEVEL_HACK
  2471. if (pud_present(*pud)) /* Another has populated it */
  2472. pmd_free(mm, new);
  2473. else
  2474. pud_populate(mm, pud, new);
  2475. #else
  2476. if (pgd_present(*pud)) /* Another has populated it */
  2477. pmd_free(mm, new);
  2478. else
  2479. pgd_populate(mm, pud, new);
  2480. #endif /* __ARCH_HAS_4LEVEL_HACK */
  2481. spin_unlock(&mm->page_table_lock);
  2482. return 0;
  2483. }
  2484. #endif /* __PAGETABLE_PMD_FOLDED */
  2485. int make_pages_present(unsigned long addr, unsigned long end)
  2486. {
  2487. int ret, len, write;
  2488. struct vm_area_struct * vma;
  2489. vma = find_vma(current->mm, addr);
  2490. if (!vma)
  2491. return -1;
  2492. write = (vma->vm_flags & VM_WRITE) != 0;
  2493. BUG_ON(addr >= end);
  2494. BUG_ON(end > vma->vm_end);
  2495. len = DIV_ROUND_UP(end, PAGE_SIZE) - addr/PAGE_SIZE;
  2496. ret = get_user_pages(current, current->mm, addr,
  2497. len, write, 0, NULL, NULL);
  2498. if (ret < 0)
  2499. return ret;
  2500. return ret == len ? 0 : -1;
  2501. }
  2502. #if !defined(__HAVE_ARCH_GATE_AREA)
  2503. #if defined(AT_SYSINFO_EHDR)
  2504. static struct vm_area_struct gate_vma;
  2505. static int __init gate_vma_init(void)
  2506. {
  2507. gate_vma.vm_mm = NULL;
  2508. gate_vma.vm_start = FIXADDR_USER_START;
  2509. gate_vma.vm_end = FIXADDR_USER_END;
  2510. gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC;
  2511. gate_vma.vm_page_prot = __P101;
  2512. /*
  2513. * Make sure the vDSO gets into every core dump.
  2514. * Dumping its contents makes post-mortem fully interpretable later
  2515. * without matching up the same kernel and hardware config to see
  2516. * what PC values meant.
  2517. */
  2518. gate_vma.vm_flags |= VM_ALWAYSDUMP;
  2519. return 0;
  2520. }
  2521. __initcall(gate_vma_init);
  2522. #endif
  2523. struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
  2524. {
  2525. #ifdef AT_SYSINFO_EHDR
  2526. return &gate_vma;
  2527. #else
  2528. return NULL;
  2529. #endif
  2530. }
  2531. int in_gate_area_no_task(unsigned long addr)
  2532. {
  2533. #ifdef AT_SYSINFO_EHDR
  2534. if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END))
  2535. return 1;
  2536. #endif
  2537. return 0;
  2538. }
  2539. #endif /* __HAVE_ARCH_GATE_AREA */
  2540. #ifdef CONFIG_HAVE_IOREMAP_PROT
  2541. static resource_size_t follow_phys(struct vm_area_struct *vma,
  2542. unsigned long address, unsigned int flags,
  2543. unsigned long *prot)
  2544. {
  2545. pgd_t *pgd;
  2546. pud_t *pud;
  2547. pmd_t *pmd;
  2548. pte_t *ptep, pte;
  2549. spinlock_t *ptl;
  2550. resource_size_t phys_addr = 0;
  2551. struct mm_struct *mm = vma->vm_mm;
  2552. VM_BUG_ON(!(vma->vm_flags & (VM_IO | VM_PFNMAP)));
  2553. pgd = pgd_offset(mm, address);
  2554. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  2555. goto no_page_table;
  2556. pud = pud_offset(pgd, address);
  2557. if (pud_none(*pud) || unlikely(pud_bad(*pud)))
  2558. goto no_page_table;
  2559. pmd = pmd_offset(pud, address);
  2560. if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
  2561. goto no_page_table;
  2562. /* We cannot handle huge page PFN maps. Luckily they don't exist. */
  2563. if (pmd_huge(*pmd))
  2564. goto no_page_table;
  2565. ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
  2566. if (!ptep)
  2567. goto out;
  2568. pte = *ptep;
  2569. if (!pte_present(pte))
  2570. goto unlock;
  2571. if ((flags & FOLL_WRITE) && !pte_write(pte))
  2572. goto unlock;
  2573. phys_addr = pte_pfn(pte);
  2574. phys_addr <<= PAGE_SHIFT; /* Shift here to avoid overflow on PAE */
  2575. *prot = pgprot_val(pte_pgprot(pte));
  2576. unlock:
  2577. pte_unmap_unlock(ptep, ptl);
  2578. out:
  2579. return phys_addr;
  2580. no_page_table:
  2581. return 0;
  2582. }
  2583. int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
  2584. void *buf, int len, int write)
  2585. {
  2586. resource_size_t phys_addr;
  2587. unsigned long prot = 0;
  2588. void *maddr;
  2589. int offset = addr & (PAGE_SIZE-1);
  2590. if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
  2591. return -EINVAL;
  2592. phys_addr = follow_phys(vma, addr, write, &prot);
  2593. if (!phys_addr)
  2594. return -EINVAL;
  2595. maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot);
  2596. if (write)
  2597. memcpy_toio(maddr + offset, buf, len);
  2598. else
  2599. memcpy_fromio(buf, maddr + offset, len);
  2600. iounmap(maddr);
  2601. return len;
  2602. }
  2603. #endif
  2604. /*
  2605. * Access another process' address space.
  2606. * Source/target buffer must be kernel space,
  2607. * Do not walk the page table directly, use get_user_pages
  2608. */
  2609. int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
  2610. {
  2611. struct mm_struct *mm;
  2612. struct vm_area_struct *vma;
  2613. void *old_buf = buf;
  2614. mm = get_task_mm(tsk);
  2615. if (!mm)
  2616. return 0;
  2617. down_read(&mm->mmap_sem);
  2618. /* ignore errors, just check how much was successfully transferred */
  2619. while (len) {
  2620. int bytes, ret, offset;
  2621. void *maddr;
  2622. struct page *page = NULL;
  2623. ret = get_user_pages(tsk, mm, addr, 1,
  2624. write, 1, &page, &vma);
  2625. if (ret <= 0) {
  2626. /*
  2627. * Check if this is a VM_IO | VM_PFNMAP VMA, which
  2628. * we can access using slightly different code.
  2629. */
  2630. #ifdef CONFIG_HAVE_IOREMAP_PROT
  2631. vma = find_vma(mm, addr);
  2632. if (!vma)
  2633. break;
  2634. if (vma->vm_ops && vma->vm_ops->access)
  2635. ret = vma->vm_ops->access(vma, addr, buf,
  2636. len, write);
  2637. if (ret <= 0)
  2638. #endif
  2639. break;
  2640. bytes = ret;
  2641. } else {
  2642. bytes = len;
  2643. offset = addr & (PAGE_SIZE-1);
  2644. if (bytes > PAGE_SIZE-offset)
  2645. bytes = PAGE_SIZE-offset;
  2646. maddr = kmap(page);
  2647. if (write) {
  2648. copy_to_user_page(vma, page, addr,
  2649. maddr + offset, buf, bytes);
  2650. set_page_dirty_lock(page);
  2651. } else {
  2652. copy_from_user_page(vma, page, addr,
  2653. buf, maddr + offset, bytes);
  2654. }
  2655. kunmap(page);
  2656. page_cache_release(page);
  2657. }
  2658. len -= bytes;
  2659. buf += bytes;
  2660. addr += bytes;
  2661. }
  2662. up_read(&mm->mmap_sem);
  2663. mmput(mm);
  2664. return buf - old_buf;
  2665. }
  2666. /*
  2667. * Print the name of a VMA.
  2668. */
  2669. void print_vma_addr(char *prefix, unsigned long ip)
  2670. {
  2671. struct mm_struct *mm = current->mm;
  2672. struct vm_area_struct *vma;
  2673. /*
  2674. * Do not print if we are in atomic
  2675. * contexts (in exception stacks, etc.):
  2676. */
  2677. if (preempt_count())
  2678. return;
  2679. down_read(&mm->mmap_sem);
  2680. vma = find_vma(mm, ip);
  2681. if (vma && vma->vm_file) {
  2682. struct file *f = vma->vm_file;
  2683. char *buf = (char *)__get_free_page(GFP_KERNEL);
  2684. if (buf) {
  2685. char *p, *s;
  2686. p = d_path(&f->f_path, buf, PAGE_SIZE);
  2687. if (IS_ERR(p))
  2688. p = "?";
  2689. s = strrchr(p, '/');
  2690. if (s)
  2691. p = s+1;
  2692. printk("%s%s[%lx+%lx]", prefix, p,
  2693. vma->vm_start,
  2694. vma->vm_end - vma->vm_start);
  2695. free_page((unsigned long)buf);
  2696. }
  2697. }
  2698. up_read(&current->mm->mmap_sem);
  2699. }