memory.c 79 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962
  1. /*
  2. * linux/mm/memory.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. */
  6. /*
  7. * demand-loading started 01.12.91 - seems it is high on the list of
  8. * things wanted, and it should be easy to implement. - Linus
  9. */
  10. /*
  11. * Ok, demand-loading was easy, shared pages a little bit tricker. Shared
  12. * pages started 02.12.91, seems to work. - Linus.
  13. *
  14. * Tested sharing by executing about 30 /bin/sh: under the old kernel it
  15. * would have taken more than the 6M I have free, but it worked well as
  16. * far as I could see.
  17. *
  18. * Also corrected some "invalidate()"s - I wasn't doing enough of them.
  19. */
  20. /*
  21. * Real VM (paging to/from disk) started 18.12.91. Much more work and
  22. * thought has to go into this. Oh, well..
  23. * 19.12.91 - works, somewhat. Sometimes I get faults, don't know why.
  24. * Found it. Everything seems to work now.
  25. * 20.12.91 - Ok, making the swap-device changeable like the root.
  26. */
  27. /*
  28. * 05.04.94 - Multi-page memory management added for v1.1.
  29. * Idea by Alex Bligh (alex@cconcepts.co.uk)
  30. *
  31. * 16.07.99 - Support of BIGMEM added by Gerhard Wichert, Siemens AG
  32. * (Gerhard.Wichert@pdb.siemens.de)
  33. *
  34. * Aug/Sep 2004 Changed to four level page tables (Andi Kleen)
  35. */
  36. #include <linux/kernel_stat.h>
  37. #include <linux/mm.h>
  38. #include <linux/hugetlb.h>
  39. #include <linux/mman.h>
  40. #include <linux/swap.h>
  41. #include <linux/highmem.h>
  42. #include <linux/pagemap.h>
  43. #include <linux/rmap.h>
  44. #include <linux/module.h>
  45. #include <linux/delayacct.h>
  46. #include <linux/init.h>
  47. #include <linux/writeback.h>
  48. #include <linux/memcontrol.h>
  49. #include <asm/pgalloc.h>
  50. #include <asm/uaccess.h>
  51. #include <asm/tlb.h>
  52. #include <asm/tlbflush.h>
  53. #include <asm/pgtable.h>
  54. #include <linux/swapops.h>
  55. #include <linux/elf.h>
  56. #include "internal.h"
  57. #ifndef CONFIG_NEED_MULTIPLE_NODES
  58. /* use the per-pgdat data instead for discontigmem - mbligh */
  59. unsigned long max_mapnr;
  60. struct page *mem_map;
  61. EXPORT_SYMBOL(max_mapnr);
  62. EXPORT_SYMBOL(mem_map);
  63. #endif
  64. unsigned long num_physpages;
  65. /*
  66. * A number of key systems in x86 including ioremap() rely on the assumption
  67. * that high_memory defines the upper bound on direct map memory, then end
  68. * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
  69. * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
  70. * and ZONE_HIGHMEM.
  71. */
  72. void * high_memory;
  73. EXPORT_SYMBOL(num_physpages);
  74. EXPORT_SYMBOL(high_memory);
  75. /*
  76. * Randomize the address space (stacks, mmaps, brk, etc.).
  77. *
  78. * ( When CONFIG_COMPAT_BRK=y we exclude brk from randomization,
  79. * as ancient (libc5 based) binaries can segfault. )
  80. */
  81. int randomize_va_space __read_mostly =
  82. #ifdef CONFIG_COMPAT_BRK
  83. 1;
  84. #else
  85. 2;
  86. #endif
  87. static int __init disable_randmaps(char *s)
  88. {
  89. randomize_va_space = 0;
  90. return 1;
  91. }
  92. __setup("norandmaps", disable_randmaps);
  93. /*
  94. * If a p?d_bad entry is found while walking page tables, report
  95. * the error, before resetting entry to p?d_none. Usually (but
  96. * very seldom) called out from the p?d_none_or_clear_bad macros.
  97. */
  98. void pgd_clear_bad(pgd_t *pgd)
  99. {
  100. pgd_ERROR(*pgd);
  101. pgd_clear(pgd);
  102. }
  103. void pud_clear_bad(pud_t *pud)
  104. {
  105. pud_ERROR(*pud);
  106. pud_clear(pud);
  107. }
  108. void pmd_clear_bad(pmd_t *pmd)
  109. {
  110. pmd_ERROR(*pmd);
  111. pmd_clear(pmd);
  112. }
  113. /*
  114. * Note: this doesn't free the actual pages themselves. That
  115. * has been handled earlier when unmapping all the memory regions.
  116. */
  117. static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd)
  118. {
  119. pgtable_t token = pmd_pgtable(*pmd);
  120. pmd_clear(pmd);
  121. pte_free_tlb(tlb, token);
  122. tlb->mm->nr_ptes--;
  123. }
  124. static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
  125. unsigned long addr, unsigned long end,
  126. unsigned long floor, unsigned long ceiling)
  127. {
  128. pmd_t *pmd;
  129. unsigned long next;
  130. unsigned long start;
  131. start = addr;
  132. pmd = pmd_offset(pud, addr);
  133. do {
  134. next = pmd_addr_end(addr, end);
  135. if (pmd_none_or_clear_bad(pmd))
  136. continue;
  137. free_pte_range(tlb, pmd);
  138. } while (pmd++, addr = next, addr != end);
  139. start &= PUD_MASK;
  140. if (start < floor)
  141. return;
  142. if (ceiling) {
  143. ceiling &= PUD_MASK;
  144. if (!ceiling)
  145. return;
  146. }
  147. if (end - 1 > ceiling - 1)
  148. return;
  149. pmd = pmd_offset(pud, start);
  150. pud_clear(pud);
  151. pmd_free_tlb(tlb, pmd);
  152. }
  153. static inline void free_pud_range(struct mmu_gather *tlb, pgd_t *pgd,
  154. unsigned long addr, unsigned long end,
  155. unsigned long floor, unsigned long ceiling)
  156. {
  157. pud_t *pud;
  158. unsigned long next;
  159. unsigned long start;
  160. start = addr;
  161. pud = pud_offset(pgd, addr);
  162. do {
  163. next = pud_addr_end(addr, end);
  164. if (pud_none_or_clear_bad(pud))
  165. continue;
  166. free_pmd_range(tlb, pud, addr, next, floor, ceiling);
  167. } while (pud++, addr = next, addr != end);
  168. start &= PGDIR_MASK;
  169. if (start < floor)
  170. return;
  171. if (ceiling) {
  172. ceiling &= PGDIR_MASK;
  173. if (!ceiling)
  174. return;
  175. }
  176. if (end - 1 > ceiling - 1)
  177. return;
  178. pud = pud_offset(pgd, start);
  179. pgd_clear(pgd);
  180. pud_free_tlb(tlb, pud);
  181. }
  182. /*
  183. * This function frees user-level page tables of a process.
  184. *
  185. * Must be called with pagetable lock held.
  186. */
  187. void free_pgd_range(struct mmu_gather *tlb,
  188. unsigned long addr, unsigned long end,
  189. unsigned long floor, unsigned long ceiling)
  190. {
  191. pgd_t *pgd;
  192. unsigned long next;
  193. unsigned long start;
  194. /*
  195. * The next few lines have given us lots of grief...
  196. *
  197. * Why are we testing PMD* at this top level? Because often
  198. * there will be no work to do at all, and we'd prefer not to
  199. * go all the way down to the bottom just to discover that.
  200. *
  201. * Why all these "- 1"s? Because 0 represents both the bottom
  202. * of the address space and the top of it (using -1 for the
  203. * top wouldn't help much: the masks would do the wrong thing).
  204. * The rule is that addr 0 and floor 0 refer to the bottom of
  205. * the address space, but end 0 and ceiling 0 refer to the top
  206. * Comparisons need to use "end - 1" and "ceiling - 1" (though
  207. * that end 0 case should be mythical).
  208. *
  209. * Wherever addr is brought up or ceiling brought down, we must
  210. * be careful to reject "the opposite 0" before it confuses the
  211. * subsequent tests. But what about where end is brought down
  212. * by PMD_SIZE below? no, end can't go down to 0 there.
  213. *
  214. * Whereas we round start (addr) and ceiling down, by different
  215. * masks at different levels, in order to test whether a table
  216. * now has no other vmas using it, so can be freed, we don't
  217. * bother to round floor or end up - the tests don't need that.
  218. */
  219. addr &= PMD_MASK;
  220. if (addr < floor) {
  221. addr += PMD_SIZE;
  222. if (!addr)
  223. return;
  224. }
  225. if (ceiling) {
  226. ceiling &= PMD_MASK;
  227. if (!ceiling)
  228. return;
  229. }
  230. if (end - 1 > ceiling - 1)
  231. end -= PMD_SIZE;
  232. if (addr > end - 1)
  233. return;
  234. start = addr;
  235. pgd = pgd_offset(tlb->mm, addr);
  236. do {
  237. next = pgd_addr_end(addr, end);
  238. if (pgd_none_or_clear_bad(pgd))
  239. continue;
  240. free_pud_range(tlb, pgd, addr, next, floor, ceiling);
  241. } while (pgd++, addr = next, addr != end);
  242. }
  243. void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
  244. unsigned long floor, unsigned long ceiling)
  245. {
  246. while (vma) {
  247. struct vm_area_struct *next = vma->vm_next;
  248. unsigned long addr = vma->vm_start;
  249. /*
  250. * Hide vma from rmap and vmtruncate before freeing pgtables
  251. */
  252. anon_vma_unlink(vma);
  253. unlink_file_vma(vma);
  254. if (is_vm_hugetlb_page(vma)) {
  255. hugetlb_free_pgd_range(tlb, addr, vma->vm_end,
  256. floor, next? next->vm_start: ceiling);
  257. } else {
  258. /*
  259. * Optimization: gather nearby vmas into one call down
  260. */
  261. while (next && next->vm_start <= vma->vm_end + PMD_SIZE
  262. && !is_vm_hugetlb_page(next)) {
  263. vma = next;
  264. next = vma->vm_next;
  265. anon_vma_unlink(vma);
  266. unlink_file_vma(vma);
  267. }
  268. free_pgd_range(tlb, addr, vma->vm_end,
  269. floor, next? next->vm_start: ceiling);
  270. }
  271. vma = next;
  272. }
  273. }
  274. int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
  275. {
  276. pgtable_t new = pte_alloc_one(mm, address);
  277. if (!new)
  278. return -ENOMEM;
  279. /*
  280. * Ensure all pte setup (eg. pte page lock and page clearing) are
  281. * visible before the pte is made visible to other CPUs by being
  282. * put into page tables.
  283. *
  284. * The other side of the story is the pointer chasing in the page
  285. * table walking code (when walking the page table without locking;
  286. * ie. most of the time). Fortunately, these data accesses consist
  287. * of a chain of data-dependent loads, meaning most CPUs (alpha
  288. * being the notable exception) will already guarantee loads are
  289. * seen in-order. See the alpha page table accessors for the
  290. * smp_read_barrier_depends() barriers in page table walking code.
  291. */
  292. smp_wmb(); /* Could be smp_wmb__xxx(before|after)_spin_lock */
  293. spin_lock(&mm->page_table_lock);
  294. if (!pmd_present(*pmd)) { /* Has another populated it ? */
  295. mm->nr_ptes++;
  296. pmd_populate(mm, pmd, new);
  297. new = NULL;
  298. }
  299. spin_unlock(&mm->page_table_lock);
  300. if (new)
  301. pte_free(mm, new);
  302. return 0;
  303. }
  304. int __pte_alloc_kernel(pmd_t *pmd, unsigned long address)
  305. {
  306. pte_t *new = pte_alloc_one_kernel(&init_mm, address);
  307. if (!new)
  308. return -ENOMEM;
  309. smp_wmb(); /* See comment in __pte_alloc */
  310. spin_lock(&init_mm.page_table_lock);
  311. if (!pmd_present(*pmd)) { /* Has another populated it ? */
  312. pmd_populate_kernel(&init_mm, pmd, new);
  313. new = NULL;
  314. }
  315. spin_unlock(&init_mm.page_table_lock);
  316. if (new)
  317. pte_free_kernel(&init_mm, new);
  318. return 0;
  319. }
  320. static inline void add_mm_rss(struct mm_struct *mm, int file_rss, int anon_rss)
  321. {
  322. if (file_rss)
  323. add_mm_counter(mm, file_rss, file_rss);
  324. if (anon_rss)
  325. add_mm_counter(mm, anon_rss, anon_rss);
  326. }
  327. /*
  328. * This function is called to print an error when a bad pte
  329. * is found. For example, we might have a PFN-mapped pte in
  330. * a region that doesn't allow it.
  331. *
  332. * The calling function must still handle the error.
  333. */
  334. static void print_bad_pte(struct vm_area_struct *vma, pte_t pte,
  335. unsigned long vaddr)
  336. {
  337. printk(KERN_ERR "Bad pte = %08llx, process = %s, "
  338. "vm_flags = %lx, vaddr = %lx\n",
  339. (long long)pte_val(pte),
  340. (vma->vm_mm == current->mm ? current->comm : "???"),
  341. vma->vm_flags, vaddr);
  342. dump_stack();
  343. }
  344. static inline int is_cow_mapping(unsigned int flags)
  345. {
  346. return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
  347. }
  348. /*
  349. * vm_normal_page -- This function gets the "struct page" associated with a pte.
  350. *
  351. * "Special" mappings do not wish to be associated with a "struct page" (either
  352. * it doesn't exist, or it exists but they don't want to touch it). In this
  353. * case, NULL is returned here. "Normal" mappings do have a struct page.
  354. *
  355. * There are 2 broad cases. Firstly, an architecture may define a pte_special()
  356. * pte bit, in which case this function is trivial. Secondly, an architecture
  357. * may not have a spare pte bit, which requires a more complicated scheme,
  358. * described below.
  359. *
  360. * A raw VM_PFNMAP mapping (ie. one that is not COWed) is always considered a
  361. * special mapping (even if there are underlying and valid "struct pages").
  362. * COWed pages of a VM_PFNMAP are always normal.
  363. *
  364. * The way we recognize COWed pages within VM_PFNMAP mappings is through the
  365. * rules set up by "remap_pfn_range()": the vma will have the VM_PFNMAP bit
  366. * set, and the vm_pgoff will point to the first PFN mapped: thus every special
  367. * mapping will always honor the rule
  368. *
  369. * pfn_of_page == vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT)
  370. *
  371. * And for normal mappings this is false.
  372. *
  373. * This restricts such mappings to be a linear translation from virtual address
  374. * to pfn. To get around this restriction, we allow arbitrary mappings so long
  375. * as the vma is not a COW mapping; in that case, we know that all ptes are
  376. * special (because none can have been COWed).
  377. *
  378. *
  379. * In order to support COW of arbitrary special mappings, we have VM_MIXEDMAP.
  380. *
  381. * VM_MIXEDMAP mappings can likewise contain memory with or without "struct
  382. * page" backing, however the difference is that _all_ pages with a struct
  383. * page (that is, those where pfn_valid is true) are refcounted and considered
  384. * normal pages by the VM. The disadvantage is that pages are refcounted
  385. * (which can be slower and simply not an option for some PFNMAP users). The
  386. * advantage is that we don't have to follow the strict linearity rule of
  387. * PFNMAP mappings in order to support COWable mappings.
  388. *
  389. */
  390. #ifdef __HAVE_ARCH_PTE_SPECIAL
  391. # define HAVE_PTE_SPECIAL 1
  392. #else
  393. # define HAVE_PTE_SPECIAL 0
  394. #endif
  395. struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
  396. pte_t pte)
  397. {
  398. unsigned long pfn;
  399. if (HAVE_PTE_SPECIAL) {
  400. if (likely(!pte_special(pte))) {
  401. VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
  402. return pte_page(pte);
  403. }
  404. VM_BUG_ON(!(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)));
  405. return NULL;
  406. }
  407. /* !HAVE_PTE_SPECIAL case follows: */
  408. pfn = pte_pfn(pte);
  409. if (unlikely(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))) {
  410. if (vma->vm_flags & VM_MIXEDMAP) {
  411. if (!pfn_valid(pfn))
  412. return NULL;
  413. goto out;
  414. } else {
  415. unsigned long off;
  416. off = (addr - vma->vm_start) >> PAGE_SHIFT;
  417. if (pfn == vma->vm_pgoff + off)
  418. return NULL;
  419. if (!is_cow_mapping(vma->vm_flags))
  420. return NULL;
  421. }
  422. }
  423. VM_BUG_ON(!pfn_valid(pfn));
  424. /*
  425. * NOTE! We still have PageReserved() pages in the page tables.
  426. *
  427. * eg. VDSO mappings can cause them to exist.
  428. */
  429. out:
  430. return pfn_to_page(pfn);
  431. }
  432. /*
  433. * copy one vm_area from one task to the other. Assumes the page tables
  434. * already present in the new task to be cleared in the whole range
  435. * covered by this vma.
  436. */
  437. static inline void
  438. copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  439. pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
  440. unsigned long addr, int *rss)
  441. {
  442. unsigned long vm_flags = vma->vm_flags;
  443. pte_t pte = *src_pte;
  444. struct page *page;
  445. /* pte contains position in swap or file, so copy. */
  446. if (unlikely(!pte_present(pte))) {
  447. if (!pte_file(pte)) {
  448. swp_entry_t entry = pte_to_swp_entry(pte);
  449. swap_duplicate(entry);
  450. /* make sure dst_mm is on swapoff's mmlist. */
  451. if (unlikely(list_empty(&dst_mm->mmlist))) {
  452. spin_lock(&mmlist_lock);
  453. if (list_empty(&dst_mm->mmlist))
  454. list_add(&dst_mm->mmlist,
  455. &src_mm->mmlist);
  456. spin_unlock(&mmlist_lock);
  457. }
  458. if (is_write_migration_entry(entry) &&
  459. is_cow_mapping(vm_flags)) {
  460. /*
  461. * COW mappings require pages in both parent
  462. * and child to be set to read.
  463. */
  464. make_migration_entry_read(&entry);
  465. pte = swp_entry_to_pte(entry);
  466. set_pte_at(src_mm, addr, src_pte, pte);
  467. }
  468. }
  469. goto out_set_pte;
  470. }
  471. /*
  472. * If it's a COW mapping, write protect it both
  473. * in the parent and the child
  474. */
  475. if (is_cow_mapping(vm_flags)) {
  476. ptep_set_wrprotect(src_mm, addr, src_pte);
  477. pte = pte_wrprotect(pte);
  478. }
  479. /*
  480. * If it's a shared mapping, mark it clean in
  481. * the child
  482. */
  483. if (vm_flags & VM_SHARED)
  484. pte = pte_mkclean(pte);
  485. pte = pte_mkold(pte);
  486. page = vm_normal_page(vma, addr, pte);
  487. if (page) {
  488. get_page(page);
  489. page_dup_rmap(page, vma, addr);
  490. rss[!!PageAnon(page)]++;
  491. }
  492. out_set_pte:
  493. set_pte_at(dst_mm, addr, dst_pte, pte);
  494. }
  495. static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  496. pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma,
  497. unsigned long addr, unsigned long end)
  498. {
  499. pte_t *src_pte, *dst_pte;
  500. spinlock_t *src_ptl, *dst_ptl;
  501. int progress = 0;
  502. int rss[2];
  503. again:
  504. rss[1] = rss[0] = 0;
  505. dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
  506. if (!dst_pte)
  507. return -ENOMEM;
  508. src_pte = pte_offset_map_nested(src_pmd, addr);
  509. src_ptl = pte_lockptr(src_mm, src_pmd);
  510. spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
  511. arch_enter_lazy_mmu_mode();
  512. do {
  513. /*
  514. * We are holding two locks at this point - either of them
  515. * could generate latencies in another task on another CPU.
  516. */
  517. if (progress >= 32) {
  518. progress = 0;
  519. if (need_resched() ||
  520. spin_needbreak(src_ptl) || spin_needbreak(dst_ptl))
  521. break;
  522. }
  523. if (pte_none(*src_pte)) {
  524. progress++;
  525. continue;
  526. }
  527. copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss);
  528. progress += 8;
  529. } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
  530. arch_leave_lazy_mmu_mode();
  531. spin_unlock(src_ptl);
  532. pte_unmap_nested(src_pte - 1);
  533. add_mm_rss(dst_mm, rss[0], rss[1]);
  534. pte_unmap_unlock(dst_pte - 1, dst_ptl);
  535. cond_resched();
  536. if (addr != end)
  537. goto again;
  538. return 0;
  539. }
  540. static inline int copy_pmd_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  541. pud_t *dst_pud, pud_t *src_pud, struct vm_area_struct *vma,
  542. unsigned long addr, unsigned long end)
  543. {
  544. pmd_t *src_pmd, *dst_pmd;
  545. unsigned long next;
  546. dst_pmd = pmd_alloc(dst_mm, dst_pud, addr);
  547. if (!dst_pmd)
  548. return -ENOMEM;
  549. src_pmd = pmd_offset(src_pud, addr);
  550. do {
  551. next = pmd_addr_end(addr, end);
  552. if (pmd_none_or_clear_bad(src_pmd))
  553. continue;
  554. if (copy_pte_range(dst_mm, src_mm, dst_pmd, src_pmd,
  555. vma, addr, next))
  556. return -ENOMEM;
  557. } while (dst_pmd++, src_pmd++, addr = next, addr != end);
  558. return 0;
  559. }
  560. static inline int copy_pud_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  561. pgd_t *dst_pgd, pgd_t *src_pgd, struct vm_area_struct *vma,
  562. unsigned long addr, unsigned long end)
  563. {
  564. pud_t *src_pud, *dst_pud;
  565. unsigned long next;
  566. dst_pud = pud_alloc(dst_mm, dst_pgd, addr);
  567. if (!dst_pud)
  568. return -ENOMEM;
  569. src_pud = pud_offset(src_pgd, addr);
  570. do {
  571. next = pud_addr_end(addr, end);
  572. if (pud_none_or_clear_bad(src_pud))
  573. continue;
  574. if (copy_pmd_range(dst_mm, src_mm, dst_pud, src_pud,
  575. vma, addr, next))
  576. return -ENOMEM;
  577. } while (dst_pud++, src_pud++, addr = next, addr != end);
  578. return 0;
  579. }
  580. int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  581. struct vm_area_struct *vma)
  582. {
  583. pgd_t *src_pgd, *dst_pgd;
  584. unsigned long next;
  585. unsigned long addr = vma->vm_start;
  586. unsigned long end = vma->vm_end;
  587. /*
  588. * Don't copy ptes where a page fault will fill them correctly.
  589. * Fork becomes much lighter when there are big shared or private
  590. * readonly mappings. The tradeoff is that copy_page_range is more
  591. * efficient than faulting.
  592. */
  593. if (!(vma->vm_flags & (VM_HUGETLB|VM_NONLINEAR|VM_PFNMAP|VM_INSERTPAGE))) {
  594. if (!vma->anon_vma)
  595. return 0;
  596. }
  597. if (is_vm_hugetlb_page(vma))
  598. return copy_hugetlb_page_range(dst_mm, src_mm, vma);
  599. dst_pgd = pgd_offset(dst_mm, addr);
  600. src_pgd = pgd_offset(src_mm, addr);
  601. do {
  602. next = pgd_addr_end(addr, end);
  603. if (pgd_none_or_clear_bad(src_pgd))
  604. continue;
  605. if (copy_pud_range(dst_mm, src_mm, dst_pgd, src_pgd,
  606. vma, addr, next))
  607. return -ENOMEM;
  608. } while (dst_pgd++, src_pgd++, addr = next, addr != end);
  609. return 0;
  610. }
  611. static unsigned long zap_pte_range(struct mmu_gather *tlb,
  612. struct vm_area_struct *vma, pmd_t *pmd,
  613. unsigned long addr, unsigned long end,
  614. long *zap_work, struct zap_details *details)
  615. {
  616. struct mm_struct *mm = tlb->mm;
  617. pte_t *pte;
  618. spinlock_t *ptl;
  619. int file_rss = 0;
  620. int anon_rss = 0;
  621. pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
  622. arch_enter_lazy_mmu_mode();
  623. do {
  624. pte_t ptent = *pte;
  625. if (pte_none(ptent)) {
  626. (*zap_work)--;
  627. continue;
  628. }
  629. (*zap_work) -= PAGE_SIZE;
  630. if (pte_present(ptent)) {
  631. struct page *page;
  632. page = vm_normal_page(vma, addr, ptent);
  633. if (unlikely(details) && page) {
  634. /*
  635. * unmap_shared_mapping_pages() wants to
  636. * invalidate cache without truncating:
  637. * unmap shared but keep private pages.
  638. */
  639. if (details->check_mapping &&
  640. details->check_mapping != page->mapping)
  641. continue;
  642. /*
  643. * Each page->index must be checked when
  644. * invalidating or truncating nonlinear.
  645. */
  646. if (details->nonlinear_vma &&
  647. (page->index < details->first_index ||
  648. page->index > details->last_index))
  649. continue;
  650. }
  651. ptent = ptep_get_and_clear_full(mm, addr, pte,
  652. tlb->fullmm);
  653. tlb_remove_tlb_entry(tlb, pte, addr);
  654. if (unlikely(!page))
  655. continue;
  656. if (unlikely(details) && details->nonlinear_vma
  657. && linear_page_index(details->nonlinear_vma,
  658. addr) != page->index)
  659. set_pte_at(mm, addr, pte,
  660. pgoff_to_pte(page->index));
  661. if (PageAnon(page))
  662. anon_rss--;
  663. else {
  664. if (pte_dirty(ptent))
  665. set_page_dirty(page);
  666. if (pte_young(ptent))
  667. SetPageReferenced(page);
  668. file_rss--;
  669. }
  670. page_remove_rmap(page, vma);
  671. tlb_remove_page(tlb, page);
  672. continue;
  673. }
  674. /*
  675. * If details->check_mapping, we leave swap entries;
  676. * if details->nonlinear_vma, we leave file entries.
  677. */
  678. if (unlikely(details))
  679. continue;
  680. if (!pte_file(ptent))
  681. free_swap_and_cache(pte_to_swp_entry(ptent));
  682. pte_clear_not_present_full(mm, addr, pte, tlb->fullmm);
  683. } while (pte++, addr += PAGE_SIZE, (addr != end && *zap_work > 0));
  684. add_mm_rss(mm, file_rss, anon_rss);
  685. arch_leave_lazy_mmu_mode();
  686. pte_unmap_unlock(pte - 1, ptl);
  687. return addr;
  688. }
  689. static inline unsigned long zap_pmd_range(struct mmu_gather *tlb,
  690. struct vm_area_struct *vma, pud_t *pud,
  691. unsigned long addr, unsigned long end,
  692. long *zap_work, struct zap_details *details)
  693. {
  694. pmd_t *pmd;
  695. unsigned long next;
  696. pmd = pmd_offset(pud, addr);
  697. do {
  698. next = pmd_addr_end(addr, end);
  699. if (pmd_none_or_clear_bad(pmd)) {
  700. (*zap_work)--;
  701. continue;
  702. }
  703. next = zap_pte_range(tlb, vma, pmd, addr, next,
  704. zap_work, details);
  705. } while (pmd++, addr = next, (addr != end && *zap_work > 0));
  706. return addr;
  707. }
  708. static inline unsigned long zap_pud_range(struct mmu_gather *tlb,
  709. struct vm_area_struct *vma, pgd_t *pgd,
  710. unsigned long addr, unsigned long end,
  711. long *zap_work, struct zap_details *details)
  712. {
  713. pud_t *pud;
  714. unsigned long next;
  715. pud = pud_offset(pgd, addr);
  716. do {
  717. next = pud_addr_end(addr, end);
  718. if (pud_none_or_clear_bad(pud)) {
  719. (*zap_work)--;
  720. continue;
  721. }
  722. next = zap_pmd_range(tlb, vma, pud, addr, next,
  723. zap_work, details);
  724. } while (pud++, addr = next, (addr != end && *zap_work > 0));
  725. return addr;
  726. }
  727. static unsigned long unmap_page_range(struct mmu_gather *tlb,
  728. struct vm_area_struct *vma,
  729. unsigned long addr, unsigned long end,
  730. long *zap_work, struct zap_details *details)
  731. {
  732. pgd_t *pgd;
  733. unsigned long next;
  734. if (details && !details->check_mapping && !details->nonlinear_vma)
  735. details = NULL;
  736. BUG_ON(addr >= end);
  737. tlb_start_vma(tlb, vma);
  738. pgd = pgd_offset(vma->vm_mm, addr);
  739. do {
  740. next = pgd_addr_end(addr, end);
  741. if (pgd_none_or_clear_bad(pgd)) {
  742. (*zap_work)--;
  743. continue;
  744. }
  745. next = zap_pud_range(tlb, vma, pgd, addr, next,
  746. zap_work, details);
  747. } while (pgd++, addr = next, (addr != end && *zap_work > 0));
  748. tlb_end_vma(tlb, vma);
  749. return addr;
  750. }
  751. #ifdef CONFIG_PREEMPT
  752. # define ZAP_BLOCK_SIZE (8 * PAGE_SIZE)
  753. #else
  754. /* No preempt: go for improved straight-line efficiency */
  755. # define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE)
  756. #endif
  757. /**
  758. * unmap_vmas - unmap a range of memory covered by a list of vma's
  759. * @tlbp: address of the caller's struct mmu_gather
  760. * @vma: the starting vma
  761. * @start_addr: virtual address at which to start unmapping
  762. * @end_addr: virtual address at which to end unmapping
  763. * @nr_accounted: Place number of unmapped pages in vm-accountable vma's here
  764. * @details: details of nonlinear truncation or shared cache invalidation
  765. *
  766. * Returns the end address of the unmapping (restart addr if interrupted).
  767. *
  768. * Unmap all pages in the vma list.
  769. *
  770. * We aim to not hold locks for too long (for scheduling latency reasons).
  771. * So zap pages in ZAP_BLOCK_SIZE bytecounts. This means we need to
  772. * return the ending mmu_gather to the caller.
  773. *
  774. * Only addresses between `start' and `end' will be unmapped.
  775. *
  776. * The VMA list must be sorted in ascending virtual address order.
  777. *
  778. * unmap_vmas() assumes that the caller will flush the whole unmapped address
  779. * range after unmap_vmas() returns. So the only responsibility here is to
  780. * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
  781. * drops the lock and schedules.
  782. */
  783. unsigned long unmap_vmas(struct mmu_gather **tlbp,
  784. struct vm_area_struct *vma, unsigned long start_addr,
  785. unsigned long end_addr, unsigned long *nr_accounted,
  786. struct zap_details *details)
  787. {
  788. long zap_work = ZAP_BLOCK_SIZE;
  789. unsigned long tlb_start = 0; /* For tlb_finish_mmu */
  790. int tlb_start_valid = 0;
  791. unsigned long start = start_addr;
  792. spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
  793. int fullmm = (*tlbp)->fullmm;
  794. for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
  795. unsigned long end;
  796. start = max(vma->vm_start, start_addr);
  797. if (start >= vma->vm_end)
  798. continue;
  799. end = min(vma->vm_end, end_addr);
  800. if (end <= vma->vm_start)
  801. continue;
  802. if (vma->vm_flags & VM_ACCOUNT)
  803. *nr_accounted += (end - start) >> PAGE_SHIFT;
  804. while (start != end) {
  805. if (!tlb_start_valid) {
  806. tlb_start = start;
  807. tlb_start_valid = 1;
  808. }
  809. if (unlikely(is_vm_hugetlb_page(vma))) {
  810. /*
  811. * It is undesirable to test vma->vm_file as it
  812. * should be non-null for valid hugetlb area.
  813. * However, vm_file will be NULL in the error
  814. * cleanup path of do_mmap_pgoff. When
  815. * hugetlbfs ->mmap method fails,
  816. * do_mmap_pgoff() nullifies vma->vm_file
  817. * before calling this function to clean up.
  818. * Since no pte has actually been setup, it is
  819. * safe to do nothing in this case.
  820. */
  821. if (vma->vm_file) {
  822. unmap_hugepage_range(vma, start, end, NULL);
  823. zap_work -= (end - start) /
  824. pages_per_huge_page(hstate_vma(vma));
  825. }
  826. start = end;
  827. } else
  828. start = unmap_page_range(*tlbp, vma,
  829. start, end, &zap_work, details);
  830. if (zap_work > 0) {
  831. BUG_ON(start != end);
  832. break;
  833. }
  834. tlb_finish_mmu(*tlbp, tlb_start, start);
  835. if (need_resched() ||
  836. (i_mmap_lock && spin_needbreak(i_mmap_lock))) {
  837. if (i_mmap_lock) {
  838. *tlbp = NULL;
  839. goto out;
  840. }
  841. cond_resched();
  842. }
  843. *tlbp = tlb_gather_mmu(vma->vm_mm, fullmm);
  844. tlb_start_valid = 0;
  845. zap_work = ZAP_BLOCK_SIZE;
  846. }
  847. }
  848. out:
  849. return start; /* which is now the end (or restart) address */
  850. }
  851. /**
  852. * zap_page_range - remove user pages in a given range
  853. * @vma: vm_area_struct holding the applicable pages
  854. * @address: starting address of pages to zap
  855. * @size: number of bytes to zap
  856. * @details: details of nonlinear truncation or shared cache invalidation
  857. */
  858. unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
  859. unsigned long size, struct zap_details *details)
  860. {
  861. struct mm_struct *mm = vma->vm_mm;
  862. struct mmu_gather *tlb;
  863. unsigned long end = address + size;
  864. unsigned long nr_accounted = 0;
  865. lru_add_drain();
  866. tlb = tlb_gather_mmu(mm, 0);
  867. update_hiwater_rss(mm);
  868. end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
  869. if (tlb)
  870. tlb_finish_mmu(tlb, address, end);
  871. return end;
  872. }
  873. /*
  874. * Do a quick page-table lookup for a single page.
  875. */
  876. struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
  877. unsigned int flags)
  878. {
  879. pgd_t *pgd;
  880. pud_t *pud;
  881. pmd_t *pmd;
  882. pte_t *ptep, pte;
  883. spinlock_t *ptl;
  884. struct page *page;
  885. struct mm_struct *mm = vma->vm_mm;
  886. page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
  887. if (!IS_ERR(page)) {
  888. BUG_ON(flags & FOLL_GET);
  889. goto out;
  890. }
  891. page = NULL;
  892. pgd = pgd_offset(mm, address);
  893. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  894. goto no_page_table;
  895. pud = pud_offset(pgd, address);
  896. if (pud_none(*pud))
  897. goto no_page_table;
  898. if (pud_huge(*pud)) {
  899. BUG_ON(flags & FOLL_GET);
  900. page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE);
  901. goto out;
  902. }
  903. if (unlikely(pud_bad(*pud)))
  904. goto no_page_table;
  905. pmd = pmd_offset(pud, address);
  906. if (pmd_none(*pmd))
  907. goto no_page_table;
  908. if (pmd_huge(*pmd)) {
  909. BUG_ON(flags & FOLL_GET);
  910. page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
  911. goto out;
  912. }
  913. if (unlikely(pmd_bad(*pmd)))
  914. goto no_page_table;
  915. ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
  916. pte = *ptep;
  917. if (!pte_present(pte))
  918. goto no_page;
  919. if ((flags & FOLL_WRITE) && !pte_write(pte))
  920. goto unlock;
  921. page = vm_normal_page(vma, address, pte);
  922. if (unlikely(!page))
  923. goto bad_page;
  924. if (flags & FOLL_GET)
  925. get_page(page);
  926. if (flags & FOLL_TOUCH) {
  927. if ((flags & FOLL_WRITE) &&
  928. !pte_dirty(pte) && !PageDirty(page))
  929. set_page_dirty(page);
  930. mark_page_accessed(page);
  931. }
  932. unlock:
  933. pte_unmap_unlock(ptep, ptl);
  934. out:
  935. return page;
  936. bad_page:
  937. pte_unmap_unlock(ptep, ptl);
  938. return ERR_PTR(-EFAULT);
  939. no_page:
  940. pte_unmap_unlock(ptep, ptl);
  941. if (!pte_none(pte))
  942. return page;
  943. /* Fall through to ZERO_PAGE handling */
  944. no_page_table:
  945. /*
  946. * When core dumping an enormous anonymous area that nobody
  947. * has touched so far, we don't want to allocate page tables.
  948. */
  949. if (flags & FOLL_ANON) {
  950. page = ZERO_PAGE(0);
  951. if (flags & FOLL_GET)
  952. get_page(page);
  953. BUG_ON(flags & FOLL_WRITE);
  954. }
  955. return page;
  956. }
  957. /* Can we do the FOLL_ANON optimization? */
  958. static inline int use_zero_page(struct vm_area_struct *vma)
  959. {
  960. /*
  961. * We don't want to optimize FOLL_ANON for make_pages_present()
  962. * when it tries to page in a VM_LOCKED region. As to VM_SHARED,
  963. * we want to get the page from the page tables to make sure
  964. * that we serialize and update with any other user of that
  965. * mapping.
  966. */
  967. if (vma->vm_flags & (VM_LOCKED | VM_SHARED))
  968. return 0;
  969. /*
  970. * And if we have a fault routine, it's not an anonymous region.
  971. */
  972. return !vma->vm_ops || !vma->vm_ops->fault;
  973. }
  974. int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
  975. unsigned long start, int len, int write, int force,
  976. struct page **pages, struct vm_area_struct **vmas)
  977. {
  978. int i;
  979. unsigned int vm_flags;
  980. if (len <= 0)
  981. return 0;
  982. /*
  983. * Require read or write permissions.
  984. * If 'force' is set, we only require the "MAY" flags.
  985. */
  986. vm_flags = write ? (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
  987. vm_flags &= force ? (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
  988. i = 0;
  989. do {
  990. struct vm_area_struct *vma;
  991. unsigned int foll_flags;
  992. vma = find_extend_vma(mm, start);
  993. if (!vma && in_gate_area(tsk, start)) {
  994. unsigned long pg = start & PAGE_MASK;
  995. struct vm_area_struct *gate_vma = get_gate_vma(tsk);
  996. pgd_t *pgd;
  997. pud_t *pud;
  998. pmd_t *pmd;
  999. pte_t *pte;
  1000. if (write) /* user gate pages are read-only */
  1001. return i ? : -EFAULT;
  1002. if (pg > TASK_SIZE)
  1003. pgd = pgd_offset_k(pg);
  1004. else
  1005. pgd = pgd_offset_gate(mm, pg);
  1006. BUG_ON(pgd_none(*pgd));
  1007. pud = pud_offset(pgd, pg);
  1008. BUG_ON(pud_none(*pud));
  1009. pmd = pmd_offset(pud, pg);
  1010. if (pmd_none(*pmd))
  1011. return i ? : -EFAULT;
  1012. pte = pte_offset_map(pmd, pg);
  1013. if (pte_none(*pte)) {
  1014. pte_unmap(pte);
  1015. return i ? : -EFAULT;
  1016. }
  1017. if (pages) {
  1018. struct page *page = vm_normal_page(gate_vma, start, *pte);
  1019. pages[i] = page;
  1020. if (page)
  1021. get_page(page);
  1022. }
  1023. pte_unmap(pte);
  1024. if (vmas)
  1025. vmas[i] = gate_vma;
  1026. i++;
  1027. start += PAGE_SIZE;
  1028. len--;
  1029. continue;
  1030. }
  1031. if (!vma || (vma->vm_flags & (VM_IO | VM_PFNMAP))
  1032. || !(vm_flags & vma->vm_flags))
  1033. return i ? : -EFAULT;
  1034. if (is_vm_hugetlb_page(vma)) {
  1035. i = follow_hugetlb_page(mm, vma, pages, vmas,
  1036. &start, &len, i, write);
  1037. continue;
  1038. }
  1039. foll_flags = FOLL_TOUCH;
  1040. if (pages)
  1041. foll_flags |= FOLL_GET;
  1042. if (!write && use_zero_page(vma))
  1043. foll_flags |= FOLL_ANON;
  1044. do {
  1045. struct page *page;
  1046. /*
  1047. * If tsk is ooming, cut off its access to large memory
  1048. * allocations. It has a pending SIGKILL, but it can't
  1049. * be processed until returning to user space.
  1050. */
  1051. if (unlikely(test_tsk_thread_flag(tsk, TIF_MEMDIE)))
  1052. return i ? i : -ENOMEM;
  1053. if (write)
  1054. foll_flags |= FOLL_WRITE;
  1055. cond_resched();
  1056. while (!(page = follow_page(vma, start, foll_flags))) {
  1057. int ret;
  1058. ret = handle_mm_fault(mm, vma, start,
  1059. foll_flags & FOLL_WRITE);
  1060. if (ret & VM_FAULT_ERROR) {
  1061. if (ret & VM_FAULT_OOM)
  1062. return i ? i : -ENOMEM;
  1063. else if (ret & VM_FAULT_SIGBUS)
  1064. return i ? i : -EFAULT;
  1065. BUG();
  1066. }
  1067. if (ret & VM_FAULT_MAJOR)
  1068. tsk->maj_flt++;
  1069. else
  1070. tsk->min_flt++;
  1071. /*
  1072. * The VM_FAULT_WRITE bit tells us that
  1073. * do_wp_page has broken COW when necessary,
  1074. * even if maybe_mkwrite decided not to set
  1075. * pte_write. We can thus safely do subsequent
  1076. * page lookups as if they were reads.
  1077. */
  1078. if (ret & VM_FAULT_WRITE)
  1079. foll_flags &= ~FOLL_WRITE;
  1080. cond_resched();
  1081. }
  1082. if (IS_ERR(page))
  1083. return i ? i : PTR_ERR(page);
  1084. if (pages) {
  1085. pages[i] = page;
  1086. flush_anon_page(vma, page, start);
  1087. flush_dcache_page(page);
  1088. }
  1089. if (vmas)
  1090. vmas[i] = vma;
  1091. i++;
  1092. start += PAGE_SIZE;
  1093. len--;
  1094. } while (len && start < vma->vm_end);
  1095. } while (len);
  1096. return i;
  1097. }
  1098. EXPORT_SYMBOL(get_user_pages);
  1099. pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
  1100. spinlock_t **ptl)
  1101. {
  1102. pgd_t * pgd = pgd_offset(mm, addr);
  1103. pud_t * pud = pud_alloc(mm, pgd, addr);
  1104. if (pud) {
  1105. pmd_t * pmd = pmd_alloc(mm, pud, addr);
  1106. if (pmd)
  1107. return pte_alloc_map_lock(mm, pmd, addr, ptl);
  1108. }
  1109. return NULL;
  1110. }
  1111. /*
  1112. * This is the old fallback for page remapping.
  1113. *
  1114. * For historical reasons, it only allows reserved pages. Only
  1115. * old drivers should use this, and they needed to mark their
  1116. * pages reserved for the old functions anyway.
  1117. */
  1118. static int insert_page(struct vm_area_struct *vma, unsigned long addr,
  1119. struct page *page, pgprot_t prot)
  1120. {
  1121. struct mm_struct *mm = vma->vm_mm;
  1122. int retval;
  1123. pte_t *pte;
  1124. spinlock_t *ptl;
  1125. retval = mem_cgroup_charge(page, mm, GFP_KERNEL);
  1126. if (retval)
  1127. goto out;
  1128. retval = -EINVAL;
  1129. if (PageAnon(page))
  1130. goto out_uncharge;
  1131. retval = -ENOMEM;
  1132. flush_dcache_page(page);
  1133. pte = get_locked_pte(mm, addr, &ptl);
  1134. if (!pte)
  1135. goto out_uncharge;
  1136. retval = -EBUSY;
  1137. if (!pte_none(*pte))
  1138. goto out_unlock;
  1139. /* Ok, finally just insert the thing.. */
  1140. get_page(page);
  1141. inc_mm_counter(mm, file_rss);
  1142. page_add_file_rmap(page);
  1143. set_pte_at(mm, addr, pte, mk_pte(page, prot));
  1144. retval = 0;
  1145. pte_unmap_unlock(pte, ptl);
  1146. return retval;
  1147. out_unlock:
  1148. pte_unmap_unlock(pte, ptl);
  1149. out_uncharge:
  1150. mem_cgroup_uncharge_page(page);
  1151. out:
  1152. return retval;
  1153. }
  1154. /**
  1155. * vm_insert_page - insert single page into user vma
  1156. * @vma: user vma to map to
  1157. * @addr: target user address of this page
  1158. * @page: source kernel page
  1159. *
  1160. * This allows drivers to insert individual pages they've allocated
  1161. * into a user vma.
  1162. *
  1163. * The page has to be a nice clean _individual_ kernel allocation.
  1164. * If you allocate a compound page, you need to have marked it as
  1165. * such (__GFP_COMP), or manually just split the page up yourself
  1166. * (see split_page()).
  1167. *
  1168. * NOTE! Traditionally this was done with "remap_pfn_range()" which
  1169. * took an arbitrary page protection parameter. This doesn't allow
  1170. * that. Your vma protection will have to be set up correctly, which
  1171. * means that if you want a shared writable mapping, you'd better
  1172. * ask for a shared writable mapping!
  1173. *
  1174. * The page does not need to be reserved.
  1175. */
  1176. int vm_insert_page(struct vm_area_struct *vma, unsigned long addr,
  1177. struct page *page)
  1178. {
  1179. if (addr < vma->vm_start || addr >= vma->vm_end)
  1180. return -EFAULT;
  1181. if (!page_count(page))
  1182. return -EINVAL;
  1183. vma->vm_flags |= VM_INSERTPAGE;
  1184. return insert_page(vma, addr, page, vma->vm_page_prot);
  1185. }
  1186. EXPORT_SYMBOL(vm_insert_page);
  1187. static int insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1188. unsigned long pfn, pgprot_t prot)
  1189. {
  1190. struct mm_struct *mm = vma->vm_mm;
  1191. int retval;
  1192. pte_t *pte, entry;
  1193. spinlock_t *ptl;
  1194. retval = -ENOMEM;
  1195. pte = get_locked_pte(mm, addr, &ptl);
  1196. if (!pte)
  1197. goto out;
  1198. retval = -EBUSY;
  1199. if (!pte_none(*pte))
  1200. goto out_unlock;
  1201. /* Ok, finally just insert the thing.. */
  1202. entry = pte_mkspecial(pfn_pte(pfn, prot));
  1203. set_pte_at(mm, addr, pte, entry);
  1204. update_mmu_cache(vma, addr, entry); /* XXX: why not for insert_page? */
  1205. retval = 0;
  1206. out_unlock:
  1207. pte_unmap_unlock(pte, ptl);
  1208. out:
  1209. return retval;
  1210. }
  1211. /**
  1212. * vm_insert_pfn - insert single pfn into user vma
  1213. * @vma: user vma to map to
  1214. * @addr: target user address of this page
  1215. * @pfn: source kernel pfn
  1216. *
  1217. * Similar to vm_inert_page, this allows drivers to insert individual pages
  1218. * they've allocated into a user vma. Same comments apply.
  1219. *
  1220. * This function should only be called from a vm_ops->fault handler, and
  1221. * in that case the handler should return NULL.
  1222. *
  1223. * vma cannot be a COW mapping.
  1224. *
  1225. * As this is called only for pages that do not currently exist, we
  1226. * do not need to flush old virtual caches or the TLB.
  1227. */
  1228. int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
  1229. unsigned long pfn)
  1230. {
  1231. /*
  1232. * Technically, architectures with pte_special can avoid all these
  1233. * restrictions (same for remap_pfn_range). However we would like
  1234. * consistency in testing and feature parity among all, so we should
  1235. * try to keep these invariants in place for everybody.
  1236. */
  1237. BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
  1238. BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
  1239. (VM_PFNMAP|VM_MIXEDMAP));
  1240. BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
  1241. BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
  1242. if (addr < vma->vm_start || addr >= vma->vm_end)
  1243. return -EFAULT;
  1244. return insert_pfn(vma, addr, pfn, vma->vm_page_prot);
  1245. }
  1246. EXPORT_SYMBOL(vm_insert_pfn);
  1247. int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
  1248. unsigned long pfn)
  1249. {
  1250. BUG_ON(!(vma->vm_flags & VM_MIXEDMAP));
  1251. if (addr < vma->vm_start || addr >= vma->vm_end)
  1252. return -EFAULT;
  1253. /*
  1254. * If we don't have pte special, then we have to use the pfn_valid()
  1255. * based VM_MIXEDMAP scheme (see vm_normal_page), and thus we *must*
  1256. * refcount the page if pfn_valid is true (hence insert_page rather
  1257. * than insert_pfn).
  1258. */
  1259. if (!HAVE_PTE_SPECIAL && pfn_valid(pfn)) {
  1260. struct page *page;
  1261. page = pfn_to_page(pfn);
  1262. return insert_page(vma, addr, page, vma->vm_page_prot);
  1263. }
  1264. return insert_pfn(vma, addr, pfn, vma->vm_page_prot);
  1265. }
  1266. EXPORT_SYMBOL(vm_insert_mixed);
  1267. /*
  1268. * maps a range of physical memory into the requested pages. the old
  1269. * mappings are removed. any references to nonexistent pages results
  1270. * in null mappings (currently treated as "copy-on-access")
  1271. */
  1272. static int remap_pte_range(struct mm_struct *mm, pmd_t *pmd,
  1273. unsigned long addr, unsigned long end,
  1274. unsigned long pfn, pgprot_t prot)
  1275. {
  1276. pte_t *pte;
  1277. spinlock_t *ptl;
  1278. pte = pte_alloc_map_lock(mm, pmd, addr, &ptl);
  1279. if (!pte)
  1280. return -ENOMEM;
  1281. arch_enter_lazy_mmu_mode();
  1282. do {
  1283. BUG_ON(!pte_none(*pte));
  1284. set_pte_at(mm, addr, pte, pte_mkspecial(pfn_pte(pfn, prot)));
  1285. pfn++;
  1286. } while (pte++, addr += PAGE_SIZE, addr != end);
  1287. arch_leave_lazy_mmu_mode();
  1288. pte_unmap_unlock(pte - 1, ptl);
  1289. return 0;
  1290. }
  1291. static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud,
  1292. unsigned long addr, unsigned long end,
  1293. unsigned long pfn, pgprot_t prot)
  1294. {
  1295. pmd_t *pmd;
  1296. unsigned long next;
  1297. pfn -= addr >> PAGE_SHIFT;
  1298. pmd = pmd_alloc(mm, pud, addr);
  1299. if (!pmd)
  1300. return -ENOMEM;
  1301. do {
  1302. next = pmd_addr_end(addr, end);
  1303. if (remap_pte_range(mm, pmd, addr, next,
  1304. pfn + (addr >> PAGE_SHIFT), prot))
  1305. return -ENOMEM;
  1306. } while (pmd++, addr = next, addr != end);
  1307. return 0;
  1308. }
  1309. static inline int remap_pud_range(struct mm_struct *mm, pgd_t *pgd,
  1310. unsigned long addr, unsigned long end,
  1311. unsigned long pfn, pgprot_t prot)
  1312. {
  1313. pud_t *pud;
  1314. unsigned long next;
  1315. pfn -= addr >> PAGE_SHIFT;
  1316. pud = pud_alloc(mm, pgd, addr);
  1317. if (!pud)
  1318. return -ENOMEM;
  1319. do {
  1320. next = pud_addr_end(addr, end);
  1321. if (remap_pmd_range(mm, pud, addr, next,
  1322. pfn + (addr >> PAGE_SHIFT), prot))
  1323. return -ENOMEM;
  1324. } while (pud++, addr = next, addr != end);
  1325. return 0;
  1326. }
  1327. /**
  1328. * remap_pfn_range - remap kernel memory to userspace
  1329. * @vma: user vma to map to
  1330. * @addr: target user address to start at
  1331. * @pfn: physical address of kernel memory
  1332. * @size: size of map area
  1333. * @prot: page protection flags for this mapping
  1334. *
  1335. * Note: this is only safe if the mm semaphore is held when called.
  1336. */
  1337. int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
  1338. unsigned long pfn, unsigned long size, pgprot_t prot)
  1339. {
  1340. pgd_t *pgd;
  1341. unsigned long next;
  1342. unsigned long end = addr + PAGE_ALIGN(size);
  1343. struct mm_struct *mm = vma->vm_mm;
  1344. int err;
  1345. /*
  1346. * Physically remapped pages are special. Tell the
  1347. * rest of the world about it:
  1348. * VM_IO tells people not to look at these pages
  1349. * (accesses can have side effects).
  1350. * VM_RESERVED is specified all over the place, because
  1351. * in 2.4 it kept swapout's vma scan off this vma; but
  1352. * in 2.6 the LRU scan won't even find its pages, so this
  1353. * flag means no more than count its pages in reserved_vm,
  1354. * and omit it from core dump, even when VM_IO turned off.
  1355. * VM_PFNMAP tells the core MM that the base pages are just
  1356. * raw PFN mappings, and do not have a "struct page" associated
  1357. * with them.
  1358. *
  1359. * There's a horrible special case to handle copy-on-write
  1360. * behaviour that some programs depend on. We mark the "original"
  1361. * un-COW'ed pages by matching them up with "vma->vm_pgoff".
  1362. */
  1363. if (is_cow_mapping(vma->vm_flags)) {
  1364. if (addr != vma->vm_start || end != vma->vm_end)
  1365. return -EINVAL;
  1366. vma->vm_pgoff = pfn;
  1367. }
  1368. vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP;
  1369. BUG_ON(addr >= end);
  1370. pfn -= addr >> PAGE_SHIFT;
  1371. pgd = pgd_offset(mm, addr);
  1372. flush_cache_range(vma, addr, end);
  1373. do {
  1374. next = pgd_addr_end(addr, end);
  1375. err = remap_pud_range(mm, pgd, addr, next,
  1376. pfn + (addr >> PAGE_SHIFT), prot);
  1377. if (err)
  1378. break;
  1379. } while (pgd++, addr = next, addr != end);
  1380. return err;
  1381. }
  1382. EXPORT_SYMBOL(remap_pfn_range);
  1383. static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd,
  1384. unsigned long addr, unsigned long end,
  1385. pte_fn_t fn, void *data)
  1386. {
  1387. pte_t *pte;
  1388. int err;
  1389. pgtable_t token;
  1390. spinlock_t *uninitialized_var(ptl);
  1391. pte = (mm == &init_mm) ?
  1392. pte_alloc_kernel(pmd, addr) :
  1393. pte_alloc_map_lock(mm, pmd, addr, &ptl);
  1394. if (!pte)
  1395. return -ENOMEM;
  1396. BUG_ON(pmd_huge(*pmd));
  1397. token = pmd_pgtable(*pmd);
  1398. do {
  1399. err = fn(pte, token, addr, data);
  1400. if (err)
  1401. break;
  1402. } while (pte++, addr += PAGE_SIZE, addr != end);
  1403. if (mm != &init_mm)
  1404. pte_unmap_unlock(pte-1, ptl);
  1405. return err;
  1406. }
  1407. static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
  1408. unsigned long addr, unsigned long end,
  1409. pte_fn_t fn, void *data)
  1410. {
  1411. pmd_t *pmd;
  1412. unsigned long next;
  1413. int err;
  1414. BUG_ON(pud_huge(*pud));
  1415. pmd = pmd_alloc(mm, pud, addr);
  1416. if (!pmd)
  1417. return -ENOMEM;
  1418. do {
  1419. next = pmd_addr_end(addr, end);
  1420. err = apply_to_pte_range(mm, pmd, addr, next, fn, data);
  1421. if (err)
  1422. break;
  1423. } while (pmd++, addr = next, addr != end);
  1424. return err;
  1425. }
  1426. static int apply_to_pud_range(struct mm_struct *mm, pgd_t *pgd,
  1427. unsigned long addr, unsigned long end,
  1428. pte_fn_t fn, void *data)
  1429. {
  1430. pud_t *pud;
  1431. unsigned long next;
  1432. int err;
  1433. pud = pud_alloc(mm, pgd, addr);
  1434. if (!pud)
  1435. return -ENOMEM;
  1436. do {
  1437. next = pud_addr_end(addr, end);
  1438. err = apply_to_pmd_range(mm, pud, addr, next, fn, data);
  1439. if (err)
  1440. break;
  1441. } while (pud++, addr = next, addr != end);
  1442. return err;
  1443. }
  1444. /*
  1445. * Scan a region of virtual memory, filling in page tables as necessary
  1446. * and calling a provided function on each leaf page table.
  1447. */
  1448. int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
  1449. unsigned long size, pte_fn_t fn, void *data)
  1450. {
  1451. pgd_t *pgd;
  1452. unsigned long next;
  1453. unsigned long end = addr + size;
  1454. int err;
  1455. BUG_ON(addr >= end);
  1456. pgd = pgd_offset(mm, addr);
  1457. do {
  1458. next = pgd_addr_end(addr, end);
  1459. err = apply_to_pud_range(mm, pgd, addr, next, fn, data);
  1460. if (err)
  1461. break;
  1462. } while (pgd++, addr = next, addr != end);
  1463. return err;
  1464. }
  1465. EXPORT_SYMBOL_GPL(apply_to_page_range);
  1466. /*
  1467. * handle_pte_fault chooses page fault handler according to an entry
  1468. * which was read non-atomically. Before making any commitment, on
  1469. * those architectures or configurations (e.g. i386 with PAE) which
  1470. * might give a mix of unmatched parts, do_swap_page and do_file_page
  1471. * must check under lock before unmapping the pte and proceeding
  1472. * (but do_wp_page is only called after already making such a check;
  1473. * and do_anonymous_page and do_no_page can safely check later on).
  1474. */
  1475. static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
  1476. pte_t *page_table, pte_t orig_pte)
  1477. {
  1478. int same = 1;
  1479. #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT)
  1480. if (sizeof(pte_t) > sizeof(unsigned long)) {
  1481. spinlock_t *ptl = pte_lockptr(mm, pmd);
  1482. spin_lock(ptl);
  1483. same = pte_same(*page_table, orig_pte);
  1484. spin_unlock(ptl);
  1485. }
  1486. #endif
  1487. pte_unmap(page_table);
  1488. return same;
  1489. }
  1490. /*
  1491. * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
  1492. * servicing faults for write access. In the normal case, do always want
  1493. * pte_mkwrite. But get_user_pages can cause write faults for mappings
  1494. * that do not have writing enabled, when used by access_process_vm.
  1495. */
  1496. static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
  1497. {
  1498. if (likely(vma->vm_flags & VM_WRITE))
  1499. pte = pte_mkwrite(pte);
  1500. return pte;
  1501. }
  1502. static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
  1503. {
  1504. /*
  1505. * If the source page was a PFN mapping, we don't have
  1506. * a "struct page" for it. We do a best-effort copy by
  1507. * just copying from the original user address. If that
  1508. * fails, we just zero-fill it. Live with it.
  1509. */
  1510. if (unlikely(!src)) {
  1511. void *kaddr = kmap_atomic(dst, KM_USER0);
  1512. void __user *uaddr = (void __user *)(va & PAGE_MASK);
  1513. /*
  1514. * This really shouldn't fail, because the page is there
  1515. * in the page tables. But it might just be unreadable,
  1516. * in which case we just give up and fill the result with
  1517. * zeroes.
  1518. */
  1519. if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
  1520. memset(kaddr, 0, PAGE_SIZE);
  1521. kunmap_atomic(kaddr, KM_USER0);
  1522. flush_dcache_page(dst);
  1523. } else
  1524. copy_user_highpage(dst, src, va, vma);
  1525. }
  1526. /*
  1527. * This routine handles present pages, when users try to write
  1528. * to a shared page. It is done by copying the page to a new address
  1529. * and decrementing the shared-page counter for the old page.
  1530. *
  1531. * Note that this routine assumes that the protection checks have been
  1532. * done by the caller (the low-level page fault routine in most cases).
  1533. * Thus we can safely just mark it writable once we've done any necessary
  1534. * COW.
  1535. *
  1536. * We also mark the page dirty at this point even though the page will
  1537. * change only once the write actually happens. This avoids a few races,
  1538. * and potentially makes it more efficient.
  1539. *
  1540. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  1541. * but allow concurrent faults), with pte both mapped and locked.
  1542. * We return with mmap_sem still held, but pte unmapped and unlocked.
  1543. */
  1544. static int do_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
  1545. unsigned long address, pte_t *page_table, pmd_t *pmd,
  1546. spinlock_t *ptl, pte_t orig_pte)
  1547. {
  1548. struct page *old_page, *new_page;
  1549. pte_t entry;
  1550. int reuse = 0, ret = 0;
  1551. int page_mkwrite = 0;
  1552. struct page *dirty_page = NULL;
  1553. old_page = vm_normal_page(vma, address, orig_pte);
  1554. if (!old_page) {
  1555. /*
  1556. * VM_MIXEDMAP !pfn_valid() case
  1557. *
  1558. * We should not cow pages in a shared writeable mapping.
  1559. * Just mark the pages writable as we can't do any dirty
  1560. * accounting on raw pfn maps.
  1561. */
  1562. if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
  1563. (VM_WRITE|VM_SHARED))
  1564. goto reuse;
  1565. goto gotten;
  1566. }
  1567. /*
  1568. * Take out anonymous pages first, anonymous shared vmas are
  1569. * not dirty accountable.
  1570. */
  1571. if (PageAnon(old_page)) {
  1572. if (!TestSetPageLocked(old_page)) {
  1573. reuse = can_share_swap_page(old_page);
  1574. unlock_page(old_page);
  1575. }
  1576. } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
  1577. (VM_WRITE|VM_SHARED))) {
  1578. /*
  1579. * Only catch write-faults on shared writable pages,
  1580. * read-only shared pages can get COWed by
  1581. * get_user_pages(.write=1, .force=1).
  1582. */
  1583. if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
  1584. /*
  1585. * Notify the address space that the page is about to
  1586. * become writable so that it can prohibit this or wait
  1587. * for the page to get into an appropriate state.
  1588. *
  1589. * We do this without the lock held, so that it can
  1590. * sleep if it needs to.
  1591. */
  1592. page_cache_get(old_page);
  1593. pte_unmap_unlock(page_table, ptl);
  1594. if (vma->vm_ops->page_mkwrite(vma, old_page) < 0)
  1595. goto unwritable_page;
  1596. /*
  1597. * Since we dropped the lock we need to revalidate
  1598. * the PTE as someone else may have changed it. If
  1599. * they did, we just return, as we can count on the
  1600. * MMU to tell us if they didn't also make it writable.
  1601. */
  1602. page_table = pte_offset_map_lock(mm, pmd, address,
  1603. &ptl);
  1604. page_cache_release(old_page);
  1605. if (!pte_same(*page_table, orig_pte))
  1606. goto unlock;
  1607. page_mkwrite = 1;
  1608. }
  1609. dirty_page = old_page;
  1610. get_page(dirty_page);
  1611. reuse = 1;
  1612. }
  1613. if (reuse) {
  1614. reuse:
  1615. flush_cache_page(vma, address, pte_pfn(orig_pte));
  1616. entry = pte_mkyoung(orig_pte);
  1617. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1618. if (ptep_set_access_flags(vma, address, page_table, entry,1))
  1619. update_mmu_cache(vma, address, entry);
  1620. ret |= VM_FAULT_WRITE;
  1621. goto unlock;
  1622. }
  1623. /*
  1624. * Ok, we need to copy. Oh, well..
  1625. */
  1626. page_cache_get(old_page);
  1627. gotten:
  1628. pte_unmap_unlock(page_table, ptl);
  1629. if (unlikely(anon_vma_prepare(vma)))
  1630. goto oom;
  1631. VM_BUG_ON(old_page == ZERO_PAGE(0));
  1632. new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
  1633. if (!new_page)
  1634. goto oom;
  1635. cow_user_page(new_page, old_page, address, vma);
  1636. __SetPageUptodate(new_page);
  1637. if (mem_cgroup_charge(new_page, mm, GFP_KERNEL))
  1638. goto oom_free_new;
  1639. /*
  1640. * Re-check the pte - we dropped the lock
  1641. */
  1642. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  1643. if (likely(pte_same(*page_table, orig_pte))) {
  1644. if (old_page) {
  1645. if (!PageAnon(old_page)) {
  1646. dec_mm_counter(mm, file_rss);
  1647. inc_mm_counter(mm, anon_rss);
  1648. }
  1649. } else
  1650. inc_mm_counter(mm, anon_rss);
  1651. flush_cache_page(vma, address, pte_pfn(orig_pte));
  1652. entry = mk_pte(new_page, vma->vm_page_prot);
  1653. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  1654. /*
  1655. * Clear the pte entry and flush it first, before updating the
  1656. * pte with the new entry. This will avoid a race condition
  1657. * seen in the presence of one thread doing SMC and another
  1658. * thread doing COW.
  1659. */
  1660. ptep_clear_flush(vma, address, page_table);
  1661. set_pte_at(mm, address, page_table, entry);
  1662. update_mmu_cache(vma, address, entry);
  1663. lru_cache_add_active(new_page);
  1664. page_add_new_anon_rmap(new_page, vma, address);
  1665. if (old_page) {
  1666. /*
  1667. * Only after switching the pte to the new page may
  1668. * we remove the mapcount here. Otherwise another
  1669. * process may come and find the rmap count decremented
  1670. * before the pte is switched to the new page, and
  1671. * "reuse" the old page writing into it while our pte
  1672. * here still points into it and can be read by other
  1673. * threads.
  1674. *
  1675. * The critical issue is to order this
  1676. * page_remove_rmap with the ptp_clear_flush above.
  1677. * Those stores are ordered by (if nothing else,)
  1678. * the barrier present in the atomic_add_negative
  1679. * in page_remove_rmap.
  1680. *
  1681. * Then the TLB flush in ptep_clear_flush ensures that
  1682. * no process can access the old page before the
  1683. * decremented mapcount is visible. And the old page
  1684. * cannot be reused until after the decremented
  1685. * mapcount is visible. So transitively, TLBs to
  1686. * old page will be flushed before it can be reused.
  1687. */
  1688. page_remove_rmap(old_page, vma);
  1689. }
  1690. /* Free the old page.. */
  1691. new_page = old_page;
  1692. ret |= VM_FAULT_WRITE;
  1693. } else
  1694. mem_cgroup_uncharge_page(new_page);
  1695. if (new_page)
  1696. page_cache_release(new_page);
  1697. if (old_page)
  1698. page_cache_release(old_page);
  1699. unlock:
  1700. pte_unmap_unlock(page_table, ptl);
  1701. if (dirty_page) {
  1702. if (vma->vm_file)
  1703. file_update_time(vma->vm_file);
  1704. /*
  1705. * Yes, Virginia, this is actually required to prevent a race
  1706. * with clear_page_dirty_for_io() from clearing the page dirty
  1707. * bit after it clear all dirty ptes, but before a racing
  1708. * do_wp_page installs a dirty pte.
  1709. *
  1710. * do_no_page is protected similarly.
  1711. */
  1712. wait_on_page_locked(dirty_page);
  1713. set_page_dirty_balance(dirty_page, page_mkwrite);
  1714. put_page(dirty_page);
  1715. }
  1716. return ret;
  1717. oom_free_new:
  1718. page_cache_release(new_page);
  1719. oom:
  1720. if (old_page)
  1721. page_cache_release(old_page);
  1722. return VM_FAULT_OOM;
  1723. unwritable_page:
  1724. page_cache_release(old_page);
  1725. return VM_FAULT_SIGBUS;
  1726. }
  1727. /*
  1728. * Helper functions for unmap_mapping_range().
  1729. *
  1730. * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __
  1731. *
  1732. * We have to restart searching the prio_tree whenever we drop the lock,
  1733. * since the iterator is only valid while the lock is held, and anyway
  1734. * a later vma might be split and reinserted earlier while lock dropped.
  1735. *
  1736. * The list of nonlinear vmas could be handled more efficiently, using
  1737. * a placeholder, but handle it in the same way until a need is shown.
  1738. * It is important to search the prio_tree before nonlinear list: a vma
  1739. * may become nonlinear and be shifted from prio_tree to nonlinear list
  1740. * while the lock is dropped; but never shifted from list to prio_tree.
  1741. *
  1742. * In order to make forward progress despite restarting the search,
  1743. * vm_truncate_count is used to mark a vma as now dealt with, so we can
  1744. * quickly skip it next time around. Since the prio_tree search only
  1745. * shows us those vmas affected by unmapping the range in question, we
  1746. * can't efficiently keep all vmas in step with mapping->truncate_count:
  1747. * so instead reset them all whenever it wraps back to 0 (then go to 1).
  1748. * mapping->truncate_count and vma->vm_truncate_count are protected by
  1749. * i_mmap_lock.
  1750. *
  1751. * In order to make forward progress despite repeatedly restarting some
  1752. * large vma, note the restart_addr from unmap_vmas when it breaks out:
  1753. * and restart from that address when we reach that vma again. It might
  1754. * have been split or merged, shrunk or extended, but never shifted: so
  1755. * restart_addr remains valid so long as it remains in the vma's range.
  1756. * unmap_mapping_range forces truncate_count to leap over page-aligned
  1757. * values so we can save vma's restart_addr in its truncate_count field.
  1758. */
  1759. #define is_restart_addr(truncate_count) (!((truncate_count) & ~PAGE_MASK))
  1760. static void reset_vma_truncate_counts(struct address_space *mapping)
  1761. {
  1762. struct vm_area_struct *vma;
  1763. struct prio_tree_iter iter;
  1764. vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX)
  1765. vma->vm_truncate_count = 0;
  1766. list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list)
  1767. vma->vm_truncate_count = 0;
  1768. }
  1769. static int unmap_mapping_range_vma(struct vm_area_struct *vma,
  1770. unsigned long start_addr, unsigned long end_addr,
  1771. struct zap_details *details)
  1772. {
  1773. unsigned long restart_addr;
  1774. int need_break;
  1775. /*
  1776. * files that support invalidating or truncating portions of the
  1777. * file from under mmaped areas must have their ->fault function
  1778. * return a locked page (and set VM_FAULT_LOCKED in the return).
  1779. * This provides synchronisation against concurrent unmapping here.
  1780. */
  1781. again:
  1782. restart_addr = vma->vm_truncate_count;
  1783. if (is_restart_addr(restart_addr) && start_addr < restart_addr) {
  1784. start_addr = restart_addr;
  1785. if (start_addr >= end_addr) {
  1786. /* Top of vma has been split off since last time */
  1787. vma->vm_truncate_count = details->truncate_count;
  1788. return 0;
  1789. }
  1790. }
  1791. restart_addr = zap_page_range(vma, start_addr,
  1792. end_addr - start_addr, details);
  1793. need_break = need_resched() || spin_needbreak(details->i_mmap_lock);
  1794. if (restart_addr >= end_addr) {
  1795. /* We have now completed this vma: mark it so */
  1796. vma->vm_truncate_count = details->truncate_count;
  1797. if (!need_break)
  1798. return 0;
  1799. } else {
  1800. /* Note restart_addr in vma's truncate_count field */
  1801. vma->vm_truncate_count = restart_addr;
  1802. if (!need_break)
  1803. goto again;
  1804. }
  1805. spin_unlock(details->i_mmap_lock);
  1806. cond_resched();
  1807. spin_lock(details->i_mmap_lock);
  1808. return -EINTR;
  1809. }
  1810. static inline void unmap_mapping_range_tree(struct prio_tree_root *root,
  1811. struct zap_details *details)
  1812. {
  1813. struct vm_area_struct *vma;
  1814. struct prio_tree_iter iter;
  1815. pgoff_t vba, vea, zba, zea;
  1816. restart:
  1817. vma_prio_tree_foreach(vma, &iter, root,
  1818. details->first_index, details->last_index) {
  1819. /* Skip quickly over those we have already dealt with */
  1820. if (vma->vm_truncate_count == details->truncate_count)
  1821. continue;
  1822. vba = vma->vm_pgoff;
  1823. vea = vba + ((vma->vm_end - vma->vm_start) >> PAGE_SHIFT) - 1;
  1824. /* Assume for now that PAGE_CACHE_SHIFT == PAGE_SHIFT */
  1825. zba = details->first_index;
  1826. if (zba < vba)
  1827. zba = vba;
  1828. zea = details->last_index;
  1829. if (zea > vea)
  1830. zea = vea;
  1831. if (unmap_mapping_range_vma(vma,
  1832. ((zba - vba) << PAGE_SHIFT) + vma->vm_start,
  1833. ((zea - vba + 1) << PAGE_SHIFT) + vma->vm_start,
  1834. details) < 0)
  1835. goto restart;
  1836. }
  1837. }
  1838. static inline void unmap_mapping_range_list(struct list_head *head,
  1839. struct zap_details *details)
  1840. {
  1841. struct vm_area_struct *vma;
  1842. /*
  1843. * In nonlinear VMAs there is no correspondence between virtual address
  1844. * offset and file offset. So we must perform an exhaustive search
  1845. * across *all* the pages in each nonlinear VMA, not just the pages
  1846. * whose virtual address lies outside the file truncation point.
  1847. */
  1848. restart:
  1849. list_for_each_entry(vma, head, shared.vm_set.list) {
  1850. /* Skip quickly over those we have already dealt with */
  1851. if (vma->vm_truncate_count == details->truncate_count)
  1852. continue;
  1853. details->nonlinear_vma = vma;
  1854. if (unmap_mapping_range_vma(vma, vma->vm_start,
  1855. vma->vm_end, details) < 0)
  1856. goto restart;
  1857. }
  1858. }
  1859. /**
  1860. * unmap_mapping_range - unmap the portion of all mmaps in the specified address_space corresponding to the specified page range in the underlying file.
  1861. * @mapping: the address space containing mmaps to be unmapped.
  1862. * @holebegin: byte in first page to unmap, relative to the start of
  1863. * the underlying file. This will be rounded down to a PAGE_SIZE
  1864. * boundary. Note that this is different from vmtruncate(), which
  1865. * must keep the partial page. In contrast, we must get rid of
  1866. * partial pages.
  1867. * @holelen: size of prospective hole in bytes. This will be rounded
  1868. * up to a PAGE_SIZE boundary. A holelen of zero truncates to the
  1869. * end of the file.
  1870. * @even_cows: 1 when truncating a file, unmap even private COWed pages;
  1871. * but 0 when invalidating pagecache, don't throw away private data.
  1872. */
  1873. void unmap_mapping_range(struct address_space *mapping,
  1874. loff_t const holebegin, loff_t const holelen, int even_cows)
  1875. {
  1876. struct zap_details details;
  1877. pgoff_t hba = holebegin >> PAGE_SHIFT;
  1878. pgoff_t hlen = (holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1879. /* Check for overflow. */
  1880. if (sizeof(holelen) > sizeof(hlen)) {
  1881. long long holeend =
  1882. (holebegin + holelen + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1883. if (holeend & ~(long long)ULONG_MAX)
  1884. hlen = ULONG_MAX - hba + 1;
  1885. }
  1886. details.check_mapping = even_cows? NULL: mapping;
  1887. details.nonlinear_vma = NULL;
  1888. details.first_index = hba;
  1889. details.last_index = hba + hlen - 1;
  1890. if (details.last_index < details.first_index)
  1891. details.last_index = ULONG_MAX;
  1892. details.i_mmap_lock = &mapping->i_mmap_lock;
  1893. spin_lock(&mapping->i_mmap_lock);
  1894. /* Protect against endless unmapping loops */
  1895. mapping->truncate_count++;
  1896. if (unlikely(is_restart_addr(mapping->truncate_count))) {
  1897. if (mapping->truncate_count == 0)
  1898. reset_vma_truncate_counts(mapping);
  1899. mapping->truncate_count++;
  1900. }
  1901. details.truncate_count = mapping->truncate_count;
  1902. if (unlikely(!prio_tree_empty(&mapping->i_mmap)))
  1903. unmap_mapping_range_tree(&mapping->i_mmap, &details);
  1904. if (unlikely(!list_empty(&mapping->i_mmap_nonlinear)))
  1905. unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details);
  1906. spin_unlock(&mapping->i_mmap_lock);
  1907. }
  1908. EXPORT_SYMBOL(unmap_mapping_range);
  1909. /**
  1910. * vmtruncate - unmap mappings "freed" by truncate() syscall
  1911. * @inode: inode of the file used
  1912. * @offset: file offset to start truncating
  1913. *
  1914. * NOTE! We have to be ready to update the memory sharing
  1915. * between the file and the memory map for a potential last
  1916. * incomplete page. Ugly, but necessary.
  1917. */
  1918. int vmtruncate(struct inode * inode, loff_t offset)
  1919. {
  1920. if (inode->i_size < offset) {
  1921. unsigned long limit;
  1922. limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
  1923. if (limit != RLIM_INFINITY && offset > limit)
  1924. goto out_sig;
  1925. if (offset > inode->i_sb->s_maxbytes)
  1926. goto out_big;
  1927. i_size_write(inode, offset);
  1928. } else {
  1929. struct address_space *mapping = inode->i_mapping;
  1930. /*
  1931. * truncation of in-use swapfiles is disallowed - it would
  1932. * cause subsequent swapout to scribble on the now-freed
  1933. * blocks.
  1934. */
  1935. if (IS_SWAPFILE(inode))
  1936. return -ETXTBSY;
  1937. i_size_write(inode, offset);
  1938. /*
  1939. * unmap_mapping_range is called twice, first simply for
  1940. * efficiency so that truncate_inode_pages does fewer
  1941. * single-page unmaps. However after this first call, and
  1942. * before truncate_inode_pages finishes, it is possible for
  1943. * private pages to be COWed, which remain after
  1944. * truncate_inode_pages finishes, hence the second
  1945. * unmap_mapping_range call must be made for correctness.
  1946. */
  1947. unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1);
  1948. truncate_inode_pages(mapping, offset);
  1949. unmap_mapping_range(mapping, offset + PAGE_SIZE - 1, 0, 1);
  1950. }
  1951. if (inode->i_op && inode->i_op->truncate)
  1952. inode->i_op->truncate(inode);
  1953. return 0;
  1954. out_sig:
  1955. send_sig(SIGXFSZ, current, 0);
  1956. out_big:
  1957. return -EFBIG;
  1958. }
  1959. EXPORT_SYMBOL(vmtruncate);
  1960. int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end)
  1961. {
  1962. struct address_space *mapping = inode->i_mapping;
  1963. /*
  1964. * If the underlying filesystem is not going to provide
  1965. * a way to truncate a range of blocks (punch a hole) -
  1966. * we should return failure right now.
  1967. */
  1968. if (!inode->i_op || !inode->i_op->truncate_range)
  1969. return -ENOSYS;
  1970. mutex_lock(&inode->i_mutex);
  1971. down_write(&inode->i_alloc_sem);
  1972. unmap_mapping_range(mapping, offset, (end - offset), 1);
  1973. truncate_inode_pages_range(mapping, offset, end);
  1974. unmap_mapping_range(mapping, offset, (end - offset), 1);
  1975. inode->i_op->truncate_range(inode, offset, end);
  1976. up_write(&inode->i_alloc_sem);
  1977. mutex_unlock(&inode->i_mutex);
  1978. return 0;
  1979. }
  1980. /*
  1981. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  1982. * but allow concurrent faults), and pte mapped but not yet locked.
  1983. * We return with mmap_sem still held, but pte unmapped and unlocked.
  1984. */
  1985. static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
  1986. unsigned long address, pte_t *page_table, pmd_t *pmd,
  1987. int write_access, pte_t orig_pte)
  1988. {
  1989. spinlock_t *ptl;
  1990. struct page *page;
  1991. swp_entry_t entry;
  1992. pte_t pte;
  1993. int ret = 0;
  1994. if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
  1995. goto out;
  1996. entry = pte_to_swp_entry(orig_pte);
  1997. if (is_migration_entry(entry)) {
  1998. migration_entry_wait(mm, pmd, address);
  1999. goto out;
  2000. }
  2001. delayacct_set_flag(DELAYACCT_PF_SWAPIN);
  2002. page = lookup_swap_cache(entry);
  2003. if (!page) {
  2004. grab_swap_token(); /* Contend for token _before_ read-in */
  2005. page = swapin_readahead(entry,
  2006. GFP_HIGHUSER_MOVABLE, vma, address);
  2007. if (!page) {
  2008. /*
  2009. * Back out if somebody else faulted in this pte
  2010. * while we released the pte lock.
  2011. */
  2012. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2013. if (likely(pte_same(*page_table, orig_pte)))
  2014. ret = VM_FAULT_OOM;
  2015. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2016. goto unlock;
  2017. }
  2018. /* Had to read the page from swap area: Major fault */
  2019. ret = VM_FAULT_MAJOR;
  2020. count_vm_event(PGMAJFAULT);
  2021. }
  2022. if (mem_cgroup_charge(page, mm, GFP_KERNEL)) {
  2023. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2024. ret = VM_FAULT_OOM;
  2025. goto out;
  2026. }
  2027. mark_page_accessed(page);
  2028. lock_page(page);
  2029. delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
  2030. /*
  2031. * Back out if somebody else already faulted in this pte.
  2032. */
  2033. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2034. if (unlikely(!pte_same(*page_table, orig_pte)))
  2035. goto out_nomap;
  2036. if (unlikely(!PageUptodate(page))) {
  2037. ret = VM_FAULT_SIGBUS;
  2038. goto out_nomap;
  2039. }
  2040. /* The page isn't present yet, go ahead with the fault. */
  2041. inc_mm_counter(mm, anon_rss);
  2042. pte = mk_pte(page, vma->vm_page_prot);
  2043. if (write_access && can_share_swap_page(page)) {
  2044. pte = maybe_mkwrite(pte_mkdirty(pte), vma);
  2045. write_access = 0;
  2046. }
  2047. flush_icache_page(vma, page);
  2048. set_pte_at(mm, address, page_table, pte);
  2049. page_add_anon_rmap(page, vma, address);
  2050. swap_free(entry);
  2051. if (vm_swap_full())
  2052. remove_exclusive_swap_page(page);
  2053. unlock_page(page);
  2054. if (write_access) {
  2055. ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte);
  2056. if (ret & VM_FAULT_ERROR)
  2057. ret &= VM_FAULT_ERROR;
  2058. goto out;
  2059. }
  2060. /* No need to invalidate - it was non-present before */
  2061. update_mmu_cache(vma, address, pte);
  2062. unlock:
  2063. pte_unmap_unlock(page_table, ptl);
  2064. out:
  2065. return ret;
  2066. out_nomap:
  2067. mem_cgroup_uncharge_page(page);
  2068. pte_unmap_unlock(page_table, ptl);
  2069. unlock_page(page);
  2070. page_cache_release(page);
  2071. return ret;
  2072. }
  2073. /*
  2074. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2075. * but allow concurrent faults), and pte mapped but not yet locked.
  2076. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2077. */
  2078. static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
  2079. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2080. int write_access)
  2081. {
  2082. struct page *page;
  2083. spinlock_t *ptl;
  2084. pte_t entry;
  2085. /* Allocate our own private page. */
  2086. pte_unmap(page_table);
  2087. if (unlikely(anon_vma_prepare(vma)))
  2088. goto oom;
  2089. page = alloc_zeroed_user_highpage_movable(vma, address);
  2090. if (!page)
  2091. goto oom;
  2092. __SetPageUptodate(page);
  2093. if (mem_cgroup_charge(page, mm, GFP_KERNEL))
  2094. goto oom_free_page;
  2095. entry = mk_pte(page, vma->vm_page_prot);
  2096. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  2097. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2098. if (!pte_none(*page_table))
  2099. goto release;
  2100. inc_mm_counter(mm, anon_rss);
  2101. lru_cache_add_active(page);
  2102. page_add_new_anon_rmap(page, vma, address);
  2103. set_pte_at(mm, address, page_table, entry);
  2104. /* No need to invalidate - it was non-present before */
  2105. update_mmu_cache(vma, address, entry);
  2106. unlock:
  2107. pte_unmap_unlock(page_table, ptl);
  2108. return 0;
  2109. release:
  2110. mem_cgroup_uncharge_page(page);
  2111. page_cache_release(page);
  2112. goto unlock;
  2113. oom_free_page:
  2114. page_cache_release(page);
  2115. oom:
  2116. return VM_FAULT_OOM;
  2117. }
  2118. /*
  2119. * __do_fault() tries to create a new page mapping. It aggressively
  2120. * tries to share with existing pages, but makes a separate copy if
  2121. * the FAULT_FLAG_WRITE is set in the flags parameter in order to avoid
  2122. * the next page fault.
  2123. *
  2124. * As this is called only for pages that do not currently exist, we
  2125. * do not need to flush old virtual caches or the TLB.
  2126. *
  2127. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2128. * but allow concurrent faults), and pte neither mapped nor locked.
  2129. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2130. */
  2131. static int __do_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2132. unsigned long address, pmd_t *pmd,
  2133. pgoff_t pgoff, unsigned int flags, pte_t orig_pte)
  2134. {
  2135. pte_t *page_table;
  2136. spinlock_t *ptl;
  2137. struct page *page;
  2138. pte_t entry;
  2139. int anon = 0;
  2140. struct page *dirty_page = NULL;
  2141. struct vm_fault vmf;
  2142. int ret;
  2143. int page_mkwrite = 0;
  2144. vmf.virtual_address = (void __user *)(address & PAGE_MASK);
  2145. vmf.pgoff = pgoff;
  2146. vmf.flags = flags;
  2147. vmf.page = NULL;
  2148. ret = vma->vm_ops->fault(vma, &vmf);
  2149. if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))
  2150. return ret;
  2151. /*
  2152. * For consistency in subsequent calls, make the faulted page always
  2153. * locked.
  2154. */
  2155. if (unlikely(!(ret & VM_FAULT_LOCKED)))
  2156. lock_page(vmf.page);
  2157. else
  2158. VM_BUG_ON(!PageLocked(vmf.page));
  2159. /*
  2160. * Should we do an early C-O-W break?
  2161. */
  2162. page = vmf.page;
  2163. if (flags & FAULT_FLAG_WRITE) {
  2164. if (!(vma->vm_flags & VM_SHARED)) {
  2165. anon = 1;
  2166. if (unlikely(anon_vma_prepare(vma))) {
  2167. ret = VM_FAULT_OOM;
  2168. goto out;
  2169. }
  2170. page = alloc_page_vma(GFP_HIGHUSER_MOVABLE,
  2171. vma, address);
  2172. if (!page) {
  2173. ret = VM_FAULT_OOM;
  2174. goto out;
  2175. }
  2176. copy_user_highpage(page, vmf.page, address, vma);
  2177. __SetPageUptodate(page);
  2178. } else {
  2179. /*
  2180. * If the page will be shareable, see if the backing
  2181. * address space wants to know that the page is about
  2182. * to become writable
  2183. */
  2184. if (vma->vm_ops->page_mkwrite) {
  2185. unlock_page(page);
  2186. if (vma->vm_ops->page_mkwrite(vma, page) < 0) {
  2187. ret = VM_FAULT_SIGBUS;
  2188. anon = 1; /* no anon but release vmf.page */
  2189. goto out_unlocked;
  2190. }
  2191. lock_page(page);
  2192. /*
  2193. * XXX: this is not quite right (racy vs
  2194. * invalidate) to unlock and relock the page
  2195. * like this, however a better fix requires
  2196. * reworking page_mkwrite locking API, which
  2197. * is better done later.
  2198. */
  2199. if (!page->mapping) {
  2200. ret = 0;
  2201. anon = 1; /* no anon but release vmf.page */
  2202. goto out;
  2203. }
  2204. page_mkwrite = 1;
  2205. }
  2206. }
  2207. }
  2208. if (mem_cgroup_charge(page, mm, GFP_KERNEL)) {
  2209. ret = VM_FAULT_OOM;
  2210. goto out;
  2211. }
  2212. page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
  2213. /*
  2214. * This silly early PAGE_DIRTY setting removes a race
  2215. * due to the bad i386 page protection. But it's valid
  2216. * for other architectures too.
  2217. *
  2218. * Note that if write_access is true, we either now have
  2219. * an exclusive copy of the page, or this is a shared mapping,
  2220. * so we can make it writable and dirty to avoid having to
  2221. * handle that later.
  2222. */
  2223. /* Only go through if we didn't race with anybody else... */
  2224. if (likely(pte_same(*page_table, orig_pte))) {
  2225. flush_icache_page(vma, page);
  2226. entry = mk_pte(page, vma->vm_page_prot);
  2227. if (flags & FAULT_FLAG_WRITE)
  2228. entry = maybe_mkwrite(pte_mkdirty(entry), vma);
  2229. set_pte_at(mm, address, page_table, entry);
  2230. if (anon) {
  2231. inc_mm_counter(mm, anon_rss);
  2232. lru_cache_add_active(page);
  2233. page_add_new_anon_rmap(page, vma, address);
  2234. } else {
  2235. inc_mm_counter(mm, file_rss);
  2236. page_add_file_rmap(page);
  2237. if (flags & FAULT_FLAG_WRITE) {
  2238. dirty_page = page;
  2239. get_page(dirty_page);
  2240. }
  2241. }
  2242. /* no need to invalidate: a not-present page won't be cached */
  2243. update_mmu_cache(vma, address, entry);
  2244. } else {
  2245. mem_cgroup_uncharge_page(page);
  2246. if (anon)
  2247. page_cache_release(page);
  2248. else
  2249. anon = 1; /* no anon but release faulted_page */
  2250. }
  2251. pte_unmap_unlock(page_table, ptl);
  2252. out:
  2253. unlock_page(vmf.page);
  2254. out_unlocked:
  2255. if (anon)
  2256. page_cache_release(vmf.page);
  2257. else if (dirty_page) {
  2258. if (vma->vm_file)
  2259. file_update_time(vma->vm_file);
  2260. set_page_dirty_balance(dirty_page, page_mkwrite);
  2261. put_page(dirty_page);
  2262. }
  2263. return ret;
  2264. }
  2265. static int do_linear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2266. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2267. int write_access, pte_t orig_pte)
  2268. {
  2269. pgoff_t pgoff = (((address & PAGE_MASK)
  2270. - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
  2271. unsigned int flags = (write_access ? FAULT_FLAG_WRITE : 0);
  2272. pte_unmap(page_table);
  2273. return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte);
  2274. }
  2275. /*
  2276. * Fault of a previously existing named mapping. Repopulate the pte
  2277. * from the encoded file_pte if possible. This enables swappable
  2278. * nonlinear vmas.
  2279. *
  2280. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2281. * but allow concurrent faults), and pte mapped but not yet locked.
  2282. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2283. */
  2284. static int do_nonlinear_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2285. unsigned long address, pte_t *page_table, pmd_t *pmd,
  2286. int write_access, pte_t orig_pte)
  2287. {
  2288. unsigned int flags = FAULT_FLAG_NONLINEAR |
  2289. (write_access ? FAULT_FLAG_WRITE : 0);
  2290. pgoff_t pgoff;
  2291. if (!pte_unmap_same(mm, pmd, page_table, orig_pte))
  2292. return 0;
  2293. if (unlikely(!(vma->vm_flags & VM_NONLINEAR) ||
  2294. !(vma->vm_flags & VM_CAN_NONLINEAR))) {
  2295. /*
  2296. * Page table corrupted: show pte and kill process.
  2297. */
  2298. print_bad_pte(vma, orig_pte, address);
  2299. return VM_FAULT_OOM;
  2300. }
  2301. pgoff = pte_to_pgoff(orig_pte);
  2302. return __do_fault(mm, vma, address, pmd, pgoff, flags, orig_pte);
  2303. }
  2304. /*
  2305. * These routines also need to handle stuff like marking pages dirty
  2306. * and/or accessed for architectures that don't do it in hardware (most
  2307. * RISC architectures). The early dirtying is also good on the i386.
  2308. *
  2309. * There is also a hook called "update_mmu_cache()" that architectures
  2310. * with external mmu caches can use to update those (ie the Sparc or
  2311. * PowerPC hashed page tables that act as extended TLBs).
  2312. *
  2313. * We enter with non-exclusive mmap_sem (to exclude vma changes,
  2314. * but allow concurrent faults), and pte mapped but not yet locked.
  2315. * We return with mmap_sem still held, but pte unmapped and unlocked.
  2316. */
  2317. static inline int handle_pte_fault(struct mm_struct *mm,
  2318. struct vm_area_struct *vma, unsigned long address,
  2319. pte_t *pte, pmd_t *pmd, int write_access)
  2320. {
  2321. pte_t entry;
  2322. spinlock_t *ptl;
  2323. entry = *pte;
  2324. if (!pte_present(entry)) {
  2325. if (pte_none(entry)) {
  2326. if (vma->vm_ops) {
  2327. if (likely(vma->vm_ops->fault))
  2328. return do_linear_fault(mm, vma, address,
  2329. pte, pmd, write_access, entry);
  2330. }
  2331. return do_anonymous_page(mm, vma, address,
  2332. pte, pmd, write_access);
  2333. }
  2334. if (pte_file(entry))
  2335. return do_nonlinear_fault(mm, vma, address,
  2336. pte, pmd, write_access, entry);
  2337. return do_swap_page(mm, vma, address,
  2338. pte, pmd, write_access, entry);
  2339. }
  2340. ptl = pte_lockptr(mm, pmd);
  2341. spin_lock(ptl);
  2342. if (unlikely(!pte_same(*pte, entry)))
  2343. goto unlock;
  2344. if (write_access) {
  2345. if (!pte_write(entry))
  2346. return do_wp_page(mm, vma, address,
  2347. pte, pmd, ptl, entry);
  2348. entry = pte_mkdirty(entry);
  2349. }
  2350. entry = pte_mkyoung(entry);
  2351. if (ptep_set_access_flags(vma, address, pte, entry, write_access)) {
  2352. update_mmu_cache(vma, address, entry);
  2353. } else {
  2354. /*
  2355. * This is needed only for protection faults but the arch code
  2356. * is not yet telling us if this is a protection fault or not.
  2357. * This still avoids useless tlb flushes for .text page faults
  2358. * with threads.
  2359. */
  2360. if (write_access)
  2361. flush_tlb_page(vma, address);
  2362. }
  2363. unlock:
  2364. pte_unmap_unlock(pte, ptl);
  2365. return 0;
  2366. }
  2367. /*
  2368. * By the time we get here, we already hold the mm semaphore
  2369. */
  2370. int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
  2371. unsigned long address, int write_access)
  2372. {
  2373. pgd_t *pgd;
  2374. pud_t *pud;
  2375. pmd_t *pmd;
  2376. pte_t *pte;
  2377. __set_current_state(TASK_RUNNING);
  2378. count_vm_event(PGFAULT);
  2379. if (unlikely(is_vm_hugetlb_page(vma)))
  2380. return hugetlb_fault(mm, vma, address, write_access);
  2381. pgd = pgd_offset(mm, address);
  2382. pud = pud_alloc(mm, pgd, address);
  2383. if (!pud)
  2384. return VM_FAULT_OOM;
  2385. pmd = pmd_alloc(mm, pud, address);
  2386. if (!pmd)
  2387. return VM_FAULT_OOM;
  2388. pte = pte_alloc_map(mm, pmd, address);
  2389. if (!pte)
  2390. return VM_FAULT_OOM;
  2391. return handle_pte_fault(mm, vma, address, pte, pmd, write_access);
  2392. }
  2393. #ifndef __PAGETABLE_PUD_FOLDED
  2394. /*
  2395. * Allocate page upper directory.
  2396. * We've already handled the fast-path in-line.
  2397. */
  2398. int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
  2399. {
  2400. pud_t *new = pud_alloc_one(mm, address);
  2401. if (!new)
  2402. return -ENOMEM;
  2403. smp_wmb(); /* See comment in __pte_alloc */
  2404. spin_lock(&mm->page_table_lock);
  2405. if (pgd_present(*pgd)) /* Another has populated it */
  2406. pud_free(mm, new);
  2407. else
  2408. pgd_populate(mm, pgd, new);
  2409. spin_unlock(&mm->page_table_lock);
  2410. return 0;
  2411. }
  2412. #endif /* __PAGETABLE_PUD_FOLDED */
  2413. #ifndef __PAGETABLE_PMD_FOLDED
  2414. /*
  2415. * Allocate page middle directory.
  2416. * We've already handled the fast-path in-line.
  2417. */
  2418. int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
  2419. {
  2420. pmd_t *new = pmd_alloc_one(mm, address);
  2421. if (!new)
  2422. return -ENOMEM;
  2423. smp_wmb(); /* See comment in __pte_alloc */
  2424. spin_lock(&mm->page_table_lock);
  2425. #ifndef __ARCH_HAS_4LEVEL_HACK
  2426. if (pud_present(*pud)) /* Another has populated it */
  2427. pmd_free(mm, new);
  2428. else
  2429. pud_populate(mm, pud, new);
  2430. #else
  2431. if (pgd_present(*pud)) /* Another has populated it */
  2432. pmd_free(mm, new);
  2433. else
  2434. pgd_populate(mm, pud, new);
  2435. #endif /* __ARCH_HAS_4LEVEL_HACK */
  2436. spin_unlock(&mm->page_table_lock);
  2437. return 0;
  2438. }
  2439. #endif /* __PAGETABLE_PMD_FOLDED */
  2440. int make_pages_present(unsigned long addr, unsigned long end)
  2441. {
  2442. int ret, len, write;
  2443. struct vm_area_struct * vma;
  2444. vma = find_vma(current->mm, addr);
  2445. if (!vma)
  2446. return -1;
  2447. write = (vma->vm_flags & VM_WRITE) != 0;
  2448. BUG_ON(addr >= end);
  2449. BUG_ON(end > vma->vm_end);
  2450. len = DIV_ROUND_UP(end, PAGE_SIZE) - addr/PAGE_SIZE;
  2451. ret = get_user_pages(current, current->mm, addr,
  2452. len, write, 0, NULL, NULL);
  2453. if (ret < 0)
  2454. return ret;
  2455. return ret == len ? 0 : -1;
  2456. }
  2457. #if !defined(__HAVE_ARCH_GATE_AREA)
  2458. #if defined(AT_SYSINFO_EHDR)
  2459. static struct vm_area_struct gate_vma;
  2460. static int __init gate_vma_init(void)
  2461. {
  2462. gate_vma.vm_mm = NULL;
  2463. gate_vma.vm_start = FIXADDR_USER_START;
  2464. gate_vma.vm_end = FIXADDR_USER_END;
  2465. gate_vma.vm_flags = VM_READ | VM_MAYREAD | VM_EXEC | VM_MAYEXEC;
  2466. gate_vma.vm_page_prot = __P101;
  2467. /*
  2468. * Make sure the vDSO gets into every core dump.
  2469. * Dumping its contents makes post-mortem fully interpretable later
  2470. * without matching up the same kernel and hardware config to see
  2471. * what PC values meant.
  2472. */
  2473. gate_vma.vm_flags |= VM_ALWAYSDUMP;
  2474. return 0;
  2475. }
  2476. __initcall(gate_vma_init);
  2477. #endif
  2478. struct vm_area_struct *get_gate_vma(struct task_struct *tsk)
  2479. {
  2480. #ifdef AT_SYSINFO_EHDR
  2481. return &gate_vma;
  2482. #else
  2483. return NULL;
  2484. #endif
  2485. }
  2486. int in_gate_area_no_task(unsigned long addr)
  2487. {
  2488. #ifdef AT_SYSINFO_EHDR
  2489. if ((addr >= FIXADDR_USER_START) && (addr < FIXADDR_USER_END))
  2490. return 1;
  2491. #endif
  2492. return 0;
  2493. }
  2494. #endif /* __HAVE_ARCH_GATE_AREA */
  2495. #ifdef CONFIG_HAVE_IOREMAP_PROT
  2496. static resource_size_t follow_phys(struct vm_area_struct *vma,
  2497. unsigned long address, unsigned int flags,
  2498. unsigned long *prot)
  2499. {
  2500. pgd_t *pgd;
  2501. pud_t *pud;
  2502. pmd_t *pmd;
  2503. pte_t *ptep, pte;
  2504. spinlock_t *ptl;
  2505. resource_size_t phys_addr = 0;
  2506. struct mm_struct *mm = vma->vm_mm;
  2507. VM_BUG_ON(!(vma->vm_flags & (VM_IO | VM_PFNMAP)));
  2508. pgd = pgd_offset(mm, address);
  2509. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  2510. goto no_page_table;
  2511. pud = pud_offset(pgd, address);
  2512. if (pud_none(*pud) || unlikely(pud_bad(*pud)))
  2513. goto no_page_table;
  2514. pmd = pmd_offset(pud, address);
  2515. if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
  2516. goto no_page_table;
  2517. /* We cannot handle huge page PFN maps. Luckily they don't exist. */
  2518. if (pmd_huge(*pmd))
  2519. goto no_page_table;
  2520. ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
  2521. if (!ptep)
  2522. goto out;
  2523. pte = *ptep;
  2524. if (!pte_present(pte))
  2525. goto unlock;
  2526. if ((flags & FOLL_WRITE) && !pte_write(pte))
  2527. goto unlock;
  2528. phys_addr = pte_pfn(pte);
  2529. phys_addr <<= PAGE_SHIFT; /* Shift here to avoid overflow on PAE */
  2530. *prot = pgprot_val(pte_pgprot(pte));
  2531. unlock:
  2532. pte_unmap_unlock(ptep, ptl);
  2533. out:
  2534. return phys_addr;
  2535. no_page_table:
  2536. return 0;
  2537. }
  2538. int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
  2539. void *buf, int len, int write)
  2540. {
  2541. resource_size_t phys_addr;
  2542. unsigned long prot = 0;
  2543. void *maddr;
  2544. int offset = addr & (PAGE_SIZE-1);
  2545. if (!(vma->vm_flags & (VM_IO | VM_PFNMAP)))
  2546. return -EINVAL;
  2547. phys_addr = follow_phys(vma, addr, write, &prot);
  2548. if (!phys_addr)
  2549. return -EINVAL;
  2550. maddr = ioremap_prot(phys_addr, PAGE_SIZE, prot);
  2551. if (write)
  2552. memcpy_toio(maddr + offset, buf, len);
  2553. else
  2554. memcpy_fromio(buf, maddr + offset, len);
  2555. iounmap(maddr);
  2556. return len;
  2557. }
  2558. #endif
  2559. /*
  2560. * Access another process' address space.
  2561. * Source/target buffer must be kernel space,
  2562. * Do not walk the page table directly, use get_user_pages
  2563. */
  2564. int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write)
  2565. {
  2566. struct mm_struct *mm;
  2567. struct vm_area_struct *vma;
  2568. void *old_buf = buf;
  2569. mm = get_task_mm(tsk);
  2570. if (!mm)
  2571. return 0;
  2572. down_read(&mm->mmap_sem);
  2573. /* ignore errors, just check how much was successfully transferred */
  2574. while (len) {
  2575. int bytes, ret, offset;
  2576. void *maddr;
  2577. struct page *page = NULL;
  2578. ret = get_user_pages(tsk, mm, addr, 1,
  2579. write, 1, &page, &vma);
  2580. if (ret <= 0) {
  2581. /*
  2582. * Check if this is a VM_IO | VM_PFNMAP VMA, which
  2583. * we can access using slightly different code.
  2584. */
  2585. #ifdef CONFIG_HAVE_IOREMAP_PROT
  2586. vma = find_vma(mm, addr);
  2587. if (!vma)
  2588. break;
  2589. if (vma->vm_ops && vma->vm_ops->access)
  2590. ret = vma->vm_ops->access(vma, addr, buf,
  2591. len, write);
  2592. if (ret <= 0)
  2593. #endif
  2594. break;
  2595. bytes = ret;
  2596. } else {
  2597. bytes = len;
  2598. offset = addr & (PAGE_SIZE-1);
  2599. if (bytes > PAGE_SIZE-offset)
  2600. bytes = PAGE_SIZE-offset;
  2601. maddr = kmap(page);
  2602. if (write) {
  2603. copy_to_user_page(vma, page, addr,
  2604. maddr + offset, buf, bytes);
  2605. set_page_dirty_lock(page);
  2606. } else {
  2607. copy_from_user_page(vma, page, addr,
  2608. buf, maddr + offset, bytes);
  2609. }
  2610. kunmap(page);
  2611. page_cache_release(page);
  2612. }
  2613. len -= bytes;
  2614. buf += bytes;
  2615. addr += bytes;
  2616. }
  2617. up_read(&mm->mmap_sem);
  2618. mmput(mm);
  2619. return buf - old_buf;
  2620. }
  2621. /*
  2622. * Print the name of a VMA.
  2623. */
  2624. void print_vma_addr(char *prefix, unsigned long ip)
  2625. {
  2626. struct mm_struct *mm = current->mm;
  2627. struct vm_area_struct *vma;
  2628. /*
  2629. * Do not print if we are in atomic
  2630. * contexts (in exception stacks, etc.):
  2631. */
  2632. if (preempt_count())
  2633. return;
  2634. down_read(&mm->mmap_sem);
  2635. vma = find_vma(mm, ip);
  2636. if (vma && vma->vm_file) {
  2637. struct file *f = vma->vm_file;
  2638. char *buf = (char *)__get_free_page(GFP_KERNEL);
  2639. if (buf) {
  2640. char *p, *s;
  2641. p = d_path(&f->f_path, buf, PAGE_SIZE);
  2642. if (IS_ERR(p))
  2643. p = "?";
  2644. s = strrchr(p, '/');
  2645. if (s)
  2646. p = s+1;
  2647. printk("%s%s[%lx+%lx]", prefix, p,
  2648. vma->vm_start,
  2649. vma->vm_end - vma->vm_start);
  2650. free_page((unsigned long)buf);
  2651. }
  2652. }
  2653. up_read(&current->mm->mmap_sem);
  2654. }