vmalloc.c 68 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719
  1. /*
  2. * linux/mm/vmalloc.c
  3. *
  4. * Copyright (C) 1993 Linus Torvalds
  5. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  6. * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
  7. * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
  8. * Numa awareness, Christoph Lameter, SGI, June 2005
  9. */
  10. #include <linux/vmalloc.h>
  11. #include <linux/mm.h>
  12. #include <linux/module.h>
  13. #include <linux/highmem.h>
  14. #include <linux/sched.h>
  15. #include <linux/slab.h>
  16. #include <linux/spinlock.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/proc_fs.h>
  19. #include <linux/seq_file.h>
  20. #include <linux/debugobjects.h>
  21. #include <linux/kallsyms.h>
  22. #include <linux/list.h>
  23. #include <linux/rbtree.h>
  24. #include <linux/radix-tree.h>
  25. #include <linux/rcupdate.h>
  26. #include <linux/pfn.h>
  27. #include <linux/kmemleak.h>
  28. #include <linux/atomic.h>
  29. #include <linux/llist.h>
  30. #include <asm/uaccess.h>
  31. #include <asm/tlbflush.h>
  32. #include <asm/shmparam.h>
  33. struct vfree_deferred {
  34. struct llist_head list;
  35. struct work_struct wq;
  36. };
  37. static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
  38. static void __vunmap(const void *, int);
  39. static void free_work(struct work_struct *w)
  40. {
  41. struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
  42. struct llist_node *llnode = llist_del_all(&p->list);
  43. while (llnode) {
  44. void *p = llnode;
  45. llnode = llist_next(llnode);
  46. __vunmap(p, 1);
  47. }
  48. }
  49. /*** Page table manipulation functions ***/
  50. static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
  51. {
  52. pte_t *pte;
  53. pte = pte_offset_kernel(pmd, addr);
  54. do {
  55. pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
  56. WARN_ON(!pte_none(ptent) && !pte_present(ptent));
  57. } while (pte++, addr += PAGE_SIZE, addr != end);
  58. }
  59. static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
  60. {
  61. pmd_t *pmd;
  62. unsigned long next;
  63. pmd = pmd_offset(pud, addr);
  64. do {
  65. next = pmd_addr_end(addr, end);
  66. if (pmd_none_or_clear_bad(pmd))
  67. continue;
  68. vunmap_pte_range(pmd, addr, next);
  69. } while (pmd++, addr = next, addr != end);
  70. }
  71. static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
  72. {
  73. pud_t *pud;
  74. unsigned long next;
  75. pud = pud_offset(pgd, addr);
  76. do {
  77. next = pud_addr_end(addr, end);
  78. if (pud_none_or_clear_bad(pud))
  79. continue;
  80. vunmap_pmd_range(pud, addr, next);
  81. } while (pud++, addr = next, addr != end);
  82. }
  83. static void vunmap_page_range(unsigned long addr, unsigned long end)
  84. {
  85. pgd_t *pgd;
  86. unsigned long next;
  87. BUG_ON(addr >= end);
  88. pgd = pgd_offset_k(addr);
  89. do {
  90. next = pgd_addr_end(addr, end);
  91. if (pgd_none_or_clear_bad(pgd))
  92. continue;
  93. vunmap_pud_range(pgd, addr, next);
  94. } while (pgd++, addr = next, addr != end);
  95. }
  96. static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
  97. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  98. {
  99. pte_t *pte;
  100. /*
  101. * nr is a running index into the array which helps higher level
  102. * callers keep track of where we're up to.
  103. */
  104. pte = pte_alloc_kernel(pmd, addr);
  105. if (!pte)
  106. return -ENOMEM;
  107. do {
  108. struct page *page = pages[*nr];
  109. if (WARN_ON(!pte_none(*pte)))
  110. return -EBUSY;
  111. if (WARN_ON(!page))
  112. return -ENOMEM;
  113. set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
  114. (*nr)++;
  115. } while (pte++, addr += PAGE_SIZE, addr != end);
  116. return 0;
  117. }
  118. static int vmap_pmd_range(pud_t *pud, unsigned long addr,
  119. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  120. {
  121. pmd_t *pmd;
  122. unsigned long next;
  123. pmd = pmd_alloc(&init_mm, pud, addr);
  124. if (!pmd)
  125. return -ENOMEM;
  126. do {
  127. next = pmd_addr_end(addr, end);
  128. if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
  129. return -ENOMEM;
  130. } while (pmd++, addr = next, addr != end);
  131. return 0;
  132. }
  133. static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
  134. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  135. {
  136. pud_t *pud;
  137. unsigned long next;
  138. pud = pud_alloc(&init_mm, pgd, addr);
  139. if (!pud)
  140. return -ENOMEM;
  141. do {
  142. next = pud_addr_end(addr, end);
  143. if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
  144. return -ENOMEM;
  145. } while (pud++, addr = next, addr != end);
  146. return 0;
  147. }
  148. /*
  149. * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
  150. * will have pfns corresponding to the "pages" array.
  151. *
  152. * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
  153. */
  154. static int vmap_page_range_noflush(unsigned long start, unsigned long end,
  155. pgprot_t prot, struct page **pages)
  156. {
  157. pgd_t *pgd;
  158. unsigned long next;
  159. unsigned long addr = start;
  160. int err = 0;
  161. int nr = 0;
  162. BUG_ON(addr >= end);
  163. pgd = pgd_offset_k(addr);
  164. do {
  165. next = pgd_addr_end(addr, end);
  166. err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
  167. if (err)
  168. return err;
  169. } while (pgd++, addr = next, addr != end);
  170. return nr;
  171. }
  172. static int vmap_page_range(unsigned long start, unsigned long end,
  173. pgprot_t prot, struct page **pages)
  174. {
  175. int ret;
  176. ret = vmap_page_range_noflush(start, end, prot, pages);
  177. flush_cache_vmap(start, end);
  178. return ret;
  179. }
  180. int is_vmalloc_or_module_addr(const void *x)
  181. {
  182. /*
  183. * ARM, x86-64 and sparc64 put modules in a special place,
  184. * and fall back on vmalloc() if that fails. Others
  185. * just put it in the vmalloc space.
  186. */
  187. #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
  188. unsigned long addr = (unsigned long)x;
  189. if (addr >= MODULES_VADDR && addr < MODULES_END)
  190. return 1;
  191. #endif
  192. return is_vmalloc_addr(x);
  193. }
  194. /*
  195. * Walk a vmap address to the struct page it maps.
  196. */
  197. struct page *vmalloc_to_page(const void *vmalloc_addr)
  198. {
  199. unsigned long addr = (unsigned long) vmalloc_addr;
  200. struct page *page = NULL;
  201. pgd_t *pgd = pgd_offset_k(addr);
  202. /*
  203. * XXX we might need to change this if we add VIRTUAL_BUG_ON for
  204. * architectures that do not vmalloc module space
  205. */
  206. VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
  207. if (!pgd_none(*pgd)) {
  208. pud_t *pud = pud_offset(pgd, addr);
  209. if (!pud_none(*pud)) {
  210. pmd_t *pmd = pmd_offset(pud, addr);
  211. if (!pmd_none(*pmd)) {
  212. pte_t *ptep, pte;
  213. ptep = pte_offset_map(pmd, addr);
  214. pte = *ptep;
  215. if (pte_present(pte))
  216. page = pte_page(pte);
  217. pte_unmap(ptep);
  218. }
  219. }
  220. }
  221. return page;
  222. }
  223. EXPORT_SYMBOL(vmalloc_to_page);
  224. /*
  225. * Map a vmalloc()-space virtual address to the physical page frame number.
  226. */
  227. unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
  228. {
  229. return page_to_pfn(vmalloc_to_page(vmalloc_addr));
  230. }
  231. EXPORT_SYMBOL(vmalloc_to_pfn);
  232. /*** Global kva allocator ***/
  233. #define VM_LAZY_FREE 0x01
  234. #define VM_LAZY_FREEING 0x02
  235. #define VM_VM_AREA 0x04
  236. static DEFINE_SPINLOCK(vmap_area_lock);
  237. /* Export for kexec only */
  238. LIST_HEAD(vmap_area_list);
  239. static struct rb_root vmap_area_root = RB_ROOT;
  240. /* The vmap cache globals are protected by vmap_area_lock */
  241. static struct rb_node *free_vmap_cache;
  242. static unsigned long cached_hole_size;
  243. static unsigned long cached_vstart;
  244. static unsigned long cached_align;
  245. static unsigned long vmap_area_pcpu_hole;
  246. static struct vmap_area *__find_vmap_area(unsigned long addr)
  247. {
  248. struct rb_node *n = vmap_area_root.rb_node;
  249. while (n) {
  250. struct vmap_area *va;
  251. va = rb_entry(n, struct vmap_area, rb_node);
  252. if (addr < va->va_start)
  253. n = n->rb_left;
  254. else if (addr >= va->va_end)
  255. n = n->rb_right;
  256. else
  257. return va;
  258. }
  259. return NULL;
  260. }
  261. static void __insert_vmap_area(struct vmap_area *va)
  262. {
  263. struct rb_node **p = &vmap_area_root.rb_node;
  264. struct rb_node *parent = NULL;
  265. struct rb_node *tmp;
  266. while (*p) {
  267. struct vmap_area *tmp_va;
  268. parent = *p;
  269. tmp_va = rb_entry(parent, struct vmap_area, rb_node);
  270. if (va->va_start < tmp_va->va_end)
  271. p = &(*p)->rb_left;
  272. else if (va->va_end > tmp_va->va_start)
  273. p = &(*p)->rb_right;
  274. else
  275. BUG();
  276. }
  277. rb_link_node(&va->rb_node, parent, p);
  278. rb_insert_color(&va->rb_node, &vmap_area_root);
  279. /* address-sort this list */
  280. tmp = rb_prev(&va->rb_node);
  281. if (tmp) {
  282. struct vmap_area *prev;
  283. prev = rb_entry(tmp, struct vmap_area, rb_node);
  284. list_add_rcu(&va->list, &prev->list);
  285. } else
  286. list_add_rcu(&va->list, &vmap_area_list);
  287. }
  288. static void purge_vmap_area_lazy(void);
  289. /*
  290. * Allocate a region of KVA of the specified size and alignment, within the
  291. * vstart and vend.
  292. */
  293. static struct vmap_area *alloc_vmap_area(unsigned long size,
  294. unsigned long align,
  295. unsigned long vstart, unsigned long vend,
  296. int node, gfp_t gfp_mask)
  297. {
  298. struct vmap_area *va;
  299. struct rb_node *n;
  300. unsigned long addr;
  301. int purged = 0;
  302. struct vmap_area *first;
  303. BUG_ON(!size);
  304. BUG_ON(size & ~PAGE_MASK);
  305. BUG_ON(!is_power_of_2(align));
  306. va = kmalloc_node(sizeof(struct vmap_area),
  307. gfp_mask & GFP_RECLAIM_MASK, node);
  308. if (unlikely(!va))
  309. return ERR_PTR(-ENOMEM);
  310. retry:
  311. spin_lock(&vmap_area_lock);
  312. /*
  313. * Invalidate cache if we have more permissive parameters.
  314. * cached_hole_size notes the largest hole noticed _below_
  315. * the vmap_area cached in free_vmap_cache: if size fits
  316. * into that hole, we want to scan from vstart to reuse
  317. * the hole instead of allocating above free_vmap_cache.
  318. * Note that __free_vmap_area may update free_vmap_cache
  319. * without updating cached_hole_size or cached_align.
  320. */
  321. if (!free_vmap_cache ||
  322. size < cached_hole_size ||
  323. vstart < cached_vstart ||
  324. align < cached_align) {
  325. nocache:
  326. cached_hole_size = 0;
  327. free_vmap_cache = NULL;
  328. }
  329. /* record if we encounter less permissive parameters */
  330. cached_vstart = vstart;
  331. cached_align = align;
  332. /* find starting point for our search */
  333. if (free_vmap_cache) {
  334. first = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
  335. addr = ALIGN(first->va_end, align);
  336. if (addr < vstart)
  337. goto nocache;
  338. if (addr + size < addr)
  339. goto overflow;
  340. } else {
  341. addr = ALIGN(vstart, align);
  342. if (addr + size < addr)
  343. goto overflow;
  344. n = vmap_area_root.rb_node;
  345. first = NULL;
  346. while (n) {
  347. struct vmap_area *tmp;
  348. tmp = rb_entry(n, struct vmap_area, rb_node);
  349. if (tmp->va_end >= addr) {
  350. first = tmp;
  351. if (tmp->va_start <= addr)
  352. break;
  353. n = n->rb_left;
  354. } else
  355. n = n->rb_right;
  356. }
  357. if (!first)
  358. goto found;
  359. }
  360. /* from the starting point, walk areas until a suitable hole is found */
  361. while (addr + size > first->va_start && addr + size <= vend) {
  362. if (addr + cached_hole_size < first->va_start)
  363. cached_hole_size = first->va_start - addr;
  364. addr = ALIGN(first->va_end, align);
  365. if (addr + size < addr)
  366. goto overflow;
  367. if (list_is_last(&first->list, &vmap_area_list))
  368. goto found;
  369. first = list_entry(first->list.next,
  370. struct vmap_area, list);
  371. }
  372. found:
  373. if (addr + size > vend)
  374. goto overflow;
  375. va->va_start = addr;
  376. va->va_end = addr + size;
  377. va->flags = 0;
  378. __insert_vmap_area(va);
  379. free_vmap_cache = &va->rb_node;
  380. spin_unlock(&vmap_area_lock);
  381. BUG_ON(va->va_start & (align-1));
  382. BUG_ON(va->va_start < vstart);
  383. BUG_ON(va->va_end > vend);
  384. return va;
  385. overflow:
  386. spin_unlock(&vmap_area_lock);
  387. if (!purged) {
  388. purge_vmap_area_lazy();
  389. purged = 1;
  390. goto retry;
  391. }
  392. if (printk_ratelimit())
  393. printk(KERN_WARNING
  394. "vmap allocation for size %lu failed: "
  395. "use vmalloc=<size> to increase size.\n", size);
  396. kfree(va);
  397. return ERR_PTR(-EBUSY);
  398. }
  399. static void __free_vmap_area(struct vmap_area *va)
  400. {
  401. BUG_ON(RB_EMPTY_NODE(&va->rb_node));
  402. if (free_vmap_cache) {
  403. if (va->va_end < cached_vstart) {
  404. free_vmap_cache = NULL;
  405. } else {
  406. struct vmap_area *cache;
  407. cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
  408. if (va->va_start <= cache->va_start) {
  409. free_vmap_cache = rb_prev(&va->rb_node);
  410. /*
  411. * We don't try to update cached_hole_size or
  412. * cached_align, but it won't go very wrong.
  413. */
  414. }
  415. }
  416. }
  417. rb_erase(&va->rb_node, &vmap_area_root);
  418. RB_CLEAR_NODE(&va->rb_node);
  419. list_del_rcu(&va->list);
  420. /*
  421. * Track the highest possible candidate for pcpu area
  422. * allocation. Areas outside of vmalloc area can be returned
  423. * here too, consider only end addresses which fall inside
  424. * vmalloc area proper.
  425. */
  426. if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
  427. vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
  428. kfree_rcu(va, rcu_head);
  429. }
  430. /*
  431. * Free a region of KVA allocated by alloc_vmap_area
  432. */
  433. static void free_vmap_area(struct vmap_area *va)
  434. {
  435. spin_lock(&vmap_area_lock);
  436. __free_vmap_area(va);
  437. spin_unlock(&vmap_area_lock);
  438. }
  439. /*
  440. * Clear the pagetable entries of a given vmap_area
  441. */
  442. static void unmap_vmap_area(struct vmap_area *va)
  443. {
  444. vunmap_page_range(va->va_start, va->va_end);
  445. }
  446. static void vmap_debug_free_range(unsigned long start, unsigned long end)
  447. {
  448. /*
  449. * Unmap page tables and force a TLB flush immediately if
  450. * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free
  451. * bugs similarly to those in linear kernel virtual address
  452. * space after a page has been freed.
  453. *
  454. * All the lazy freeing logic is still retained, in order to
  455. * minimise intrusiveness of this debugging feature.
  456. *
  457. * This is going to be *slow* (linear kernel virtual address
  458. * debugging doesn't do a broadcast TLB flush so it is a lot
  459. * faster).
  460. */
  461. #ifdef CONFIG_DEBUG_PAGEALLOC
  462. vunmap_page_range(start, end);
  463. flush_tlb_kernel_range(start, end);
  464. #endif
  465. }
  466. /*
  467. * lazy_max_pages is the maximum amount of virtual address space we gather up
  468. * before attempting to purge with a TLB flush.
  469. *
  470. * There is a tradeoff here: a larger number will cover more kernel page tables
  471. * and take slightly longer to purge, but it will linearly reduce the number of
  472. * global TLB flushes that must be performed. It would seem natural to scale
  473. * this number up linearly with the number of CPUs (because vmapping activity
  474. * could also scale linearly with the number of CPUs), however it is likely
  475. * that in practice, workloads might be constrained in other ways that mean
  476. * vmap activity will not scale linearly with CPUs. Also, I want to be
  477. * conservative and not introduce a big latency on huge systems, so go with
  478. * a less aggressive log scale. It will still be an improvement over the old
  479. * code, and it will be simple to change the scale factor if we find that it
  480. * becomes a problem on bigger systems.
  481. */
  482. static unsigned long lazy_max_pages(void)
  483. {
  484. unsigned int log;
  485. log = fls(num_online_cpus());
  486. return log * (32UL * 1024 * 1024 / PAGE_SIZE);
  487. }
  488. static atomic_t vmap_lazy_nr = ATOMIC_INIT(0);
  489. /* for per-CPU blocks */
  490. static void purge_fragmented_blocks_allcpus(void);
  491. /*
  492. * called before a call to iounmap() if the caller wants vm_area_struct's
  493. * immediately freed.
  494. */
  495. void set_iounmap_nonlazy(void)
  496. {
  497. atomic_set(&vmap_lazy_nr, lazy_max_pages()+1);
  498. }
  499. /*
  500. * Purges all lazily-freed vmap areas.
  501. *
  502. * If sync is 0 then don't purge if there is already a purge in progress.
  503. * If force_flush is 1, then flush kernel TLBs between *start and *end even
  504. * if we found no lazy vmap areas to unmap (callers can use this to optimise
  505. * their own TLB flushing).
  506. * Returns with *start = min(*start, lowest purged address)
  507. * *end = max(*end, highest purged address)
  508. */
  509. static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
  510. int sync, int force_flush)
  511. {
  512. static DEFINE_SPINLOCK(purge_lock);
  513. LIST_HEAD(valist);
  514. struct vmap_area *va;
  515. struct vmap_area *n_va;
  516. int nr = 0;
  517. /*
  518. * If sync is 0 but force_flush is 1, we'll go sync anyway but callers
  519. * should not expect such behaviour. This just simplifies locking for
  520. * the case that isn't actually used at the moment anyway.
  521. */
  522. if (!sync && !force_flush) {
  523. if (!spin_trylock(&purge_lock))
  524. return;
  525. } else
  526. spin_lock(&purge_lock);
  527. if (sync)
  528. purge_fragmented_blocks_allcpus();
  529. rcu_read_lock();
  530. list_for_each_entry_rcu(va, &vmap_area_list, list) {
  531. if (va->flags & VM_LAZY_FREE) {
  532. if (va->va_start < *start)
  533. *start = va->va_start;
  534. if (va->va_end > *end)
  535. *end = va->va_end;
  536. nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
  537. list_add_tail(&va->purge_list, &valist);
  538. va->flags |= VM_LAZY_FREEING;
  539. va->flags &= ~VM_LAZY_FREE;
  540. }
  541. }
  542. rcu_read_unlock();
  543. if (nr)
  544. atomic_sub(nr, &vmap_lazy_nr);
  545. if (nr || force_flush)
  546. flush_tlb_kernel_range(*start, *end);
  547. if (nr) {
  548. spin_lock(&vmap_area_lock);
  549. list_for_each_entry_safe(va, n_va, &valist, purge_list)
  550. __free_vmap_area(va);
  551. spin_unlock(&vmap_area_lock);
  552. }
  553. spin_unlock(&purge_lock);
  554. }
  555. /*
  556. * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
  557. * is already purging.
  558. */
  559. static void try_purge_vmap_area_lazy(void)
  560. {
  561. unsigned long start = ULONG_MAX, end = 0;
  562. __purge_vmap_area_lazy(&start, &end, 0, 0);
  563. }
  564. /*
  565. * Kick off a purge of the outstanding lazy areas.
  566. */
  567. static void purge_vmap_area_lazy(void)
  568. {
  569. unsigned long start = ULONG_MAX, end = 0;
  570. __purge_vmap_area_lazy(&start, &end, 1, 0);
  571. }
  572. /*
  573. * Free a vmap area, caller ensuring that the area has been unmapped
  574. * and flush_cache_vunmap had been called for the correct range
  575. * previously.
  576. */
  577. static void free_vmap_area_noflush(struct vmap_area *va)
  578. {
  579. va->flags |= VM_LAZY_FREE;
  580. atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
  581. if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages()))
  582. try_purge_vmap_area_lazy();
  583. }
  584. /*
  585. * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
  586. * called for the correct range previously.
  587. */
  588. static void free_unmap_vmap_area_noflush(struct vmap_area *va)
  589. {
  590. unmap_vmap_area(va);
  591. free_vmap_area_noflush(va);
  592. }
  593. /*
  594. * Free and unmap a vmap area
  595. */
  596. static void free_unmap_vmap_area(struct vmap_area *va)
  597. {
  598. flush_cache_vunmap(va->va_start, va->va_end);
  599. free_unmap_vmap_area_noflush(va);
  600. }
  601. static struct vmap_area *find_vmap_area(unsigned long addr)
  602. {
  603. struct vmap_area *va;
  604. spin_lock(&vmap_area_lock);
  605. va = __find_vmap_area(addr);
  606. spin_unlock(&vmap_area_lock);
  607. return va;
  608. }
  609. static void free_unmap_vmap_area_addr(unsigned long addr)
  610. {
  611. struct vmap_area *va;
  612. va = find_vmap_area(addr);
  613. BUG_ON(!va);
  614. free_unmap_vmap_area(va);
  615. }
  616. /*** Per cpu kva allocator ***/
  617. /*
  618. * vmap space is limited especially on 32 bit architectures. Ensure there is
  619. * room for at least 16 percpu vmap blocks per CPU.
  620. */
  621. /*
  622. * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
  623. * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
  624. * instead (we just need a rough idea)
  625. */
  626. #if BITS_PER_LONG == 32
  627. #define VMALLOC_SPACE (128UL*1024*1024)
  628. #else
  629. #define VMALLOC_SPACE (128UL*1024*1024*1024)
  630. #endif
  631. #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
  632. #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
  633. #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
  634. #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
  635. #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
  636. #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
  637. #define VMAP_BBMAP_BITS \
  638. VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
  639. VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
  640. VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
  641. #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
  642. static bool vmap_initialized __read_mostly = false;
  643. struct vmap_block_queue {
  644. spinlock_t lock;
  645. struct list_head free;
  646. };
  647. struct vmap_block {
  648. spinlock_t lock;
  649. struct vmap_area *va;
  650. unsigned long free, dirty;
  651. DECLARE_BITMAP(dirty_map, VMAP_BBMAP_BITS);
  652. struct list_head free_list;
  653. struct rcu_head rcu_head;
  654. struct list_head purge;
  655. };
  656. /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
  657. static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
  658. /*
  659. * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
  660. * in the free path. Could get rid of this if we change the API to return a
  661. * "cookie" from alloc, to be passed to free. But no big deal yet.
  662. */
  663. static DEFINE_SPINLOCK(vmap_block_tree_lock);
  664. static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
  665. /*
  666. * We should probably have a fallback mechanism to allocate virtual memory
  667. * out of partially filled vmap blocks. However vmap block sizing should be
  668. * fairly reasonable according to the vmalloc size, so it shouldn't be a
  669. * big problem.
  670. */
  671. static unsigned long addr_to_vb_idx(unsigned long addr)
  672. {
  673. addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
  674. addr /= VMAP_BLOCK_SIZE;
  675. return addr;
  676. }
  677. static struct vmap_block *new_vmap_block(gfp_t gfp_mask)
  678. {
  679. struct vmap_block_queue *vbq;
  680. struct vmap_block *vb;
  681. struct vmap_area *va;
  682. unsigned long vb_idx;
  683. int node, err;
  684. node = numa_node_id();
  685. vb = kmalloc_node(sizeof(struct vmap_block),
  686. gfp_mask & GFP_RECLAIM_MASK, node);
  687. if (unlikely(!vb))
  688. return ERR_PTR(-ENOMEM);
  689. va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
  690. VMALLOC_START, VMALLOC_END,
  691. node, gfp_mask);
  692. if (IS_ERR(va)) {
  693. kfree(vb);
  694. return ERR_CAST(va);
  695. }
  696. err = radix_tree_preload(gfp_mask);
  697. if (unlikely(err)) {
  698. kfree(vb);
  699. free_vmap_area(va);
  700. return ERR_PTR(err);
  701. }
  702. spin_lock_init(&vb->lock);
  703. vb->va = va;
  704. vb->free = VMAP_BBMAP_BITS;
  705. vb->dirty = 0;
  706. bitmap_zero(vb->dirty_map, VMAP_BBMAP_BITS);
  707. INIT_LIST_HEAD(&vb->free_list);
  708. vb_idx = addr_to_vb_idx(va->va_start);
  709. spin_lock(&vmap_block_tree_lock);
  710. err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
  711. spin_unlock(&vmap_block_tree_lock);
  712. BUG_ON(err);
  713. radix_tree_preload_end();
  714. vbq = &get_cpu_var(vmap_block_queue);
  715. spin_lock(&vbq->lock);
  716. list_add_rcu(&vb->free_list, &vbq->free);
  717. spin_unlock(&vbq->lock);
  718. put_cpu_var(vmap_block_queue);
  719. return vb;
  720. }
  721. static void free_vmap_block(struct vmap_block *vb)
  722. {
  723. struct vmap_block *tmp;
  724. unsigned long vb_idx;
  725. vb_idx = addr_to_vb_idx(vb->va->va_start);
  726. spin_lock(&vmap_block_tree_lock);
  727. tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
  728. spin_unlock(&vmap_block_tree_lock);
  729. BUG_ON(tmp != vb);
  730. free_vmap_area_noflush(vb->va);
  731. kfree_rcu(vb, rcu_head);
  732. }
  733. static void purge_fragmented_blocks(int cpu)
  734. {
  735. LIST_HEAD(purge);
  736. struct vmap_block *vb;
  737. struct vmap_block *n_vb;
  738. struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
  739. rcu_read_lock();
  740. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  741. if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
  742. continue;
  743. spin_lock(&vb->lock);
  744. if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
  745. vb->free = 0; /* prevent further allocs after releasing lock */
  746. vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
  747. bitmap_fill(vb->dirty_map, VMAP_BBMAP_BITS);
  748. spin_lock(&vbq->lock);
  749. list_del_rcu(&vb->free_list);
  750. spin_unlock(&vbq->lock);
  751. spin_unlock(&vb->lock);
  752. list_add_tail(&vb->purge, &purge);
  753. } else
  754. spin_unlock(&vb->lock);
  755. }
  756. rcu_read_unlock();
  757. list_for_each_entry_safe(vb, n_vb, &purge, purge) {
  758. list_del(&vb->purge);
  759. free_vmap_block(vb);
  760. }
  761. }
  762. static void purge_fragmented_blocks_allcpus(void)
  763. {
  764. int cpu;
  765. for_each_possible_cpu(cpu)
  766. purge_fragmented_blocks(cpu);
  767. }
  768. static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
  769. {
  770. struct vmap_block_queue *vbq;
  771. struct vmap_block *vb;
  772. unsigned long addr = 0;
  773. unsigned int order;
  774. BUG_ON(size & ~PAGE_MASK);
  775. BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
  776. if (WARN_ON(size == 0)) {
  777. /*
  778. * Allocating 0 bytes isn't what caller wants since
  779. * get_order(0) returns funny result. Just warn and terminate
  780. * early.
  781. */
  782. return NULL;
  783. }
  784. order = get_order(size);
  785. again:
  786. rcu_read_lock();
  787. vbq = &get_cpu_var(vmap_block_queue);
  788. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  789. int i;
  790. spin_lock(&vb->lock);
  791. if (vb->free < 1UL << order)
  792. goto next;
  793. i = VMAP_BBMAP_BITS - vb->free;
  794. addr = vb->va->va_start + (i << PAGE_SHIFT);
  795. BUG_ON(addr_to_vb_idx(addr) !=
  796. addr_to_vb_idx(vb->va->va_start));
  797. vb->free -= 1UL << order;
  798. if (vb->free == 0) {
  799. spin_lock(&vbq->lock);
  800. list_del_rcu(&vb->free_list);
  801. spin_unlock(&vbq->lock);
  802. }
  803. spin_unlock(&vb->lock);
  804. break;
  805. next:
  806. spin_unlock(&vb->lock);
  807. }
  808. put_cpu_var(vmap_block_queue);
  809. rcu_read_unlock();
  810. if (!addr) {
  811. vb = new_vmap_block(gfp_mask);
  812. if (IS_ERR(vb))
  813. return vb;
  814. goto again;
  815. }
  816. return (void *)addr;
  817. }
  818. static void vb_free(const void *addr, unsigned long size)
  819. {
  820. unsigned long offset;
  821. unsigned long vb_idx;
  822. unsigned int order;
  823. struct vmap_block *vb;
  824. BUG_ON(size & ~PAGE_MASK);
  825. BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
  826. flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
  827. order = get_order(size);
  828. offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
  829. vb_idx = addr_to_vb_idx((unsigned long)addr);
  830. rcu_read_lock();
  831. vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
  832. rcu_read_unlock();
  833. BUG_ON(!vb);
  834. vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
  835. spin_lock(&vb->lock);
  836. BUG_ON(bitmap_allocate_region(vb->dirty_map, offset >> PAGE_SHIFT, order));
  837. vb->dirty += 1UL << order;
  838. if (vb->dirty == VMAP_BBMAP_BITS) {
  839. BUG_ON(vb->free);
  840. spin_unlock(&vb->lock);
  841. free_vmap_block(vb);
  842. } else
  843. spin_unlock(&vb->lock);
  844. }
  845. /**
  846. * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
  847. *
  848. * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
  849. * to amortize TLB flushing overheads. What this means is that any page you
  850. * have now, may, in a former life, have been mapped into kernel virtual
  851. * address by the vmap layer and so there might be some CPUs with TLB entries
  852. * still referencing that page (additional to the regular 1:1 kernel mapping).
  853. *
  854. * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
  855. * be sure that none of the pages we have control over will have any aliases
  856. * from the vmap layer.
  857. */
  858. void vm_unmap_aliases(void)
  859. {
  860. unsigned long start = ULONG_MAX, end = 0;
  861. int cpu;
  862. int flush = 0;
  863. if (unlikely(!vmap_initialized))
  864. return;
  865. for_each_possible_cpu(cpu) {
  866. struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
  867. struct vmap_block *vb;
  868. rcu_read_lock();
  869. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  870. int i, j;
  871. spin_lock(&vb->lock);
  872. i = find_first_bit(vb->dirty_map, VMAP_BBMAP_BITS);
  873. if (i < VMAP_BBMAP_BITS) {
  874. unsigned long s, e;
  875. j = find_last_bit(vb->dirty_map,
  876. VMAP_BBMAP_BITS);
  877. j = j + 1; /* need exclusive index */
  878. s = vb->va->va_start + (i << PAGE_SHIFT);
  879. e = vb->va->va_start + (j << PAGE_SHIFT);
  880. flush = 1;
  881. if (s < start)
  882. start = s;
  883. if (e > end)
  884. end = e;
  885. }
  886. spin_unlock(&vb->lock);
  887. }
  888. rcu_read_unlock();
  889. }
  890. __purge_vmap_area_lazy(&start, &end, 1, flush);
  891. }
  892. EXPORT_SYMBOL_GPL(vm_unmap_aliases);
  893. /**
  894. * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
  895. * @mem: the pointer returned by vm_map_ram
  896. * @count: the count passed to that vm_map_ram call (cannot unmap partial)
  897. */
  898. void vm_unmap_ram(const void *mem, unsigned int count)
  899. {
  900. unsigned long size = count << PAGE_SHIFT;
  901. unsigned long addr = (unsigned long)mem;
  902. BUG_ON(!addr);
  903. BUG_ON(addr < VMALLOC_START);
  904. BUG_ON(addr > VMALLOC_END);
  905. BUG_ON(addr & (PAGE_SIZE-1));
  906. debug_check_no_locks_freed(mem, size);
  907. vmap_debug_free_range(addr, addr+size);
  908. if (likely(count <= VMAP_MAX_ALLOC))
  909. vb_free(mem, size);
  910. else
  911. free_unmap_vmap_area_addr(addr);
  912. }
  913. EXPORT_SYMBOL(vm_unmap_ram);
  914. /**
  915. * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
  916. * @pages: an array of pointers to the pages to be mapped
  917. * @count: number of pages
  918. * @node: prefer to allocate data structures on this node
  919. * @prot: memory protection to use. PAGE_KERNEL for regular RAM
  920. *
  921. * Returns: a pointer to the address that has been mapped, or %NULL on failure
  922. */
  923. void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
  924. {
  925. unsigned long size = count << PAGE_SHIFT;
  926. unsigned long addr;
  927. void *mem;
  928. if (likely(count <= VMAP_MAX_ALLOC)) {
  929. mem = vb_alloc(size, GFP_KERNEL);
  930. if (IS_ERR(mem))
  931. return NULL;
  932. addr = (unsigned long)mem;
  933. } else {
  934. struct vmap_area *va;
  935. va = alloc_vmap_area(size, PAGE_SIZE,
  936. VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
  937. if (IS_ERR(va))
  938. return NULL;
  939. addr = va->va_start;
  940. mem = (void *)addr;
  941. }
  942. if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
  943. vm_unmap_ram(mem, count);
  944. return NULL;
  945. }
  946. return mem;
  947. }
  948. EXPORT_SYMBOL(vm_map_ram);
  949. static struct vm_struct *vmlist __initdata;
  950. /**
  951. * vm_area_add_early - add vmap area early during boot
  952. * @vm: vm_struct to add
  953. *
  954. * This function is used to add fixed kernel vm area to vmlist before
  955. * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
  956. * should contain proper values and the other fields should be zero.
  957. *
  958. * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
  959. */
  960. void __init vm_area_add_early(struct vm_struct *vm)
  961. {
  962. struct vm_struct *tmp, **p;
  963. BUG_ON(vmap_initialized);
  964. for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
  965. if (tmp->addr >= vm->addr) {
  966. BUG_ON(tmp->addr < vm->addr + vm->size);
  967. break;
  968. } else
  969. BUG_ON(tmp->addr + tmp->size > vm->addr);
  970. }
  971. vm->next = *p;
  972. *p = vm;
  973. }
  974. /**
  975. * vm_area_register_early - register vmap area early during boot
  976. * @vm: vm_struct to register
  977. * @align: requested alignment
  978. *
  979. * This function is used to register kernel vm area before
  980. * vmalloc_init() is called. @vm->size and @vm->flags should contain
  981. * proper values on entry and other fields should be zero. On return,
  982. * vm->addr contains the allocated address.
  983. *
  984. * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
  985. */
  986. void __init vm_area_register_early(struct vm_struct *vm, size_t align)
  987. {
  988. static size_t vm_init_off __initdata;
  989. unsigned long addr;
  990. addr = ALIGN(VMALLOC_START + vm_init_off, align);
  991. vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
  992. vm->addr = (void *)addr;
  993. vm_area_add_early(vm);
  994. }
  995. void __init vmalloc_init(void)
  996. {
  997. struct vmap_area *va;
  998. struct vm_struct *tmp;
  999. int i;
  1000. for_each_possible_cpu(i) {
  1001. struct vmap_block_queue *vbq;
  1002. struct vfree_deferred *p;
  1003. vbq = &per_cpu(vmap_block_queue, i);
  1004. spin_lock_init(&vbq->lock);
  1005. INIT_LIST_HEAD(&vbq->free);
  1006. p = &per_cpu(vfree_deferred, i);
  1007. init_llist_head(&p->list);
  1008. INIT_WORK(&p->wq, free_work);
  1009. }
  1010. /* Import existing vmlist entries. */
  1011. for (tmp = vmlist; tmp; tmp = tmp->next) {
  1012. va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
  1013. va->flags = VM_VM_AREA;
  1014. va->va_start = (unsigned long)tmp->addr;
  1015. va->va_end = va->va_start + tmp->size;
  1016. va->vm = tmp;
  1017. __insert_vmap_area(va);
  1018. }
  1019. vmap_area_pcpu_hole = VMALLOC_END;
  1020. vmap_initialized = true;
  1021. }
  1022. /**
  1023. * map_kernel_range_noflush - map kernel VM area with the specified pages
  1024. * @addr: start of the VM area to map
  1025. * @size: size of the VM area to map
  1026. * @prot: page protection flags to use
  1027. * @pages: pages to map
  1028. *
  1029. * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size
  1030. * specify should have been allocated using get_vm_area() and its
  1031. * friends.
  1032. *
  1033. * NOTE:
  1034. * This function does NOT do any cache flushing. The caller is
  1035. * responsible for calling flush_cache_vmap() on to-be-mapped areas
  1036. * before calling this function.
  1037. *
  1038. * RETURNS:
  1039. * The number of pages mapped on success, -errno on failure.
  1040. */
  1041. int map_kernel_range_noflush(unsigned long addr, unsigned long size,
  1042. pgprot_t prot, struct page **pages)
  1043. {
  1044. return vmap_page_range_noflush(addr, addr + size, prot, pages);
  1045. }
  1046. /**
  1047. * unmap_kernel_range_noflush - unmap kernel VM area
  1048. * @addr: start of the VM area to unmap
  1049. * @size: size of the VM area to unmap
  1050. *
  1051. * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size
  1052. * specify should have been allocated using get_vm_area() and its
  1053. * friends.
  1054. *
  1055. * NOTE:
  1056. * This function does NOT do any cache flushing. The caller is
  1057. * responsible for calling flush_cache_vunmap() on to-be-mapped areas
  1058. * before calling this function and flush_tlb_kernel_range() after.
  1059. */
  1060. void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
  1061. {
  1062. vunmap_page_range(addr, addr + size);
  1063. }
  1064. EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush);
  1065. /**
  1066. * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
  1067. * @addr: start of the VM area to unmap
  1068. * @size: size of the VM area to unmap
  1069. *
  1070. * Similar to unmap_kernel_range_noflush() but flushes vcache before
  1071. * the unmapping and tlb after.
  1072. */
  1073. void unmap_kernel_range(unsigned long addr, unsigned long size)
  1074. {
  1075. unsigned long end = addr + size;
  1076. flush_cache_vunmap(addr, end);
  1077. vunmap_page_range(addr, end);
  1078. flush_tlb_kernel_range(addr, end);
  1079. }
  1080. int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
  1081. {
  1082. unsigned long addr = (unsigned long)area->addr;
  1083. unsigned long end = addr + area->size - PAGE_SIZE;
  1084. int err;
  1085. err = vmap_page_range(addr, end, prot, *pages);
  1086. if (err > 0) {
  1087. *pages += err;
  1088. err = 0;
  1089. }
  1090. return err;
  1091. }
  1092. EXPORT_SYMBOL_GPL(map_vm_area);
  1093. static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
  1094. unsigned long flags, const void *caller)
  1095. {
  1096. spin_lock(&vmap_area_lock);
  1097. vm->flags = flags;
  1098. vm->addr = (void *)va->va_start;
  1099. vm->size = va->va_end - va->va_start;
  1100. vm->caller = caller;
  1101. va->vm = vm;
  1102. va->flags |= VM_VM_AREA;
  1103. spin_unlock(&vmap_area_lock);
  1104. }
  1105. static void clear_vm_uninitialized_flag(struct vm_struct *vm)
  1106. {
  1107. /*
  1108. * Before removing VM_UNINITIALIZED,
  1109. * we should make sure that vm has proper values.
  1110. * Pair with smp_rmb() in show_numa_info().
  1111. */
  1112. smp_wmb();
  1113. vm->flags &= ~VM_UNINITIALIZED;
  1114. }
  1115. static struct vm_struct *__get_vm_area_node(unsigned long size,
  1116. unsigned long align, unsigned long flags, unsigned long start,
  1117. unsigned long end, int node, gfp_t gfp_mask, const void *caller)
  1118. {
  1119. struct vmap_area *va;
  1120. struct vm_struct *area;
  1121. BUG_ON(in_interrupt());
  1122. if (flags & VM_IOREMAP)
  1123. align = 1ul << clamp(fls(size), PAGE_SHIFT, IOREMAP_MAX_ORDER);
  1124. size = PAGE_ALIGN(size);
  1125. if (unlikely(!size))
  1126. return NULL;
  1127. area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
  1128. if (unlikely(!area))
  1129. return NULL;
  1130. /*
  1131. * We always allocate a guard page.
  1132. */
  1133. size += PAGE_SIZE;
  1134. va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
  1135. if (IS_ERR(va)) {
  1136. kfree(area);
  1137. return NULL;
  1138. }
  1139. setup_vmalloc_vm(area, va, flags, caller);
  1140. return area;
  1141. }
  1142. struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
  1143. unsigned long start, unsigned long end)
  1144. {
  1145. return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
  1146. GFP_KERNEL, __builtin_return_address(0));
  1147. }
  1148. EXPORT_SYMBOL_GPL(__get_vm_area);
  1149. struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
  1150. unsigned long start, unsigned long end,
  1151. const void *caller)
  1152. {
  1153. return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
  1154. GFP_KERNEL, caller);
  1155. }
  1156. /**
  1157. * get_vm_area - reserve a contiguous kernel virtual area
  1158. * @size: size of the area
  1159. * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
  1160. *
  1161. * Search an area of @size in the kernel virtual mapping area,
  1162. * and reserved it for out purposes. Returns the area descriptor
  1163. * on success or %NULL on failure.
  1164. */
  1165. struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
  1166. {
  1167. return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
  1168. NUMA_NO_NODE, GFP_KERNEL,
  1169. __builtin_return_address(0));
  1170. }
  1171. struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
  1172. const void *caller)
  1173. {
  1174. return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
  1175. NUMA_NO_NODE, GFP_KERNEL, caller);
  1176. }
  1177. /**
  1178. * find_vm_area - find a continuous kernel virtual area
  1179. * @addr: base address
  1180. *
  1181. * Search for the kernel VM area starting at @addr, and return it.
  1182. * It is up to the caller to do all required locking to keep the returned
  1183. * pointer valid.
  1184. */
  1185. struct vm_struct *find_vm_area(const void *addr)
  1186. {
  1187. struct vmap_area *va;
  1188. va = find_vmap_area((unsigned long)addr);
  1189. if (va && va->flags & VM_VM_AREA)
  1190. return va->vm;
  1191. return NULL;
  1192. }
  1193. /**
  1194. * remove_vm_area - find and remove a continuous kernel virtual area
  1195. * @addr: base address
  1196. *
  1197. * Search for the kernel VM area starting at @addr, and remove it.
  1198. * This function returns the found VM area, but using it is NOT safe
  1199. * on SMP machines, except for its size or flags.
  1200. */
  1201. struct vm_struct *remove_vm_area(const void *addr)
  1202. {
  1203. struct vmap_area *va;
  1204. va = find_vmap_area((unsigned long)addr);
  1205. if (va && va->flags & VM_VM_AREA) {
  1206. struct vm_struct *vm = va->vm;
  1207. spin_lock(&vmap_area_lock);
  1208. va->vm = NULL;
  1209. va->flags &= ~VM_VM_AREA;
  1210. spin_unlock(&vmap_area_lock);
  1211. vmap_debug_free_range(va->va_start, va->va_end);
  1212. free_unmap_vmap_area(va);
  1213. vm->size -= PAGE_SIZE;
  1214. return vm;
  1215. }
  1216. return NULL;
  1217. }
  1218. static void __vunmap(const void *addr, int deallocate_pages)
  1219. {
  1220. struct vm_struct *area;
  1221. if (!addr)
  1222. return;
  1223. if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
  1224. addr))
  1225. return;
  1226. area = remove_vm_area(addr);
  1227. if (unlikely(!area)) {
  1228. WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
  1229. addr);
  1230. return;
  1231. }
  1232. debug_check_no_locks_freed(addr, area->size);
  1233. debug_check_no_obj_freed(addr, area->size);
  1234. if (deallocate_pages) {
  1235. int i;
  1236. for (i = 0; i < area->nr_pages; i++) {
  1237. struct page *page = area->pages[i];
  1238. BUG_ON(!page);
  1239. __free_page(page);
  1240. }
  1241. if (area->flags & VM_VPAGES)
  1242. vfree(area->pages);
  1243. else
  1244. kfree(area->pages);
  1245. }
  1246. kfree(area);
  1247. return;
  1248. }
  1249. /**
  1250. * vfree - release memory allocated by vmalloc()
  1251. * @addr: memory base address
  1252. *
  1253. * Free the virtually continuous memory area starting at @addr, as
  1254. * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
  1255. * NULL, no operation is performed.
  1256. *
  1257. * Must not be called in NMI context (strictly speaking, only if we don't
  1258. * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
  1259. * conventions for vfree() arch-depenedent would be a really bad idea)
  1260. *
  1261. * NOTE: assumes that the object at *addr has a size >= sizeof(llist_node)
  1262. */
  1263. void vfree(const void *addr)
  1264. {
  1265. BUG_ON(in_nmi());
  1266. kmemleak_free(addr);
  1267. if (!addr)
  1268. return;
  1269. if (unlikely(in_interrupt())) {
  1270. struct vfree_deferred *p = &__get_cpu_var(vfree_deferred);
  1271. if (llist_add((struct llist_node *)addr, &p->list))
  1272. schedule_work(&p->wq);
  1273. } else
  1274. __vunmap(addr, 1);
  1275. }
  1276. EXPORT_SYMBOL(vfree);
  1277. /**
  1278. * vunmap - release virtual mapping obtained by vmap()
  1279. * @addr: memory base address
  1280. *
  1281. * Free the virtually contiguous memory area starting at @addr,
  1282. * which was created from the page array passed to vmap().
  1283. *
  1284. * Must not be called in interrupt context.
  1285. */
  1286. void vunmap(const void *addr)
  1287. {
  1288. BUG_ON(in_interrupt());
  1289. might_sleep();
  1290. if (addr)
  1291. __vunmap(addr, 0);
  1292. }
  1293. EXPORT_SYMBOL(vunmap);
  1294. /**
  1295. * vmap - map an array of pages into virtually contiguous space
  1296. * @pages: array of page pointers
  1297. * @count: number of pages to map
  1298. * @flags: vm_area->flags
  1299. * @prot: page protection for the mapping
  1300. *
  1301. * Maps @count pages from @pages into contiguous kernel virtual
  1302. * space.
  1303. */
  1304. void *vmap(struct page **pages, unsigned int count,
  1305. unsigned long flags, pgprot_t prot)
  1306. {
  1307. struct vm_struct *area;
  1308. might_sleep();
  1309. if (count > totalram_pages)
  1310. return NULL;
  1311. area = get_vm_area_caller((count << PAGE_SHIFT), flags,
  1312. __builtin_return_address(0));
  1313. if (!area)
  1314. return NULL;
  1315. if (map_vm_area(area, prot, &pages)) {
  1316. vunmap(area->addr);
  1317. return NULL;
  1318. }
  1319. return area->addr;
  1320. }
  1321. EXPORT_SYMBOL(vmap);
  1322. static void *__vmalloc_node(unsigned long size, unsigned long align,
  1323. gfp_t gfp_mask, pgprot_t prot,
  1324. int node, const void *caller);
  1325. static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
  1326. pgprot_t prot, int node, const void *caller)
  1327. {
  1328. const int order = 0;
  1329. struct page **pages;
  1330. unsigned int nr_pages, array_size, i;
  1331. gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
  1332. nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
  1333. array_size = (nr_pages * sizeof(struct page *));
  1334. area->nr_pages = nr_pages;
  1335. /* Please note that the recursion is strictly bounded. */
  1336. if (array_size > PAGE_SIZE) {
  1337. pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM,
  1338. PAGE_KERNEL, node, caller);
  1339. area->flags |= VM_VPAGES;
  1340. } else {
  1341. pages = kmalloc_node(array_size, nested_gfp, node);
  1342. }
  1343. area->pages = pages;
  1344. area->caller = caller;
  1345. if (!area->pages) {
  1346. remove_vm_area(area->addr);
  1347. kfree(area);
  1348. return NULL;
  1349. }
  1350. for (i = 0; i < area->nr_pages; i++) {
  1351. struct page *page;
  1352. gfp_t tmp_mask = gfp_mask | __GFP_NOWARN;
  1353. if (node < 0)
  1354. page = alloc_page(tmp_mask);
  1355. else
  1356. page = alloc_pages_node(node, tmp_mask, order);
  1357. if (unlikely(!page)) {
  1358. /* Successfully allocated i pages, free them in __vunmap() */
  1359. area->nr_pages = i;
  1360. goto fail;
  1361. }
  1362. area->pages[i] = page;
  1363. }
  1364. if (map_vm_area(area, prot, &pages))
  1365. goto fail;
  1366. return area->addr;
  1367. fail:
  1368. warn_alloc_failed(gfp_mask, order,
  1369. "vmalloc: allocation failure, allocated %ld of %ld bytes\n",
  1370. (area->nr_pages*PAGE_SIZE), area->size);
  1371. vfree(area->addr);
  1372. return NULL;
  1373. }
  1374. /**
  1375. * __vmalloc_node_range - allocate virtually contiguous memory
  1376. * @size: allocation size
  1377. * @align: desired alignment
  1378. * @start: vm area range start
  1379. * @end: vm area range end
  1380. * @gfp_mask: flags for the page level allocator
  1381. * @prot: protection mask for the allocated pages
  1382. * @node: node to use for allocation or NUMA_NO_NODE
  1383. * @caller: caller's return address
  1384. *
  1385. * Allocate enough pages to cover @size from the page level
  1386. * allocator with @gfp_mask flags. Map them into contiguous
  1387. * kernel virtual space, using a pagetable protection of @prot.
  1388. */
  1389. void *__vmalloc_node_range(unsigned long size, unsigned long align,
  1390. unsigned long start, unsigned long end, gfp_t gfp_mask,
  1391. pgprot_t prot, int node, const void *caller)
  1392. {
  1393. struct vm_struct *area;
  1394. void *addr;
  1395. unsigned long real_size = size;
  1396. size = PAGE_ALIGN(size);
  1397. if (!size || (size >> PAGE_SHIFT) > totalram_pages)
  1398. goto fail;
  1399. area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED,
  1400. start, end, node, gfp_mask, caller);
  1401. if (!area)
  1402. goto fail;
  1403. addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
  1404. if (!addr)
  1405. goto fail;
  1406. /*
  1407. * In this function, newly allocated vm_struct has VM_UNINITIALIZED
  1408. * flag. It means that vm_struct is not fully initialized.
  1409. * Now, it is fully initialized, so remove this flag here.
  1410. */
  1411. clear_vm_uninitialized_flag(area);
  1412. /*
  1413. * A ref_count = 3 is needed because the vm_struct and vmap_area
  1414. * structures allocated in the __get_vm_area_node() function contain
  1415. * references to the virtual address of the vmalloc'ed block.
  1416. */
  1417. kmemleak_alloc(addr, real_size, 3, gfp_mask);
  1418. return addr;
  1419. fail:
  1420. warn_alloc_failed(gfp_mask, 0,
  1421. "vmalloc: allocation failure: %lu bytes\n",
  1422. real_size);
  1423. return NULL;
  1424. }
  1425. /**
  1426. * __vmalloc_node - allocate virtually contiguous memory
  1427. * @size: allocation size
  1428. * @align: desired alignment
  1429. * @gfp_mask: flags for the page level allocator
  1430. * @prot: protection mask for the allocated pages
  1431. * @node: node to use for allocation or NUMA_NO_NODE
  1432. * @caller: caller's return address
  1433. *
  1434. * Allocate enough pages to cover @size from the page level
  1435. * allocator with @gfp_mask flags. Map them into contiguous
  1436. * kernel virtual space, using a pagetable protection of @prot.
  1437. */
  1438. static void *__vmalloc_node(unsigned long size, unsigned long align,
  1439. gfp_t gfp_mask, pgprot_t prot,
  1440. int node, const void *caller)
  1441. {
  1442. return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
  1443. gfp_mask, prot, node, caller);
  1444. }
  1445. void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
  1446. {
  1447. return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
  1448. __builtin_return_address(0));
  1449. }
  1450. EXPORT_SYMBOL(__vmalloc);
  1451. static inline void *__vmalloc_node_flags(unsigned long size,
  1452. int node, gfp_t flags)
  1453. {
  1454. return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
  1455. node, __builtin_return_address(0));
  1456. }
  1457. /**
  1458. * vmalloc - allocate virtually contiguous memory
  1459. * @size: allocation size
  1460. * Allocate enough pages to cover @size from the page level
  1461. * allocator and map them into contiguous kernel virtual space.
  1462. *
  1463. * For tight control over page level allocator and protection flags
  1464. * use __vmalloc() instead.
  1465. */
  1466. void *vmalloc(unsigned long size)
  1467. {
  1468. return __vmalloc_node_flags(size, NUMA_NO_NODE,
  1469. GFP_KERNEL | __GFP_HIGHMEM);
  1470. }
  1471. EXPORT_SYMBOL(vmalloc);
  1472. /**
  1473. * vzalloc - allocate virtually contiguous memory with zero fill
  1474. * @size: allocation size
  1475. * Allocate enough pages to cover @size from the page level
  1476. * allocator and map them into contiguous kernel virtual space.
  1477. * The memory allocated is set to zero.
  1478. *
  1479. * For tight control over page level allocator and protection flags
  1480. * use __vmalloc() instead.
  1481. */
  1482. void *vzalloc(unsigned long size)
  1483. {
  1484. return __vmalloc_node_flags(size, NUMA_NO_NODE,
  1485. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
  1486. }
  1487. EXPORT_SYMBOL(vzalloc);
  1488. /**
  1489. * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
  1490. * @size: allocation size
  1491. *
  1492. * The resulting memory area is zeroed so it can be mapped to userspace
  1493. * without leaking data.
  1494. */
  1495. void *vmalloc_user(unsigned long size)
  1496. {
  1497. struct vm_struct *area;
  1498. void *ret;
  1499. ret = __vmalloc_node(size, SHMLBA,
  1500. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
  1501. PAGE_KERNEL, NUMA_NO_NODE,
  1502. __builtin_return_address(0));
  1503. if (ret) {
  1504. area = find_vm_area(ret);
  1505. area->flags |= VM_USERMAP;
  1506. }
  1507. return ret;
  1508. }
  1509. EXPORT_SYMBOL(vmalloc_user);
  1510. /**
  1511. * vmalloc_node - allocate memory on a specific node
  1512. * @size: allocation size
  1513. * @node: numa node
  1514. *
  1515. * Allocate enough pages to cover @size from the page level
  1516. * allocator and map them into contiguous kernel virtual space.
  1517. *
  1518. * For tight control over page level allocator and protection flags
  1519. * use __vmalloc() instead.
  1520. */
  1521. void *vmalloc_node(unsigned long size, int node)
  1522. {
  1523. return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
  1524. node, __builtin_return_address(0));
  1525. }
  1526. EXPORT_SYMBOL(vmalloc_node);
  1527. /**
  1528. * vzalloc_node - allocate memory on a specific node with zero fill
  1529. * @size: allocation size
  1530. * @node: numa node
  1531. *
  1532. * Allocate enough pages to cover @size from the page level
  1533. * allocator and map them into contiguous kernel virtual space.
  1534. * The memory allocated is set to zero.
  1535. *
  1536. * For tight control over page level allocator and protection flags
  1537. * use __vmalloc_node() instead.
  1538. */
  1539. void *vzalloc_node(unsigned long size, int node)
  1540. {
  1541. return __vmalloc_node_flags(size, node,
  1542. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
  1543. }
  1544. EXPORT_SYMBOL(vzalloc_node);
  1545. #ifndef PAGE_KERNEL_EXEC
  1546. # define PAGE_KERNEL_EXEC PAGE_KERNEL
  1547. #endif
  1548. /**
  1549. * vmalloc_exec - allocate virtually contiguous, executable memory
  1550. * @size: allocation size
  1551. *
  1552. * Kernel-internal function to allocate enough pages to cover @size
  1553. * the page level allocator and map them into contiguous and
  1554. * executable kernel virtual space.
  1555. *
  1556. * For tight control over page level allocator and protection flags
  1557. * use __vmalloc() instead.
  1558. */
  1559. void *vmalloc_exec(unsigned long size)
  1560. {
  1561. return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
  1562. NUMA_NO_NODE, __builtin_return_address(0));
  1563. }
  1564. #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
  1565. #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
  1566. #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
  1567. #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
  1568. #else
  1569. #define GFP_VMALLOC32 GFP_KERNEL
  1570. #endif
  1571. /**
  1572. * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
  1573. * @size: allocation size
  1574. *
  1575. * Allocate enough 32bit PA addressable pages to cover @size from the
  1576. * page level allocator and map them into contiguous kernel virtual space.
  1577. */
  1578. void *vmalloc_32(unsigned long size)
  1579. {
  1580. return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
  1581. NUMA_NO_NODE, __builtin_return_address(0));
  1582. }
  1583. EXPORT_SYMBOL(vmalloc_32);
  1584. /**
  1585. * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
  1586. * @size: allocation size
  1587. *
  1588. * The resulting memory area is 32bit addressable and zeroed so it can be
  1589. * mapped to userspace without leaking data.
  1590. */
  1591. void *vmalloc_32_user(unsigned long size)
  1592. {
  1593. struct vm_struct *area;
  1594. void *ret;
  1595. ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
  1596. NUMA_NO_NODE, __builtin_return_address(0));
  1597. if (ret) {
  1598. area = find_vm_area(ret);
  1599. area->flags |= VM_USERMAP;
  1600. }
  1601. return ret;
  1602. }
  1603. EXPORT_SYMBOL(vmalloc_32_user);
  1604. /*
  1605. * small helper routine , copy contents to buf from addr.
  1606. * If the page is not present, fill zero.
  1607. */
  1608. static int aligned_vread(char *buf, char *addr, unsigned long count)
  1609. {
  1610. struct page *p;
  1611. int copied = 0;
  1612. while (count) {
  1613. unsigned long offset, length;
  1614. offset = (unsigned long)addr & ~PAGE_MASK;
  1615. length = PAGE_SIZE - offset;
  1616. if (length > count)
  1617. length = count;
  1618. p = vmalloc_to_page(addr);
  1619. /*
  1620. * To do safe access to this _mapped_ area, we need
  1621. * lock. But adding lock here means that we need to add
  1622. * overhead of vmalloc()/vfree() calles for this _debug_
  1623. * interface, rarely used. Instead of that, we'll use
  1624. * kmap() and get small overhead in this access function.
  1625. */
  1626. if (p) {
  1627. /*
  1628. * we can expect USER0 is not used (see vread/vwrite's
  1629. * function description)
  1630. */
  1631. void *map = kmap_atomic(p);
  1632. memcpy(buf, map + offset, length);
  1633. kunmap_atomic(map);
  1634. } else
  1635. memset(buf, 0, length);
  1636. addr += length;
  1637. buf += length;
  1638. copied += length;
  1639. count -= length;
  1640. }
  1641. return copied;
  1642. }
  1643. static int aligned_vwrite(char *buf, char *addr, unsigned long count)
  1644. {
  1645. struct page *p;
  1646. int copied = 0;
  1647. while (count) {
  1648. unsigned long offset, length;
  1649. offset = (unsigned long)addr & ~PAGE_MASK;
  1650. length = PAGE_SIZE - offset;
  1651. if (length > count)
  1652. length = count;
  1653. p = vmalloc_to_page(addr);
  1654. /*
  1655. * To do safe access to this _mapped_ area, we need
  1656. * lock. But adding lock here means that we need to add
  1657. * overhead of vmalloc()/vfree() calles for this _debug_
  1658. * interface, rarely used. Instead of that, we'll use
  1659. * kmap() and get small overhead in this access function.
  1660. */
  1661. if (p) {
  1662. /*
  1663. * we can expect USER0 is not used (see vread/vwrite's
  1664. * function description)
  1665. */
  1666. void *map = kmap_atomic(p);
  1667. memcpy(map + offset, buf, length);
  1668. kunmap_atomic(map);
  1669. }
  1670. addr += length;
  1671. buf += length;
  1672. copied += length;
  1673. count -= length;
  1674. }
  1675. return copied;
  1676. }
  1677. /**
  1678. * vread() - read vmalloc area in a safe way.
  1679. * @buf: buffer for reading data
  1680. * @addr: vm address.
  1681. * @count: number of bytes to be read.
  1682. *
  1683. * Returns # of bytes which addr and buf should be increased.
  1684. * (same number to @count). Returns 0 if [addr...addr+count) doesn't
  1685. * includes any intersect with alive vmalloc area.
  1686. *
  1687. * This function checks that addr is a valid vmalloc'ed area, and
  1688. * copy data from that area to a given buffer. If the given memory range
  1689. * of [addr...addr+count) includes some valid address, data is copied to
  1690. * proper area of @buf. If there are memory holes, they'll be zero-filled.
  1691. * IOREMAP area is treated as memory hole and no copy is done.
  1692. *
  1693. * If [addr...addr+count) doesn't includes any intersects with alive
  1694. * vm_struct area, returns 0. @buf should be kernel's buffer.
  1695. *
  1696. * Note: In usual ops, vread() is never necessary because the caller
  1697. * should know vmalloc() area is valid and can use memcpy().
  1698. * This is for routines which have to access vmalloc area without
  1699. * any informaion, as /dev/kmem.
  1700. *
  1701. */
  1702. long vread(char *buf, char *addr, unsigned long count)
  1703. {
  1704. struct vmap_area *va;
  1705. struct vm_struct *vm;
  1706. char *vaddr, *buf_start = buf;
  1707. unsigned long buflen = count;
  1708. unsigned long n;
  1709. /* Don't allow overflow */
  1710. if ((unsigned long) addr + count < count)
  1711. count = -(unsigned long) addr;
  1712. spin_lock(&vmap_area_lock);
  1713. list_for_each_entry(va, &vmap_area_list, list) {
  1714. if (!count)
  1715. break;
  1716. if (!(va->flags & VM_VM_AREA))
  1717. continue;
  1718. vm = va->vm;
  1719. vaddr = (char *) vm->addr;
  1720. if (addr >= vaddr + vm->size - PAGE_SIZE)
  1721. continue;
  1722. while (addr < vaddr) {
  1723. if (count == 0)
  1724. goto finished;
  1725. *buf = '\0';
  1726. buf++;
  1727. addr++;
  1728. count--;
  1729. }
  1730. n = vaddr + vm->size - PAGE_SIZE - addr;
  1731. if (n > count)
  1732. n = count;
  1733. if (!(vm->flags & VM_IOREMAP))
  1734. aligned_vread(buf, addr, n);
  1735. else /* IOREMAP area is treated as memory hole */
  1736. memset(buf, 0, n);
  1737. buf += n;
  1738. addr += n;
  1739. count -= n;
  1740. }
  1741. finished:
  1742. spin_unlock(&vmap_area_lock);
  1743. if (buf == buf_start)
  1744. return 0;
  1745. /* zero-fill memory holes */
  1746. if (buf != buf_start + buflen)
  1747. memset(buf, 0, buflen - (buf - buf_start));
  1748. return buflen;
  1749. }
  1750. /**
  1751. * vwrite() - write vmalloc area in a safe way.
  1752. * @buf: buffer for source data
  1753. * @addr: vm address.
  1754. * @count: number of bytes to be read.
  1755. *
  1756. * Returns # of bytes which addr and buf should be incresed.
  1757. * (same number to @count).
  1758. * If [addr...addr+count) doesn't includes any intersect with valid
  1759. * vmalloc area, returns 0.
  1760. *
  1761. * This function checks that addr is a valid vmalloc'ed area, and
  1762. * copy data from a buffer to the given addr. If specified range of
  1763. * [addr...addr+count) includes some valid address, data is copied from
  1764. * proper area of @buf. If there are memory holes, no copy to hole.
  1765. * IOREMAP area is treated as memory hole and no copy is done.
  1766. *
  1767. * If [addr...addr+count) doesn't includes any intersects with alive
  1768. * vm_struct area, returns 0. @buf should be kernel's buffer.
  1769. *
  1770. * Note: In usual ops, vwrite() is never necessary because the caller
  1771. * should know vmalloc() area is valid and can use memcpy().
  1772. * This is for routines which have to access vmalloc area without
  1773. * any informaion, as /dev/kmem.
  1774. */
  1775. long vwrite(char *buf, char *addr, unsigned long count)
  1776. {
  1777. struct vmap_area *va;
  1778. struct vm_struct *vm;
  1779. char *vaddr;
  1780. unsigned long n, buflen;
  1781. int copied = 0;
  1782. /* Don't allow overflow */
  1783. if ((unsigned long) addr + count < count)
  1784. count = -(unsigned long) addr;
  1785. buflen = count;
  1786. spin_lock(&vmap_area_lock);
  1787. list_for_each_entry(va, &vmap_area_list, list) {
  1788. if (!count)
  1789. break;
  1790. if (!(va->flags & VM_VM_AREA))
  1791. continue;
  1792. vm = va->vm;
  1793. vaddr = (char *) vm->addr;
  1794. if (addr >= vaddr + vm->size - PAGE_SIZE)
  1795. continue;
  1796. while (addr < vaddr) {
  1797. if (count == 0)
  1798. goto finished;
  1799. buf++;
  1800. addr++;
  1801. count--;
  1802. }
  1803. n = vaddr + vm->size - PAGE_SIZE - addr;
  1804. if (n > count)
  1805. n = count;
  1806. if (!(vm->flags & VM_IOREMAP)) {
  1807. aligned_vwrite(buf, addr, n);
  1808. copied++;
  1809. }
  1810. buf += n;
  1811. addr += n;
  1812. count -= n;
  1813. }
  1814. finished:
  1815. spin_unlock(&vmap_area_lock);
  1816. if (!copied)
  1817. return 0;
  1818. return buflen;
  1819. }
  1820. /**
  1821. * remap_vmalloc_range_partial - map vmalloc pages to userspace
  1822. * @vma: vma to cover
  1823. * @uaddr: target user address to start at
  1824. * @kaddr: virtual address of vmalloc kernel memory
  1825. * @size: size of map area
  1826. *
  1827. * Returns: 0 for success, -Exxx on failure
  1828. *
  1829. * This function checks that @kaddr is a valid vmalloc'ed area,
  1830. * and that it is big enough to cover the range starting at
  1831. * @uaddr in @vma. Will return failure if that criteria isn't
  1832. * met.
  1833. *
  1834. * Similar to remap_pfn_range() (see mm/memory.c)
  1835. */
  1836. int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
  1837. void *kaddr, unsigned long size)
  1838. {
  1839. struct vm_struct *area;
  1840. size = PAGE_ALIGN(size);
  1841. if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
  1842. return -EINVAL;
  1843. area = find_vm_area(kaddr);
  1844. if (!area)
  1845. return -EINVAL;
  1846. if (!(area->flags & VM_USERMAP))
  1847. return -EINVAL;
  1848. if (kaddr + size > area->addr + area->size)
  1849. return -EINVAL;
  1850. do {
  1851. struct page *page = vmalloc_to_page(kaddr);
  1852. int ret;
  1853. ret = vm_insert_page(vma, uaddr, page);
  1854. if (ret)
  1855. return ret;
  1856. uaddr += PAGE_SIZE;
  1857. kaddr += PAGE_SIZE;
  1858. size -= PAGE_SIZE;
  1859. } while (size > 0);
  1860. vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
  1861. return 0;
  1862. }
  1863. EXPORT_SYMBOL(remap_vmalloc_range_partial);
  1864. /**
  1865. * remap_vmalloc_range - map vmalloc pages to userspace
  1866. * @vma: vma to cover (map full range of vma)
  1867. * @addr: vmalloc memory
  1868. * @pgoff: number of pages into addr before first page to map
  1869. *
  1870. * Returns: 0 for success, -Exxx on failure
  1871. *
  1872. * This function checks that addr is a valid vmalloc'ed area, and
  1873. * that it is big enough to cover the vma. Will return failure if
  1874. * that criteria isn't met.
  1875. *
  1876. * Similar to remap_pfn_range() (see mm/memory.c)
  1877. */
  1878. int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
  1879. unsigned long pgoff)
  1880. {
  1881. return remap_vmalloc_range_partial(vma, vma->vm_start,
  1882. addr + (pgoff << PAGE_SHIFT),
  1883. vma->vm_end - vma->vm_start);
  1884. }
  1885. EXPORT_SYMBOL(remap_vmalloc_range);
  1886. /*
  1887. * Implement a stub for vmalloc_sync_all() if the architecture chose not to
  1888. * have one.
  1889. */
  1890. void __attribute__((weak)) vmalloc_sync_all(void)
  1891. {
  1892. }
  1893. static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
  1894. {
  1895. pte_t ***p = data;
  1896. if (p) {
  1897. *(*p) = pte;
  1898. (*p)++;
  1899. }
  1900. return 0;
  1901. }
  1902. /**
  1903. * alloc_vm_area - allocate a range of kernel address space
  1904. * @size: size of the area
  1905. * @ptes: returns the PTEs for the address space
  1906. *
  1907. * Returns: NULL on failure, vm_struct on success
  1908. *
  1909. * This function reserves a range of kernel address space, and
  1910. * allocates pagetables to map that range. No actual mappings
  1911. * are created.
  1912. *
  1913. * If @ptes is non-NULL, pointers to the PTEs (in init_mm)
  1914. * allocated for the VM area are returned.
  1915. */
  1916. struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
  1917. {
  1918. struct vm_struct *area;
  1919. area = get_vm_area_caller(size, VM_IOREMAP,
  1920. __builtin_return_address(0));
  1921. if (area == NULL)
  1922. return NULL;
  1923. /*
  1924. * This ensures that page tables are constructed for this region
  1925. * of kernel virtual address space and mapped into init_mm.
  1926. */
  1927. if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
  1928. size, f, ptes ? &ptes : NULL)) {
  1929. free_vm_area(area);
  1930. return NULL;
  1931. }
  1932. return area;
  1933. }
  1934. EXPORT_SYMBOL_GPL(alloc_vm_area);
  1935. void free_vm_area(struct vm_struct *area)
  1936. {
  1937. struct vm_struct *ret;
  1938. ret = remove_vm_area(area->addr);
  1939. BUG_ON(ret != area);
  1940. kfree(area);
  1941. }
  1942. EXPORT_SYMBOL_GPL(free_vm_area);
  1943. #ifdef CONFIG_SMP
  1944. static struct vmap_area *node_to_va(struct rb_node *n)
  1945. {
  1946. return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
  1947. }
  1948. /**
  1949. * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
  1950. * @end: target address
  1951. * @pnext: out arg for the next vmap_area
  1952. * @pprev: out arg for the previous vmap_area
  1953. *
  1954. * Returns: %true if either or both of next and prev are found,
  1955. * %false if no vmap_area exists
  1956. *
  1957. * Find vmap_areas end addresses of which enclose @end. ie. if not
  1958. * NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
  1959. */
  1960. static bool pvm_find_next_prev(unsigned long end,
  1961. struct vmap_area **pnext,
  1962. struct vmap_area **pprev)
  1963. {
  1964. struct rb_node *n = vmap_area_root.rb_node;
  1965. struct vmap_area *va = NULL;
  1966. while (n) {
  1967. va = rb_entry(n, struct vmap_area, rb_node);
  1968. if (end < va->va_end)
  1969. n = n->rb_left;
  1970. else if (end > va->va_end)
  1971. n = n->rb_right;
  1972. else
  1973. break;
  1974. }
  1975. if (!va)
  1976. return false;
  1977. if (va->va_end > end) {
  1978. *pnext = va;
  1979. *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
  1980. } else {
  1981. *pprev = va;
  1982. *pnext = node_to_va(rb_next(&(*pprev)->rb_node));
  1983. }
  1984. return true;
  1985. }
  1986. /**
  1987. * pvm_determine_end - find the highest aligned address between two vmap_areas
  1988. * @pnext: in/out arg for the next vmap_area
  1989. * @pprev: in/out arg for the previous vmap_area
  1990. * @align: alignment
  1991. *
  1992. * Returns: determined end address
  1993. *
  1994. * Find the highest aligned address between *@pnext and *@pprev below
  1995. * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned
  1996. * down address is between the end addresses of the two vmap_areas.
  1997. *
  1998. * Please note that the address returned by this function may fall
  1999. * inside *@pnext vmap_area. The caller is responsible for checking
  2000. * that.
  2001. */
  2002. static unsigned long pvm_determine_end(struct vmap_area **pnext,
  2003. struct vmap_area **pprev,
  2004. unsigned long align)
  2005. {
  2006. const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
  2007. unsigned long addr;
  2008. if (*pnext)
  2009. addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
  2010. else
  2011. addr = vmalloc_end;
  2012. while (*pprev && (*pprev)->va_end > addr) {
  2013. *pnext = *pprev;
  2014. *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
  2015. }
  2016. return addr;
  2017. }
  2018. /**
  2019. * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
  2020. * @offsets: array containing offset of each area
  2021. * @sizes: array containing size of each area
  2022. * @nr_vms: the number of areas to allocate
  2023. * @align: alignment, all entries in @offsets and @sizes must be aligned to this
  2024. *
  2025. * Returns: kmalloc'd vm_struct pointer array pointing to allocated
  2026. * vm_structs on success, %NULL on failure
  2027. *
  2028. * Percpu allocator wants to use congruent vm areas so that it can
  2029. * maintain the offsets among percpu areas. This function allocates
  2030. * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
  2031. * be scattered pretty far, distance between two areas easily going up
  2032. * to gigabytes. To avoid interacting with regular vmallocs, these
  2033. * areas are allocated from top.
  2034. *
  2035. * Despite its complicated look, this allocator is rather simple. It
  2036. * does everything top-down and scans areas from the end looking for
  2037. * matching slot. While scanning, if any of the areas overlaps with
  2038. * existing vmap_area, the base address is pulled down to fit the
  2039. * area. Scanning is repeated till all the areas fit and then all
  2040. * necessary data structres are inserted and the result is returned.
  2041. */
  2042. struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
  2043. const size_t *sizes, int nr_vms,
  2044. size_t align)
  2045. {
  2046. const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
  2047. const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
  2048. struct vmap_area **vas, *prev, *next;
  2049. struct vm_struct **vms;
  2050. int area, area2, last_area, term_area;
  2051. unsigned long base, start, end, last_end;
  2052. bool purged = false;
  2053. /* verify parameters and allocate data structures */
  2054. BUG_ON(align & ~PAGE_MASK || !is_power_of_2(align));
  2055. for (last_area = 0, area = 0; area < nr_vms; area++) {
  2056. start = offsets[area];
  2057. end = start + sizes[area];
  2058. /* is everything aligned properly? */
  2059. BUG_ON(!IS_ALIGNED(offsets[area], align));
  2060. BUG_ON(!IS_ALIGNED(sizes[area], align));
  2061. /* detect the area with the highest address */
  2062. if (start > offsets[last_area])
  2063. last_area = area;
  2064. for (area2 = 0; area2 < nr_vms; area2++) {
  2065. unsigned long start2 = offsets[area2];
  2066. unsigned long end2 = start2 + sizes[area2];
  2067. if (area2 == area)
  2068. continue;
  2069. BUG_ON(start2 >= start && start2 < end);
  2070. BUG_ON(end2 <= end && end2 > start);
  2071. }
  2072. }
  2073. last_end = offsets[last_area] + sizes[last_area];
  2074. if (vmalloc_end - vmalloc_start < last_end) {
  2075. WARN_ON(true);
  2076. return NULL;
  2077. }
  2078. vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
  2079. vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
  2080. if (!vas || !vms)
  2081. goto err_free2;
  2082. for (area = 0; area < nr_vms; area++) {
  2083. vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL);
  2084. vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
  2085. if (!vas[area] || !vms[area])
  2086. goto err_free;
  2087. }
  2088. retry:
  2089. spin_lock(&vmap_area_lock);
  2090. /* start scanning - we scan from the top, begin with the last area */
  2091. area = term_area = last_area;
  2092. start = offsets[area];
  2093. end = start + sizes[area];
  2094. if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
  2095. base = vmalloc_end - last_end;
  2096. goto found;
  2097. }
  2098. base = pvm_determine_end(&next, &prev, align) - end;
  2099. while (true) {
  2100. BUG_ON(next && next->va_end <= base + end);
  2101. BUG_ON(prev && prev->va_end > base + end);
  2102. /*
  2103. * base might have underflowed, add last_end before
  2104. * comparing.
  2105. */
  2106. if (base + last_end < vmalloc_start + last_end) {
  2107. spin_unlock(&vmap_area_lock);
  2108. if (!purged) {
  2109. purge_vmap_area_lazy();
  2110. purged = true;
  2111. goto retry;
  2112. }
  2113. goto err_free;
  2114. }
  2115. /*
  2116. * If next overlaps, move base downwards so that it's
  2117. * right below next and then recheck.
  2118. */
  2119. if (next && next->va_start < base + end) {
  2120. base = pvm_determine_end(&next, &prev, align) - end;
  2121. term_area = area;
  2122. continue;
  2123. }
  2124. /*
  2125. * If prev overlaps, shift down next and prev and move
  2126. * base so that it's right below new next and then
  2127. * recheck.
  2128. */
  2129. if (prev && prev->va_end > base + start) {
  2130. next = prev;
  2131. prev = node_to_va(rb_prev(&next->rb_node));
  2132. base = pvm_determine_end(&next, &prev, align) - end;
  2133. term_area = area;
  2134. continue;
  2135. }
  2136. /*
  2137. * This area fits, move on to the previous one. If
  2138. * the previous one is the terminal one, we're done.
  2139. */
  2140. area = (area + nr_vms - 1) % nr_vms;
  2141. if (area == term_area)
  2142. break;
  2143. start = offsets[area];
  2144. end = start + sizes[area];
  2145. pvm_find_next_prev(base + end, &next, &prev);
  2146. }
  2147. found:
  2148. /* we've found a fitting base, insert all va's */
  2149. for (area = 0; area < nr_vms; area++) {
  2150. struct vmap_area *va = vas[area];
  2151. va->va_start = base + offsets[area];
  2152. va->va_end = va->va_start + sizes[area];
  2153. __insert_vmap_area(va);
  2154. }
  2155. vmap_area_pcpu_hole = base + offsets[last_area];
  2156. spin_unlock(&vmap_area_lock);
  2157. /* insert all vm's */
  2158. for (area = 0; area < nr_vms; area++)
  2159. setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
  2160. pcpu_get_vm_areas);
  2161. kfree(vas);
  2162. return vms;
  2163. err_free:
  2164. for (area = 0; area < nr_vms; area++) {
  2165. kfree(vas[area]);
  2166. kfree(vms[area]);
  2167. }
  2168. err_free2:
  2169. kfree(vas);
  2170. kfree(vms);
  2171. return NULL;
  2172. }
  2173. /**
  2174. * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
  2175. * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
  2176. * @nr_vms: the number of allocated areas
  2177. *
  2178. * Free vm_structs and the array allocated by pcpu_get_vm_areas().
  2179. */
  2180. void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
  2181. {
  2182. int i;
  2183. for (i = 0; i < nr_vms; i++)
  2184. free_vm_area(vms[i]);
  2185. kfree(vms);
  2186. }
  2187. #endif /* CONFIG_SMP */
  2188. #ifdef CONFIG_PROC_FS
  2189. static void *s_start(struct seq_file *m, loff_t *pos)
  2190. __acquires(&vmap_area_lock)
  2191. {
  2192. loff_t n = *pos;
  2193. struct vmap_area *va;
  2194. spin_lock(&vmap_area_lock);
  2195. va = list_entry((&vmap_area_list)->next, typeof(*va), list);
  2196. while (n > 0 && &va->list != &vmap_area_list) {
  2197. n--;
  2198. va = list_entry(va->list.next, typeof(*va), list);
  2199. }
  2200. if (!n && &va->list != &vmap_area_list)
  2201. return va;
  2202. return NULL;
  2203. }
  2204. static void *s_next(struct seq_file *m, void *p, loff_t *pos)
  2205. {
  2206. struct vmap_area *va = p, *next;
  2207. ++*pos;
  2208. next = list_entry(va->list.next, typeof(*va), list);
  2209. if (&next->list != &vmap_area_list)
  2210. return next;
  2211. return NULL;
  2212. }
  2213. static void s_stop(struct seq_file *m, void *p)
  2214. __releases(&vmap_area_lock)
  2215. {
  2216. spin_unlock(&vmap_area_lock);
  2217. }
  2218. static void show_numa_info(struct seq_file *m, struct vm_struct *v)
  2219. {
  2220. if (IS_ENABLED(CONFIG_NUMA)) {
  2221. unsigned int nr, *counters = m->private;
  2222. if (!counters)
  2223. return;
  2224. memset(counters, 0, nr_node_ids * sizeof(unsigned int));
  2225. for (nr = 0; nr < v->nr_pages; nr++)
  2226. counters[page_to_nid(v->pages[nr])]++;
  2227. for_each_node_state(nr, N_HIGH_MEMORY)
  2228. if (counters[nr])
  2229. seq_printf(m, " N%u=%u", nr, counters[nr]);
  2230. }
  2231. }
  2232. static int s_show(struct seq_file *m, void *p)
  2233. {
  2234. struct vmap_area *va = p;
  2235. struct vm_struct *v;
  2236. if (va->flags & (VM_LAZY_FREE | VM_LAZY_FREEING))
  2237. return 0;
  2238. if (!(va->flags & VM_VM_AREA)) {
  2239. seq_printf(m, "0x%pK-0x%pK %7ld vm_map_ram\n",
  2240. (void *)va->va_start, (void *)va->va_end,
  2241. va->va_end - va->va_start);
  2242. return 0;
  2243. }
  2244. v = va->vm;
  2245. /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
  2246. smp_rmb();
  2247. if (v->flags & VM_UNINITIALIZED)
  2248. return 0;
  2249. seq_printf(m, "0x%pK-0x%pK %7ld",
  2250. v->addr, v->addr + v->size, v->size);
  2251. if (v->caller)
  2252. seq_printf(m, " %pS", v->caller);
  2253. if (v->nr_pages)
  2254. seq_printf(m, " pages=%d", v->nr_pages);
  2255. if (v->phys_addr)
  2256. seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr);
  2257. if (v->flags & VM_IOREMAP)
  2258. seq_printf(m, " ioremap");
  2259. if (v->flags & VM_ALLOC)
  2260. seq_printf(m, " vmalloc");
  2261. if (v->flags & VM_MAP)
  2262. seq_printf(m, " vmap");
  2263. if (v->flags & VM_USERMAP)
  2264. seq_printf(m, " user");
  2265. if (v->flags & VM_VPAGES)
  2266. seq_printf(m, " vpages");
  2267. show_numa_info(m, v);
  2268. seq_putc(m, '\n');
  2269. return 0;
  2270. }
  2271. static const struct seq_operations vmalloc_op = {
  2272. .start = s_start,
  2273. .next = s_next,
  2274. .stop = s_stop,
  2275. .show = s_show,
  2276. };
  2277. static int vmalloc_open(struct inode *inode, struct file *file)
  2278. {
  2279. unsigned int *ptr = NULL;
  2280. int ret;
  2281. if (IS_ENABLED(CONFIG_NUMA)) {
  2282. ptr = kmalloc(nr_node_ids * sizeof(unsigned int), GFP_KERNEL);
  2283. if (ptr == NULL)
  2284. return -ENOMEM;
  2285. }
  2286. ret = seq_open(file, &vmalloc_op);
  2287. if (!ret) {
  2288. struct seq_file *m = file->private_data;
  2289. m->private = ptr;
  2290. } else
  2291. kfree(ptr);
  2292. return ret;
  2293. }
  2294. static const struct file_operations proc_vmalloc_operations = {
  2295. .open = vmalloc_open,
  2296. .read = seq_read,
  2297. .llseek = seq_lseek,
  2298. .release = seq_release_private,
  2299. };
  2300. static int __init proc_vmalloc_init(void)
  2301. {
  2302. proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations);
  2303. return 0;
  2304. }
  2305. module_init(proc_vmalloc_init);
  2306. void get_vmalloc_info(struct vmalloc_info *vmi)
  2307. {
  2308. struct vmap_area *va;
  2309. unsigned long free_area_size;
  2310. unsigned long prev_end;
  2311. vmi->used = 0;
  2312. vmi->largest_chunk = 0;
  2313. prev_end = VMALLOC_START;
  2314. spin_lock(&vmap_area_lock);
  2315. if (list_empty(&vmap_area_list)) {
  2316. vmi->largest_chunk = VMALLOC_TOTAL;
  2317. goto out;
  2318. }
  2319. list_for_each_entry(va, &vmap_area_list, list) {
  2320. unsigned long addr = va->va_start;
  2321. /*
  2322. * Some archs keep another range for modules in vmalloc space
  2323. */
  2324. if (addr < VMALLOC_START)
  2325. continue;
  2326. if (addr >= VMALLOC_END)
  2327. break;
  2328. if (va->flags & (VM_LAZY_FREE | VM_LAZY_FREEING))
  2329. continue;
  2330. vmi->used += (va->va_end - va->va_start);
  2331. free_area_size = addr - prev_end;
  2332. if (vmi->largest_chunk < free_area_size)
  2333. vmi->largest_chunk = free_area_size;
  2334. prev_end = va->va_end;
  2335. }
  2336. if (VMALLOC_END - prev_end > vmi->largest_chunk)
  2337. vmi->largest_chunk = VMALLOC_END - prev_end;
  2338. out:
  2339. spin_unlock(&vmap_area_lock);
  2340. }
  2341. #endif