vmalloc.c 68 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721
  1. /*
  2. * linux/mm/vmalloc.c
  3. *
  4. * Copyright (C) 1993 Linus Torvalds
  5. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  6. * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
  7. * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
  8. * Numa awareness, Christoph Lameter, SGI, June 2005
  9. */
  10. #include <linux/vmalloc.h>
  11. #include <linux/mm.h>
  12. #include <linux/module.h>
  13. #include <linux/highmem.h>
  14. #include <linux/sched.h>
  15. #include <linux/slab.h>
  16. #include <linux/spinlock.h>
  17. #include <linux/interrupt.h>
  18. #include <linux/proc_fs.h>
  19. #include <linux/seq_file.h>
  20. #include <linux/debugobjects.h>
  21. #include <linux/kallsyms.h>
  22. #include <linux/list.h>
  23. #include <linux/rbtree.h>
  24. #include <linux/radix-tree.h>
  25. #include <linux/rcupdate.h>
  26. #include <linux/pfn.h>
  27. #include <linux/kmemleak.h>
  28. #include <linux/atomic.h>
  29. #include <linux/llist.h>
  30. #include <asm/uaccess.h>
  31. #include <asm/tlbflush.h>
  32. #include <asm/shmparam.h>
  33. struct vfree_deferred {
  34. struct llist_head list;
  35. struct work_struct wq;
  36. };
  37. static DEFINE_PER_CPU(struct vfree_deferred, vfree_deferred);
  38. static void __vunmap(const void *, int);
  39. static void free_work(struct work_struct *w)
  40. {
  41. struct vfree_deferred *p = container_of(w, struct vfree_deferred, wq);
  42. struct llist_node *llnode = llist_del_all(&p->list);
  43. while (llnode) {
  44. void *p = llnode;
  45. llnode = llist_next(llnode);
  46. __vunmap(p, 1);
  47. }
  48. }
  49. /*** Page table manipulation functions ***/
  50. static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
  51. {
  52. pte_t *pte;
  53. pte = pte_offset_kernel(pmd, addr);
  54. do {
  55. pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
  56. WARN_ON(!pte_none(ptent) && !pte_present(ptent));
  57. } while (pte++, addr += PAGE_SIZE, addr != end);
  58. }
  59. static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end)
  60. {
  61. pmd_t *pmd;
  62. unsigned long next;
  63. pmd = pmd_offset(pud, addr);
  64. do {
  65. next = pmd_addr_end(addr, end);
  66. if (pmd_none_or_clear_bad(pmd))
  67. continue;
  68. vunmap_pte_range(pmd, addr, next);
  69. } while (pmd++, addr = next, addr != end);
  70. }
  71. static void vunmap_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end)
  72. {
  73. pud_t *pud;
  74. unsigned long next;
  75. pud = pud_offset(pgd, addr);
  76. do {
  77. next = pud_addr_end(addr, end);
  78. if (pud_none_or_clear_bad(pud))
  79. continue;
  80. vunmap_pmd_range(pud, addr, next);
  81. } while (pud++, addr = next, addr != end);
  82. }
  83. static void vunmap_page_range(unsigned long addr, unsigned long end)
  84. {
  85. pgd_t *pgd;
  86. unsigned long next;
  87. BUG_ON(addr >= end);
  88. pgd = pgd_offset_k(addr);
  89. do {
  90. next = pgd_addr_end(addr, end);
  91. if (pgd_none_or_clear_bad(pgd))
  92. continue;
  93. vunmap_pud_range(pgd, addr, next);
  94. } while (pgd++, addr = next, addr != end);
  95. }
  96. static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
  97. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  98. {
  99. pte_t *pte;
  100. /*
  101. * nr is a running index into the array which helps higher level
  102. * callers keep track of where we're up to.
  103. */
  104. pte = pte_alloc_kernel(pmd, addr);
  105. if (!pte)
  106. return -ENOMEM;
  107. do {
  108. struct page *page = pages[*nr];
  109. if (WARN_ON(!pte_none(*pte)))
  110. return -EBUSY;
  111. if (WARN_ON(!page))
  112. return -ENOMEM;
  113. set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
  114. (*nr)++;
  115. } while (pte++, addr += PAGE_SIZE, addr != end);
  116. return 0;
  117. }
  118. static int vmap_pmd_range(pud_t *pud, unsigned long addr,
  119. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  120. {
  121. pmd_t *pmd;
  122. unsigned long next;
  123. pmd = pmd_alloc(&init_mm, pud, addr);
  124. if (!pmd)
  125. return -ENOMEM;
  126. do {
  127. next = pmd_addr_end(addr, end);
  128. if (vmap_pte_range(pmd, addr, next, prot, pages, nr))
  129. return -ENOMEM;
  130. } while (pmd++, addr = next, addr != end);
  131. return 0;
  132. }
  133. static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
  134. unsigned long end, pgprot_t prot, struct page **pages, int *nr)
  135. {
  136. pud_t *pud;
  137. unsigned long next;
  138. pud = pud_alloc(&init_mm, pgd, addr);
  139. if (!pud)
  140. return -ENOMEM;
  141. do {
  142. next = pud_addr_end(addr, end);
  143. if (vmap_pmd_range(pud, addr, next, prot, pages, nr))
  144. return -ENOMEM;
  145. } while (pud++, addr = next, addr != end);
  146. return 0;
  147. }
  148. /*
  149. * Set up page tables in kva (addr, end). The ptes shall have prot "prot", and
  150. * will have pfns corresponding to the "pages" array.
  151. *
  152. * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
  153. */
  154. static int vmap_page_range_noflush(unsigned long start, unsigned long end,
  155. pgprot_t prot, struct page **pages)
  156. {
  157. pgd_t *pgd;
  158. unsigned long next;
  159. unsigned long addr = start;
  160. int err = 0;
  161. int nr = 0;
  162. BUG_ON(addr >= end);
  163. pgd = pgd_offset_k(addr);
  164. do {
  165. next = pgd_addr_end(addr, end);
  166. err = vmap_pud_range(pgd, addr, next, prot, pages, &nr);
  167. if (err)
  168. return err;
  169. } while (pgd++, addr = next, addr != end);
  170. return nr;
  171. }
  172. static int vmap_page_range(unsigned long start, unsigned long end,
  173. pgprot_t prot, struct page **pages)
  174. {
  175. int ret;
  176. ret = vmap_page_range_noflush(start, end, prot, pages);
  177. flush_cache_vmap(start, end);
  178. return ret;
  179. }
  180. int is_vmalloc_or_module_addr(const void *x)
  181. {
  182. /*
  183. * ARM, x86-64 and sparc64 put modules in a special place,
  184. * and fall back on vmalloc() if that fails. Others
  185. * just put it in the vmalloc space.
  186. */
  187. #if defined(CONFIG_MODULES) && defined(MODULES_VADDR)
  188. unsigned long addr = (unsigned long)x;
  189. if (addr >= MODULES_VADDR && addr < MODULES_END)
  190. return 1;
  191. #endif
  192. return is_vmalloc_addr(x);
  193. }
  194. /*
  195. * Walk a vmap address to the struct page it maps.
  196. */
  197. struct page *vmalloc_to_page(const void *vmalloc_addr)
  198. {
  199. unsigned long addr = (unsigned long) vmalloc_addr;
  200. struct page *page = NULL;
  201. pgd_t *pgd = pgd_offset_k(addr);
  202. /*
  203. * XXX we might need to change this if we add VIRTUAL_BUG_ON for
  204. * architectures that do not vmalloc module space
  205. */
  206. VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr));
  207. if (!pgd_none(*pgd)) {
  208. pud_t *pud = pud_offset(pgd, addr);
  209. if (!pud_none(*pud)) {
  210. pmd_t *pmd = pmd_offset(pud, addr);
  211. if (!pmd_none(*pmd)) {
  212. pte_t *ptep, pte;
  213. ptep = pte_offset_map(pmd, addr);
  214. pte = *ptep;
  215. if (pte_present(pte))
  216. page = pte_page(pte);
  217. pte_unmap(ptep);
  218. }
  219. }
  220. }
  221. return page;
  222. }
  223. EXPORT_SYMBOL(vmalloc_to_page);
  224. /*
  225. * Map a vmalloc()-space virtual address to the physical page frame number.
  226. */
  227. unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
  228. {
  229. return page_to_pfn(vmalloc_to_page(vmalloc_addr));
  230. }
  231. EXPORT_SYMBOL(vmalloc_to_pfn);
  232. /*** Global kva allocator ***/
  233. #define VM_LAZY_FREE 0x01
  234. #define VM_LAZY_FREEING 0x02
  235. #define VM_VM_AREA 0x04
  236. static DEFINE_SPINLOCK(vmap_area_lock);
  237. /* Export for kexec only */
  238. LIST_HEAD(vmap_area_list);
  239. static struct rb_root vmap_area_root = RB_ROOT;
  240. /* The vmap cache globals are protected by vmap_area_lock */
  241. static struct rb_node *free_vmap_cache;
  242. static unsigned long cached_hole_size;
  243. static unsigned long cached_vstart;
  244. static unsigned long cached_align;
  245. static unsigned long vmap_area_pcpu_hole;
  246. static struct vmap_area *__find_vmap_area(unsigned long addr)
  247. {
  248. struct rb_node *n = vmap_area_root.rb_node;
  249. while (n) {
  250. struct vmap_area *va;
  251. va = rb_entry(n, struct vmap_area, rb_node);
  252. if (addr < va->va_start)
  253. n = n->rb_left;
  254. else if (addr >= va->va_end)
  255. n = n->rb_right;
  256. else
  257. return va;
  258. }
  259. return NULL;
  260. }
  261. static void __insert_vmap_area(struct vmap_area *va)
  262. {
  263. struct rb_node **p = &vmap_area_root.rb_node;
  264. struct rb_node *parent = NULL;
  265. struct rb_node *tmp;
  266. while (*p) {
  267. struct vmap_area *tmp_va;
  268. parent = *p;
  269. tmp_va = rb_entry(parent, struct vmap_area, rb_node);
  270. if (va->va_start < tmp_va->va_end)
  271. p = &(*p)->rb_left;
  272. else if (va->va_end > tmp_va->va_start)
  273. p = &(*p)->rb_right;
  274. else
  275. BUG();
  276. }
  277. rb_link_node(&va->rb_node, parent, p);
  278. rb_insert_color(&va->rb_node, &vmap_area_root);
  279. /* address-sort this list */
  280. tmp = rb_prev(&va->rb_node);
  281. if (tmp) {
  282. struct vmap_area *prev;
  283. prev = rb_entry(tmp, struct vmap_area, rb_node);
  284. list_add_rcu(&va->list, &prev->list);
  285. } else
  286. list_add_rcu(&va->list, &vmap_area_list);
  287. }
  288. static void purge_vmap_area_lazy(void);
  289. /*
  290. * Allocate a region of KVA of the specified size and alignment, within the
  291. * vstart and vend.
  292. */
  293. static struct vmap_area *alloc_vmap_area(unsigned long size,
  294. unsigned long align,
  295. unsigned long vstart, unsigned long vend,
  296. int node, gfp_t gfp_mask)
  297. {
  298. struct vmap_area *va;
  299. struct rb_node *n;
  300. unsigned long addr;
  301. int purged = 0;
  302. struct vmap_area *first;
  303. BUG_ON(!size);
  304. BUG_ON(size & ~PAGE_MASK);
  305. BUG_ON(!is_power_of_2(align));
  306. va = kmalloc_node(sizeof(struct vmap_area),
  307. gfp_mask & GFP_RECLAIM_MASK, node);
  308. if (unlikely(!va))
  309. return ERR_PTR(-ENOMEM);
  310. /*
  311. * Only scan the relevant parts containing pointers to other objects
  312. * to avoid false negatives.
  313. */
  314. kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);
  315. retry:
  316. spin_lock(&vmap_area_lock);
  317. /*
  318. * Invalidate cache if we have more permissive parameters.
  319. * cached_hole_size notes the largest hole noticed _below_
  320. * the vmap_area cached in free_vmap_cache: if size fits
  321. * into that hole, we want to scan from vstart to reuse
  322. * the hole instead of allocating above free_vmap_cache.
  323. * Note that __free_vmap_area may update free_vmap_cache
  324. * without updating cached_hole_size or cached_align.
  325. */
  326. if (!free_vmap_cache ||
  327. size < cached_hole_size ||
  328. vstart < cached_vstart ||
  329. align < cached_align) {
  330. nocache:
  331. cached_hole_size = 0;
  332. free_vmap_cache = NULL;
  333. }
  334. /* record if we encounter less permissive parameters */
  335. cached_vstart = vstart;
  336. cached_align = align;
  337. /* find starting point for our search */
  338. if (free_vmap_cache) {
  339. first = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
  340. addr = ALIGN(first->va_end, align);
  341. if (addr < vstart)
  342. goto nocache;
  343. if (addr + size < addr)
  344. goto overflow;
  345. } else {
  346. addr = ALIGN(vstart, align);
  347. if (addr + size < addr)
  348. goto overflow;
  349. n = vmap_area_root.rb_node;
  350. first = NULL;
  351. while (n) {
  352. struct vmap_area *tmp;
  353. tmp = rb_entry(n, struct vmap_area, rb_node);
  354. if (tmp->va_end >= addr) {
  355. first = tmp;
  356. if (tmp->va_start <= addr)
  357. break;
  358. n = n->rb_left;
  359. } else
  360. n = n->rb_right;
  361. }
  362. if (!first)
  363. goto found;
  364. }
  365. /* from the starting point, walk areas until a suitable hole is found */
  366. while (addr + size > first->va_start && addr + size <= vend) {
  367. if (addr + cached_hole_size < first->va_start)
  368. cached_hole_size = first->va_start - addr;
  369. addr = ALIGN(first->va_end, align);
  370. if (addr + size < addr)
  371. goto overflow;
  372. if (list_is_last(&first->list, &vmap_area_list))
  373. goto found;
  374. first = list_entry(first->list.next,
  375. struct vmap_area, list);
  376. }
  377. found:
  378. if (addr + size > vend)
  379. goto overflow;
  380. va->va_start = addr;
  381. va->va_end = addr + size;
  382. va->flags = 0;
  383. __insert_vmap_area(va);
  384. free_vmap_cache = &va->rb_node;
  385. spin_unlock(&vmap_area_lock);
  386. BUG_ON(va->va_start & (align-1));
  387. BUG_ON(va->va_start < vstart);
  388. BUG_ON(va->va_end > vend);
  389. return va;
  390. overflow:
  391. spin_unlock(&vmap_area_lock);
  392. if (!purged) {
  393. purge_vmap_area_lazy();
  394. purged = 1;
  395. goto retry;
  396. }
  397. if (printk_ratelimit())
  398. printk(KERN_WARNING
  399. "vmap allocation for size %lu failed: "
  400. "use vmalloc=<size> to increase size.\n", size);
  401. kfree(va);
  402. return ERR_PTR(-EBUSY);
  403. }
  404. static void __free_vmap_area(struct vmap_area *va)
  405. {
  406. BUG_ON(RB_EMPTY_NODE(&va->rb_node));
  407. if (free_vmap_cache) {
  408. if (va->va_end < cached_vstart) {
  409. free_vmap_cache = NULL;
  410. } else {
  411. struct vmap_area *cache;
  412. cache = rb_entry(free_vmap_cache, struct vmap_area, rb_node);
  413. if (va->va_start <= cache->va_start) {
  414. free_vmap_cache = rb_prev(&va->rb_node);
  415. /*
  416. * We don't try to update cached_hole_size or
  417. * cached_align, but it won't go very wrong.
  418. */
  419. }
  420. }
  421. }
  422. rb_erase(&va->rb_node, &vmap_area_root);
  423. RB_CLEAR_NODE(&va->rb_node);
  424. list_del_rcu(&va->list);
  425. /*
  426. * Track the highest possible candidate for pcpu area
  427. * allocation. Areas outside of vmalloc area can be returned
  428. * here too, consider only end addresses which fall inside
  429. * vmalloc area proper.
  430. */
  431. if (va->va_end > VMALLOC_START && va->va_end <= VMALLOC_END)
  432. vmap_area_pcpu_hole = max(vmap_area_pcpu_hole, va->va_end);
  433. kfree_rcu(va, rcu_head);
  434. }
  435. /*
  436. * Free a region of KVA allocated by alloc_vmap_area
  437. */
  438. static void free_vmap_area(struct vmap_area *va)
  439. {
  440. spin_lock(&vmap_area_lock);
  441. __free_vmap_area(va);
  442. spin_unlock(&vmap_area_lock);
  443. }
  444. /*
  445. * Clear the pagetable entries of a given vmap_area
  446. */
  447. static void unmap_vmap_area(struct vmap_area *va)
  448. {
  449. vunmap_page_range(va->va_start, va->va_end);
  450. }
  451. static void vmap_debug_free_range(unsigned long start, unsigned long end)
  452. {
  453. /*
  454. * Unmap page tables and force a TLB flush immediately if
  455. * CONFIG_DEBUG_PAGEALLOC is set. This catches use after free
  456. * bugs similarly to those in linear kernel virtual address
  457. * space after a page has been freed.
  458. *
  459. * All the lazy freeing logic is still retained, in order to
  460. * minimise intrusiveness of this debugging feature.
  461. *
  462. * This is going to be *slow* (linear kernel virtual address
  463. * debugging doesn't do a broadcast TLB flush so it is a lot
  464. * faster).
  465. */
  466. #ifdef CONFIG_DEBUG_PAGEALLOC
  467. vunmap_page_range(start, end);
  468. flush_tlb_kernel_range(start, end);
  469. #endif
  470. }
  471. /*
  472. * lazy_max_pages is the maximum amount of virtual address space we gather up
  473. * before attempting to purge with a TLB flush.
  474. *
  475. * There is a tradeoff here: a larger number will cover more kernel page tables
  476. * and take slightly longer to purge, but it will linearly reduce the number of
  477. * global TLB flushes that must be performed. It would seem natural to scale
  478. * this number up linearly with the number of CPUs (because vmapping activity
  479. * could also scale linearly with the number of CPUs), however it is likely
  480. * that in practice, workloads might be constrained in other ways that mean
  481. * vmap activity will not scale linearly with CPUs. Also, I want to be
  482. * conservative and not introduce a big latency on huge systems, so go with
  483. * a less aggressive log scale. It will still be an improvement over the old
  484. * code, and it will be simple to change the scale factor if we find that it
  485. * becomes a problem on bigger systems.
  486. */
  487. static unsigned long lazy_max_pages(void)
  488. {
  489. unsigned int log;
  490. log = fls(num_online_cpus());
  491. return log * (32UL * 1024 * 1024 / PAGE_SIZE);
  492. }
  493. static atomic_t vmap_lazy_nr = ATOMIC_INIT(0);
  494. /* for per-CPU blocks */
  495. static void purge_fragmented_blocks_allcpus(void);
  496. /*
  497. * called before a call to iounmap() if the caller wants vm_area_struct's
  498. * immediately freed.
  499. */
  500. void set_iounmap_nonlazy(void)
  501. {
  502. atomic_set(&vmap_lazy_nr, lazy_max_pages()+1);
  503. }
  504. /*
  505. * Purges all lazily-freed vmap areas.
  506. *
  507. * If sync is 0 then don't purge if there is already a purge in progress.
  508. * If force_flush is 1, then flush kernel TLBs between *start and *end even
  509. * if we found no lazy vmap areas to unmap (callers can use this to optimise
  510. * their own TLB flushing).
  511. * Returns with *start = min(*start, lowest purged address)
  512. * *end = max(*end, highest purged address)
  513. */
  514. static void __purge_vmap_area_lazy(unsigned long *start, unsigned long *end,
  515. int sync, int force_flush)
  516. {
  517. static DEFINE_SPINLOCK(purge_lock);
  518. LIST_HEAD(valist);
  519. struct vmap_area *va;
  520. struct vmap_area *n_va;
  521. int nr = 0;
  522. /*
  523. * If sync is 0 but force_flush is 1, we'll go sync anyway but callers
  524. * should not expect such behaviour. This just simplifies locking for
  525. * the case that isn't actually used at the moment anyway.
  526. */
  527. if (!sync && !force_flush) {
  528. if (!spin_trylock(&purge_lock))
  529. return;
  530. } else
  531. spin_lock(&purge_lock);
  532. if (sync)
  533. purge_fragmented_blocks_allcpus();
  534. rcu_read_lock();
  535. list_for_each_entry_rcu(va, &vmap_area_list, list) {
  536. if (va->flags & VM_LAZY_FREE) {
  537. if (va->va_start < *start)
  538. *start = va->va_start;
  539. if (va->va_end > *end)
  540. *end = va->va_end;
  541. nr += (va->va_end - va->va_start) >> PAGE_SHIFT;
  542. list_add_tail(&va->purge_list, &valist);
  543. va->flags |= VM_LAZY_FREEING;
  544. va->flags &= ~VM_LAZY_FREE;
  545. }
  546. }
  547. rcu_read_unlock();
  548. if (nr)
  549. atomic_sub(nr, &vmap_lazy_nr);
  550. if (nr || force_flush)
  551. flush_tlb_kernel_range(*start, *end);
  552. if (nr) {
  553. spin_lock(&vmap_area_lock);
  554. list_for_each_entry_safe(va, n_va, &valist, purge_list)
  555. __free_vmap_area(va);
  556. spin_unlock(&vmap_area_lock);
  557. }
  558. spin_unlock(&purge_lock);
  559. }
  560. /*
  561. * Kick off a purge of the outstanding lazy areas. Don't bother if somebody
  562. * is already purging.
  563. */
  564. static void try_purge_vmap_area_lazy(void)
  565. {
  566. unsigned long start = ULONG_MAX, end = 0;
  567. __purge_vmap_area_lazy(&start, &end, 0, 0);
  568. }
  569. /*
  570. * Kick off a purge of the outstanding lazy areas.
  571. */
  572. static void purge_vmap_area_lazy(void)
  573. {
  574. unsigned long start = ULONG_MAX, end = 0;
  575. __purge_vmap_area_lazy(&start, &end, 1, 0);
  576. }
  577. /*
  578. * Free a vmap area, caller ensuring that the area has been unmapped
  579. * and flush_cache_vunmap had been called for the correct range
  580. * previously.
  581. */
  582. static void free_vmap_area_noflush(struct vmap_area *va)
  583. {
  584. va->flags |= VM_LAZY_FREE;
  585. atomic_add((va->va_end - va->va_start) >> PAGE_SHIFT, &vmap_lazy_nr);
  586. if (unlikely(atomic_read(&vmap_lazy_nr) > lazy_max_pages()))
  587. try_purge_vmap_area_lazy();
  588. }
  589. /*
  590. * Free and unmap a vmap area, caller ensuring flush_cache_vunmap had been
  591. * called for the correct range previously.
  592. */
  593. static void free_unmap_vmap_area_noflush(struct vmap_area *va)
  594. {
  595. unmap_vmap_area(va);
  596. free_vmap_area_noflush(va);
  597. }
  598. /*
  599. * Free and unmap a vmap area
  600. */
  601. static void free_unmap_vmap_area(struct vmap_area *va)
  602. {
  603. flush_cache_vunmap(va->va_start, va->va_end);
  604. free_unmap_vmap_area_noflush(va);
  605. }
  606. static struct vmap_area *find_vmap_area(unsigned long addr)
  607. {
  608. struct vmap_area *va;
  609. spin_lock(&vmap_area_lock);
  610. va = __find_vmap_area(addr);
  611. spin_unlock(&vmap_area_lock);
  612. return va;
  613. }
  614. static void free_unmap_vmap_area_addr(unsigned long addr)
  615. {
  616. struct vmap_area *va;
  617. va = find_vmap_area(addr);
  618. BUG_ON(!va);
  619. free_unmap_vmap_area(va);
  620. }
  621. /*** Per cpu kva allocator ***/
  622. /*
  623. * vmap space is limited especially on 32 bit architectures. Ensure there is
  624. * room for at least 16 percpu vmap blocks per CPU.
  625. */
  626. /*
  627. * If we had a constant VMALLOC_START and VMALLOC_END, we'd like to be able
  628. * to #define VMALLOC_SPACE (VMALLOC_END-VMALLOC_START). Guess
  629. * instead (we just need a rough idea)
  630. */
  631. #if BITS_PER_LONG == 32
  632. #define VMALLOC_SPACE (128UL*1024*1024)
  633. #else
  634. #define VMALLOC_SPACE (128UL*1024*1024*1024)
  635. #endif
  636. #define VMALLOC_PAGES (VMALLOC_SPACE / PAGE_SIZE)
  637. #define VMAP_MAX_ALLOC BITS_PER_LONG /* 256K with 4K pages */
  638. #define VMAP_BBMAP_BITS_MAX 1024 /* 4MB with 4K pages */
  639. #define VMAP_BBMAP_BITS_MIN (VMAP_MAX_ALLOC*2)
  640. #define VMAP_MIN(x, y) ((x) < (y) ? (x) : (y)) /* can't use min() */
  641. #define VMAP_MAX(x, y) ((x) > (y) ? (x) : (y)) /* can't use max() */
  642. #define VMAP_BBMAP_BITS \
  643. VMAP_MIN(VMAP_BBMAP_BITS_MAX, \
  644. VMAP_MAX(VMAP_BBMAP_BITS_MIN, \
  645. VMALLOC_PAGES / roundup_pow_of_two(NR_CPUS) / 16))
  646. #define VMAP_BLOCK_SIZE (VMAP_BBMAP_BITS * PAGE_SIZE)
  647. static bool vmap_initialized __read_mostly = false;
  648. struct vmap_block_queue {
  649. spinlock_t lock;
  650. struct list_head free;
  651. };
  652. struct vmap_block {
  653. spinlock_t lock;
  654. struct vmap_area *va;
  655. unsigned long free, dirty;
  656. DECLARE_BITMAP(dirty_map, VMAP_BBMAP_BITS);
  657. struct list_head free_list;
  658. struct rcu_head rcu_head;
  659. struct list_head purge;
  660. };
  661. /* Queue of free and dirty vmap blocks, for allocation and flushing purposes */
  662. static DEFINE_PER_CPU(struct vmap_block_queue, vmap_block_queue);
  663. /*
  664. * Radix tree of vmap blocks, indexed by address, to quickly find a vmap block
  665. * in the free path. Could get rid of this if we change the API to return a
  666. * "cookie" from alloc, to be passed to free. But no big deal yet.
  667. */
  668. static DEFINE_SPINLOCK(vmap_block_tree_lock);
  669. static RADIX_TREE(vmap_block_tree, GFP_ATOMIC);
  670. /*
  671. * We should probably have a fallback mechanism to allocate virtual memory
  672. * out of partially filled vmap blocks. However vmap block sizing should be
  673. * fairly reasonable according to the vmalloc size, so it shouldn't be a
  674. * big problem.
  675. */
  676. static unsigned long addr_to_vb_idx(unsigned long addr)
  677. {
  678. addr -= VMALLOC_START & ~(VMAP_BLOCK_SIZE-1);
  679. addr /= VMAP_BLOCK_SIZE;
  680. return addr;
  681. }
  682. static struct vmap_block *new_vmap_block(gfp_t gfp_mask)
  683. {
  684. struct vmap_block_queue *vbq;
  685. struct vmap_block *vb;
  686. struct vmap_area *va;
  687. unsigned long vb_idx;
  688. int node, err;
  689. node = numa_node_id();
  690. vb = kmalloc_node(sizeof(struct vmap_block),
  691. gfp_mask & GFP_RECLAIM_MASK, node);
  692. if (unlikely(!vb))
  693. return ERR_PTR(-ENOMEM);
  694. va = alloc_vmap_area(VMAP_BLOCK_SIZE, VMAP_BLOCK_SIZE,
  695. VMALLOC_START, VMALLOC_END,
  696. node, gfp_mask);
  697. if (IS_ERR(va)) {
  698. kfree(vb);
  699. return ERR_CAST(va);
  700. }
  701. err = radix_tree_preload(gfp_mask);
  702. if (unlikely(err)) {
  703. kfree(vb);
  704. free_vmap_area(va);
  705. return ERR_PTR(err);
  706. }
  707. spin_lock_init(&vb->lock);
  708. vb->va = va;
  709. vb->free = VMAP_BBMAP_BITS;
  710. vb->dirty = 0;
  711. bitmap_zero(vb->dirty_map, VMAP_BBMAP_BITS);
  712. INIT_LIST_HEAD(&vb->free_list);
  713. vb_idx = addr_to_vb_idx(va->va_start);
  714. spin_lock(&vmap_block_tree_lock);
  715. err = radix_tree_insert(&vmap_block_tree, vb_idx, vb);
  716. spin_unlock(&vmap_block_tree_lock);
  717. BUG_ON(err);
  718. radix_tree_preload_end();
  719. vbq = &get_cpu_var(vmap_block_queue);
  720. spin_lock(&vbq->lock);
  721. list_add_rcu(&vb->free_list, &vbq->free);
  722. spin_unlock(&vbq->lock);
  723. put_cpu_var(vmap_block_queue);
  724. return vb;
  725. }
  726. static void free_vmap_block(struct vmap_block *vb)
  727. {
  728. struct vmap_block *tmp;
  729. unsigned long vb_idx;
  730. vb_idx = addr_to_vb_idx(vb->va->va_start);
  731. spin_lock(&vmap_block_tree_lock);
  732. tmp = radix_tree_delete(&vmap_block_tree, vb_idx);
  733. spin_unlock(&vmap_block_tree_lock);
  734. BUG_ON(tmp != vb);
  735. free_vmap_area_noflush(vb->va);
  736. kfree_rcu(vb, rcu_head);
  737. }
  738. static void purge_fragmented_blocks(int cpu)
  739. {
  740. LIST_HEAD(purge);
  741. struct vmap_block *vb;
  742. struct vmap_block *n_vb;
  743. struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
  744. rcu_read_lock();
  745. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  746. if (!(vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS))
  747. continue;
  748. spin_lock(&vb->lock);
  749. if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
  750. vb->free = 0; /* prevent further allocs after releasing lock */
  751. vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
  752. bitmap_fill(vb->dirty_map, VMAP_BBMAP_BITS);
  753. spin_lock(&vbq->lock);
  754. list_del_rcu(&vb->free_list);
  755. spin_unlock(&vbq->lock);
  756. spin_unlock(&vb->lock);
  757. list_add_tail(&vb->purge, &purge);
  758. } else
  759. spin_unlock(&vb->lock);
  760. }
  761. rcu_read_unlock();
  762. list_for_each_entry_safe(vb, n_vb, &purge, purge) {
  763. list_del(&vb->purge);
  764. free_vmap_block(vb);
  765. }
  766. }
  767. static void purge_fragmented_blocks_allcpus(void)
  768. {
  769. int cpu;
  770. for_each_possible_cpu(cpu)
  771. purge_fragmented_blocks(cpu);
  772. }
  773. static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
  774. {
  775. struct vmap_block_queue *vbq;
  776. struct vmap_block *vb;
  777. unsigned long addr = 0;
  778. unsigned int order;
  779. BUG_ON(size & ~PAGE_MASK);
  780. BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
  781. if (WARN_ON(size == 0)) {
  782. /*
  783. * Allocating 0 bytes isn't what caller wants since
  784. * get_order(0) returns funny result. Just warn and terminate
  785. * early.
  786. */
  787. return NULL;
  788. }
  789. order = get_order(size);
  790. again:
  791. rcu_read_lock();
  792. vbq = &get_cpu_var(vmap_block_queue);
  793. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  794. int i;
  795. spin_lock(&vb->lock);
  796. if (vb->free < 1UL << order)
  797. goto next;
  798. i = VMAP_BBMAP_BITS - vb->free;
  799. addr = vb->va->va_start + (i << PAGE_SHIFT);
  800. BUG_ON(addr_to_vb_idx(addr) !=
  801. addr_to_vb_idx(vb->va->va_start));
  802. vb->free -= 1UL << order;
  803. if (vb->free == 0) {
  804. spin_lock(&vbq->lock);
  805. list_del_rcu(&vb->free_list);
  806. spin_unlock(&vbq->lock);
  807. }
  808. spin_unlock(&vb->lock);
  809. break;
  810. next:
  811. spin_unlock(&vb->lock);
  812. }
  813. put_cpu_var(vmap_block_queue);
  814. rcu_read_unlock();
  815. if (!addr) {
  816. vb = new_vmap_block(gfp_mask);
  817. if (IS_ERR(vb))
  818. return vb;
  819. goto again;
  820. }
  821. return (void *)addr;
  822. }
  823. static void vb_free(const void *addr, unsigned long size)
  824. {
  825. unsigned long offset;
  826. unsigned long vb_idx;
  827. unsigned int order;
  828. struct vmap_block *vb;
  829. BUG_ON(size & ~PAGE_MASK);
  830. BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC);
  831. flush_cache_vunmap((unsigned long)addr, (unsigned long)addr + size);
  832. order = get_order(size);
  833. offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
  834. vb_idx = addr_to_vb_idx((unsigned long)addr);
  835. rcu_read_lock();
  836. vb = radix_tree_lookup(&vmap_block_tree, vb_idx);
  837. rcu_read_unlock();
  838. BUG_ON(!vb);
  839. vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
  840. spin_lock(&vb->lock);
  841. BUG_ON(bitmap_allocate_region(vb->dirty_map, offset >> PAGE_SHIFT, order));
  842. vb->dirty += 1UL << order;
  843. if (vb->dirty == VMAP_BBMAP_BITS) {
  844. BUG_ON(vb->free);
  845. spin_unlock(&vb->lock);
  846. free_vmap_block(vb);
  847. } else
  848. spin_unlock(&vb->lock);
  849. }
  850. /**
  851. * vm_unmap_aliases - unmap outstanding lazy aliases in the vmap layer
  852. *
  853. * The vmap/vmalloc layer lazily flushes kernel virtual mappings primarily
  854. * to amortize TLB flushing overheads. What this means is that any page you
  855. * have now, may, in a former life, have been mapped into kernel virtual
  856. * address by the vmap layer and so there might be some CPUs with TLB entries
  857. * still referencing that page (additional to the regular 1:1 kernel mapping).
  858. *
  859. * vm_unmap_aliases flushes all such lazy mappings. After it returns, we can
  860. * be sure that none of the pages we have control over will have any aliases
  861. * from the vmap layer.
  862. */
  863. void vm_unmap_aliases(void)
  864. {
  865. unsigned long start = ULONG_MAX, end = 0;
  866. int cpu;
  867. int flush = 0;
  868. if (unlikely(!vmap_initialized))
  869. return;
  870. for_each_possible_cpu(cpu) {
  871. struct vmap_block_queue *vbq = &per_cpu(vmap_block_queue, cpu);
  872. struct vmap_block *vb;
  873. rcu_read_lock();
  874. list_for_each_entry_rcu(vb, &vbq->free, free_list) {
  875. int i, j;
  876. spin_lock(&vb->lock);
  877. i = find_first_bit(vb->dirty_map, VMAP_BBMAP_BITS);
  878. if (i < VMAP_BBMAP_BITS) {
  879. unsigned long s, e;
  880. j = find_last_bit(vb->dirty_map,
  881. VMAP_BBMAP_BITS);
  882. j = j + 1; /* need exclusive index */
  883. s = vb->va->va_start + (i << PAGE_SHIFT);
  884. e = vb->va->va_start + (j << PAGE_SHIFT);
  885. flush = 1;
  886. if (s < start)
  887. start = s;
  888. if (e > end)
  889. end = e;
  890. }
  891. spin_unlock(&vb->lock);
  892. }
  893. rcu_read_unlock();
  894. }
  895. __purge_vmap_area_lazy(&start, &end, 1, flush);
  896. }
  897. EXPORT_SYMBOL_GPL(vm_unmap_aliases);
  898. /**
  899. * vm_unmap_ram - unmap linear kernel address space set up by vm_map_ram
  900. * @mem: the pointer returned by vm_map_ram
  901. * @count: the count passed to that vm_map_ram call (cannot unmap partial)
  902. */
  903. void vm_unmap_ram(const void *mem, unsigned int count)
  904. {
  905. unsigned long size = count << PAGE_SHIFT;
  906. unsigned long addr = (unsigned long)mem;
  907. BUG_ON(!addr);
  908. BUG_ON(addr < VMALLOC_START);
  909. BUG_ON(addr > VMALLOC_END);
  910. BUG_ON(addr & (PAGE_SIZE-1));
  911. debug_check_no_locks_freed(mem, size);
  912. vmap_debug_free_range(addr, addr+size);
  913. if (likely(count <= VMAP_MAX_ALLOC))
  914. vb_free(mem, size);
  915. else
  916. free_unmap_vmap_area_addr(addr);
  917. }
  918. EXPORT_SYMBOL(vm_unmap_ram);
  919. /**
  920. * vm_map_ram - map pages linearly into kernel virtual address (vmalloc space)
  921. * @pages: an array of pointers to the pages to be mapped
  922. * @count: number of pages
  923. * @node: prefer to allocate data structures on this node
  924. * @prot: memory protection to use. PAGE_KERNEL for regular RAM
  925. *
  926. * Returns: a pointer to the address that has been mapped, or %NULL on failure
  927. */
  928. void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot)
  929. {
  930. unsigned long size = count << PAGE_SHIFT;
  931. unsigned long addr;
  932. void *mem;
  933. if (likely(count <= VMAP_MAX_ALLOC)) {
  934. mem = vb_alloc(size, GFP_KERNEL);
  935. if (IS_ERR(mem))
  936. return NULL;
  937. addr = (unsigned long)mem;
  938. } else {
  939. struct vmap_area *va;
  940. va = alloc_vmap_area(size, PAGE_SIZE,
  941. VMALLOC_START, VMALLOC_END, node, GFP_KERNEL);
  942. if (IS_ERR(va))
  943. return NULL;
  944. addr = va->va_start;
  945. mem = (void *)addr;
  946. }
  947. if (vmap_page_range(addr, addr + size, prot, pages) < 0) {
  948. vm_unmap_ram(mem, count);
  949. return NULL;
  950. }
  951. return mem;
  952. }
  953. EXPORT_SYMBOL(vm_map_ram);
  954. static struct vm_struct *vmlist __initdata;
  955. /**
  956. * vm_area_add_early - add vmap area early during boot
  957. * @vm: vm_struct to add
  958. *
  959. * This function is used to add fixed kernel vm area to vmlist before
  960. * vmalloc_init() is called. @vm->addr, @vm->size, and @vm->flags
  961. * should contain proper values and the other fields should be zero.
  962. *
  963. * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
  964. */
  965. void __init vm_area_add_early(struct vm_struct *vm)
  966. {
  967. struct vm_struct *tmp, **p;
  968. BUG_ON(vmap_initialized);
  969. for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
  970. if (tmp->addr >= vm->addr) {
  971. BUG_ON(tmp->addr < vm->addr + vm->size);
  972. break;
  973. } else
  974. BUG_ON(tmp->addr + tmp->size > vm->addr);
  975. }
  976. vm->next = *p;
  977. *p = vm;
  978. }
  979. /**
  980. * vm_area_register_early - register vmap area early during boot
  981. * @vm: vm_struct to register
  982. * @align: requested alignment
  983. *
  984. * This function is used to register kernel vm area before
  985. * vmalloc_init() is called. @vm->size and @vm->flags should contain
  986. * proper values on entry and other fields should be zero. On return,
  987. * vm->addr contains the allocated address.
  988. *
  989. * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
  990. */
  991. void __init vm_area_register_early(struct vm_struct *vm, size_t align)
  992. {
  993. static size_t vm_init_off __initdata;
  994. unsigned long addr;
  995. addr = ALIGN(VMALLOC_START + vm_init_off, align);
  996. vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
  997. vm->addr = (void *)addr;
  998. vm_area_add_early(vm);
  999. }
  1000. void __init vmalloc_init(void)
  1001. {
  1002. struct vmap_area *va;
  1003. struct vm_struct *tmp;
  1004. int i;
  1005. for_each_possible_cpu(i) {
  1006. struct vmap_block_queue *vbq;
  1007. struct vfree_deferred *p;
  1008. vbq = &per_cpu(vmap_block_queue, i);
  1009. spin_lock_init(&vbq->lock);
  1010. INIT_LIST_HEAD(&vbq->free);
  1011. p = &per_cpu(vfree_deferred, i);
  1012. init_llist_head(&p->list);
  1013. INIT_WORK(&p->wq, free_work);
  1014. }
  1015. /* Import existing vmlist entries. */
  1016. for (tmp = vmlist; tmp; tmp = tmp->next) {
  1017. va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
  1018. va->flags = VM_VM_AREA;
  1019. va->va_start = (unsigned long)tmp->addr;
  1020. va->va_end = va->va_start + tmp->size;
  1021. va->vm = tmp;
  1022. __insert_vmap_area(va);
  1023. }
  1024. vmap_area_pcpu_hole = VMALLOC_END;
  1025. vmap_initialized = true;
  1026. }
  1027. /**
  1028. * map_kernel_range_noflush - map kernel VM area with the specified pages
  1029. * @addr: start of the VM area to map
  1030. * @size: size of the VM area to map
  1031. * @prot: page protection flags to use
  1032. * @pages: pages to map
  1033. *
  1034. * Map PFN_UP(@size) pages at @addr. The VM area @addr and @size
  1035. * specify should have been allocated using get_vm_area() and its
  1036. * friends.
  1037. *
  1038. * NOTE:
  1039. * This function does NOT do any cache flushing. The caller is
  1040. * responsible for calling flush_cache_vmap() on to-be-mapped areas
  1041. * before calling this function.
  1042. *
  1043. * RETURNS:
  1044. * The number of pages mapped on success, -errno on failure.
  1045. */
  1046. int map_kernel_range_noflush(unsigned long addr, unsigned long size,
  1047. pgprot_t prot, struct page **pages)
  1048. {
  1049. return vmap_page_range_noflush(addr, addr + size, prot, pages);
  1050. }
  1051. /**
  1052. * unmap_kernel_range_noflush - unmap kernel VM area
  1053. * @addr: start of the VM area to unmap
  1054. * @size: size of the VM area to unmap
  1055. *
  1056. * Unmap PFN_UP(@size) pages at @addr. The VM area @addr and @size
  1057. * specify should have been allocated using get_vm_area() and its
  1058. * friends.
  1059. *
  1060. * NOTE:
  1061. * This function does NOT do any cache flushing. The caller is
  1062. * responsible for calling flush_cache_vunmap() on to-be-mapped areas
  1063. * before calling this function and flush_tlb_kernel_range() after.
  1064. */
  1065. void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
  1066. {
  1067. vunmap_page_range(addr, addr + size);
  1068. }
  1069. EXPORT_SYMBOL_GPL(unmap_kernel_range_noflush);
  1070. /**
  1071. * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
  1072. * @addr: start of the VM area to unmap
  1073. * @size: size of the VM area to unmap
  1074. *
  1075. * Similar to unmap_kernel_range_noflush() but flushes vcache before
  1076. * the unmapping and tlb after.
  1077. */
  1078. void unmap_kernel_range(unsigned long addr, unsigned long size)
  1079. {
  1080. unsigned long end = addr + size;
  1081. flush_cache_vunmap(addr, end);
  1082. vunmap_page_range(addr, end);
  1083. flush_tlb_kernel_range(addr, end);
  1084. }
  1085. int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
  1086. {
  1087. unsigned long addr = (unsigned long)area->addr;
  1088. unsigned long end = addr + get_vm_area_size(area);
  1089. int err;
  1090. err = vmap_page_range(addr, end, prot, *pages);
  1091. if (err > 0) {
  1092. *pages += err;
  1093. err = 0;
  1094. }
  1095. return err;
  1096. }
  1097. EXPORT_SYMBOL_GPL(map_vm_area);
  1098. static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
  1099. unsigned long flags, const void *caller)
  1100. {
  1101. spin_lock(&vmap_area_lock);
  1102. vm->flags = flags;
  1103. vm->addr = (void *)va->va_start;
  1104. vm->size = va->va_end - va->va_start;
  1105. vm->caller = caller;
  1106. va->vm = vm;
  1107. va->flags |= VM_VM_AREA;
  1108. spin_unlock(&vmap_area_lock);
  1109. }
  1110. static void clear_vm_uninitialized_flag(struct vm_struct *vm)
  1111. {
  1112. /*
  1113. * Before removing VM_UNINITIALIZED,
  1114. * we should make sure that vm has proper values.
  1115. * Pair with smp_rmb() in show_numa_info().
  1116. */
  1117. smp_wmb();
  1118. vm->flags &= ~VM_UNINITIALIZED;
  1119. }
  1120. static struct vm_struct *__get_vm_area_node(unsigned long size,
  1121. unsigned long align, unsigned long flags, unsigned long start,
  1122. unsigned long end, int node, gfp_t gfp_mask, const void *caller)
  1123. {
  1124. struct vmap_area *va;
  1125. struct vm_struct *area;
  1126. BUG_ON(in_interrupt());
  1127. if (flags & VM_IOREMAP)
  1128. align = 1ul << clamp(fls(size), PAGE_SHIFT, IOREMAP_MAX_ORDER);
  1129. size = PAGE_ALIGN(size);
  1130. if (unlikely(!size))
  1131. return NULL;
  1132. area = kzalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
  1133. if (unlikely(!area))
  1134. return NULL;
  1135. /*
  1136. * We always allocate a guard page.
  1137. */
  1138. size += PAGE_SIZE;
  1139. va = alloc_vmap_area(size, align, start, end, node, gfp_mask);
  1140. if (IS_ERR(va)) {
  1141. kfree(area);
  1142. return NULL;
  1143. }
  1144. setup_vmalloc_vm(area, va, flags, caller);
  1145. return area;
  1146. }
  1147. struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
  1148. unsigned long start, unsigned long end)
  1149. {
  1150. return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
  1151. GFP_KERNEL, __builtin_return_address(0));
  1152. }
  1153. EXPORT_SYMBOL_GPL(__get_vm_area);
  1154. struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
  1155. unsigned long start, unsigned long end,
  1156. const void *caller)
  1157. {
  1158. return __get_vm_area_node(size, 1, flags, start, end, NUMA_NO_NODE,
  1159. GFP_KERNEL, caller);
  1160. }
  1161. /**
  1162. * get_vm_area - reserve a contiguous kernel virtual area
  1163. * @size: size of the area
  1164. * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
  1165. *
  1166. * Search an area of @size in the kernel virtual mapping area,
  1167. * and reserved it for out purposes. Returns the area descriptor
  1168. * on success or %NULL on failure.
  1169. */
  1170. struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
  1171. {
  1172. return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
  1173. NUMA_NO_NODE, GFP_KERNEL,
  1174. __builtin_return_address(0));
  1175. }
  1176. struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
  1177. const void *caller)
  1178. {
  1179. return __get_vm_area_node(size, 1, flags, VMALLOC_START, VMALLOC_END,
  1180. NUMA_NO_NODE, GFP_KERNEL, caller);
  1181. }
  1182. /**
  1183. * find_vm_area - find a continuous kernel virtual area
  1184. * @addr: base address
  1185. *
  1186. * Search for the kernel VM area starting at @addr, and return it.
  1187. * It is up to the caller to do all required locking to keep the returned
  1188. * pointer valid.
  1189. */
  1190. struct vm_struct *find_vm_area(const void *addr)
  1191. {
  1192. struct vmap_area *va;
  1193. va = find_vmap_area((unsigned long)addr);
  1194. if (va && va->flags & VM_VM_AREA)
  1195. return va->vm;
  1196. return NULL;
  1197. }
  1198. /**
  1199. * remove_vm_area - find and remove a continuous kernel virtual area
  1200. * @addr: base address
  1201. *
  1202. * Search for the kernel VM area starting at @addr, and remove it.
  1203. * This function returns the found VM area, but using it is NOT safe
  1204. * on SMP machines, except for its size or flags.
  1205. */
  1206. struct vm_struct *remove_vm_area(const void *addr)
  1207. {
  1208. struct vmap_area *va;
  1209. va = find_vmap_area((unsigned long)addr);
  1210. if (va && va->flags & VM_VM_AREA) {
  1211. struct vm_struct *vm = va->vm;
  1212. spin_lock(&vmap_area_lock);
  1213. va->vm = NULL;
  1214. va->flags &= ~VM_VM_AREA;
  1215. spin_unlock(&vmap_area_lock);
  1216. vmap_debug_free_range(va->va_start, va->va_end);
  1217. free_unmap_vmap_area(va);
  1218. vm->size -= PAGE_SIZE;
  1219. return vm;
  1220. }
  1221. return NULL;
  1222. }
  1223. static void __vunmap(const void *addr, int deallocate_pages)
  1224. {
  1225. struct vm_struct *area;
  1226. if (!addr)
  1227. return;
  1228. if (WARN(!PAGE_ALIGNED(addr), "Trying to vfree() bad address (%p)\n",
  1229. addr))
  1230. return;
  1231. area = remove_vm_area(addr);
  1232. if (unlikely(!area)) {
  1233. WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
  1234. addr);
  1235. return;
  1236. }
  1237. debug_check_no_locks_freed(addr, area->size);
  1238. debug_check_no_obj_freed(addr, area->size);
  1239. if (deallocate_pages) {
  1240. int i;
  1241. for (i = 0; i < area->nr_pages; i++) {
  1242. struct page *page = area->pages[i];
  1243. BUG_ON(!page);
  1244. __free_page(page);
  1245. }
  1246. if (area->flags & VM_VPAGES)
  1247. vfree(area->pages);
  1248. else
  1249. kfree(area->pages);
  1250. }
  1251. kfree(area);
  1252. return;
  1253. }
  1254. /**
  1255. * vfree - release memory allocated by vmalloc()
  1256. * @addr: memory base address
  1257. *
  1258. * Free the virtually continuous memory area starting at @addr, as
  1259. * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
  1260. * NULL, no operation is performed.
  1261. *
  1262. * Must not be called in NMI context (strictly speaking, only if we don't
  1263. * have CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG, but making the calling
  1264. * conventions for vfree() arch-depenedent would be a really bad idea)
  1265. *
  1266. * NOTE: assumes that the object at *addr has a size >= sizeof(llist_node)
  1267. */
  1268. void vfree(const void *addr)
  1269. {
  1270. BUG_ON(in_nmi());
  1271. kmemleak_free(addr);
  1272. if (!addr)
  1273. return;
  1274. if (unlikely(in_interrupt())) {
  1275. struct vfree_deferred *p = &__get_cpu_var(vfree_deferred);
  1276. if (llist_add((struct llist_node *)addr, &p->list))
  1277. schedule_work(&p->wq);
  1278. } else
  1279. __vunmap(addr, 1);
  1280. }
  1281. EXPORT_SYMBOL(vfree);
  1282. /**
  1283. * vunmap - release virtual mapping obtained by vmap()
  1284. * @addr: memory base address
  1285. *
  1286. * Free the virtually contiguous memory area starting at @addr,
  1287. * which was created from the page array passed to vmap().
  1288. *
  1289. * Must not be called in interrupt context.
  1290. */
  1291. void vunmap(const void *addr)
  1292. {
  1293. BUG_ON(in_interrupt());
  1294. might_sleep();
  1295. if (addr)
  1296. __vunmap(addr, 0);
  1297. }
  1298. EXPORT_SYMBOL(vunmap);
  1299. /**
  1300. * vmap - map an array of pages into virtually contiguous space
  1301. * @pages: array of page pointers
  1302. * @count: number of pages to map
  1303. * @flags: vm_area->flags
  1304. * @prot: page protection for the mapping
  1305. *
  1306. * Maps @count pages from @pages into contiguous kernel virtual
  1307. * space.
  1308. */
  1309. void *vmap(struct page **pages, unsigned int count,
  1310. unsigned long flags, pgprot_t prot)
  1311. {
  1312. struct vm_struct *area;
  1313. might_sleep();
  1314. if (count > totalram_pages)
  1315. return NULL;
  1316. area = get_vm_area_caller((count << PAGE_SHIFT), flags,
  1317. __builtin_return_address(0));
  1318. if (!area)
  1319. return NULL;
  1320. if (map_vm_area(area, prot, &pages)) {
  1321. vunmap(area->addr);
  1322. return NULL;
  1323. }
  1324. return area->addr;
  1325. }
  1326. EXPORT_SYMBOL(vmap);
  1327. static void *__vmalloc_node(unsigned long size, unsigned long align,
  1328. gfp_t gfp_mask, pgprot_t prot,
  1329. int node, const void *caller);
  1330. static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
  1331. pgprot_t prot, int node)
  1332. {
  1333. const int order = 0;
  1334. struct page **pages;
  1335. unsigned int nr_pages, array_size, i;
  1336. gfp_t nested_gfp = (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO;
  1337. nr_pages = get_vm_area_size(area) >> PAGE_SHIFT;
  1338. array_size = (nr_pages * sizeof(struct page *));
  1339. area->nr_pages = nr_pages;
  1340. /* Please note that the recursion is strictly bounded. */
  1341. if (array_size > PAGE_SIZE) {
  1342. pages = __vmalloc_node(array_size, 1, nested_gfp|__GFP_HIGHMEM,
  1343. PAGE_KERNEL, node, area->caller);
  1344. area->flags |= VM_VPAGES;
  1345. } else {
  1346. pages = kmalloc_node(array_size, nested_gfp, node);
  1347. }
  1348. area->pages = pages;
  1349. if (!area->pages) {
  1350. remove_vm_area(area->addr);
  1351. kfree(area);
  1352. return NULL;
  1353. }
  1354. for (i = 0; i < area->nr_pages; i++) {
  1355. struct page *page;
  1356. gfp_t tmp_mask = gfp_mask | __GFP_NOWARN;
  1357. if (node == NUMA_NO_NODE)
  1358. page = alloc_page(tmp_mask);
  1359. else
  1360. page = alloc_pages_node(node, tmp_mask, order);
  1361. if (unlikely(!page)) {
  1362. /* Successfully allocated i pages, free them in __vunmap() */
  1363. area->nr_pages = i;
  1364. goto fail;
  1365. }
  1366. area->pages[i] = page;
  1367. }
  1368. if (map_vm_area(area, prot, &pages))
  1369. goto fail;
  1370. return area->addr;
  1371. fail:
  1372. warn_alloc_failed(gfp_mask, order,
  1373. "vmalloc: allocation failure, allocated %ld of %ld bytes\n",
  1374. (area->nr_pages*PAGE_SIZE), area->size);
  1375. vfree(area->addr);
  1376. return NULL;
  1377. }
  1378. /**
  1379. * __vmalloc_node_range - allocate virtually contiguous memory
  1380. * @size: allocation size
  1381. * @align: desired alignment
  1382. * @start: vm area range start
  1383. * @end: vm area range end
  1384. * @gfp_mask: flags for the page level allocator
  1385. * @prot: protection mask for the allocated pages
  1386. * @node: node to use for allocation or NUMA_NO_NODE
  1387. * @caller: caller's return address
  1388. *
  1389. * Allocate enough pages to cover @size from the page level
  1390. * allocator with @gfp_mask flags. Map them into contiguous
  1391. * kernel virtual space, using a pagetable protection of @prot.
  1392. */
  1393. void *__vmalloc_node_range(unsigned long size, unsigned long align,
  1394. unsigned long start, unsigned long end, gfp_t gfp_mask,
  1395. pgprot_t prot, int node, const void *caller)
  1396. {
  1397. struct vm_struct *area;
  1398. void *addr;
  1399. unsigned long real_size = size;
  1400. size = PAGE_ALIGN(size);
  1401. if (!size || (size >> PAGE_SHIFT) > totalram_pages)
  1402. goto fail;
  1403. area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED,
  1404. start, end, node, gfp_mask, caller);
  1405. if (!area)
  1406. goto fail;
  1407. addr = __vmalloc_area_node(area, gfp_mask, prot, node);
  1408. if (!addr)
  1409. return NULL;
  1410. /*
  1411. * In this function, newly allocated vm_struct has VM_UNINITIALIZED
  1412. * flag. It means that vm_struct is not fully initialized.
  1413. * Now, it is fully initialized, so remove this flag here.
  1414. */
  1415. clear_vm_uninitialized_flag(area);
  1416. /*
  1417. * A ref_count = 2 is needed because vm_struct allocated in
  1418. * __get_vm_area_node() contains a reference to the virtual address of
  1419. * the vmalloc'ed block.
  1420. */
  1421. kmemleak_alloc(addr, real_size, 2, gfp_mask);
  1422. return addr;
  1423. fail:
  1424. warn_alloc_failed(gfp_mask, 0,
  1425. "vmalloc: allocation failure: %lu bytes\n",
  1426. real_size);
  1427. return NULL;
  1428. }
  1429. /**
  1430. * __vmalloc_node - allocate virtually contiguous memory
  1431. * @size: allocation size
  1432. * @align: desired alignment
  1433. * @gfp_mask: flags for the page level allocator
  1434. * @prot: protection mask for the allocated pages
  1435. * @node: node to use for allocation or NUMA_NO_NODE
  1436. * @caller: caller's return address
  1437. *
  1438. * Allocate enough pages to cover @size from the page level
  1439. * allocator with @gfp_mask flags. Map them into contiguous
  1440. * kernel virtual space, using a pagetable protection of @prot.
  1441. */
  1442. static void *__vmalloc_node(unsigned long size, unsigned long align,
  1443. gfp_t gfp_mask, pgprot_t prot,
  1444. int node, const void *caller)
  1445. {
  1446. return __vmalloc_node_range(size, align, VMALLOC_START, VMALLOC_END,
  1447. gfp_mask, prot, node, caller);
  1448. }
  1449. void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
  1450. {
  1451. return __vmalloc_node(size, 1, gfp_mask, prot, NUMA_NO_NODE,
  1452. __builtin_return_address(0));
  1453. }
  1454. EXPORT_SYMBOL(__vmalloc);
  1455. static inline void *__vmalloc_node_flags(unsigned long size,
  1456. int node, gfp_t flags)
  1457. {
  1458. return __vmalloc_node(size, 1, flags, PAGE_KERNEL,
  1459. node, __builtin_return_address(0));
  1460. }
  1461. /**
  1462. * vmalloc - allocate virtually contiguous memory
  1463. * @size: allocation size
  1464. * Allocate enough pages to cover @size from the page level
  1465. * allocator and map them into contiguous kernel virtual space.
  1466. *
  1467. * For tight control over page level allocator and protection flags
  1468. * use __vmalloc() instead.
  1469. */
  1470. void *vmalloc(unsigned long size)
  1471. {
  1472. return __vmalloc_node_flags(size, NUMA_NO_NODE,
  1473. GFP_KERNEL | __GFP_HIGHMEM);
  1474. }
  1475. EXPORT_SYMBOL(vmalloc);
  1476. /**
  1477. * vzalloc - allocate virtually contiguous memory with zero fill
  1478. * @size: allocation size
  1479. * Allocate enough pages to cover @size from the page level
  1480. * allocator and map them into contiguous kernel virtual space.
  1481. * The memory allocated is set to zero.
  1482. *
  1483. * For tight control over page level allocator and protection flags
  1484. * use __vmalloc() instead.
  1485. */
  1486. void *vzalloc(unsigned long size)
  1487. {
  1488. return __vmalloc_node_flags(size, NUMA_NO_NODE,
  1489. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
  1490. }
  1491. EXPORT_SYMBOL(vzalloc);
  1492. /**
  1493. * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
  1494. * @size: allocation size
  1495. *
  1496. * The resulting memory area is zeroed so it can be mapped to userspace
  1497. * without leaking data.
  1498. */
  1499. void *vmalloc_user(unsigned long size)
  1500. {
  1501. struct vm_struct *area;
  1502. void *ret;
  1503. ret = __vmalloc_node(size, SHMLBA,
  1504. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
  1505. PAGE_KERNEL, NUMA_NO_NODE,
  1506. __builtin_return_address(0));
  1507. if (ret) {
  1508. area = find_vm_area(ret);
  1509. area->flags |= VM_USERMAP;
  1510. }
  1511. return ret;
  1512. }
  1513. EXPORT_SYMBOL(vmalloc_user);
  1514. /**
  1515. * vmalloc_node - allocate memory on a specific node
  1516. * @size: allocation size
  1517. * @node: numa node
  1518. *
  1519. * Allocate enough pages to cover @size from the page level
  1520. * allocator and map them into contiguous kernel virtual space.
  1521. *
  1522. * For tight control over page level allocator and protection flags
  1523. * use __vmalloc() instead.
  1524. */
  1525. void *vmalloc_node(unsigned long size, int node)
  1526. {
  1527. return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
  1528. node, __builtin_return_address(0));
  1529. }
  1530. EXPORT_SYMBOL(vmalloc_node);
  1531. /**
  1532. * vzalloc_node - allocate memory on a specific node with zero fill
  1533. * @size: allocation size
  1534. * @node: numa node
  1535. *
  1536. * Allocate enough pages to cover @size from the page level
  1537. * allocator and map them into contiguous kernel virtual space.
  1538. * The memory allocated is set to zero.
  1539. *
  1540. * For tight control over page level allocator and protection flags
  1541. * use __vmalloc_node() instead.
  1542. */
  1543. void *vzalloc_node(unsigned long size, int node)
  1544. {
  1545. return __vmalloc_node_flags(size, node,
  1546. GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO);
  1547. }
  1548. EXPORT_SYMBOL(vzalloc_node);
  1549. #ifndef PAGE_KERNEL_EXEC
  1550. # define PAGE_KERNEL_EXEC PAGE_KERNEL
  1551. #endif
  1552. /**
  1553. * vmalloc_exec - allocate virtually contiguous, executable memory
  1554. * @size: allocation size
  1555. *
  1556. * Kernel-internal function to allocate enough pages to cover @size
  1557. * the page level allocator and map them into contiguous and
  1558. * executable kernel virtual space.
  1559. *
  1560. * For tight control over page level allocator and protection flags
  1561. * use __vmalloc() instead.
  1562. */
  1563. void *vmalloc_exec(unsigned long size)
  1564. {
  1565. return __vmalloc_node(size, 1, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC,
  1566. NUMA_NO_NODE, __builtin_return_address(0));
  1567. }
  1568. #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
  1569. #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
  1570. #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
  1571. #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
  1572. #else
  1573. #define GFP_VMALLOC32 GFP_KERNEL
  1574. #endif
  1575. /**
  1576. * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
  1577. * @size: allocation size
  1578. *
  1579. * Allocate enough 32bit PA addressable pages to cover @size from the
  1580. * page level allocator and map them into contiguous kernel virtual space.
  1581. */
  1582. void *vmalloc_32(unsigned long size)
  1583. {
  1584. return __vmalloc_node(size, 1, GFP_VMALLOC32, PAGE_KERNEL,
  1585. NUMA_NO_NODE, __builtin_return_address(0));
  1586. }
  1587. EXPORT_SYMBOL(vmalloc_32);
  1588. /**
  1589. * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
  1590. * @size: allocation size
  1591. *
  1592. * The resulting memory area is 32bit addressable and zeroed so it can be
  1593. * mapped to userspace without leaking data.
  1594. */
  1595. void *vmalloc_32_user(unsigned long size)
  1596. {
  1597. struct vm_struct *area;
  1598. void *ret;
  1599. ret = __vmalloc_node(size, 1, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL,
  1600. NUMA_NO_NODE, __builtin_return_address(0));
  1601. if (ret) {
  1602. area = find_vm_area(ret);
  1603. area->flags |= VM_USERMAP;
  1604. }
  1605. return ret;
  1606. }
  1607. EXPORT_SYMBOL(vmalloc_32_user);
  1608. /*
  1609. * small helper routine , copy contents to buf from addr.
  1610. * If the page is not present, fill zero.
  1611. */
  1612. static int aligned_vread(char *buf, char *addr, unsigned long count)
  1613. {
  1614. struct page *p;
  1615. int copied = 0;
  1616. while (count) {
  1617. unsigned long offset, length;
  1618. offset = (unsigned long)addr & ~PAGE_MASK;
  1619. length = PAGE_SIZE - offset;
  1620. if (length > count)
  1621. length = count;
  1622. p = vmalloc_to_page(addr);
  1623. /*
  1624. * To do safe access to this _mapped_ area, we need
  1625. * lock. But adding lock here means that we need to add
  1626. * overhead of vmalloc()/vfree() calles for this _debug_
  1627. * interface, rarely used. Instead of that, we'll use
  1628. * kmap() and get small overhead in this access function.
  1629. */
  1630. if (p) {
  1631. /*
  1632. * we can expect USER0 is not used (see vread/vwrite's
  1633. * function description)
  1634. */
  1635. void *map = kmap_atomic(p);
  1636. memcpy(buf, map + offset, length);
  1637. kunmap_atomic(map);
  1638. } else
  1639. memset(buf, 0, length);
  1640. addr += length;
  1641. buf += length;
  1642. copied += length;
  1643. count -= length;
  1644. }
  1645. return copied;
  1646. }
  1647. static int aligned_vwrite(char *buf, char *addr, unsigned long count)
  1648. {
  1649. struct page *p;
  1650. int copied = 0;
  1651. while (count) {
  1652. unsigned long offset, length;
  1653. offset = (unsigned long)addr & ~PAGE_MASK;
  1654. length = PAGE_SIZE - offset;
  1655. if (length > count)
  1656. length = count;
  1657. p = vmalloc_to_page(addr);
  1658. /*
  1659. * To do safe access to this _mapped_ area, we need
  1660. * lock. But adding lock here means that we need to add
  1661. * overhead of vmalloc()/vfree() calles for this _debug_
  1662. * interface, rarely used. Instead of that, we'll use
  1663. * kmap() and get small overhead in this access function.
  1664. */
  1665. if (p) {
  1666. /*
  1667. * we can expect USER0 is not used (see vread/vwrite's
  1668. * function description)
  1669. */
  1670. void *map = kmap_atomic(p);
  1671. memcpy(map + offset, buf, length);
  1672. kunmap_atomic(map);
  1673. }
  1674. addr += length;
  1675. buf += length;
  1676. copied += length;
  1677. count -= length;
  1678. }
  1679. return copied;
  1680. }
  1681. /**
  1682. * vread() - read vmalloc area in a safe way.
  1683. * @buf: buffer for reading data
  1684. * @addr: vm address.
  1685. * @count: number of bytes to be read.
  1686. *
  1687. * Returns # of bytes which addr and buf should be increased.
  1688. * (same number to @count). Returns 0 if [addr...addr+count) doesn't
  1689. * includes any intersect with alive vmalloc area.
  1690. *
  1691. * This function checks that addr is a valid vmalloc'ed area, and
  1692. * copy data from that area to a given buffer. If the given memory range
  1693. * of [addr...addr+count) includes some valid address, data is copied to
  1694. * proper area of @buf. If there are memory holes, they'll be zero-filled.
  1695. * IOREMAP area is treated as memory hole and no copy is done.
  1696. *
  1697. * If [addr...addr+count) doesn't includes any intersects with alive
  1698. * vm_struct area, returns 0. @buf should be kernel's buffer.
  1699. *
  1700. * Note: In usual ops, vread() is never necessary because the caller
  1701. * should know vmalloc() area is valid and can use memcpy().
  1702. * This is for routines which have to access vmalloc area without
  1703. * any informaion, as /dev/kmem.
  1704. *
  1705. */
  1706. long vread(char *buf, char *addr, unsigned long count)
  1707. {
  1708. struct vmap_area *va;
  1709. struct vm_struct *vm;
  1710. char *vaddr, *buf_start = buf;
  1711. unsigned long buflen = count;
  1712. unsigned long n;
  1713. /* Don't allow overflow */
  1714. if ((unsigned long) addr + count < count)
  1715. count = -(unsigned long) addr;
  1716. spin_lock(&vmap_area_lock);
  1717. list_for_each_entry(va, &vmap_area_list, list) {
  1718. if (!count)
  1719. break;
  1720. if (!(va->flags & VM_VM_AREA))
  1721. continue;
  1722. vm = va->vm;
  1723. vaddr = (char *) vm->addr;
  1724. if (addr >= vaddr + get_vm_area_size(vm))
  1725. continue;
  1726. while (addr < vaddr) {
  1727. if (count == 0)
  1728. goto finished;
  1729. *buf = '\0';
  1730. buf++;
  1731. addr++;
  1732. count--;
  1733. }
  1734. n = vaddr + get_vm_area_size(vm) - addr;
  1735. if (n > count)
  1736. n = count;
  1737. if (!(vm->flags & VM_IOREMAP))
  1738. aligned_vread(buf, addr, n);
  1739. else /* IOREMAP area is treated as memory hole */
  1740. memset(buf, 0, n);
  1741. buf += n;
  1742. addr += n;
  1743. count -= n;
  1744. }
  1745. finished:
  1746. spin_unlock(&vmap_area_lock);
  1747. if (buf == buf_start)
  1748. return 0;
  1749. /* zero-fill memory holes */
  1750. if (buf != buf_start + buflen)
  1751. memset(buf, 0, buflen - (buf - buf_start));
  1752. return buflen;
  1753. }
  1754. /**
  1755. * vwrite() - write vmalloc area in a safe way.
  1756. * @buf: buffer for source data
  1757. * @addr: vm address.
  1758. * @count: number of bytes to be read.
  1759. *
  1760. * Returns # of bytes which addr and buf should be incresed.
  1761. * (same number to @count).
  1762. * If [addr...addr+count) doesn't includes any intersect with valid
  1763. * vmalloc area, returns 0.
  1764. *
  1765. * This function checks that addr is a valid vmalloc'ed area, and
  1766. * copy data from a buffer to the given addr. If specified range of
  1767. * [addr...addr+count) includes some valid address, data is copied from
  1768. * proper area of @buf. If there are memory holes, no copy to hole.
  1769. * IOREMAP area is treated as memory hole and no copy is done.
  1770. *
  1771. * If [addr...addr+count) doesn't includes any intersects with alive
  1772. * vm_struct area, returns 0. @buf should be kernel's buffer.
  1773. *
  1774. * Note: In usual ops, vwrite() is never necessary because the caller
  1775. * should know vmalloc() area is valid and can use memcpy().
  1776. * This is for routines which have to access vmalloc area without
  1777. * any informaion, as /dev/kmem.
  1778. */
  1779. long vwrite(char *buf, char *addr, unsigned long count)
  1780. {
  1781. struct vmap_area *va;
  1782. struct vm_struct *vm;
  1783. char *vaddr;
  1784. unsigned long n, buflen;
  1785. int copied = 0;
  1786. /* Don't allow overflow */
  1787. if ((unsigned long) addr + count < count)
  1788. count = -(unsigned long) addr;
  1789. buflen = count;
  1790. spin_lock(&vmap_area_lock);
  1791. list_for_each_entry(va, &vmap_area_list, list) {
  1792. if (!count)
  1793. break;
  1794. if (!(va->flags & VM_VM_AREA))
  1795. continue;
  1796. vm = va->vm;
  1797. vaddr = (char *) vm->addr;
  1798. if (addr >= vaddr + get_vm_area_size(vm))
  1799. continue;
  1800. while (addr < vaddr) {
  1801. if (count == 0)
  1802. goto finished;
  1803. buf++;
  1804. addr++;
  1805. count--;
  1806. }
  1807. n = vaddr + get_vm_area_size(vm) - addr;
  1808. if (n > count)
  1809. n = count;
  1810. if (!(vm->flags & VM_IOREMAP)) {
  1811. aligned_vwrite(buf, addr, n);
  1812. copied++;
  1813. }
  1814. buf += n;
  1815. addr += n;
  1816. count -= n;
  1817. }
  1818. finished:
  1819. spin_unlock(&vmap_area_lock);
  1820. if (!copied)
  1821. return 0;
  1822. return buflen;
  1823. }
  1824. /**
  1825. * remap_vmalloc_range_partial - map vmalloc pages to userspace
  1826. * @vma: vma to cover
  1827. * @uaddr: target user address to start at
  1828. * @kaddr: virtual address of vmalloc kernel memory
  1829. * @size: size of map area
  1830. *
  1831. * Returns: 0 for success, -Exxx on failure
  1832. *
  1833. * This function checks that @kaddr is a valid vmalloc'ed area,
  1834. * and that it is big enough to cover the range starting at
  1835. * @uaddr in @vma. Will return failure if that criteria isn't
  1836. * met.
  1837. *
  1838. * Similar to remap_pfn_range() (see mm/memory.c)
  1839. */
  1840. int remap_vmalloc_range_partial(struct vm_area_struct *vma, unsigned long uaddr,
  1841. void *kaddr, unsigned long size)
  1842. {
  1843. struct vm_struct *area;
  1844. size = PAGE_ALIGN(size);
  1845. if (!PAGE_ALIGNED(uaddr) || !PAGE_ALIGNED(kaddr))
  1846. return -EINVAL;
  1847. area = find_vm_area(kaddr);
  1848. if (!area)
  1849. return -EINVAL;
  1850. if (!(area->flags & VM_USERMAP))
  1851. return -EINVAL;
  1852. if (kaddr + size > area->addr + area->size)
  1853. return -EINVAL;
  1854. do {
  1855. struct page *page = vmalloc_to_page(kaddr);
  1856. int ret;
  1857. ret = vm_insert_page(vma, uaddr, page);
  1858. if (ret)
  1859. return ret;
  1860. uaddr += PAGE_SIZE;
  1861. kaddr += PAGE_SIZE;
  1862. size -= PAGE_SIZE;
  1863. } while (size > 0);
  1864. vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
  1865. return 0;
  1866. }
  1867. EXPORT_SYMBOL(remap_vmalloc_range_partial);
  1868. /**
  1869. * remap_vmalloc_range - map vmalloc pages to userspace
  1870. * @vma: vma to cover (map full range of vma)
  1871. * @addr: vmalloc memory
  1872. * @pgoff: number of pages into addr before first page to map
  1873. *
  1874. * Returns: 0 for success, -Exxx on failure
  1875. *
  1876. * This function checks that addr is a valid vmalloc'ed area, and
  1877. * that it is big enough to cover the vma. Will return failure if
  1878. * that criteria isn't met.
  1879. *
  1880. * Similar to remap_pfn_range() (see mm/memory.c)
  1881. */
  1882. int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
  1883. unsigned long pgoff)
  1884. {
  1885. return remap_vmalloc_range_partial(vma, vma->vm_start,
  1886. addr + (pgoff << PAGE_SHIFT),
  1887. vma->vm_end - vma->vm_start);
  1888. }
  1889. EXPORT_SYMBOL(remap_vmalloc_range);
  1890. /*
  1891. * Implement a stub for vmalloc_sync_all() if the architecture chose not to
  1892. * have one.
  1893. */
  1894. void __attribute__((weak)) vmalloc_sync_all(void)
  1895. {
  1896. }
  1897. static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
  1898. {
  1899. pte_t ***p = data;
  1900. if (p) {
  1901. *(*p) = pte;
  1902. (*p)++;
  1903. }
  1904. return 0;
  1905. }
  1906. /**
  1907. * alloc_vm_area - allocate a range of kernel address space
  1908. * @size: size of the area
  1909. * @ptes: returns the PTEs for the address space
  1910. *
  1911. * Returns: NULL on failure, vm_struct on success
  1912. *
  1913. * This function reserves a range of kernel address space, and
  1914. * allocates pagetables to map that range. No actual mappings
  1915. * are created.
  1916. *
  1917. * If @ptes is non-NULL, pointers to the PTEs (in init_mm)
  1918. * allocated for the VM area are returned.
  1919. */
  1920. struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
  1921. {
  1922. struct vm_struct *area;
  1923. area = get_vm_area_caller(size, VM_IOREMAP,
  1924. __builtin_return_address(0));
  1925. if (area == NULL)
  1926. return NULL;
  1927. /*
  1928. * This ensures that page tables are constructed for this region
  1929. * of kernel virtual address space and mapped into init_mm.
  1930. */
  1931. if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
  1932. size, f, ptes ? &ptes : NULL)) {
  1933. free_vm_area(area);
  1934. return NULL;
  1935. }
  1936. return area;
  1937. }
  1938. EXPORT_SYMBOL_GPL(alloc_vm_area);
  1939. void free_vm_area(struct vm_struct *area)
  1940. {
  1941. struct vm_struct *ret;
  1942. ret = remove_vm_area(area->addr);
  1943. BUG_ON(ret != area);
  1944. kfree(area);
  1945. }
  1946. EXPORT_SYMBOL_GPL(free_vm_area);
  1947. #ifdef CONFIG_SMP
  1948. static struct vmap_area *node_to_va(struct rb_node *n)
  1949. {
  1950. return n ? rb_entry(n, struct vmap_area, rb_node) : NULL;
  1951. }
  1952. /**
  1953. * pvm_find_next_prev - find the next and prev vmap_area surrounding @end
  1954. * @end: target address
  1955. * @pnext: out arg for the next vmap_area
  1956. * @pprev: out arg for the previous vmap_area
  1957. *
  1958. * Returns: %true if either or both of next and prev are found,
  1959. * %false if no vmap_area exists
  1960. *
  1961. * Find vmap_areas end addresses of which enclose @end. ie. if not
  1962. * NULL, *pnext->va_end > @end and *pprev->va_end <= @end.
  1963. */
  1964. static bool pvm_find_next_prev(unsigned long end,
  1965. struct vmap_area **pnext,
  1966. struct vmap_area **pprev)
  1967. {
  1968. struct rb_node *n = vmap_area_root.rb_node;
  1969. struct vmap_area *va = NULL;
  1970. while (n) {
  1971. va = rb_entry(n, struct vmap_area, rb_node);
  1972. if (end < va->va_end)
  1973. n = n->rb_left;
  1974. else if (end > va->va_end)
  1975. n = n->rb_right;
  1976. else
  1977. break;
  1978. }
  1979. if (!va)
  1980. return false;
  1981. if (va->va_end > end) {
  1982. *pnext = va;
  1983. *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
  1984. } else {
  1985. *pprev = va;
  1986. *pnext = node_to_va(rb_next(&(*pprev)->rb_node));
  1987. }
  1988. return true;
  1989. }
  1990. /**
  1991. * pvm_determine_end - find the highest aligned address between two vmap_areas
  1992. * @pnext: in/out arg for the next vmap_area
  1993. * @pprev: in/out arg for the previous vmap_area
  1994. * @align: alignment
  1995. *
  1996. * Returns: determined end address
  1997. *
  1998. * Find the highest aligned address between *@pnext and *@pprev below
  1999. * VMALLOC_END. *@pnext and *@pprev are adjusted so that the aligned
  2000. * down address is between the end addresses of the two vmap_areas.
  2001. *
  2002. * Please note that the address returned by this function may fall
  2003. * inside *@pnext vmap_area. The caller is responsible for checking
  2004. * that.
  2005. */
  2006. static unsigned long pvm_determine_end(struct vmap_area **pnext,
  2007. struct vmap_area **pprev,
  2008. unsigned long align)
  2009. {
  2010. const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
  2011. unsigned long addr;
  2012. if (*pnext)
  2013. addr = min((*pnext)->va_start & ~(align - 1), vmalloc_end);
  2014. else
  2015. addr = vmalloc_end;
  2016. while (*pprev && (*pprev)->va_end > addr) {
  2017. *pnext = *pprev;
  2018. *pprev = node_to_va(rb_prev(&(*pnext)->rb_node));
  2019. }
  2020. return addr;
  2021. }
  2022. /**
  2023. * pcpu_get_vm_areas - allocate vmalloc areas for percpu allocator
  2024. * @offsets: array containing offset of each area
  2025. * @sizes: array containing size of each area
  2026. * @nr_vms: the number of areas to allocate
  2027. * @align: alignment, all entries in @offsets and @sizes must be aligned to this
  2028. *
  2029. * Returns: kmalloc'd vm_struct pointer array pointing to allocated
  2030. * vm_structs on success, %NULL on failure
  2031. *
  2032. * Percpu allocator wants to use congruent vm areas so that it can
  2033. * maintain the offsets among percpu areas. This function allocates
  2034. * congruent vmalloc areas for it with GFP_KERNEL. These areas tend to
  2035. * be scattered pretty far, distance between two areas easily going up
  2036. * to gigabytes. To avoid interacting with regular vmallocs, these
  2037. * areas are allocated from top.
  2038. *
  2039. * Despite its complicated look, this allocator is rather simple. It
  2040. * does everything top-down and scans areas from the end looking for
  2041. * matching slot. While scanning, if any of the areas overlaps with
  2042. * existing vmap_area, the base address is pulled down to fit the
  2043. * area. Scanning is repeated till all the areas fit and then all
  2044. * necessary data structres are inserted and the result is returned.
  2045. */
  2046. struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets,
  2047. const size_t *sizes, int nr_vms,
  2048. size_t align)
  2049. {
  2050. const unsigned long vmalloc_start = ALIGN(VMALLOC_START, align);
  2051. const unsigned long vmalloc_end = VMALLOC_END & ~(align - 1);
  2052. struct vmap_area **vas, *prev, *next;
  2053. struct vm_struct **vms;
  2054. int area, area2, last_area, term_area;
  2055. unsigned long base, start, end, last_end;
  2056. bool purged = false;
  2057. /* verify parameters and allocate data structures */
  2058. BUG_ON(align & ~PAGE_MASK || !is_power_of_2(align));
  2059. for (last_area = 0, area = 0; area < nr_vms; area++) {
  2060. start = offsets[area];
  2061. end = start + sizes[area];
  2062. /* is everything aligned properly? */
  2063. BUG_ON(!IS_ALIGNED(offsets[area], align));
  2064. BUG_ON(!IS_ALIGNED(sizes[area], align));
  2065. /* detect the area with the highest address */
  2066. if (start > offsets[last_area])
  2067. last_area = area;
  2068. for (area2 = 0; area2 < nr_vms; area2++) {
  2069. unsigned long start2 = offsets[area2];
  2070. unsigned long end2 = start2 + sizes[area2];
  2071. if (area2 == area)
  2072. continue;
  2073. BUG_ON(start2 >= start && start2 < end);
  2074. BUG_ON(end2 <= end && end2 > start);
  2075. }
  2076. }
  2077. last_end = offsets[last_area] + sizes[last_area];
  2078. if (vmalloc_end - vmalloc_start < last_end) {
  2079. WARN_ON(true);
  2080. return NULL;
  2081. }
  2082. vms = kcalloc(nr_vms, sizeof(vms[0]), GFP_KERNEL);
  2083. vas = kcalloc(nr_vms, sizeof(vas[0]), GFP_KERNEL);
  2084. if (!vas || !vms)
  2085. goto err_free2;
  2086. for (area = 0; area < nr_vms; area++) {
  2087. vas[area] = kzalloc(sizeof(struct vmap_area), GFP_KERNEL);
  2088. vms[area] = kzalloc(sizeof(struct vm_struct), GFP_KERNEL);
  2089. if (!vas[area] || !vms[area])
  2090. goto err_free;
  2091. }
  2092. retry:
  2093. spin_lock(&vmap_area_lock);
  2094. /* start scanning - we scan from the top, begin with the last area */
  2095. area = term_area = last_area;
  2096. start = offsets[area];
  2097. end = start + sizes[area];
  2098. if (!pvm_find_next_prev(vmap_area_pcpu_hole, &next, &prev)) {
  2099. base = vmalloc_end - last_end;
  2100. goto found;
  2101. }
  2102. base = pvm_determine_end(&next, &prev, align) - end;
  2103. while (true) {
  2104. BUG_ON(next && next->va_end <= base + end);
  2105. BUG_ON(prev && prev->va_end > base + end);
  2106. /*
  2107. * base might have underflowed, add last_end before
  2108. * comparing.
  2109. */
  2110. if (base + last_end < vmalloc_start + last_end) {
  2111. spin_unlock(&vmap_area_lock);
  2112. if (!purged) {
  2113. purge_vmap_area_lazy();
  2114. purged = true;
  2115. goto retry;
  2116. }
  2117. goto err_free;
  2118. }
  2119. /*
  2120. * If next overlaps, move base downwards so that it's
  2121. * right below next and then recheck.
  2122. */
  2123. if (next && next->va_start < base + end) {
  2124. base = pvm_determine_end(&next, &prev, align) - end;
  2125. term_area = area;
  2126. continue;
  2127. }
  2128. /*
  2129. * If prev overlaps, shift down next and prev and move
  2130. * base so that it's right below new next and then
  2131. * recheck.
  2132. */
  2133. if (prev && prev->va_end > base + start) {
  2134. next = prev;
  2135. prev = node_to_va(rb_prev(&next->rb_node));
  2136. base = pvm_determine_end(&next, &prev, align) - end;
  2137. term_area = area;
  2138. continue;
  2139. }
  2140. /*
  2141. * This area fits, move on to the previous one. If
  2142. * the previous one is the terminal one, we're done.
  2143. */
  2144. area = (area + nr_vms - 1) % nr_vms;
  2145. if (area == term_area)
  2146. break;
  2147. start = offsets[area];
  2148. end = start + sizes[area];
  2149. pvm_find_next_prev(base + end, &next, &prev);
  2150. }
  2151. found:
  2152. /* we've found a fitting base, insert all va's */
  2153. for (area = 0; area < nr_vms; area++) {
  2154. struct vmap_area *va = vas[area];
  2155. va->va_start = base + offsets[area];
  2156. va->va_end = va->va_start + sizes[area];
  2157. __insert_vmap_area(va);
  2158. }
  2159. vmap_area_pcpu_hole = base + offsets[last_area];
  2160. spin_unlock(&vmap_area_lock);
  2161. /* insert all vm's */
  2162. for (area = 0; area < nr_vms; area++)
  2163. setup_vmalloc_vm(vms[area], vas[area], VM_ALLOC,
  2164. pcpu_get_vm_areas);
  2165. kfree(vas);
  2166. return vms;
  2167. err_free:
  2168. for (area = 0; area < nr_vms; area++) {
  2169. kfree(vas[area]);
  2170. kfree(vms[area]);
  2171. }
  2172. err_free2:
  2173. kfree(vas);
  2174. kfree(vms);
  2175. return NULL;
  2176. }
  2177. /**
  2178. * pcpu_free_vm_areas - free vmalloc areas for percpu allocator
  2179. * @vms: vm_struct pointer array returned by pcpu_get_vm_areas()
  2180. * @nr_vms: the number of allocated areas
  2181. *
  2182. * Free vm_structs and the array allocated by pcpu_get_vm_areas().
  2183. */
  2184. void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
  2185. {
  2186. int i;
  2187. for (i = 0; i < nr_vms; i++)
  2188. free_vm_area(vms[i]);
  2189. kfree(vms);
  2190. }
  2191. #endif /* CONFIG_SMP */
  2192. #ifdef CONFIG_PROC_FS
  2193. static void *s_start(struct seq_file *m, loff_t *pos)
  2194. __acquires(&vmap_area_lock)
  2195. {
  2196. loff_t n = *pos;
  2197. struct vmap_area *va;
  2198. spin_lock(&vmap_area_lock);
  2199. va = list_entry((&vmap_area_list)->next, typeof(*va), list);
  2200. while (n > 0 && &va->list != &vmap_area_list) {
  2201. n--;
  2202. va = list_entry(va->list.next, typeof(*va), list);
  2203. }
  2204. if (!n && &va->list != &vmap_area_list)
  2205. return va;
  2206. return NULL;
  2207. }
  2208. static void *s_next(struct seq_file *m, void *p, loff_t *pos)
  2209. {
  2210. struct vmap_area *va = p, *next;
  2211. ++*pos;
  2212. next = list_entry(va->list.next, typeof(*va), list);
  2213. if (&next->list != &vmap_area_list)
  2214. return next;
  2215. return NULL;
  2216. }
  2217. static void s_stop(struct seq_file *m, void *p)
  2218. __releases(&vmap_area_lock)
  2219. {
  2220. spin_unlock(&vmap_area_lock);
  2221. }
  2222. static void show_numa_info(struct seq_file *m, struct vm_struct *v)
  2223. {
  2224. if (IS_ENABLED(CONFIG_NUMA)) {
  2225. unsigned int nr, *counters = m->private;
  2226. if (!counters)
  2227. return;
  2228. /* Pair with smp_wmb() in clear_vm_uninitialized_flag() */
  2229. smp_rmb();
  2230. if (v->flags & VM_UNINITIALIZED)
  2231. return;
  2232. memset(counters, 0, nr_node_ids * sizeof(unsigned int));
  2233. for (nr = 0; nr < v->nr_pages; nr++)
  2234. counters[page_to_nid(v->pages[nr])]++;
  2235. for_each_node_state(nr, N_HIGH_MEMORY)
  2236. if (counters[nr])
  2237. seq_printf(m, " N%u=%u", nr, counters[nr]);
  2238. }
  2239. }
  2240. static int s_show(struct seq_file *m, void *p)
  2241. {
  2242. struct vmap_area *va = p;
  2243. struct vm_struct *v;
  2244. /*
  2245. * s_show can encounter race with remove_vm_area, !VM_VM_AREA on
  2246. * behalf of vmap area is being tear down or vm_map_ram allocation.
  2247. */
  2248. if (!(va->flags & VM_VM_AREA))
  2249. return 0;
  2250. v = va->vm;
  2251. seq_printf(m, "0x%pK-0x%pK %7ld",
  2252. v->addr, v->addr + v->size, v->size);
  2253. if (v->caller)
  2254. seq_printf(m, " %pS", v->caller);
  2255. if (v->nr_pages)
  2256. seq_printf(m, " pages=%d", v->nr_pages);
  2257. if (v->phys_addr)
  2258. seq_printf(m, " phys=%llx", (unsigned long long)v->phys_addr);
  2259. if (v->flags & VM_IOREMAP)
  2260. seq_printf(m, " ioremap");
  2261. if (v->flags & VM_ALLOC)
  2262. seq_printf(m, " vmalloc");
  2263. if (v->flags & VM_MAP)
  2264. seq_printf(m, " vmap");
  2265. if (v->flags & VM_USERMAP)
  2266. seq_printf(m, " user");
  2267. if (v->flags & VM_VPAGES)
  2268. seq_printf(m, " vpages");
  2269. show_numa_info(m, v);
  2270. seq_putc(m, '\n');
  2271. return 0;
  2272. }
  2273. static const struct seq_operations vmalloc_op = {
  2274. .start = s_start,
  2275. .next = s_next,
  2276. .stop = s_stop,
  2277. .show = s_show,
  2278. };
  2279. static int vmalloc_open(struct inode *inode, struct file *file)
  2280. {
  2281. unsigned int *ptr = NULL;
  2282. int ret;
  2283. if (IS_ENABLED(CONFIG_NUMA)) {
  2284. ptr = kmalloc(nr_node_ids * sizeof(unsigned int), GFP_KERNEL);
  2285. if (ptr == NULL)
  2286. return -ENOMEM;
  2287. }
  2288. ret = seq_open(file, &vmalloc_op);
  2289. if (!ret) {
  2290. struct seq_file *m = file->private_data;
  2291. m->private = ptr;
  2292. } else
  2293. kfree(ptr);
  2294. return ret;
  2295. }
  2296. static const struct file_operations proc_vmalloc_operations = {
  2297. .open = vmalloc_open,
  2298. .read = seq_read,
  2299. .llseek = seq_lseek,
  2300. .release = seq_release_private,
  2301. };
  2302. static int __init proc_vmalloc_init(void)
  2303. {
  2304. proc_create("vmallocinfo", S_IRUSR, NULL, &proc_vmalloc_operations);
  2305. return 0;
  2306. }
  2307. module_init(proc_vmalloc_init);
  2308. void get_vmalloc_info(struct vmalloc_info *vmi)
  2309. {
  2310. struct vmap_area *va;
  2311. unsigned long free_area_size;
  2312. unsigned long prev_end;
  2313. vmi->used = 0;
  2314. vmi->largest_chunk = 0;
  2315. prev_end = VMALLOC_START;
  2316. spin_lock(&vmap_area_lock);
  2317. if (list_empty(&vmap_area_list)) {
  2318. vmi->largest_chunk = VMALLOC_TOTAL;
  2319. goto out;
  2320. }
  2321. list_for_each_entry(va, &vmap_area_list, list) {
  2322. unsigned long addr = va->va_start;
  2323. /*
  2324. * Some archs keep another range for modules in vmalloc space
  2325. */
  2326. if (addr < VMALLOC_START)
  2327. continue;
  2328. if (addr >= VMALLOC_END)
  2329. break;
  2330. if (va->flags & (VM_LAZY_FREE | VM_LAZY_FREEING))
  2331. continue;
  2332. vmi->used += (va->va_end - va->va_start);
  2333. free_area_size = addr - prev_end;
  2334. if (vmi->largest_chunk < free_area_size)
  2335. vmi->largest_chunk = free_area_size;
  2336. prev_end = va->va_end;
  2337. }
  2338. if (VMALLOC_END - prev_end > vmi->largest_chunk)
  2339. vmi->largest_chunk = VMALLOC_END - prev_end;
  2340. out:
  2341. spin_unlock(&vmap_area_lock);
  2342. }
  2343. #endif