vmalloc.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980
  1. /*
  2. * linux/mm/vmalloc.c
  3. *
  4. * Copyright (C) 1993 Linus Torvalds
  5. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  6. * SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
  7. * Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
  8. * Numa awareness, Christoph Lameter, SGI, June 2005
  9. */
  10. #include <linux/mm.h>
  11. #include <linux/module.h>
  12. #include <linux/highmem.h>
  13. #include <linux/slab.h>
  14. #include <linux/spinlock.h>
  15. #include <linux/interrupt.h>
  16. #include <linux/seq_file.h>
  17. #include <linux/debugobjects.h>
  18. #include <linux/vmalloc.h>
  19. #include <linux/kallsyms.h>
  20. #include <asm/uaccess.h>
  21. #include <asm/tlbflush.h>
  22. DEFINE_RWLOCK(vmlist_lock);
  23. struct vm_struct *vmlist;
  24. static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
  25. int node, void *caller);
  26. static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
  27. {
  28. pte_t *pte;
  29. pte = pte_offset_kernel(pmd, addr);
  30. do {
  31. pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
  32. WARN_ON(!pte_none(ptent) && !pte_present(ptent));
  33. } while (pte++, addr += PAGE_SIZE, addr != end);
  34. }
  35. static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
  36. unsigned long end)
  37. {
  38. pmd_t *pmd;
  39. unsigned long next;
  40. pmd = pmd_offset(pud, addr);
  41. do {
  42. next = pmd_addr_end(addr, end);
  43. if (pmd_none_or_clear_bad(pmd))
  44. continue;
  45. vunmap_pte_range(pmd, addr, next);
  46. } while (pmd++, addr = next, addr != end);
  47. }
  48. static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
  49. unsigned long end)
  50. {
  51. pud_t *pud;
  52. unsigned long next;
  53. pud = pud_offset(pgd, addr);
  54. do {
  55. next = pud_addr_end(addr, end);
  56. if (pud_none_or_clear_bad(pud))
  57. continue;
  58. vunmap_pmd_range(pud, addr, next);
  59. } while (pud++, addr = next, addr != end);
  60. }
  61. void unmap_kernel_range(unsigned long addr, unsigned long size)
  62. {
  63. pgd_t *pgd;
  64. unsigned long next;
  65. unsigned long start = addr;
  66. unsigned long end = addr + size;
  67. BUG_ON(addr >= end);
  68. pgd = pgd_offset_k(addr);
  69. flush_cache_vunmap(addr, end);
  70. do {
  71. next = pgd_addr_end(addr, end);
  72. if (pgd_none_or_clear_bad(pgd))
  73. continue;
  74. vunmap_pud_range(pgd, addr, next);
  75. } while (pgd++, addr = next, addr != end);
  76. flush_tlb_kernel_range(start, end);
  77. }
  78. static void unmap_vm_area(struct vm_struct *area)
  79. {
  80. unmap_kernel_range((unsigned long)area->addr, area->size);
  81. }
  82. static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
  83. unsigned long end, pgprot_t prot, struct page ***pages)
  84. {
  85. pte_t *pte;
  86. pte = pte_alloc_kernel(pmd, addr);
  87. if (!pte)
  88. return -ENOMEM;
  89. do {
  90. struct page *page = **pages;
  91. WARN_ON(!pte_none(*pte));
  92. if (!page)
  93. return -ENOMEM;
  94. set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
  95. (*pages)++;
  96. } while (pte++, addr += PAGE_SIZE, addr != end);
  97. return 0;
  98. }
  99. static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
  100. unsigned long end, pgprot_t prot, struct page ***pages)
  101. {
  102. pmd_t *pmd;
  103. unsigned long next;
  104. pmd = pmd_alloc(&init_mm, pud, addr);
  105. if (!pmd)
  106. return -ENOMEM;
  107. do {
  108. next = pmd_addr_end(addr, end);
  109. if (vmap_pte_range(pmd, addr, next, prot, pages))
  110. return -ENOMEM;
  111. } while (pmd++, addr = next, addr != end);
  112. return 0;
  113. }
  114. static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
  115. unsigned long end, pgprot_t prot, struct page ***pages)
  116. {
  117. pud_t *pud;
  118. unsigned long next;
  119. pud = pud_alloc(&init_mm, pgd, addr);
  120. if (!pud)
  121. return -ENOMEM;
  122. do {
  123. next = pud_addr_end(addr, end);
  124. if (vmap_pmd_range(pud, addr, next, prot, pages))
  125. return -ENOMEM;
  126. } while (pud++, addr = next, addr != end);
  127. return 0;
  128. }
  129. int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
  130. {
  131. pgd_t *pgd;
  132. unsigned long next;
  133. unsigned long addr = (unsigned long) area->addr;
  134. unsigned long end = addr + area->size - PAGE_SIZE;
  135. int err;
  136. BUG_ON(addr >= end);
  137. pgd = pgd_offset_k(addr);
  138. do {
  139. next = pgd_addr_end(addr, end);
  140. err = vmap_pud_range(pgd, addr, next, prot, pages);
  141. if (err)
  142. break;
  143. } while (pgd++, addr = next, addr != end);
  144. flush_cache_vmap((unsigned long) area->addr, end);
  145. return err;
  146. }
  147. EXPORT_SYMBOL_GPL(map_vm_area);
  148. /*
  149. * Map a vmalloc()-space virtual address to the physical page.
  150. */
  151. struct page *vmalloc_to_page(const void *vmalloc_addr)
  152. {
  153. unsigned long addr = (unsigned long) vmalloc_addr;
  154. struct page *page = NULL;
  155. pgd_t *pgd = pgd_offset_k(addr);
  156. pud_t *pud;
  157. pmd_t *pmd;
  158. pte_t *ptep, pte;
  159. if (!pgd_none(*pgd)) {
  160. pud = pud_offset(pgd, addr);
  161. if (!pud_none(*pud)) {
  162. pmd = pmd_offset(pud, addr);
  163. if (!pmd_none(*pmd)) {
  164. ptep = pte_offset_map(pmd, addr);
  165. pte = *ptep;
  166. if (pte_present(pte))
  167. page = pte_page(pte);
  168. pte_unmap(ptep);
  169. }
  170. }
  171. }
  172. return page;
  173. }
  174. EXPORT_SYMBOL(vmalloc_to_page);
  175. /*
  176. * Map a vmalloc()-space virtual address to the physical page frame number.
  177. */
  178. unsigned long vmalloc_to_pfn(const void *vmalloc_addr)
  179. {
  180. return page_to_pfn(vmalloc_to_page(vmalloc_addr));
  181. }
  182. EXPORT_SYMBOL(vmalloc_to_pfn);
  183. static struct vm_struct *
  184. __get_vm_area_node(unsigned long size, unsigned long flags, unsigned long start,
  185. unsigned long end, int node, gfp_t gfp_mask, void *caller)
  186. {
  187. struct vm_struct **p, *tmp, *area;
  188. unsigned long align = 1;
  189. unsigned long addr;
  190. BUG_ON(in_interrupt());
  191. if (flags & VM_IOREMAP) {
  192. int bit = fls(size);
  193. if (bit > IOREMAP_MAX_ORDER)
  194. bit = IOREMAP_MAX_ORDER;
  195. else if (bit < PAGE_SHIFT)
  196. bit = PAGE_SHIFT;
  197. align = 1ul << bit;
  198. }
  199. addr = ALIGN(start, align);
  200. size = PAGE_ALIGN(size);
  201. if (unlikely(!size))
  202. return NULL;
  203. area = kmalloc_node(sizeof(*area), gfp_mask & GFP_RECLAIM_MASK, node);
  204. if (unlikely(!area))
  205. return NULL;
  206. /*
  207. * We always allocate a guard page.
  208. */
  209. size += PAGE_SIZE;
  210. write_lock(&vmlist_lock);
  211. for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
  212. if ((unsigned long)tmp->addr < addr) {
  213. if((unsigned long)tmp->addr + tmp->size >= addr)
  214. addr = ALIGN(tmp->size +
  215. (unsigned long)tmp->addr, align);
  216. continue;
  217. }
  218. if ((size + addr) < addr)
  219. goto out;
  220. if (size + addr <= (unsigned long)tmp->addr)
  221. goto found;
  222. addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
  223. if (addr > end - size)
  224. goto out;
  225. }
  226. if ((size + addr) < addr)
  227. goto out;
  228. if (addr > end - size)
  229. goto out;
  230. found:
  231. area->next = *p;
  232. *p = area;
  233. area->flags = flags;
  234. area->addr = (void *)addr;
  235. area->size = size;
  236. area->pages = NULL;
  237. area->nr_pages = 0;
  238. area->phys_addr = 0;
  239. area->caller = caller;
  240. write_unlock(&vmlist_lock);
  241. return area;
  242. out:
  243. write_unlock(&vmlist_lock);
  244. kfree(area);
  245. if (printk_ratelimit())
  246. printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
  247. return NULL;
  248. }
  249. struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
  250. unsigned long start, unsigned long end)
  251. {
  252. return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL,
  253. __builtin_return_address(0));
  254. }
  255. EXPORT_SYMBOL_GPL(__get_vm_area);
  256. /**
  257. * get_vm_area - reserve a contiguous kernel virtual area
  258. * @size: size of the area
  259. * @flags: %VM_IOREMAP for I/O mappings or VM_ALLOC
  260. *
  261. * Search an area of @size in the kernel virtual mapping area,
  262. * and reserved it for out purposes. Returns the area descriptor
  263. * on success or %NULL on failure.
  264. */
  265. struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
  266. {
  267. return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
  268. -1, GFP_KERNEL, __builtin_return_address(0));
  269. }
  270. struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags,
  271. void *caller)
  272. {
  273. return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END,
  274. -1, GFP_KERNEL, caller);
  275. }
  276. struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
  277. int node, gfp_t gfp_mask)
  278. {
  279. return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
  280. gfp_mask, __builtin_return_address(0));
  281. }
  282. /* Caller must hold vmlist_lock */
  283. static struct vm_struct *__find_vm_area(const void *addr)
  284. {
  285. struct vm_struct *tmp;
  286. for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
  287. if (tmp->addr == addr)
  288. break;
  289. }
  290. return tmp;
  291. }
  292. /* Caller must hold vmlist_lock */
  293. static struct vm_struct *__remove_vm_area(const void *addr)
  294. {
  295. struct vm_struct **p, *tmp;
  296. for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
  297. if (tmp->addr == addr)
  298. goto found;
  299. }
  300. return NULL;
  301. found:
  302. unmap_vm_area(tmp);
  303. *p = tmp->next;
  304. /*
  305. * Remove the guard page.
  306. */
  307. tmp->size -= PAGE_SIZE;
  308. return tmp;
  309. }
  310. /**
  311. * remove_vm_area - find and remove a continuous kernel virtual area
  312. * @addr: base address
  313. *
  314. * Search for the kernel VM area starting at @addr, and remove it.
  315. * This function returns the found VM area, but using it is NOT safe
  316. * on SMP machines, except for its size or flags.
  317. */
  318. struct vm_struct *remove_vm_area(const void *addr)
  319. {
  320. struct vm_struct *v;
  321. write_lock(&vmlist_lock);
  322. v = __remove_vm_area(addr);
  323. write_unlock(&vmlist_lock);
  324. return v;
  325. }
  326. static void __vunmap(const void *addr, int deallocate_pages)
  327. {
  328. struct vm_struct *area;
  329. if (!addr)
  330. return;
  331. if ((PAGE_SIZE-1) & (unsigned long)addr) {
  332. printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
  333. WARN_ON(1);
  334. return;
  335. }
  336. area = remove_vm_area(addr);
  337. if (unlikely(!area)) {
  338. printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
  339. addr);
  340. WARN_ON(1);
  341. return;
  342. }
  343. debug_check_no_locks_freed(addr, area->size);
  344. debug_check_no_obj_freed(addr, area->size);
  345. if (deallocate_pages) {
  346. int i;
  347. for (i = 0; i < area->nr_pages; i++) {
  348. struct page *page = area->pages[i];
  349. BUG_ON(!page);
  350. __free_page(page);
  351. }
  352. if (area->flags & VM_VPAGES)
  353. vfree(area->pages);
  354. else
  355. kfree(area->pages);
  356. }
  357. kfree(area);
  358. return;
  359. }
  360. /**
  361. * vfree - release memory allocated by vmalloc()
  362. * @addr: memory base address
  363. *
  364. * Free the virtually continuous memory area starting at @addr, as
  365. * obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
  366. * NULL, no operation is performed.
  367. *
  368. * Must not be called in interrupt context.
  369. */
  370. void vfree(const void *addr)
  371. {
  372. BUG_ON(in_interrupt());
  373. __vunmap(addr, 1);
  374. }
  375. EXPORT_SYMBOL(vfree);
  376. /**
  377. * vunmap - release virtual mapping obtained by vmap()
  378. * @addr: memory base address
  379. *
  380. * Free the virtually contiguous memory area starting at @addr,
  381. * which was created from the page array passed to vmap().
  382. *
  383. * Must not be called in interrupt context.
  384. */
  385. void vunmap(const void *addr)
  386. {
  387. BUG_ON(in_interrupt());
  388. __vunmap(addr, 0);
  389. }
  390. EXPORT_SYMBOL(vunmap);
  391. /**
  392. * vmap - map an array of pages into virtually contiguous space
  393. * @pages: array of page pointers
  394. * @count: number of pages to map
  395. * @flags: vm_area->flags
  396. * @prot: page protection for the mapping
  397. *
  398. * Maps @count pages from @pages into contiguous kernel virtual
  399. * space.
  400. */
  401. void *vmap(struct page **pages, unsigned int count,
  402. unsigned long flags, pgprot_t prot)
  403. {
  404. struct vm_struct *area;
  405. if (count > num_physpages)
  406. return NULL;
  407. area = get_vm_area_caller((count << PAGE_SHIFT), flags,
  408. __builtin_return_address(0));
  409. if (!area)
  410. return NULL;
  411. if (map_vm_area(area, prot, &pages)) {
  412. vunmap(area->addr);
  413. return NULL;
  414. }
  415. return area->addr;
  416. }
  417. EXPORT_SYMBOL(vmap);
  418. static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
  419. pgprot_t prot, int node, void *caller)
  420. {
  421. struct page **pages;
  422. unsigned int nr_pages, array_size, i;
  423. nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
  424. array_size = (nr_pages * sizeof(struct page *));
  425. area->nr_pages = nr_pages;
  426. /* Please note that the recursion is strictly bounded. */
  427. if (array_size > PAGE_SIZE) {
  428. pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO,
  429. PAGE_KERNEL, node, caller);
  430. area->flags |= VM_VPAGES;
  431. } else {
  432. pages = kmalloc_node(array_size,
  433. (gfp_mask & GFP_RECLAIM_MASK) | __GFP_ZERO,
  434. node);
  435. }
  436. area->pages = pages;
  437. area->caller = caller;
  438. if (!area->pages) {
  439. remove_vm_area(area->addr);
  440. kfree(area);
  441. return NULL;
  442. }
  443. for (i = 0; i < area->nr_pages; i++) {
  444. struct page *page;
  445. if (node < 0)
  446. page = alloc_page(gfp_mask);
  447. else
  448. page = alloc_pages_node(node, gfp_mask, 0);
  449. if (unlikely(!page)) {
  450. /* Successfully allocated i pages, free them in __vunmap() */
  451. area->nr_pages = i;
  452. goto fail;
  453. }
  454. area->pages[i] = page;
  455. }
  456. if (map_vm_area(area, prot, &pages))
  457. goto fail;
  458. return area->addr;
  459. fail:
  460. vfree(area->addr);
  461. return NULL;
  462. }
  463. void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
  464. {
  465. return __vmalloc_area_node(area, gfp_mask, prot, -1,
  466. __builtin_return_address(0));
  467. }
  468. /**
  469. * __vmalloc_node - allocate virtually contiguous memory
  470. * @size: allocation size
  471. * @gfp_mask: flags for the page level allocator
  472. * @prot: protection mask for the allocated pages
  473. * @node: node to use for allocation or -1
  474. *
  475. * Allocate enough pages to cover @size from the page level
  476. * allocator with @gfp_mask flags. Map them into contiguous
  477. * kernel virtual space, using a pagetable protection of @prot.
  478. */
  479. static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
  480. int node, void *caller)
  481. {
  482. struct vm_struct *area;
  483. size = PAGE_ALIGN(size);
  484. if (!size || (size >> PAGE_SHIFT) > num_physpages)
  485. return NULL;
  486. area = __get_vm_area_node(size, VM_ALLOC, VMALLOC_START, VMALLOC_END,
  487. node, gfp_mask, caller);
  488. if (!area)
  489. return NULL;
  490. return __vmalloc_area_node(area, gfp_mask, prot, node, caller);
  491. }
  492. void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
  493. {
  494. return __vmalloc_node(size, gfp_mask, prot, -1,
  495. __builtin_return_address(0));
  496. }
  497. EXPORT_SYMBOL(__vmalloc);
  498. /**
  499. * vmalloc - allocate virtually contiguous memory
  500. * @size: allocation size
  501. * Allocate enough pages to cover @size from the page level
  502. * allocator and map them into contiguous kernel virtual space.
  503. *
  504. * For tight control over page level allocator and protection flags
  505. * use __vmalloc() instead.
  506. */
  507. void *vmalloc(unsigned long size)
  508. {
  509. return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
  510. -1, __builtin_return_address(0));
  511. }
  512. EXPORT_SYMBOL(vmalloc);
  513. /**
  514. * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
  515. * @size: allocation size
  516. *
  517. * The resulting memory area is zeroed so it can be mapped to userspace
  518. * without leaking data.
  519. */
  520. void *vmalloc_user(unsigned long size)
  521. {
  522. struct vm_struct *area;
  523. void *ret;
  524. ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
  525. if (ret) {
  526. write_lock(&vmlist_lock);
  527. area = __find_vm_area(ret);
  528. area->flags |= VM_USERMAP;
  529. write_unlock(&vmlist_lock);
  530. }
  531. return ret;
  532. }
  533. EXPORT_SYMBOL(vmalloc_user);
  534. /**
  535. * vmalloc_node - allocate memory on a specific node
  536. * @size: allocation size
  537. * @node: numa node
  538. *
  539. * Allocate enough pages to cover @size from the page level
  540. * allocator and map them into contiguous kernel virtual space.
  541. *
  542. * For tight control over page level allocator and protection flags
  543. * use __vmalloc() instead.
  544. */
  545. void *vmalloc_node(unsigned long size, int node)
  546. {
  547. return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL,
  548. node, __builtin_return_address(0));
  549. }
  550. EXPORT_SYMBOL(vmalloc_node);
  551. #ifndef PAGE_KERNEL_EXEC
  552. # define PAGE_KERNEL_EXEC PAGE_KERNEL
  553. #endif
  554. /**
  555. * vmalloc_exec - allocate virtually contiguous, executable memory
  556. * @size: allocation size
  557. *
  558. * Kernel-internal function to allocate enough pages to cover @size
  559. * the page level allocator and map them into contiguous and
  560. * executable kernel virtual space.
  561. *
  562. * For tight control over page level allocator and protection flags
  563. * use __vmalloc() instead.
  564. */
  565. void *vmalloc_exec(unsigned long size)
  566. {
  567. return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
  568. }
  569. #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
  570. #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
  571. #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
  572. #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
  573. #else
  574. #define GFP_VMALLOC32 GFP_KERNEL
  575. #endif
  576. /**
  577. * vmalloc_32 - allocate virtually contiguous memory (32bit addressable)
  578. * @size: allocation size
  579. *
  580. * Allocate enough 32bit PA addressable pages to cover @size from the
  581. * page level allocator and map them into contiguous kernel virtual space.
  582. */
  583. void *vmalloc_32(unsigned long size)
  584. {
  585. return __vmalloc(size, GFP_VMALLOC32, PAGE_KERNEL);
  586. }
  587. EXPORT_SYMBOL(vmalloc_32);
  588. /**
  589. * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
  590. * @size: allocation size
  591. *
  592. * The resulting memory area is 32bit addressable and zeroed so it can be
  593. * mapped to userspace without leaking data.
  594. */
  595. void *vmalloc_32_user(unsigned long size)
  596. {
  597. struct vm_struct *area;
  598. void *ret;
  599. ret = __vmalloc(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL);
  600. if (ret) {
  601. write_lock(&vmlist_lock);
  602. area = __find_vm_area(ret);
  603. area->flags |= VM_USERMAP;
  604. write_unlock(&vmlist_lock);
  605. }
  606. return ret;
  607. }
  608. EXPORT_SYMBOL(vmalloc_32_user);
  609. long vread(char *buf, char *addr, unsigned long count)
  610. {
  611. struct vm_struct *tmp;
  612. char *vaddr, *buf_start = buf;
  613. unsigned long n;
  614. /* Don't allow overflow */
  615. if ((unsigned long) addr + count < count)
  616. count = -(unsigned long) addr;
  617. read_lock(&vmlist_lock);
  618. for (tmp = vmlist; tmp; tmp = tmp->next) {
  619. vaddr = (char *) tmp->addr;
  620. if (addr >= vaddr + tmp->size - PAGE_SIZE)
  621. continue;
  622. while (addr < vaddr) {
  623. if (count == 0)
  624. goto finished;
  625. *buf = '\0';
  626. buf++;
  627. addr++;
  628. count--;
  629. }
  630. n = vaddr + tmp->size - PAGE_SIZE - addr;
  631. do {
  632. if (count == 0)
  633. goto finished;
  634. *buf = *addr;
  635. buf++;
  636. addr++;
  637. count--;
  638. } while (--n > 0);
  639. }
  640. finished:
  641. read_unlock(&vmlist_lock);
  642. return buf - buf_start;
  643. }
  644. long vwrite(char *buf, char *addr, unsigned long count)
  645. {
  646. struct vm_struct *tmp;
  647. char *vaddr, *buf_start = buf;
  648. unsigned long n;
  649. /* Don't allow overflow */
  650. if ((unsigned long) addr + count < count)
  651. count = -(unsigned long) addr;
  652. read_lock(&vmlist_lock);
  653. for (tmp = vmlist; tmp; tmp = tmp->next) {
  654. vaddr = (char *) tmp->addr;
  655. if (addr >= vaddr + tmp->size - PAGE_SIZE)
  656. continue;
  657. while (addr < vaddr) {
  658. if (count == 0)
  659. goto finished;
  660. buf++;
  661. addr++;
  662. count--;
  663. }
  664. n = vaddr + tmp->size - PAGE_SIZE - addr;
  665. do {
  666. if (count == 0)
  667. goto finished;
  668. *addr = *buf;
  669. buf++;
  670. addr++;
  671. count--;
  672. } while (--n > 0);
  673. }
  674. finished:
  675. read_unlock(&vmlist_lock);
  676. return buf - buf_start;
  677. }
  678. /**
  679. * remap_vmalloc_range - map vmalloc pages to userspace
  680. * @vma: vma to cover (map full range of vma)
  681. * @addr: vmalloc memory
  682. * @pgoff: number of pages into addr before first page to map
  683. *
  684. * Returns: 0 for success, -Exxx on failure
  685. *
  686. * This function checks that addr is a valid vmalloc'ed area, and
  687. * that it is big enough to cover the vma. Will return failure if
  688. * that criteria isn't met.
  689. *
  690. * Similar to remap_pfn_range() (see mm/memory.c)
  691. */
  692. int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
  693. unsigned long pgoff)
  694. {
  695. struct vm_struct *area;
  696. unsigned long uaddr = vma->vm_start;
  697. unsigned long usize = vma->vm_end - vma->vm_start;
  698. int ret;
  699. if ((PAGE_SIZE-1) & (unsigned long)addr)
  700. return -EINVAL;
  701. read_lock(&vmlist_lock);
  702. area = __find_vm_area(addr);
  703. if (!area)
  704. goto out_einval_locked;
  705. if (!(area->flags & VM_USERMAP))
  706. goto out_einval_locked;
  707. if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
  708. goto out_einval_locked;
  709. read_unlock(&vmlist_lock);
  710. addr += pgoff << PAGE_SHIFT;
  711. do {
  712. struct page *page = vmalloc_to_page(addr);
  713. ret = vm_insert_page(vma, uaddr, page);
  714. if (ret)
  715. return ret;
  716. uaddr += PAGE_SIZE;
  717. addr += PAGE_SIZE;
  718. usize -= PAGE_SIZE;
  719. } while (usize > 0);
  720. /* Prevent "things" like memory migration? VM_flags need a cleanup... */
  721. vma->vm_flags |= VM_RESERVED;
  722. return ret;
  723. out_einval_locked:
  724. read_unlock(&vmlist_lock);
  725. return -EINVAL;
  726. }
  727. EXPORT_SYMBOL(remap_vmalloc_range);
  728. /*
  729. * Implement a stub for vmalloc_sync_all() if the architecture chose not to
  730. * have one.
  731. */
  732. void __attribute__((weak)) vmalloc_sync_all(void)
  733. {
  734. }
  735. static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
  736. {
  737. /* apply_to_page_range() does all the hard work. */
  738. return 0;
  739. }
  740. /**
  741. * alloc_vm_area - allocate a range of kernel address space
  742. * @size: size of the area
  743. *
  744. * Returns: NULL on failure, vm_struct on success
  745. *
  746. * This function reserves a range of kernel address space, and
  747. * allocates pagetables to map that range. No actual mappings
  748. * are created. If the kernel address space is not shared
  749. * between processes, it syncs the pagetable across all
  750. * processes.
  751. */
  752. struct vm_struct *alloc_vm_area(size_t size)
  753. {
  754. struct vm_struct *area;
  755. area = get_vm_area_caller(size, VM_IOREMAP,
  756. __builtin_return_address(0));
  757. if (area == NULL)
  758. return NULL;
  759. /*
  760. * This ensures that page tables are constructed for this region
  761. * of kernel virtual address space and mapped into init_mm.
  762. */
  763. if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
  764. area->size, f, NULL)) {
  765. free_vm_area(area);
  766. return NULL;
  767. }
  768. /* Make sure the pagetables are constructed in process kernel
  769. mappings */
  770. vmalloc_sync_all();
  771. return area;
  772. }
  773. EXPORT_SYMBOL_GPL(alloc_vm_area);
  774. void free_vm_area(struct vm_struct *area)
  775. {
  776. struct vm_struct *ret;
  777. ret = remove_vm_area(area->addr);
  778. BUG_ON(ret != area);
  779. kfree(area);
  780. }
  781. EXPORT_SYMBOL_GPL(free_vm_area);
  782. #ifdef CONFIG_PROC_FS
  783. static void *s_start(struct seq_file *m, loff_t *pos)
  784. {
  785. loff_t n = *pos;
  786. struct vm_struct *v;
  787. read_lock(&vmlist_lock);
  788. v = vmlist;
  789. while (n > 0 && v) {
  790. n--;
  791. v = v->next;
  792. }
  793. if (!n)
  794. return v;
  795. return NULL;
  796. }
  797. static void *s_next(struct seq_file *m, void *p, loff_t *pos)
  798. {
  799. struct vm_struct *v = p;
  800. ++*pos;
  801. return v->next;
  802. }
  803. static void s_stop(struct seq_file *m, void *p)
  804. {
  805. read_unlock(&vmlist_lock);
  806. }
  807. static int s_show(struct seq_file *m, void *p)
  808. {
  809. struct vm_struct *v = p;
  810. seq_printf(m, "0x%p-0x%p %7ld",
  811. v->addr, v->addr + v->size, v->size);
  812. if (v->caller) {
  813. char buff[2 * KSYM_NAME_LEN];
  814. seq_putc(m, ' ');
  815. sprint_symbol(buff, (unsigned long)v->caller);
  816. seq_puts(m, buff);
  817. }
  818. if (v->nr_pages)
  819. seq_printf(m, " pages=%d", v->nr_pages);
  820. if (v->phys_addr)
  821. seq_printf(m, " phys=%lx", v->phys_addr);
  822. if (v->flags & VM_IOREMAP)
  823. seq_printf(m, " ioremap");
  824. if (v->flags & VM_ALLOC)
  825. seq_printf(m, " vmalloc");
  826. if (v->flags & VM_MAP)
  827. seq_printf(m, " vmap");
  828. if (v->flags & VM_USERMAP)
  829. seq_printf(m, " user");
  830. if (v->flags & VM_VPAGES)
  831. seq_printf(m, " vpages");
  832. seq_putc(m, '\n');
  833. return 0;
  834. }
  835. const struct seq_operations vmalloc_op = {
  836. .start = s_start,
  837. .next = s_next,
  838. .stop = s_stop,
  839. .show = s_show,
  840. };
  841. #endif