util.c 9.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420
  1. #include <linux/mm.h>
  2. #include <linux/slab.h>
  3. #include <linux/string.h>
  4. #include <linux/export.h>
  5. #include <linux/err.h>
  6. #include <linux/sched.h>
  7. #include <linux/security.h>
  8. #include <linux/swap.h>
  9. #include <linux/swapops.h>
  10. #include <linux/mman.h>
  11. #include <linux/hugetlb.h>
  12. #include <asm/uaccess.h>
  13. #include "internal.h"
  14. #define CREATE_TRACE_POINTS
  15. #include <trace/events/kmem.h>
  16. /**
  17. * kstrdup - allocate space for and copy an existing string
  18. * @s: the string to duplicate
  19. * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  20. */
  21. char *kstrdup(const char *s, gfp_t gfp)
  22. {
  23. size_t len;
  24. char *buf;
  25. if (!s)
  26. return NULL;
  27. len = strlen(s) + 1;
  28. buf = kmalloc_track_caller(len, gfp);
  29. if (buf)
  30. memcpy(buf, s, len);
  31. return buf;
  32. }
  33. EXPORT_SYMBOL(kstrdup);
  34. /**
  35. * kstrndup - allocate space for and copy an existing string
  36. * @s: the string to duplicate
  37. * @max: read at most @max chars from @s
  38. * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  39. */
  40. char *kstrndup(const char *s, size_t max, gfp_t gfp)
  41. {
  42. size_t len;
  43. char *buf;
  44. if (!s)
  45. return NULL;
  46. len = strnlen(s, max);
  47. buf = kmalloc_track_caller(len+1, gfp);
  48. if (buf) {
  49. memcpy(buf, s, len);
  50. buf[len] = '\0';
  51. }
  52. return buf;
  53. }
  54. EXPORT_SYMBOL(kstrndup);
  55. /**
  56. * kmemdup - duplicate region of memory
  57. *
  58. * @src: memory region to duplicate
  59. * @len: memory region length
  60. * @gfp: GFP mask to use
  61. */
  62. void *kmemdup(const void *src, size_t len, gfp_t gfp)
  63. {
  64. void *p;
  65. p = kmalloc_track_caller(len, gfp);
  66. if (p)
  67. memcpy(p, src, len);
  68. return p;
  69. }
  70. EXPORT_SYMBOL(kmemdup);
  71. /**
  72. * memdup_user - duplicate memory region from user space
  73. *
  74. * @src: source address in user space
  75. * @len: number of bytes to copy
  76. *
  77. * Returns an ERR_PTR() on failure.
  78. */
  79. void *memdup_user(const void __user *src, size_t len)
  80. {
  81. void *p;
  82. /*
  83. * Always use GFP_KERNEL, since copy_from_user() can sleep and
  84. * cause pagefault, which makes it pointless to use GFP_NOFS
  85. * or GFP_ATOMIC.
  86. */
  87. p = kmalloc_track_caller(len, GFP_KERNEL);
  88. if (!p)
  89. return ERR_PTR(-ENOMEM);
  90. if (copy_from_user(p, src, len)) {
  91. kfree(p);
  92. return ERR_PTR(-EFAULT);
  93. }
  94. return p;
  95. }
  96. EXPORT_SYMBOL(memdup_user);
  97. static __always_inline void *__do_krealloc(const void *p, size_t new_size,
  98. gfp_t flags)
  99. {
  100. void *ret;
  101. size_t ks = 0;
  102. if (p)
  103. ks = ksize(p);
  104. if (ks >= new_size)
  105. return (void *)p;
  106. ret = kmalloc_track_caller(new_size, flags);
  107. if (ret && p)
  108. memcpy(ret, p, ks);
  109. return ret;
  110. }
  111. /**
  112. * __krealloc - like krealloc() but don't free @p.
  113. * @p: object to reallocate memory for.
  114. * @new_size: how many bytes of memory are required.
  115. * @flags: the type of memory to allocate.
  116. *
  117. * This function is like krealloc() except it never frees the originally
  118. * allocated buffer. Use this if you don't want to free the buffer immediately
  119. * like, for example, with RCU.
  120. */
  121. void *__krealloc(const void *p, size_t new_size, gfp_t flags)
  122. {
  123. if (unlikely(!new_size))
  124. return ZERO_SIZE_PTR;
  125. return __do_krealloc(p, new_size, flags);
  126. }
  127. EXPORT_SYMBOL(__krealloc);
  128. /**
  129. * krealloc - reallocate memory. The contents will remain unchanged.
  130. * @p: object to reallocate memory for.
  131. * @new_size: how many bytes of memory are required.
  132. * @flags: the type of memory to allocate.
  133. *
  134. * The contents of the object pointed to are preserved up to the
  135. * lesser of the new and old sizes. If @p is %NULL, krealloc()
  136. * behaves exactly like kmalloc(). If @new_size is 0 and @p is not a
  137. * %NULL pointer, the object pointed to is freed.
  138. */
  139. void *krealloc(const void *p, size_t new_size, gfp_t flags)
  140. {
  141. void *ret;
  142. if (unlikely(!new_size)) {
  143. kfree(p);
  144. return ZERO_SIZE_PTR;
  145. }
  146. ret = __do_krealloc(p, new_size, flags);
  147. if (ret && p != ret)
  148. kfree(p);
  149. return ret;
  150. }
  151. EXPORT_SYMBOL(krealloc);
  152. /**
  153. * kzfree - like kfree but zero memory
  154. * @p: object to free memory of
  155. *
  156. * The memory of the object @p points to is zeroed before freed.
  157. * If @p is %NULL, kzfree() does nothing.
  158. *
  159. * Note: this function zeroes the whole allocated buffer which can be a good
  160. * deal bigger than the requested buffer size passed to kmalloc(). So be
  161. * careful when using this function in performance sensitive code.
  162. */
  163. void kzfree(const void *p)
  164. {
  165. size_t ks;
  166. void *mem = (void *)p;
  167. if (unlikely(ZERO_OR_NULL_PTR(mem)))
  168. return;
  169. ks = ksize(mem);
  170. memset(mem, 0, ks);
  171. kfree(mem);
  172. }
  173. EXPORT_SYMBOL(kzfree);
  174. /*
  175. * strndup_user - duplicate an existing string from user space
  176. * @s: The string to duplicate
  177. * @n: Maximum number of bytes to copy, including the trailing NUL.
  178. */
  179. char *strndup_user(const char __user *s, long n)
  180. {
  181. char *p;
  182. long length;
  183. length = strnlen_user(s, n);
  184. if (!length)
  185. return ERR_PTR(-EFAULT);
  186. if (length > n)
  187. return ERR_PTR(-EINVAL);
  188. p = memdup_user(s, length);
  189. if (IS_ERR(p))
  190. return p;
  191. p[length - 1] = '\0';
  192. return p;
  193. }
  194. EXPORT_SYMBOL(strndup_user);
  195. void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
  196. struct vm_area_struct *prev, struct rb_node *rb_parent)
  197. {
  198. struct vm_area_struct *next;
  199. vma->vm_prev = prev;
  200. if (prev) {
  201. next = prev->vm_next;
  202. prev->vm_next = vma;
  203. } else {
  204. mm->mmap = vma;
  205. if (rb_parent)
  206. next = rb_entry(rb_parent,
  207. struct vm_area_struct, vm_rb);
  208. else
  209. next = NULL;
  210. }
  211. vma->vm_next = next;
  212. if (next)
  213. next->vm_prev = vma;
  214. }
  215. /* Check if the vma is being used as a stack by this task */
  216. static int vm_is_stack_for_task(struct task_struct *t,
  217. struct vm_area_struct *vma)
  218. {
  219. return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
  220. }
  221. /*
  222. * Check if the vma is being used as a stack.
  223. * If is_group is non-zero, check in the entire thread group or else
  224. * just check in the current task. Returns the pid of the task that
  225. * the vma is stack for.
  226. */
  227. pid_t vm_is_stack(struct task_struct *task,
  228. struct vm_area_struct *vma, int in_group)
  229. {
  230. pid_t ret = 0;
  231. if (vm_is_stack_for_task(task, vma))
  232. return task->pid;
  233. if (in_group) {
  234. struct task_struct *t;
  235. rcu_read_lock();
  236. if (!pid_alive(task))
  237. goto done;
  238. t = task;
  239. do {
  240. if (vm_is_stack_for_task(t, vma)) {
  241. ret = t->pid;
  242. goto done;
  243. }
  244. } while_each_thread(task, t);
  245. done:
  246. rcu_read_unlock();
  247. }
  248. return ret;
  249. }
  250. #if defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
  251. void arch_pick_mmap_layout(struct mm_struct *mm)
  252. {
  253. mm->mmap_base = TASK_UNMAPPED_BASE;
  254. mm->get_unmapped_area = arch_get_unmapped_area;
  255. }
  256. #endif
  257. /*
  258. * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
  259. * back to the regular GUP.
  260. * If the architecture not support this function, simply return with no
  261. * page pinned
  262. */
  263. int __attribute__((weak)) __get_user_pages_fast(unsigned long start,
  264. int nr_pages, int write, struct page **pages)
  265. {
  266. return 0;
  267. }
  268. EXPORT_SYMBOL_GPL(__get_user_pages_fast);
  269. /**
  270. * get_user_pages_fast() - pin user pages in memory
  271. * @start: starting user address
  272. * @nr_pages: number of pages from start to pin
  273. * @write: whether pages will be written to
  274. * @pages: array that receives pointers to the pages pinned.
  275. * Should be at least nr_pages long.
  276. *
  277. * Returns number of pages pinned. This may be fewer than the number
  278. * requested. If nr_pages is 0 or negative, returns 0. If no pages
  279. * were pinned, returns -errno.
  280. *
  281. * get_user_pages_fast provides equivalent functionality to get_user_pages,
  282. * operating on current and current->mm, with force=0 and vma=NULL. However
  283. * unlike get_user_pages, it must be called without mmap_sem held.
  284. *
  285. * get_user_pages_fast may take mmap_sem and page table locks, so no
  286. * assumptions can be made about lack of locking. get_user_pages_fast is to be
  287. * implemented in a way that is advantageous (vs get_user_pages()) when the
  288. * user memory area is already faulted in and present in ptes. However if the
  289. * pages have to be faulted in, it may turn out to be slightly slower so
  290. * callers need to carefully consider what to use. On many architectures,
  291. * get_user_pages_fast simply falls back to get_user_pages.
  292. */
  293. int __attribute__((weak)) get_user_pages_fast(unsigned long start,
  294. int nr_pages, int write, struct page **pages)
  295. {
  296. struct mm_struct *mm = current->mm;
  297. int ret;
  298. down_read(&mm->mmap_sem);
  299. ret = get_user_pages(current, mm, start, nr_pages,
  300. write, 0, pages, NULL);
  301. up_read(&mm->mmap_sem);
  302. return ret;
  303. }
  304. EXPORT_SYMBOL_GPL(get_user_pages_fast);
  305. unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
  306. unsigned long len, unsigned long prot,
  307. unsigned long flag, unsigned long pgoff)
  308. {
  309. unsigned long ret;
  310. struct mm_struct *mm = current->mm;
  311. unsigned long populate;
  312. ret = security_mmap_file(file, prot, flag);
  313. if (!ret) {
  314. down_write(&mm->mmap_sem);
  315. ret = do_mmap_pgoff(file, addr, len, prot, flag, pgoff,
  316. &populate);
  317. up_write(&mm->mmap_sem);
  318. if (populate)
  319. mm_populate(ret, populate);
  320. }
  321. return ret;
  322. }
  323. unsigned long vm_mmap(struct file *file, unsigned long addr,
  324. unsigned long len, unsigned long prot,
  325. unsigned long flag, unsigned long offset)
  326. {
  327. if (unlikely(offset + PAGE_ALIGN(len) < offset))
  328. return -EINVAL;
  329. if (unlikely(offset & ~PAGE_MASK))
  330. return -EINVAL;
  331. return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
  332. }
  333. EXPORT_SYMBOL(vm_mmap);
  334. struct address_space *page_mapping(struct page *page)
  335. {
  336. struct address_space *mapping = page->mapping;
  337. VM_BUG_ON(PageSlab(page));
  338. if (unlikely(PageSwapCache(page))) {
  339. swp_entry_t entry;
  340. entry.val = page_private(page);
  341. mapping = swap_address_space(entry);
  342. } else if ((unsigned long)mapping & PAGE_MAPPING_ANON)
  343. mapping = NULL;
  344. return mapping;
  345. }
  346. /*
  347. * Committed memory limit enforced when OVERCOMMIT_NEVER policy is used
  348. */
  349. unsigned long vm_commit_limit(void)
  350. {
  351. return ((totalram_pages - hugetlb_total_pages())
  352. * sysctl_overcommit_ratio / 100) + total_swap_pages;
  353. }
  354. /* Tracepoints definitions. */
  355. EXPORT_TRACEPOINT_SYMBOL(kmalloc);
  356. EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
  357. EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
  358. EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
  359. EXPORT_TRACEPOINT_SYMBOL(kfree);
  360. EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);