slab_common.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660
  1. /*
  2. * Slab allocator functions that are independent of the allocator strategy
  3. *
  4. * (C) 2012 Christoph Lameter <cl@linux.com>
  5. */
  6. #include <linux/slab.h>
  7. #include <linux/mm.h>
  8. #include <linux/poison.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/memory.h>
  11. #include <linux/compiler.h>
  12. #include <linux/module.h>
  13. #include <linux/cpu.h>
  14. #include <linux/uaccess.h>
  15. #include <linux/seq_file.h>
  16. #include <linux/proc_fs.h>
  17. #include <asm/cacheflush.h>
  18. #include <asm/tlbflush.h>
  19. #include <asm/page.h>
  20. #include <linux/memcontrol.h>
  21. #include <trace/events/kmem.h>
  22. #include "slab.h"
  23. enum slab_state slab_state;
  24. LIST_HEAD(slab_caches);
  25. DEFINE_MUTEX(slab_mutex);
  26. struct kmem_cache *kmem_cache;
  27. #ifdef CONFIG_DEBUG_VM
  28. static int kmem_cache_sanity_check(struct mem_cgroup *memcg, const char *name,
  29. size_t size)
  30. {
  31. struct kmem_cache *s = NULL;
  32. if (!name || in_interrupt() || size < sizeof(void *) ||
  33. size > KMALLOC_MAX_SIZE) {
  34. pr_err("kmem_cache_create(%s) integrity check failed\n", name);
  35. return -EINVAL;
  36. }
  37. list_for_each_entry(s, &slab_caches, list) {
  38. char tmp;
  39. int res;
  40. /*
  41. * This happens when the module gets unloaded and doesn't
  42. * destroy its slab cache and no-one else reuses the vmalloc
  43. * area of the module. Print a warning.
  44. */
  45. res = probe_kernel_address(s->name, tmp);
  46. if (res) {
  47. pr_err("Slab cache with size %d has lost its name\n",
  48. s->object_size);
  49. continue;
  50. }
  51. #if !defined(CONFIG_SLUB) || !defined(CONFIG_SLUB_DEBUG_ON)
  52. /*
  53. * For simplicity, we won't check this in the list of memcg
  54. * caches. We have control over memcg naming, and if there
  55. * aren't duplicates in the global list, there won't be any
  56. * duplicates in the memcg lists as well.
  57. */
  58. if (!memcg && !strcmp(s->name, name)) {
  59. pr_err("%s (%s): Cache name already exists.\n",
  60. __func__, name);
  61. dump_stack();
  62. s = NULL;
  63. return -EINVAL;
  64. }
  65. #endif
  66. }
  67. WARN_ON(strchr(name, ' ')); /* It confuses parsers */
  68. return 0;
  69. }
  70. #else
  71. static inline int kmem_cache_sanity_check(struct mem_cgroup *memcg,
  72. const char *name, size_t size)
  73. {
  74. return 0;
  75. }
  76. #endif
  77. #ifdef CONFIG_MEMCG_KMEM
  78. int memcg_update_all_caches(int num_memcgs)
  79. {
  80. struct kmem_cache *s;
  81. int ret = 0;
  82. mutex_lock(&slab_mutex);
  83. list_for_each_entry(s, &slab_caches, list) {
  84. if (!is_root_cache(s))
  85. continue;
  86. ret = memcg_update_cache_size(s, num_memcgs);
  87. /*
  88. * See comment in memcontrol.c, memcg_update_cache_size:
  89. * Instead of freeing the memory, we'll just leave the caches
  90. * up to this point in an updated state.
  91. */
  92. if (ret)
  93. goto out;
  94. }
  95. memcg_update_array_size(num_memcgs);
  96. out:
  97. mutex_unlock(&slab_mutex);
  98. return ret;
  99. }
  100. #endif
  101. /*
  102. * Figure out what the alignment of the objects will be given a set of
  103. * flags, a user specified alignment and the size of the objects.
  104. */
  105. unsigned long calculate_alignment(unsigned long flags,
  106. unsigned long align, unsigned long size)
  107. {
  108. /*
  109. * If the user wants hardware cache aligned objects then follow that
  110. * suggestion if the object is sufficiently large.
  111. *
  112. * The hardware cache alignment cannot override the specified
  113. * alignment though. If that is greater then use it.
  114. */
  115. if (flags & SLAB_HWCACHE_ALIGN) {
  116. unsigned long ralign = cache_line_size();
  117. while (size <= ralign / 2)
  118. ralign /= 2;
  119. align = max(align, ralign);
  120. }
  121. if (align < ARCH_SLAB_MINALIGN)
  122. align = ARCH_SLAB_MINALIGN;
  123. return ALIGN(align, sizeof(void *));
  124. }
  125. /*
  126. * kmem_cache_create - Create a cache.
  127. * @name: A string which is used in /proc/slabinfo to identify this cache.
  128. * @size: The size of objects to be created in this cache.
  129. * @align: The required alignment for the objects.
  130. * @flags: SLAB flags
  131. * @ctor: A constructor for the objects.
  132. *
  133. * Returns a ptr to the cache on success, NULL on failure.
  134. * Cannot be called within a interrupt, but can be interrupted.
  135. * The @ctor is run when new pages are allocated by the cache.
  136. *
  137. * The flags are
  138. *
  139. * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
  140. * to catch references to uninitialised memory.
  141. *
  142. * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
  143. * for buffer overruns.
  144. *
  145. * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
  146. * cacheline. This can be beneficial if you're counting cycles as closely
  147. * as davem.
  148. */
  149. struct kmem_cache *
  150. kmem_cache_create_memcg(struct mem_cgroup *memcg, const char *name, size_t size,
  151. size_t align, unsigned long flags, void (*ctor)(void *),
  152. struct kmem_cache *parent_cache)
  153. {
  154. struct kmem_cache *s = NULL;
  155. int err = 0;
  156. get_online_cpus();
  157. mutex_lock(&slab_mutex);
  158. if (!kmem_cache_sanity_check(memcg, name, size) == 0)
  159. goto out_locked;
  160. /*
  161. * Some allocators will constraint the set of valid flags to a subset
  162. * of all flags. We expect them to define CACHE_CREATE_MASK in this
  163. * case, and we'll just provide them with a sanitized version of the
  164. * passed flags.
  165. */
  166. flags &= CACHE_CREATE_MASK;
  167. s = __kmem_cache_alias(memcg, name, size, align, flags, ctor);
  168. if (s)
  169. goto out_locked;
  170. s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
  171. if (s) {
  172. s->object_size = s->size = size;
  173. s->align = calculate_alignment(flags, align, size);
  174. s->ctor = ctor;
  175. if (memcg_register_cache(memcg, s, parent_cache)) {
  176. kmem_cache_free(kmem_cache, s);
  177. err = -ENOMEM;
  178. goto out_locked;
  179. }
  180. s->name = kstrdup(name, GFP_KERNEL);
  181. if (!s->name) {
  182. kmem_cache_free(kmem_cache, s);
  183. err = -ENOMEM;
  184. goto out_locked;
  185. }
  186. err = __kmem_cache_create(s, flags);
  187. if (!err) {
  188. s->refcount = 1;
  189. list_add(&s->list, &slab_caches);
  190. memcg_cache_list_add(memcg, s);
  191. } else {
  192. kfree(s->name);
  193. kmem_cache_free(kmem_cache, s);
  194. }
  195. } else
  196. err = -ENOMEM;
  197. out_locked:
  198. mutex_unlock(&slab_mutex);
  199. put_online_cpus();
  200. if (err) {
  201. if (flags & SLAB_PANIC)
  202. panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n",
  203. name, err);
  204. else {
  205. printk(KERN_WARNING "kmem_cache_create(%s) failed with error %d",
  206. name, err);
  207. dump_stack();
  208. }
  209. return NULL;
  210. }
  211. return s;
  212. }
  213. struct kmem_cache *
  214. kmem_cache_create(const char *name, size_t size, size_t align,
  215. unsigned long flags, void (*ctor)(void *))
  216. {
  217. return kmem_cache_create_memcg(NULL, name, size, align, flags, ctor, NULL);
  218. }
  219. EXPORT_SYMBOL(kmem_cache_create);
  220. void kmem_cache_destroy(struct kmem_cache *s)
  221. {
  222. /* Destroy all the children caches if we aren't a memcg cache */
  223. kmem_cache_destroy_memcg_children(s);
  224. get_online_cpus();
  225. mutex_lock(&slab_mutex);
  226. s->refcount--;
  227. if (!s->refcount) {
  228. list_del(&s->list);
  229. if (!__kmem_cache_shutdown(s)) {
  230. mutex_unlock(&slab_mutex);
  231. if (s->flags & SLAB_DESTROY_BY_RCU)
  232. rcu_barrier();
  233. memcg_release_cache(s);
  234. kfree(s->name);
  235. kmem_cache_free(kmem_cache, s);
  236. } else {
  237. list_add(&s->list, &slab_caches);
  238. mutex_unlock(&slab_mutex);
  239. printk(KERN_ERR "kmem_cache_destroy %s: Slab cache still has objects\n",
  240. s->name);
  241. dump_stack();
  242. }
  243. } else {
  244. mutex_unlock(&slab_mutex);
  245. }
  246. put_online_cpus();
  247. }
  248. EXPORT_SYMBOL(kmem_cache_destroy);
  249. int slab_is_available(void)
  250. {
  251. return slab_state >= UP;
  252. }
  253. #ifndef CONFIG_SLOB
  254. /* Create a cache during boot when no slab services are available yet */
  255. void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
  256. unsigned long flags)
  257. {
  258. int err;
  259. s->name = name;
  260. s->size = s->object_size = size;
  261. s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
  262. err = __kmem_cache_create(s, flags);
  263. if (err)
  264. panic("Creation of kmalloc slab %s size=%zu failed. Reason %d\n",
  265. name, size, err);
  266. s->refcount = -1; /* Exempt from merging for now */
  267. }
  268. struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
  269. unsigned long flags)
  270. {
  271. struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
  272. if (!s)
  273. panic("Out of memory when creating slab %s\n", name);
  274. create_boot_cache(s, name, size, flags);
  275. list_add(&s->list, &slab_caches);
  276. s->refcount = 1;
  277. return s;
  278. }
  279. struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1];
  280. EXPORT_SYMBOL(kmalloc_caches);
  281. #ifdef CONFIG_ZONE_DMA
  282. struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1];
  283. EXPORT_SYMBOL(kmalloc_dma_caches);
  284. #endif
  285. /*
  286. * Conversion table for small slabs sizes / 8 to the index in the
  287. * kmalloc array. This is necessary for slabs < 192 since we have non power
  288. * of two cache sizes there. The size of larger slabs can be determined using
  289. * fls.
  290. */
  291. static s8 size_index[24] = {
  292. 3, /* 8 */
  293. 4, /* 16 */
  294. 5, /* 24 */
  295. 5, /* 32 */
  296. 6, /* 40 */
  297. 6, /* 48 */
  298. 6, /* 56 */
  299. 6, /* 64 */
  300. 1, /* 72 */
  301. 1, /* 80 */
  302. 1, /* 88 */
  303. 1, /* 96 */
  304. 7, /* 104 */
  305. 7, /* 112 */
  306. 7, /* 120 */
  307. 7, /* 128 */
  308. 2, /* 136 */
  309. 2, /* 144 */
  310. 2, /* 152 */
  311. 2, /* 160 */
  312. 2, /* 168 */
  313. 2, /* 176 */
  314. 2, /* 184 */
  315. 2 /* 192 */
  316. };
  317. static inline int size_index_elem(size_t bytes)
  318. {
  319. return (bytes - 1) / 8;
  320. }
  321. /*
  322. * Find the kmem_cache structure that serves a given size of
  323. * allocation
  324. */
  325. struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
  326. {
  327. int index;
  328. if (unlikely(size > KMALLOC_MAX_SIZE)) {
  329. WARN_ON_ONCE(!(flags & __GFP_NOWARN));
  330. return NULL;
  331. }
  332. if (size <= 192) {
  333. if (!size)
  334. return ZERO_SIZE_PTR;
  335. index = size_index[size_index_elem(size)];
  336. } else
  337. index = fls(size - 1);
  338. #ifdef CONFIG_ZONE_DMA
  339. if (unlikely((flags & GFP_DMA)))
  340. return kmalloc_dma_caches[index];
  341. #endif
  342. return kmalloc_caches[index];
  343. }
  344. /*
  345. * Create the kmalloc array. Some of the regular kmalloc arrays
  346. * may already have been created because they were needed to
  347. * enable allocations for slab creation.
  348. */
  349. void __init create_kmalloc_caches(unsigned long flags)
  350. {
  351. int i;
  352. /*
  353. * Patch up the size_index table if we have strange large alignment
  354. * requirements for the kmalloc array. This is only the case for
  355. * MIPS it seems. The standard arches will not generate any code here.
  356. *
  357. * Largest permitted alignment is 256 bytes due to the way we
  358. * handle the index determination for the smaller caches.
  359. *
  360. * Make sure that nothing crazy happens if someone starts tinkering
  361. * around with ARCH_KMALLOC_MINALIGN
  362. */
  363. BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
  364. (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
  365. for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
  366. int elem = size_index_elem(i);
  367. if (elem >= ARRAY_SIZE(size_index))
  368. break;
  369. size_index[elem] = KMALLOC_SHIFT_LOW;
  370. }
  371. if (KMALLOC_MIN_SIZE >= 64) {
  372. /*
  373. * The 96 byte size cache is not used if the alignment
  374. * is 64 byte.
  375. */
  376. for (i = 64 + 8; i <= 96; i += 8)
  377. size_index[size_index_elem(i)] = 7;
  378. }
  379. if (KMALLOC_MIN_SIZE >= 128) {
  380. /*
  381. * The 192 byte sized cache is not used if the alignment
  382. * is 128 byte. Redirect kmalloc to use the 256 byte cache
  383. * instead.
  384. */
  385. for (i = 128 + 8; i <= 192; i += 8)
  386. size_index[size_index_elem(i)] = 8;
  387. }
  388. for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
  389. if (!kmalloc_caches[i]) {
  390. kmalloc_caches[i] = create_kmalloc_cache(NULL,
  391. 1 << i, flags);
  392. }
  393. /*
  394. * Caches that are not of the two-to-the-power-of size.
  395. * These have to be created immediately after the
  396. * earlier power of two caches
  397. */
  398. if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
  399. kmalloc_caches[1] = create_kmalloc_cache(NULL, 96, flags);
  400. if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
  401. kmalloc_caches[2] = create_kmalloc_cache(NULL, 192, flags);
  402. }
  403. /* Kmalloc array is now usable */
  404. slab_state = UP;
  405. for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
  406. struct kmem_cache *s = kmalloc_caches[i];
  407. char *n;
  408. if (s) {
  409. n = kasprintf(GFP_NOWAIT, "kmalloc-%d", kmalloc_size(i));
  410. BUG_ON(!n);
  411. s->name = n;
  412. }
  413. }
  414. #ifdef CONFIG_ZONE_DMA
  415. for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
  416. struct kmem_cache *s = kmalloc_caches[i];
  417. if (s) {
  418. int size = kmalloc_size(i);
  419. char *n = kasprintf(GFP_NOWAIT,
  420. "dma-kmalloc-%d", size);
  421. BUG_ON(!n);
  422. kmalloc_dma_caches[i] = create_kmalloc_cache(n,
  423. size, SLAB_CACHE_DMA | flags);
  424. }
  425. }
  426. #endif
  427. }
  428. #endif /* !CONFIG_SLOB */
  429. #ifdef CONFIG_TRACING
  430. void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
  431. {
  432. void *ret = kmalloc_order(size, flags, order);
  433. trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
  434. return ret;
  435. }
  436. EXPORT_SYMBOL(kmalloc_order_trace);
  437. #endif
  438. #ifdef CONFIG_SLABINFO
  439. #ifdef CONFIG_SLAB
  440. #define SLABINFO_RIGHTS (S_IWUSR | S_IRUSR)
  441. #else
  442. #define SLABINFO_RIGHTS S_IRUSR
  443. #endif
  444. void print_slabinfo_header(struct seq_file *m)
  445. {
  446. /*
  447. * Output format version, so at least we can change it
  448. * without _too_ many complaints.
  449. */
  450. #ifdef CONFIG_DEBUG_SLAB
  451. seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
  452. #else
  453. seq_puts(m, "slabinfo - version: 2.1\n");
  454. #endif
  455. seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
  456. "<objperslab> <pagesperslab>");
  457. seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
  458. seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
  459. #ifdef CONFIG_DEBUG_SLAB
  460. seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
  461. "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
  462. seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
  463. #endif
  464. seq_putc(m, '\n');
  465. }
  466. static void *s_start(struct seq_file *m, loff_t *pos)
  467. {
  468. loff_t n = *pos;
  469. mutex_lock(&slab_mutex);
  470. if (!n)
  471. print_slabinfo_header(m);
  472. return seq_list_start(&slab_caches, *pos);
  473. }
  474. void *slab_next(struct seq_file *m, void *p, loff_t *pos)
  475. {
  476. return seq_list_next(p, &slab_caches, pos);
  477. }
  478. void slab_stop(struct seq_file *m, void *p)
  479. {
  480. mutex_unlock(&slab_mutex);
  481. }
  482. static void
  483. memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info)
  484. {
  485. struct kmem_cache *c;
  486. struct slabinfo sinfo;
  487. int i;
  488. if (!is_root_cache(s))
  489. return;
  490. for_each_memcg_cache_index(i) {
  491. c = cache_from_memcg_idx(s, i);
  492. if (!c)
  493. continue;
  494. memset(&sinfo, 0, sizeof(sinfo));
  495. get_slabinfo(c, &sinfo);
  496. info->active_slabs += sinfo.active_slabs;
  497. info->num_slabs += sinfo.num_slabs;
  498. info->shared_avail += sinfo.shared_avail;
  499. info->active_objs += sinfo.active_objs;
  500. info->num_objs += sinfo.num_objs;
  501. }
  502. }
  503. int cache_show(struct kmem_cache *s, struct seq_file *m)
  504. {
  505. struct slabinfo sinfo;
  506. memset(&sinfo, 0, sizeof(sinfo));
  507. get_slabinfo(s, &sinfo);
  508. memcg_accumulate_slabinfo(s, &sinfo);
  509. seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
  510. cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size,
  511. sinfo.objects_per_slab, (1 << sinfo.cache_order));
  512. seq_printf(m, " : tunables %4u %4u %4u",
  513. sinfo.limit, sinfo.batchcount, sinfo.shared);
  514. seq_printf(m, " : slabdata %6lu %6lu %6lu",
  515. sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
  516. slabinfo_show_stats(m, s);
  517. seq_putc(m, '\n');
  518. return 0;
  519. }
  520. static int s_show(struct seq_file *m, void *p)
  521. {
  522. struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
  523. if (!is_root_cache(s))
  524. return 0;
  525. return cache_show(s, m);
  526. }
  527. /*
  528. * slabinfo_op - iterator that generates /proc/slabinfo
  529. *
  530. * Output layout:
  531. * cache-name
  532. * num-active-objs
  533. * total-objs
  534. * object size
  535. * num-active-slabs
  536. * total-slabs
  537. * num-pages-per-slab
  538. * + further values on SMP and with statistics enabled
  539. */
  540. static const struct seq_operations slabinfo_op = {
  541. .start = s_start,
  542. .next = slab_next,
  543. .stop = slab_stop,
  544. .show = s_show,
  545. };
  546. static int slabinfo_open(struct inode *inode, struct file *file)
  547. {
  548. return seq_open(file, &slabinfo_op);
  549. }
  550. static const struct file_operations proc_slabinfo_operations = {
  551. .open = slabinfo_open,
  552. .read = seq_read,
  553. .write = slabinfo_write,
  554. .llseek = seq_lseek,
  555. .release = seq_release,
  556. };
  557. static int __init slab_proc_init(void)
  558. {
  559. proc_create("slabinfo", SLABINFO_RIGHTS, NULL,
  560. &proc_slabinfo_operations);
  561. return 0;
  562. }
  563. module_init(slab_proc_init);
  564. #endif /* CONFIG_SLABINFO */