slab.h 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280
  1. #ifndef MM_SLAB_H
  2. #define MM_SLAB_H
  3. /*
  4. * Internal slab definitions
  5. */
  6. /*
  7. * State of the slab allocator.
  8. *
  9. * This is used to describe the states of the allocator during bootup.
  10. * Allocators use this to gradually bootstrap themselves. Most allocators
  11. * have the problem that the structures used for managing slab caches are
  12. * allocated from slab caches themselves.
  13. */
  14. enum slab_state {
  15. DOWN, /* No slab functionality yet */
  16. PARTIAL, /* SLUB: kmem_cache_node available */
  17. PARTIAL_ARRAYCACHE, /* SLAB: kmalloc size for arraycache available */
  18. PARTIAL_NODE, /* SLAB: kmalloc size for node struct available */
  19. UP, /* Slab caches usable but not all extras yet */
  20. FULL /* Everything is working */
  21. };
  22. extern enum slab_state slab_state;
  23. /* The slab cache mutex protects the management structures during changes */
  24. extern struct mutex slab_mutex;
  25. /* The list of all slab caches on the system */
  26. extern struct list_head slab_caches;
  27. /* The slab cache that manages slab cache information */
  28. extern struct kmem_cache *kmem_cache;
  29. unsigned long calculate_alignment(unsigned long flags,
  30. unsigned long align, unsigned long size);
  31. #ifndef CONFIG_SLOB
  32. /* Kmalloc array related functions */
  33. void create_kmalloc_caches(unsigned long);
  34. /* Find the kmalloc slab corresponding for a certain size */
  35. struct kmem_cache *kmalloc_slab(size_t, gfp_t);
  36. #endif
  37. /* Functions provided by the slab allocators */
  38. extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
  39. extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
  40. unsigned long flags);
  41. extern void create_boot_cache(struct kmem_cache *, const char *name,
  42. size_t size, unsigned long flags);
  43. struct mem_cgroup;
  44. #ifdef CONFIG_SLUB
  45. struct kmem_cache *
  46. __kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
  47. size_t align, unsigned long flags, void (*ctor)(void *));
  48. #else
  49. static inline struct kmem_cache *
  50. __kmem_cache_alias(struct mem_cgroup *memcg, const char *name, size_t size,
  51. size_t align, unsigned long flags, void (*ctor)(void *))
  52. { return NULL; }
  53. #endif
  54. /* Legal flag mask for kmem_cache_create(), for various configurations */
  55. #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
  56. SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )
  57. #if defined(CONFIG_DEBUG_SLAB)
  58. #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
  59. #elif defined(CONFIG_SLUB_DEBUG)
  60. #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
  61. SLAB_TRACE | SLAB_DEBUG_FREE)
  62. #else
  63. #define SLAB_DEBUG_FLAGS (0)
  64. #endif
  65. #if defined(CONFIG_SLAB)
  66. #define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
  67. SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | SLAB_NOTRACK)
  68. #elif defined(CONFIG_SLUB)
  69. #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
  70. SLAB_TEMPORARY | SLAB_NOTRACK)
  71. #else
  72. #define SLAB_CACHE_FLAGS (0)
  73. #endif
  74. #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
  75. int __kmem_cache_shutdown(struct kmem_cache *);
  76. struct seq_file;
  77. struct file;
  78. struct slabinfo {
  79. unsigned long active_objs;
  80. unsigned long num_objs;
  81. unsigned long active_slabs;
  82. unsigned long num_slabs;
  83. unsigned long shared_avail;
  84. unsigned int limit;
  85. unsigned int batchcount;
  86. unsigned int shared;
  87. unsigned int objects_per_slab;
  88. unsigned int cache_order;
  89. };
  90. void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
  91. void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
  92. ssize_t slabinfo_write(struct file *file, const char __user *buffer,
  93. size_t count, loff_t *ppos);
  94. #ifdef CONFIG_MEMCG_KMEM
  95. static inline bool is_root_cache(struct kmem_cache *s)
  96. {
  97. return !s->memcg_params || s->memcg_params->is_root_cache;
  98. }
  99. static inline bool cache_match_memcg(struct kmem_cache *cachep,
  100. struct mem_cgroup *memcg)
  101. {
  102. return (is_root_cache(cachep) && !memcg) ||
  103. (cachep->memcg_params->memcg == memcg);
  104. }
  105. static inline void memcg_bind_pages(struct kmem_cache *s, int order)
  106. {
  107. if (!is_root_cache(s))
  108. atomic_add(1 << order, &s->memcg_params->nr_pages);
  109. }
  110. static inline void memcg_release_pages(struct kmem_cache *s, int order)
  111. {
  112. if (is_root_cache(s))
  113. return;
  114. if (atomic_sub_and_test((1 << order), &s->memcg_params->nr_pages))
  115. mem_cgroup_destroy_cache(s);
  116. }
  117. static inline bool slab_equal_or_root(struct kmem_cache *s,
  118. struct kmem_cache *p)
  119. {
  120. return (p == s) ||
  121. (s->memcg_params && (p == s->memcg_params->root_cache));
  122. }
  123. /*
  124. * We use suffixes to the name in memcg because we can't have caches
  125. * created in the system with the same name. But when we print them
  126. * locally, better refer to them with the base name
  127. */
  128. static inline const char *cache_name(struct kmem_cache *s)
  129. {
  130. if (!is_root_cache(s))
  131. return s->memcg_params->root_cache->name;
  132. return s->name;
  133. }
  134. static inline struct kmem_cache *
  135. cache_from_memcg_idx(struct kmem_cache *s, int idx)
  136. {
  137. if (!s->memcg_params)
  138. return NULL;
  139. return s->memcg_params->memcg_caches[idx];
  140. }
  141. static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
  142. {
  143. if (is_root_cache(s))
  144. return s;
  145. return s->memcg_params->root_cache;
  146. }
  147. #else
  148. static inline bool is_root_cache(struct kmem_cache *s)
  149. {
  150. return true;
  151. }
  152. static inline bool cache_match_memcg(struct kmem_cache *cachep,
  153. struct mem_cgroup *memcg)
  154. {
  155. return true;
  156. }
  157. static inline void memcg_bind_pages(struct kmem_cache *s, int order)
  158. {
  159. }
  160. static inline void memcg_release_pages(struct kmem_cache *s, int order)
  161. {
  162. }
  163. static inline bool slab_equal_or_root(struct kmem_cache *s,
  164. struct kmem_cache *p)
  165. {
  166. return true;
  167. }
  168. static inline const char *cache_name(struct kmem_cache *s)
  169. {
  170. return s->name;
  171. }
  172. static inline struct kmem_cache *
  173. cache_from_memcg_idx(struct kmem_cache *s, int idx)
  174. {
  175. return NULL;
  176. }
  177. static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
  178. {
  179. return s;
  180. }
  181. #endif
  182. static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
  183. {
  184. struct kmem_cache *cachep;
  185. struct page *page;
  186. /*
  187. * When kmemcg is not being used, both assignments should return the
  188. * same value. but we don't want to pay the assignment price in that
  189. * case. If it is not compiled in, the compiler should be smart enough
  190. * to not do even the assignment. In that case, slab_equal_or_root
  191. * will also be a constant.
  192. */
  193. if (!memcg_kmem_enabled() && !unlikely(s->flags & SLAB_DEBUG_FREE))
  194. return s;
  195. page = virt_to_head_page(x);
  196. cachep = page->slab_cache;
  197. if (slab_equal_or_root(cachep, s))
  198. return cachep;
  199. pr_err("%s: Wrong slab cache. %s but object is from %s\n",
  200. __FUNCTION__, cachep->name, s->name);
  201. WARN_ON_ONCE(1);
  202. return s;
  203. }
  204. #endif
  205. /*
  206. * The slab lists for all objects.
  207. */
  208. struct kmem_cache_node {
  209. spinlock_t list_lock;
  210. #ifdef CONFIG_SLAB
  211. struct list_head slabs_partial; /* partial list first, better asm code */
  212. struct list_head slabs_full;
  213. struct list_head slabs_free;
  214. unsigned long free_objects;
  215. unsigned int free_limit;
  216. unsigned int colour_next; /* Per-node cache coloring */
  217. struct array_cache *shared; /* shared per node */
  218. struct array_cache **alien; /* on other nodes */
  219. unsigned long next_reap; /* updated without locking */
  220. int free_touched; /* updated without locking */
  221. #endif
  222. #ifdef CONFIG_SLUB
  223. unsigned long nr_partial;
  224. struct list_head partial;
  225. #ifdef CONFIG_SLUB_DEBUG
  226. atomic_long_t nr_slabs;
  227. atomic_long_t total_objects;
  228. struct list_head full;
  229. #endif
  230. #endif
  231. };
  232. void *slab_next(struct seq_file *m, void *p, loff_t *pos);
  233. void slab_stop(struct seq_file *m, void *p);