Line data Source code
1 : #ifndef _LINUX_SLAB_DEF_H
2 : #define _LINUX_SLAB_DEF_H
3 :
4 : /*
5 : * Definitions unique to the original Linux SLAB allocator.
6 : *
7 : * What we provide here is a way to optimize the frequent kmalloc
8 : * calls in the kernel by selecting the appropriate general cache
9 : * if kmalloc was called with a size that can be established at
10 : * compile time.
11 : */
12 :
13 : #include <linux/init.h>
14 : #include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
15 : #include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
16 : #include <linux/compiler.h>
17 : #include <linux/kmemtrace.h>
18 :
19 : /*
20 : * struct kmem_cache
21 : *
22 : * manages a cache.
23 : */
24 :
25 : struct kmem_cache {
26 : /* 1) per-cpu data, touched during every alloc/free */
27 : struct array_cache *array[NR_CPUS];
28 : /* 2) Cache tunables. Protected by cache_chain_mutex */
29 : unsigned int batchcount;
30 : unsigned int limit;
31 : unsigned int shared;
32 :
33 : unsigned int buffer_size;
34 : u32 reciprocal_buffer_size;
35 : /* 3) touched by every alloc & free from the backend */
36 :
37 : unsigned int flags; /* constant flags */
38 : unsigned int num; /* # of objs per slab */
39 :
40 : /* 4) cache_grow/shrink */
41 : /* order of pgs per slab (2^n) */
42 : unsigned int gfporder;
43 :
44 : /* force GFP flags, e.g. GFP_DMA */
45 : gfp_t gfpflags;
46 :
47 : size_t colour; /* cache colouring range */
48 : unsigned int colour_off; /* colour offset */
49 : struct kmem_cache *slabp_cache;
50 : unsigned int slab_size;
51 : unsigned int dflags; /* dynamic flags */
52 :
53 : /* constructor func */
54 : void (*ctor)(void *obj);
55 :
56 : /* 5) cache creation/removal */
57 : const char *name;
58 : struct list_head next;
59 :
60 : /* 6) statistics */
61 : #ifdef CONFIG_DEBUG_SLAB
62 : unsigned long num_active;
63 : unsigned long num_allocations;
64 : unsigned long high_mark;
65 : unsigned long grown;
66 : unsigned long reaped;
67 : unsigned long errors;
68 : unsigned long max_freeable;
69 : unsigned long node_allocs;
70 : unsigned long node_frees;
71 : unsigned long node_overflow;
72 : atomic_t allochit;
73 : atomic_t allocmiss;
74 : atomic_t freehit;
75 : atomic_t freemiss;
76 :
77 : /*
78 : * If debugging is enabled, then the allocator can add additional
79 : * fields and/or padding to every object. buffer_size contains the total
80 : * object size including these internal fields, the following two
81 : * variables contain the offset to the user object and its size.
82 : */
83 : int obj_offset;
84 : int obj_size;
85 : #endif /* CONFIG_DEBUG_SLAB */
86 :
87 : /*
88 : * We put nodelists[] at the end of kmem_cache, because we want to size
89 : * this array to nr_node_ids slots instead of MAX_NUMNODES
90 : * (see kmem_cache_init())
91 : * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
92 : * is statically defined, so we reserve the max number of nodes.
93 : */
94 : struct kmem_list3 *nodelists[MAX_NUMNODES];
95 : /*
96 1 : * Do not add fields after nodelists[]
97 : */
98 : };
99 :
100 : /* Size description struct for general caches. */
101 : struct cache_sizes {
102 : size_t cs_size;
103 : struct kmem_cache *cs_cachep;
104 : #ifdef CONFIG_ZONE_DMA
105 : struct kmem_cache *cs_dmacachep;
106 : #endif
107 : };
108 1 : extern struct cache_sizes malloc_sizes[];
109 :
110 : void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
111 : void *__kmalloc(size_t size, gfp_t flags);
112 :
113 : #ifdef CONFIG_TRACING
114 : extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags);
115 : extern size_t slab_buffer_size(struct kmem_cache *cachep);
116 : #else
117 : static __always_inline void *
118 : kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
119 : {
120 0 : return kmem_cache_alloc(cachep, flags);
121 : }
122 : static inline size_t slab_buffer_size(struct kmem_cache *cachep)
123 : {
124 0 : return 0;
125 : }
126 : #endif
127 :
128 : static __always_inline void *kmalloc(size_t size, gfp_t flags)
129 : {
130 2 : struct kmem_cache *cachep;
131 2 : void *ret;
132 2 :
133 4 : if (__builtin_constant_p(size)) {
134 2 : int i = 0;
135 :
136 : if (!size)
137 : return ZERO_SIZE_PTR;
138 :
139 : #define CACHE(x) \
140 : if (size <= x) \
141 : goto found; \
142 : else \
143 : i++;
144 : #include <linux/kmalloc_sizes.h>
145 : #undef CACHE
146 : return NULL;
147 : found:
148 : #ifdef CONFIG_ZONE_DMA
149 : if (flags & GFP_DMA)
150 : cachep = malloc_sizes[i].cs_dmacachep;
151 : else
152 : #endif
153 : cachep = malloc_sizes[i].cs_cachep;
154 :
155 : ret = kmem_cache_alloc_notrace(cachep, flags);
156 :
157 : trace_kmalloc(_THIS_IP_, ret,
158 : size, slab_buffer_size(cachep), flags);
159 :
160 : return ret;
161 : }
162 4 : return __kmalloc(size, flags);
163 : }
164 :
165 : #ifdef CONFIG_NUMA
166 : extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
167 : extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
168 :
169 : #ifdef CONFIG_TRACING
170 : extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
171 : gfp_t flags,
172 : int nodeid);
173 : #else
174 : static __always_inline void *
175 : kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
176 : gfp_t flags,
177 : int nodeid)
178 : {
179 : return kmem_cache_alloc_node(cachep, flags, nodeid);
180 : }
181 : #endif
182 :
183 : static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
184 : {
185 : struct kmem_cache *cachep;
186 : void *ret;
187 :
188 : if (__builtin_constant_p(size)) {
189 : int i = 0;
190 :
191 : if (!size)
192 : return ZERO_SIZE_PTR;
193 :
194 : #define CACHE(x) \
195 : if (size <= x) \
196 : goto found; \
197 : else \
198 : i++;
199 : #include <linux/kmalloc_sizes.h>
200 : #undef CACHE
201 : return NULL;
202 : found:
203 : #ifdef CONFIG_ZONE_DMA
204 : if (flags & GFP_DMA)
205 : cachep = malloc_sizes[i].cs_dmacachep;
206 : else
207 : #endif
208 : cachep = malloc_sizes[i].cs_cachep;
209 :
210 : ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
211 :
212 : trace_kmalloc_node(_THIS_IP_, ret,
213 : size, slab_buffer_size(cachep),
214 : flags, node);
215 :
216 : return ret;
217 : }
218 : return __kmalloc_node(size, flags, node);
219 : }
220 :
221 : #endif /* CONFIG_NUMA */
222 :
223 : #endif /* _LINUX_SLAB_DEF_H */
|