unimore/kernel-hacking-2024-linux-steffo
unimore/kernel-hacking-2024-linux-steffo
1
Fork 0
Code Pull requests 5 Wiki Activity

mm/slab: enable slab allocation tagging for kmalloc and friends

Browse source 
Redefine kmalloc, krealloc, kzalloc, kcalloc, etc. to record allocations
and deallocations done by these functions.

[surenb@google.com: undo _noprof additions in the documentation]
  Link: https://lkml.kernel.org/r/20240326231453.1206227-7-surenb@google.com
[rdunlap@infradead.org: fix kcalloc() kernel-doc warnings]
  Link: https://lkml.kernel.org/r/20240327044649.9199-1-rdunlap@infradead.org
Link: https://lkml.kernel.org/r/20240321163705.3067592-26-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Co-developed-by: Kent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
Signed-off-by: Randy Dunlap <rdunlap@infradead.org>
Reviewed-by: Kees Cook <keescook@chromium.org>
Tested-by: Kees Cook <keescook@chromium.org>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Alex Gaynor <alex.gaynor@gmail.com>
Cc: Alice Ryhl <aliceryhl@google.com>
Cc: Andreas Hindborg <a.hindborg@samsung.com>
Cc: Benno Lossin <benno.lossin@proton.me>
Cc: "Björn Roy Baron" <bjorn3_gh@protonmail.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Dennis Zhou <dennis@kernel.org>
Cc: Gary Guo <gary@garyguo.net>
Cc: Miguel Ojeda <ojeda@kernel.org>
Cc: Pasha Tatashin <pasha.tatashin@soleen.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Tejun Heo <tj@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Wedson Almeida Filho <wedsonaf@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
Suren Baghdasaryan 2024-03-21 09:36:47 -07:00 committed by Andrew Morton
parent 53ed0af496
commit 7bd230a266
6 changed files with 128 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
include/linux/fortify-string.h
View file

@ -725,9 +725,9 @@ __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size)
return __real_memchr_inv(p, c, size); return __real_memchr_inv(p, c, size);
} }
extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup) extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup_noprof)
__realloc_size(2); __realloc_size(2);
__FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp) __FORTIFY_INLINE void *kmemdup_noprof(const void * const POS0 p, size_t size, gfp_t gfp)
{ {
const size_t p_size = __struct_size(p); const size_t p_size = __struct_size(p);
@ -737,6 +737,7 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp
fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size, NULL); fortify_panic(FORTIFY_FUNC_kmemdup, FORTIFY_READ, p_size, size, NULL);
return __real_kmemdup(p, size, gfp); return __real_kmemdup(p, size, gfp);
} }
#define kmemdup(...) alloc_hooks(kmemdup_noprof(__VA_ARGS__))
/** /**
* strcpy - Copy a string into another string buffer * strcpy - Copy a string into another string buffer

171
include/linux/slab.h
View file

@ -271,7 +271,10 @@ int kmem_cache_shrink(struct kmem_cache *s);
/* /*
* Common kmalloc functions provided by all allocators * Common kmalloc functions provided by all allocators
*/ */
void * __must_check krealloc(const void *objp, size_t new_size, gfp_t flags) __realloc_size(2); void * __must_check krealloc_noprof(const void *objp, size_t new_size,
gfp_t flags) __realloc_size(2);
#define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__))
void kfree(const void *objp); void kfree(const void *objp);
void kfree_sensitive(const void *objp); void kfree_sensitive(const void *objp);
size_t __ksize(const void *objp); size_t __ksize(const void *objp);
@ -523,7 +526,10 @@ static __always_inline unsigned int __kmalloc_index(size_t size,
static_assert(PAGE_SHIFT <= 20); static_assert(PAGE_SHIFT <= 20);
#define kmalloc_index(s) __kmalloc_index(s, true) #define kmalloc_index(s) __kmalloc_index(s, true)
void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1); #include <linux/alloc_tag.h>
void *__kmalloc_noprof(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_size(1);
#define __kmalloc(...) alloc_hooks(__kmalloc_noprof(__VA_ARGS__))
/** /**
* kmem_cache_alloc - Allocate an object * kmem_cache_alloc - Allocate an object
@ -535,9 +541,14 @@ void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __alloc_siz
* *
* Return: pointer to the new object or %NULL in case of error * Return: pointer to the new object or %NULL in case of error
*/ */
void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) __assume_slab_alignment __malloc; void *kmem_cache_alloc_noprof(struct kmem_cache *cachep,
void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, gfp_t flags) __assume_slab_alignment __malloc;
gfp_t gfpflags) __assume_slab_alignment __malloc; #define kmem_cache_alloc(...) alloc_hooks(kmem_cache_alloc_noprof(__VA_ARGS__))
void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
gfp_t gfpflags) __assume_slab_alignment __malloc;
#define kmem_cache_alloc_lru(...) alloc_hooks(kmem_cache_alloc_lru_noprof(__VA_ARGS__))
void kmem_cache_free(struct kmem_cache *s, void *objp); void kmem_cache_free(struct kmem_cache *s, void *objp);
/* /*
@ -548,29 +559,40 @@ void kmem_cache_free(struct kmem_cache *s, void *objp);
* Note that interrupts must be enabled when calling these functions. * Note that interrupts must be enabled when calling these functions.
*/ */
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p);
#define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__))
static __always_inline void kfree_bulk(size_t size, void **p) static __always_inline void kfree_bulk(size_t size, void **p)
{ {
kmem_cache_free_bulk(NULL, size, p); kmem_cache_free_bulk(NULL, size, p);
} }
void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment
__alloc_size(1); __alloc_size(1);
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) __assume_slab_alignment #define __kmalloc_node(...) alloc_hooks(__kmalloc_node_noprof(__VA_ARGS__))
__malloc;
void *kmalloc_trace(struct kmem_cache *s, gfp_t flags, size_t size) void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t flags,
int node) __assume_slab_alignment __malloc;
#define kmem_cache_alloc_node(...) alloc_hooks(kmem_cache_alloc_node_noprof(__VA_ARGS__))
void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t flags, size_t size)
__assume_kmalloc_alignment __alloc_size(3); __assume_kmalloc_alignment __alloc_size(3);
void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags,
int node, size_t size) __assume_kmalloc_alignment int node, size_t size) __assume_kmalloc_alignment
__alloc_size(4); __alloc_size(4);
void *kmalloc_large(size_t size, gfp_t flags) __assume_page_alignment #define kmalloc_trace(...) alloc_hooks(kmalloc_trace_noprof(__VA_ARGS__))
__alloc_size(1);
void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_alignment #define kmalloc_node_trace(...) alloc_hooks(kmalloc_node_trace_noprof(__VA_ARGS__))
void *kmalloc_large_noprof(size_t size, gfp_t flags) __assume_page_alignment
__alloc_size(1);
#define kmalloc_large(...) alloc_hooks(kmalloc_large_noprof(__VA_ARGS__))
void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) __assume_page_alignment
__alloc_size(1); __alloc_size(1);
#define kmalloc_large_node(...) alloc_hooks(kmalloc_large_node_noprof(__VA_ARGS__))
/** /**
* kmalloc - allocate kernel memory * kmalloc - allocate kernel memory
@ -626,37 +648,39 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node) __assume_page_align
* Try really hard to succeed the allocation but fail * Try really hard to succeed the allocation but fail
* eventually. * eventually.
*/ */
static __always_inline __alloc_size(1) void *kmalloc(size_t size, gfp_t flags) static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t flags)
{ {
if (__builtin_constant_p(size) && size) { if (__builtin_constant_p(size) && size) {
unsigned int index; unsigned int index;
if (size > KMALLOC_MAX_CACHE_SIZE) if (size > KMALLOC_MAX_CACHE_SIZE)
return kmalloc_large(size, flags); return kmalloc_large_noprof(size, flags);
index = kmalloc_index(size); index = kmalloc_index(size);
return kmalloc_trace( return kmalloc_trace_noprof(
kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
flags, size); flags, size);
} }
return __kmalloc(size, flags); return __kmalloc_noprof(size, flags);
} }
#define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__))
static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t flags, int node) static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node)
{ {
if (__builtin_constant_p(size) && size) { if (__builtin_constant_p(size) && size) {
unsigned int index; unsigned int index;
if (size > KMALLOC_MAX_CACHE_SIZE) if (size > KMALLOC_MAX_CACHE_SIZE)
return kmalloc_large_node(size, flags, node); return kmalloc_large_node_noprof(size, flags, node);
index = kmalloc_index(size); index = kmalloc_index(size);
return kmalloc_node_trace( return kmalloc_node_trace_noprof(
kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
flags, node, size); flags, node, size);
} }
return __kmalloc_node(size, flags, node); return __kmalloc_node_noprof(size, flags, node);
} }
#define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))
/** /**
* kmalloc_array - allocate memory for an array. * kmalloc_array - allocate memory for an array.
@ -664,16 +688,17 @@ static __always_inline __alloc_size(1) void *kmalloc_node(size_t size, gfp_t fla
* @size: element size. * @size: element size.
* @flags: the type of memory to allocate (see kmalloc). * @flags: the type of memory to allocate (see kmalloc).
*/ */
static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_t flags) static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t size, gfp_t flags)
{ {
size_t bytes; size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes))) if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL; return NULL;
if (__builtin_constant_p(n) && __builtin_constant_p(size)) if (__builtin_constant_p(n) && __builtin_constant_p(size))
return kmalloc(bytes, flags); return kmalloc_noprof(bytes, flags);
return __kmalloc(bytes, flags); return kmalloc_noprof(bytes, flags);
} }
#define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
/** /**
* krealloc_array - reallocate memory for an array. * krealloc_array - reallocate memory for an array.
@ -682,18 +707,19 @@ static inline __alloc_size(1, 2) void *kmalloc_array(size_t n, size_t size, gfp_
* @new_size: new size of a single member of the array * @new_size: new size of a single member of the array
* @flags: the type of memory to allocate (see kmalloc) * @flags: the type of memory to allocate (see kmalloc)
*/ */
static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p, static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(void *p,
size_t new_n, size_t new_n,
size_t new_size, size_t new_size,
gfp_t flags) gfp_t flags)
{ {
size_t bytes; size_t bytes;
if (unlikely(check_mul_overflow(new_n, new_size, &bytes))) if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
return NULL; return NULL;
return krealloc(p, bytes, flags); return krealloc_noprof(p, bytes, flags);
} }
#define krealloc_array(...) alloc_hooks(krealloc_array_noprof(__VA_ARGS__))
/** /**
* kcalloc - allocate memory for an array. The memory is set to zero. * kcalloc - allocate memory for an array. The memory is set to zero.
@ -701,16 +727,12 @@ static inline __realloc_size(2, 3) void * __must_check krealloc_array(void *p,
* @size: element size. * @size: element size.
* @flags: the type of memory to allocate (see kmalloc). * @flags: the type of memory to allocate (see kmalloc).
*/ */
static inline __alloc_size(1, 2) void *kcalloc(size_t n, size_t size, gfp_t flags) #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO)
{
return kmalloc_array(n, size, flags | __GFP_ZERO);
}
void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node, void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags, int node,
unsigned long caller) __alloc_size(1); unsigned long caller) __alloc_size(1);
#define kmalloc_node_track_caller(size, flags, node) \ #define kmalloc_node_track_caller(...) \
__kmalloc_node_track_caller(size, flags, node, \ alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))
_RET_IP_)
/* /*
* kmalloc_track_caller is a special version of kmalloc that records the * kmalloc_track_caller is a special version of kmalloc that records the
@ -720,11 +742,9 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t flags, int node,
* allocator where we care about the real place the memory allocation * allocator where we care about the real place the memory allocation
* request comes from. * request comes from.
*/ */
#define kmalloc_track_caller(size, flags) \ #define kmalloc_track_caller(...) kmalloc_node_track_caller(__VA_ARGS__, NUMA_NO_NODE)
__kmalloc_node_track_caller(size, flags, \
NUMA_NO_NODE, _RET_IP_)
static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size, gfp_t flags, static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags,
int node) int node)
{ {
size_t bytes; size_t bytes;
@ -732,75 +752,56 @@ static inline __alloc_size(1, 2) void *kmalloc_array_node(size_t n, size_t size,
if (unlikely(check_mul_overflow(n, size, &bytes))) if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL; return NULL;
if (__builtin_constant_p(n) && __builtin_constant_p(size)) if (__builtin_constant_p(n) && __builtin_constant_p(size))
return kmalloc_node(bytes, flags, node); return kmalloc_node_noprof(bytes, flags, node);
return __kmalloc_node(bytes, flags, node); return __kmalloc_node_noprof(bytes, flags, node);
} }
#define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))
static inline __alloc_size(1, 2) void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node) #define kcalloc_node(_n, _size, _flags, _node) \
{ kmalloc_array_node(_n, _size, (_flags) | __GFP_ZERO, _node)
return kmalloc_array_node(n, size, flags | __GFP_ZERO, node);
}
/* /*
* Shortcuts * Shortcuts
*/ */
static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) #define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO)
{
return kmem_cache_alloc(k, flags | __GFP_ZERO);
}
/** /**
* kzalloc - allocate memory. The memory is set to zero. * kzalloc - allocate memory. The memory is set to zero.
* @size: how many bytes of memory are required. * @size: how many bytes of memory are required.
* @flags: the type of memory to allocate (see kmalloc). * @flags: the type of memory to allocate (see kmalloc).
*/ */
static inline __alloc_size(1) void *kzalloc(size_t size, gfp_t flags) static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags)
{ {
return kmalloc(size, flags | __GFP_ZERO); return kmalloc_noprof(size, flags | __GFP_ZERO);
} }
#define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__))
#define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
/** extern void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node) __alloc_size(1);
* kzalloc_node - allocate zeroed memory from a particular memory node. #define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__))
* @size: how many bytes of memory are required.
* @flags: the type of memory to allocate (see kmalloc).
* @node: memory node from which to allocate
*/
static inline __alloc_size(1) void *kzalloc_node(size_t size, gfp_t flags, int node)
{
return kmalloc_node(size, flags | __GFP_ZERO, node);
}
extern void *kvmalloc_node(size_t size, gfp_t flags, int node) __alloc_size(1); #define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NODE)
static inline __alloc_size(1) void *kvmalloc(size_t size, gfp_t flags) #define kvzalloc(_size, _flags) kvmalloc(_size, _flags|__GFP_ZERO)
{
return kvmalloc_node(size, flags, NUMA_NO_NODE);
}
static inline __alloc_size(1) void *kvzalloc_node(size_t size, gfp_t flags, int node)
{
return kvmalloc_node(size, flags | __GFP_ZERO, node);
}
static inline __alloc_size(1) void *kvzalloc(size_t size, gfp_t flags)
{
return kvmalloc(size, flags | __GFP_ZERO);
}
static inline __alloc_size(1, 2) void *kvmalloc_array(size_t n, size_t size, gfp_t flags) #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, _flags|__GFP_ZERO, _node)
static inline __alloc_size(1, 2) void *kvmalloc_array_noprof(size_t n, size_t size, gfp_t flags)
{ {
size_t bytes; size_t bytes;
if (unlikely(check_mul_overflow(n, size, &bytes))) if (unlikely(check_mul_overflow(n, size, &bytes)))
return NULL; return NULL;
return kvmalloc(bytes, flags); return kvmalloc_node_noprof(bytes, flags, NUMA_NO_NODE);
} }
static inline __alloc_size(1, 2) void *kvcalloc(size_t n, size_t size, gfp_t flags) #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS__))
{ #define kvcalloc(_n, _size, _flags) kvmalloc_array(_n, _size, _flags|__GFP_ZERO)
return kvmalloc_array(n, size, flags | __GFP_ZERO);
}
extern void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags) extern void *kvrealloc_noprof(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
__realloc_size(3); __realloc_size(3);
#define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__))
extern void kvfree(const void *addr); extern void kvfree(const void *addr);
DEFINE_FREE(kvfree, void *, if (_T) kvfree(_T)) DEFINE_FREE(kvfree, void *, if (_T) kvfree(_T))

4
include/linux/string.h
View file

@ -282,7 +282,9 @@ extern void kfree_const(const void *x);
extern char *kstrdup(const char *s, gfp_t gfp) __malloc; extern char *kstrdup(const char *s, gfp_t gfp) __malloc;
extern const char *kstrdup_const(const char *s, gfp_t gfp); extern const char *kstrdup_const(const char *s, gfp_t gfp);
extern char *kstrndup(const char *s, size_t len, gfp_t gfp); extern char *kstrndup(const char *s, size_t len, gfp_t gfp);
extern void *kmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2); extern void *kmemdup_noprof(const void *src, size_t len, gfp_t gfp) __realloc_size(2);
#define kmemdup(...) alloc_hooks(kmemdup_noprof(__VA_ARGS__))
extern void *kvmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2); extern void *kvmemdup(const void *src, size_t len, gfp_t gfp) __realloc_size(2);
extern char *kmemdup_nul(const char *s, size_t len, gfp_t gfp); extern char *kmemdup_nul(const char *s, size_t len, gfp_t gfp);
extern void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp); extern void *kmemdup_array(const void *src, size_t element_size, size_t count, gfp_t gfp);

6
mm/slab_common.c
View file

@ -1189,7 +1189,7 @@ __do_krealloc(const void *p, size_t new_size, gfp_t flags)
return (void *)p; return (void *)p;
} }
ret = kmalloc_track_caller(new_size, flags); ret = kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _RET_IP_);
if (ret && p) { if (ret && p) {
/* Disable KASAN checks as the object's redzone is accessed. */ /* Disable KASAN checks as the object's redzone is accessed. */
kasan_disable_current(); kasan_disable_current();
@ -1213,7 +1213,7 @@ __do_krealloc(const void *p, size_t new_size, gfp_t flags)
* *
* Return: pointer to the allocated memory or %NULL in case of error * Return: pointer to the allocated memory or %NULL in case of error
*/ */
void *krealloc(const void *p, size_t new_size, gfp_t flags) void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags)
{ {
void *ret; void *ret;
@ -1228,7 +1228,7 @@ void *krealloc(const void *p, size_t new_size, gfp_t flags)
return ret; return ret;
} }
EXPORT_SYMBOL(krealloc); EXPORT_SYMBOL(krealloc_noprof);
/** /**
* kfree_sensitive - Clear sensitive information in memory before freeing * kfree_sensitive - Clear sensitive information in memory before freeing

50
mm/slub.c
View file

@ -4002,7 +4002,7 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list
return object; return object;
} }
void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags)
{ {
void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_,
s->object_size); s->object_size);
@ -4011,9 +4011,9 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
return ret; return ret;
} }
EXPORT_SYMBOL(kmem_cache_alloc); EXPORT_SYMBOL(kmem_cache_alloc_noprof);
void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
gfp_t gfpflags) gfp_t gfpflags)
{ {
void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_, void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_,
@ -4023,7 +4023,7 @@ void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru,
return ret; return ret;
} }
EXPORT_SYMBOL(kmem_cache_alloc_lru); EXPORT_SYMBOL(kmem_cache_alloc_lru_noprof);
/** /**
* kmem_cache_alloc_node - Allocate an object on the specified node * kmem_cache_alloc_node - Allocate an object on the specified node
@ -4038,7 +4038,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_lru);
* *
* Return: pointer to the new object or %NULL in case of error * Return: pointer to the new object or %NULL in case of error
*/ */
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node)
{ {
void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
@ -4046,7 +4046,7 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
return ret; return ret;
} }
EXPORT_SYMBOL(kmem_cache_alloc_node); EXPORT_SYMBOL(kmem_cache_alloc_node_noprof);
/* /*
* To avoid unnecessary overhead, we pass through large allocation requests * To avoid unnecessary overhead, we pass through large allocation requests
@ -4063,7 +4063,7 @@ static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
flags = kmalloc_fix_flags(flags); flags = kmalloc_fix_flags(flags);
flags |= __GFP_COMP; flags |= __GFP_COMP;
folio = (struct folio *)alloc_pages_node(node, flags, order); folio = (struct folio *)alloc_pages_node_noprof(node, flags, order);
if (folio) { if (folio) {
ptr = folio_address(folio); ptr = folio_address(folio);
lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, lruvec_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B,
@ -4078,7 +4078,7 @@ static void *__kmalloc_large_node(size_t size, gfp_t flags, int node)
return ptr; return ptr;
} }
void *kmalloc_large(size_t size, gfp_t flags) void *kmalloc_large_noprof(size_t size, gfp_t flags)
{ {
void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE); void *ret = __kmalloc_large_node(size, flags, NUMA_NO_NODE);
@ -4086,9 +4086,9 @@ void *kmalloc_large(size_t size, gfp_t flags)
flags, NUMA_NO_NODE); flags, NUMA_NO_NODE);
return ret; return ret;
} }
EXPORT_SYMBOL(kmalloc_large); EXPORT_SYMBOL(kmalloc_large_noprof);
void *kmalloc_large_node(size_t size, gfp_t flags, int node) void *kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
{ {
void *ret = __kmalloc_large_node(size, flags, node); void *ret = __kmalloc_large_node(size, flags, node);
@ -4096,7 +4096,7 @@ void *kmalloc_large_node(size_t size, gfp_t flags, int node)
flags, node); flags, node);
return ret; return ret;
} }
EXPORT_SYMBOL(kmalloc_large_node); EXPORT_SYMBOL(kmalloc_large_node_noprof);
static __always_inline static __always_inline
void *__do_kmalloc_node(size_t size, gfp_t flags, int node, void *__do_kmalloc_node(size_t size, gfp_t flags, int node,
@ -4123,26 +4123,26 @@ void *__do_kmalloc_node(size_t size, gfp_t flags, int node,
return ret; return ret;
} }
void *__kmalloc_node(size_t size, gfp_t flags, int node) void *__kmalloc_node_noprof(size_t size, gfp_t flags, int node)
{ {
return __do_kmalloc_node(size, flags, node, _RET_IP_); return __do_kmalloc_node(size, flags, node, _RET_IP_);
} }
EXPORT_SYMBOL(__kmalloc_node); EXPORT_SYMBOL(__kmalloc_node_noprof);
void *__kmalloc(size_t size, gfp_t flags) void *__kmalloc_noprof(size_t size, gfp_t flags)
{ {
return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_); return __do_kmalloc_node(size, flags, NUMA_NO_NODE, _RET_IP_);
} }
EXPORT_SYMBOL(__kmalloc); EXPORT_SYMBOL(__kmalloc_noprof);
void *__kmalloc_node_track_caller(size_t size, gfp_t flags, void *kmalloc_node_track_caller_noprof(size_t size, gfp_t flags,
int node, unsigned long caller) int node, unsigned long caller)
{ {
return __do_kmalloc_node(size, flags, node, caller); return __do_kmalloc_node(size, flags, node, caller);
} }
EXPORT_SYMBOL(__kmalloc_node_track_caller); EXPORT_SYMBOL(kmalloc_node_track_caller_noprof);
void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) void *kmalloc_trace_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{ {
void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE,
_RET_IP_, size); _RET_IP_, size);
@ -4152,9 +4152,9 @@ void *kmalloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
ret = kasan_kmalloc(s, ret, size, gfpflags); ret = kasan_kmalloc(s, ret, size, gfpflags);
return ret; return ret;
} }
EXPORT_SYMBOL(kmalloc_trace); EXPORT_SYMBOL(kmalloc_trace_noprof);
void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags, void *kmalloc_node_trace_noprof(struct kmem_cache *s, gfp_t gfpflags,
int node, size_t size) int node, size_t size)
{ {
void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
@ -4164,7 +4164,7 @@ void *kmalloc_node_trace(struct kmem_cache *s, gfp_t gfpflags,
ret = kasan_kmalloc(s, ret, size, gfpflags); ret = kasan_kmalloc(s, ret, size, gfpflags);
return ret; return ret;
} }
EXPORT_SYMBOL(kmalloc_node_trace); EXPORT_SYMBOL(kmalloc_node_trace_noprof);
static noinline void free_to_partial_list( static noinline void free_to_partial_list(
struct kmem_cache *s, struct slab *slab, struct kmem_cache *s, struct slab *slab,
@ -4769,8 +4769,8 @@ static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
#endif /* CONFIG_SLUB_TINY */ #endif /* CONFIG_SLUB_TINY */
/* Note that interrupts must be enabled when calling this function. */ /* Note that interrupts must be enabled when calling this function. */
int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
void **p) void **p)
{ {
int i; int i;
struct obj_cgroup *objcg = NULL; struct obj_cgroup *objcg = NULL;
@ -4798,7 +4798,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
return i; return i;
} }
EXPORT_SYMBOL(kmem_cache_alloc_bulk); EXPORT_SYMBOL(kmem_cache_alloc_bulk_noprof);
/* /*

16
mm/util.c
View file

@ -124,16 +124,16 @@ EXPORT_SYMBOL(kstrndup);
* Return: newly allocated copy of @src or %NULL in case of error, * Return: newly allocated copy of @src or %NULL in case of error,
* result is physically contiguous. Use kfree() to free. * result is physically contiguous. Use kfree() to free.
*/ */
void *kmemdup(const void *src, size_t len, gfp_t gfp) void *kmemdup_noprof(const void *src, size_t len, gfp_t gfp)
{ {
void *p; void *p;
p = kmalloc_track_caller(len, gfp); p = kmalloc_node_track_caller_noprof(len, gfp, NUMA_NO_NODE, _RET_IP_);
if (p) if (p)
memcpy(p, src, len); memcpy(p, src, len);
return p; return p;
} }
EXPORT_SYMBOL(kmemdup); EXPORT_SYMBOL(kmemdup_noprof);
/** /**
* kmemdup_array - duplicate a given array. * kmemdup_array - duplicate a given array.
@ -609,7 +609,7 @@ EXPORT_SYMBOL(vm_mmap);
* *
* Return: pointer to the allocated memory of %NULL in case of failure * Return: pointer to the allocated memory of %NULL in case of failure
*/ */
void *kvmalloc_node(size_t size, gfp_t flags, int node) void *kvmalloc_node_noprof(size_t size, gfp_t flags, int node)
{ {
gfp_t kmalloc_flags = flags; gfp_t kmalloc_flags = flags;
void *ret; void *ret;
@ -631,7 +631,7 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node)
kmalloc_flags &= ~__GFP_NOFAIL; kmalloc_flags &= ~__GFP_NOFAIL;
} }
ret = kmalloc_node(size, kmalloc_flags, node); ret = kmalloc_node_noprof(size, kmalloc_flags, node);
/* /*
* It doesn't really make sense to fallback to vmalloc for sub page * It doesn't really make sense to fallback to vmalloc for sub page
@ -660,7 +660,7 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node)
flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
node, __builtin_return_address(0)); node, __builtin_return_address(0));
} }
EXPORT_SYMBOL(kvmalloc_node); EXPORT_SYMBOL(kvmalloc_node_noprof);
/** /**
* kvfree() - Free memory. * kvfree() - Free memory.
@ -699,7 +699,7 @@ void kvfree_sensitive(const void *addr, size_t len)
} }
EXPORT_SYMBOL(kvfree_sensitive); EXPORT_SYMBOL(kvfree_sensitive);
void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags) void *kvrealloc_noprof(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
{ {
void *newp; void *newp;
@ -712,7 +712,7 @@ void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
kvfree(p); kvfree(p);
return newp; return newp;
} }
EXPORT_SYMBOL(kvrealloc); EXPORT_SYMBOL(kvrealloc_noprof);
/** /**
* __vmalloc_array - allocate memory for a virtually contiguous array. * __vmalloc_array - allocate memory for a virtually contiguous array.