percpu: allow pcpu_alloc() to be called with IRQs off
pcpu_alloc() and pcpu_extend_area_map() perform a series of spin_lock_irq()/spin_unlock_irq() calls, which make them unsafe with respect to being called from contexts which have IRQs off. This patch converts the code to perform save/restore of flags instead, making pcpu_alloc() (or __alloc_percpu() respectively) to be called from early kernel startup stage, where IRQs are off. This is needed for proper initialization of per-cpu rq_weight data from sched_init(). tj: added comment explaining why irqsave/restore is used in alloc path. Signed-off-by: Jiri Kosina <jkosina@suse.cz> Acked-by: Ingo Molnar <mingo@elte.hu> Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
parent
1a0c3298d6
commit
403a91b165
1 changed files with 17 additions and 13 deletions
30
mm/percpu.c
30
mm/percpu.c
|
@ -153,7 +153,10 @@ static int pcpu_reserved_chunk_limit;
|
||||||
*
|
*
|
||||||
* During allocation, pcpu_alloc_mutex is kept locked all the time and
|
* During allocation, pcpu_alloc_mutex is kept locked all the time and
|
||||||
* pcpu_lock is grabbed and released as necessary. All actual memory
|
* pcpu_lock is grabbed and released as necessary. All actual memory
|
||||||
* allocations are done using GFP_KERNEL with pcpu_lock released.
|
* allocations are done using GFP_KERNEL with pcpu_lock released. In
|
||||||
|
* general, percpu memory can't be allocated with irq off but
|
||||||
|
* irqsave/restore are still used in alloc path so that it can be used
|
||||||
|
* from early init path - sched_init() specifically.
|
||||||
*
|
*
|
||||||
* Free path accesses and alters only the index data structures, so it
|
* Free path accesses and alters only the index data structures, so it
|
||||||
* can be safely called from atomic context. When memory needs to be
|
* can be safely called from atomic context. When memory needs to be
|
||||||
|
@ -366,7 +369,7 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
|
||||||
* RETURNS:
|
* RETURNS:
|
||||||
* 0 if noop, 1 if successfully extended, -errno on failure.
|
* 0 if noop, 1 if successfully extended, -errno on failure.
|
||||||
*/
|
*/
|
||||||
static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
|
static int pcpu_extend_area_map(struct pcpu_chunk *chunk, unsigned long *flags)
|
||||||
{
|
{
|
||||||
int new_alloc;
|
int new_alloc;
|
||||||
int *new;
|
int *new;
|
||||||
|
@ -376,7 +379,7 @@ static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
|
||||||
if (chunk->map_alloc >= chunk->map_used + 2)
|
if (chunk->map_alloc >= chunk->map_used + 2)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
spin_unlock_irq(&pcpu_lock);
|
spin_unlock_irqrestore(&pcpu_lock, *flags);
|
||||||
|
|
||||||
new_alloc = PCPU_DFL_MAP_ALLOC;
|
new_alloc = PCPU_DFL_MAP_ALLOC;
|
||||||
while (new_alloc < chunk->map_used + 2)
|
while (new_alloc < chunk->map_used + 2)
|
||||||
|
@ -384,7 +387,7 @@ static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
|
||||||
|
|
||||||
new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
|
new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
|
||||||
if (!new) {
|
if (!new) {
|
||||||
spin_lock_irq(&pcpu_lock);
|
spin_lock_irqsave(&pcpu_lock, *flags);
|
||||||
return -ENOMEM;
|
return -ENOMEM;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -393,7 +396,7 @@ static int pcpu_extend_area_map(struct pcpu_chunk *chunk)
|
||||||
* could have happened inbetween, so map_used couldn't have
|
* could have happened inbetween, so map_used couldn't have
|
||||||
* grown.
|
* grown.
|
||||||
*/
|
*/
|
||||||
spin_lock_irq(&pcpu_lock);
|
spin_lock_irqsave(&pcpu_lock, *flags);
|
||||||
BUG_ON(new_alloc < chunk->map_used + 2);
|
BUG_ON(new_alloc < chunk->map_used + 2);
|
||||||
|
|
||||||
size = chunk->map_alloc * sizeof(chunk->map[0]);
|
size = chunk->map_alloc * sizeof(chunk->map[0]);
|
||||||
|
@ -1047,6 +1050,7 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
|
||||||
struct pcpu_chunk *chunk;
|
struct pcpu_chunk *chunk;
|
||||||
const char *err;
|
const char *err;
|
||||||
int slot, off;
|
int slot, off;
|
||||||
|
unsigned long flags;
|
||||||
|
|
||||||
if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
|
if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
|
||||||
WARN(true, "illegal size (%zu) or align (%zu) for "
|
WARN(true, "illegal size (%zu) or align (%zu) for "
|
||||||
|
@ -1055,13 +1059,13 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
|
||||||
}
|
}
|
||||||
|
|
||||||
mutex_lock(&pcpu_alloc_mutex);
|
mutex_lock(&pcpu_alloc_mutex);
|
||||||
spin_lock_irq(&pcpu_lock);
|
spin_lock_irqsave(&pcpu_lock, flags);
|
||||||
|
|
||||||
/* serve reserved allocations from the reserved chunk if available */
|
/* serve reserved allocations from the reserved chunk if available */
|
||||||
if (reserved && pcpu_reserved_chunk) {
|
if (reserved && pcpu_reserved_chunk) {
|
||||||
chunk = pcpu_reserved_chunk;
|
chunk = pcpu_reserved_chunk;
|
||||||
if (size > chunk->contig_hint ||
|
if (size > chunk->contig_hint ||
|
||||||
pcpu_extend_area_map(chunk) < 0) {
|
pcpu_extend_area_map(chunk, &flags) < 0) {
|
||||||
err = "failed to extend area map of reserved chunk";
|
err = "failed to extend area map of reserved chunk";
|
||||||
goto fail_unlock;
|
goto fail_unlock;
|
||||||
}
|
}
|
||||||
|
@ -1079,7 +1083,7 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
|
||||||
if (size > chunk->contig_hint)
|
if (size > chunk->contig_hint)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
switch (pcpu_extend_area_map(chunk)) {
|
switch (pcpu_extend_area_map(chunk, &flags)) {
|
||||||
case 0:
|
case 0:
|
||||||
break;
|
break;
|
||||||
case 1:
|
case 1:
|
||||||
|
@ -1096,7 +1100,7 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
|
||||||
}
|
}
|
||||||
|
|
||||||
/* hmmm... no space left, create a new chunk */
|
/* hmmm... no space left, create a new chunk */
|
||||||
spin_unlock_irq(&pcpu_lock);
|
spin_unlock_irqrestore(&pcpu_lock, flags);
|
||||||
|
|
||||||
chunk = alloc_pcpu_chunk();
|
chunk = alloc_pcpu_chunk();
|
||||||
if (!chunk) {
|
if (!chunk) {
|
||||||
|
@ -1104,16 +1108,16 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
|
||||||
goto fail_unlock_mutex;
|
goto fail_unlock_mutex;
|
||||||
}
|
}
|
||||||
|
|
||||||
spin_lock_irq(&pcpu_lock);
|
spin_lock_irqsave(&pcpu_lock, flags);
|
||||||
pcpu_chunk_relocate(chunk, -1);
|
pcpu_chunk_relocate(chunk, -1);
|
||||||
goto restart;
|
goto restart;
|
||||||
|
|
||||||
area_found:
|
area_found:
|
||||||
spin_unlock_irq(&pcpu_lock);
|
spin_unlock_irqrestore(&pcpu_lock, flags);
|
||||||
|
|
||||||
/* populate, map and clear the area */
|
/* populate, map and clear the area */
|
||||||
if (pcpu_populate_chunk(chunk, off, size)) {
|
if (pcpu_populate_chunk(chunk, off, size)) {
|
||||||
spin_lock_irq(&pcpu_lock);
|
spin_lock_irqsave(&pcpu_lock, flags);
|
||||||
pcpu_free_area(chunk, off);
|
pcpu_free_area(chunk, off);
|
||||||
err = "failed to populate";
|
err = "failed to populate";
|
||||||
goto fail_unlock;
|
goto fail_unlock;
|
||||||
|
@ -1125,7 +1129,7 @@ static void *pcpu_alloc(size_t size, size_t align, bool reserved)
|
||||||
return __addr_to_pcpu_ptr(chunk->base_addr + off);
|
return __addr_to_pcpu_ptr(chunk->base_addr + off);
|
||||||
|
|
||||||
fail_unlock:
|
fail_unlock:
|
||||||
spin_unlock_irq(&pcpu_lock);
|
spin_unlock_irqrestore(&pcpu_lock, flags);
|
||||||
fail_unlock_mutex:
|
fail_unlock_mutex:
|
||||||
mutex_unlock(&pcpu_alloc_mutex);
|
mutex_unlock(&pcpu_alloc_mutex);
|
||||||
if (warn_limit) {
|
if (warn_limit) {
|
||||||
|
|
Loading…
Reference in a new issue