unimore/kernel-hacking-2024-linux-steffo
unimore/kernel-hacking-2024-linux-steffo
1
Fork 0
Code Pull requests 5 Wiki Activity

bpf: avoid grabbing spin_locks of all cpus when no free elems

Browse source 
This patch use head->first in pcpu_freelist_head to check freelist
having free or not. If having, grab spin_lock, or check next cpu's
freelist.

Before patch: hash_map performance
./map_perf_test 1
0:hash_map_perf pre-alloc 1043397 events per sec
...
The average of the test results is around 1050000 events per sec.

hash_map the worst: no free
./run_bench_bpf_hashmap_full_update.sh
Setting up benchmark 'bpf-hashmap-ful-update'...
Benchmark 'bpf-hashmap-ful-update' started.
1:hash_map_full_perf 15687 events per sec
...
The average of the test results is around 16000 events per sec.

ftrace trace:
0)               |  htab_map_update_elem() {
0)               |      __pcpu_freelist_pop() {
0)               |        _raw_spin_lock()
0)               |        _raw_spin_unlock()
0)               |        ...
0) + 25.188 us   |      }
0) + 28.439 us   |  }

The test machine is 16C, trying to get spin_lock 17 times, in addition
to 16c, there is an extralist.

after patch: hash_map performance
./map_perf_test 1
0:hash_map_perf pre-alloc 1053298 events per sec
...
The average of the test results is around 1050000 events per sec.

hash_map worst: no free
./run_bench_bpf_hashmap_full_update.sh
Setting up benchmark 'bpf-hashmap-ful-update'...
Benchmark 'bpf-hashmap-ful-update' started.
1:hash_map_full_perf 555830 events per sec
...
The average of the test results is around 550000 events per sec.

ftrace trace:
0)               |  htab_map_update_elem() {
0)               |    alloc_htab_elem() {
0)   0.586 us    |      __pcpu_freelist_pop();
0)   0.945 us    |    }
0)   8.669 us    |  }

It can be seen that after adding this patch, the map performance is
almost not degraded, and when free=0, first check head->first instead of
directly acquiring spin_lock.

Co-developed-by: Chengming Zhou <zhouchengming@bytedance.com>
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
Signed-off-by: Feng Zhou <zhoufeng.zf@bytedance.com>
Link: https://lore.kernel.org/r/20220610023308.93798-2-zhoufeng.zf@bytedance.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
Feng Zhou 2022-06-10 10:33:07 +08:00 committed by Alexei Starovoitov
parent fe92833524
commit 54a9c3a42d
1 changed files with 14 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
kernel/bpf/percpu_freelist.c
View file

@ -31,7 +31,7 @@ static inline void pcpu_freelist_push_node(struct pcpu_freelist_head *head,
struct pcpu_freelist_node *node) struct pcpu_freelist_node *node)
{ {
node->next = head->first; node->next = head->first;
head->first = node; WRITE_ONCE(head->first, node);
} }
static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head, static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head,
@ -130,14 +130,17 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
orig_cpu = cpu = raw_smp_processor_id(); orig_cpu = cpu = raw_smp_processor_id();
while (1) { while (1) {
head = per_cpu_ptr(s->freelist, cpu); head = per_cpu_ptr(s->freelist, cpu);
if (!READ_ONCE(head->first))
goto next_cpu;
raw_spin_lock(&head->lock); raw_spin_lock(&head->lock);
node = head->first; node = head->first;
if (node) { if (node) {
head->first = node->next; WRITE_ONCE(head->first, node->next);
raw_spin_unlock(&head->lock); raw_spin_unlock(&head->lock);
return node; return node;
} }
raw_spin_unlock(&head->lock); raw_spin_unlock(&head->lock);
next_cpu:
cpu = cpumask_next(cpu, cpu_possible_mask); cpu = cpumask_next(cpu, cpu_possible_mask);
if (cpu >= nr_cpu_ids) if (cpu >= nr_cpu_ids)
cpu = 0; cpu = 0;
@ -146,10 +149,12 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
} }
/* per cpu lists are all empty, try extralist */ /* per cpu lists are all empty, try extralist */
if (!READ_ONCE(s->extralist.first))
return NULL;
raw_spin_lock(&s->extralist.lock); raw_spin_lock(&s->extralist.lock);
node = s->extralist.first; node = s->extralist.first;
if (node) if (node)
s->extralist.first = node->next; WRITE_ONCE(s->extralist.first, node->next);
raw_spin_unlock(&s->extralist.lock); raw_spin_unlock(&s->extralist.lock);
return node; return node;
} }
@ -164,15 +169,18 @@ ___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
orig_cpu = cpu = raw_smp_processor_id(); orig_cpu = cpu = raw_smp_processor_id();
while (1) { while (1) {
head = per_cpu_ptr(s->freelist, cpu); head = per_cpu_ptr(s->freelist, cpu);
if (!READ_ONCE(head->first))
goto next_cpu;
if (raw_spin_trylock(&head->lock)) { if (raw_spin_trylock(&head->lock)) {
node = head->first; node = head->first;
if (node) { if (node) {
head->first = node->next; WRITE_ONCE(head->first, node->next);
raw_spin_unlock(&head->lock); raw_spin_unlock(&head->lock);
return node; return node;
} }
raw_spin_unlock(&head->lock); raw_spin_unlock(&head->lock);
} }
next_cpu:
cpu = cpumask_next(cpu, cpu_possible_mask); cpu = cpumask_next(cpu, cpu_possible_mask);
if (cpu >= nr_cpu_ids) if (cpu >= nr_cpu_ids)
cpu = 0; cpu = 0;
@ -181,11 +189,11 @@ ___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
} }
/* cannot pop from per cpu lists, try extralist */ /* cannot pop from per cpu lists, try extralist */
if (!raw_spin_trylock(&s->extralist.lock)) if (!READ_ONCE(s->extralist.first) || !raw_spin_trylock(&s->extralist.lock))
return NULL; return NULL;
node = s->extralist.first; node = s->extralist.first;
if (node) if (node)
s->extralist.first = node->next; WRITE_ONCE(s->extralist.first, node->next);
raw_spin_unlock(&s->extralist.lock); raw_spin_unlock(&s->extralist.lock);
return node; return node;
} }