bcachefs: Refactor open_bucket code
Prep work for adding a hash table of open buckets - instead of embedding a bch_extent_ptr, we need to refer to the bucket directly so that we're not calling sector_to_bucket() in the hash table lookup code, which has an expensive divide. Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>
This commit is contained in:
parent
57af63b286
commit
abe19d458e
8 changed files with 83 additions and 67 deletions
|
@ -1066,7 +1066,7 @@ static bool bch2_dev_has_open_write_point(struct bch_fs *c, struct bch_dev *ca)
|
|||
ob++) {
|
||||
spin_lock(&ob->lock);
|
||||
if (ob->valid && !ob->on_partial_list &&
|
||||
ob->ptr.dev == ca->dev_idx)
|
||||
ob->dev == ca->dev_idx)
|
||||
ret = true;
|
||||
spin_unlock(&ob->lock);
|
||||
}
|
||||
|
@ -1213,22 +1213,3 @@ void bch2_fs_allocator_background_init(struct bch_fs *c)
|
|||
{
|
||||
spin_lock_init(&c->freelist_lock);
|
||||
}
|
||||
|
||||
void bch2_open_buckets_to_text(struct printbuf *out, struct bch_fs *c)
|
||||
{
|
||||
struct open_bucket *ob;
|
||||
|
||||
for (ob = c->open_buckets;
|
||||
ob < c->open_buckets + ARRAY_SIZE(c->open_buckets);
|
||||
ob++) {
|
||||
spin_lock(&ob->lock);
|
||||
if (ob->valid && !ob->on_partial_list) {
|
||||
pr_buf(out, "%zu ref %u type %s\n",
|
||||
ob - c->open_buckets,
|
||||
atomic_read(&ob->pin),
|
||||
bch2_data_types[ob->type]);
|
||||
}
|
||||
spin_unlock(&ob->lock);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -142,6 +142,4 @@ int bch2_dev_allocator_start(struct bch_dev *);
|
|||
int bch2_alloc_write_all(struct bch_fs *, unsigned);
|
||||
void bch2_fs_allocator_background_init(struct bch_fs *);
|
||||
|
||||
void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *);
|
||||
|
||||
#endif /* _BCACHEFS_ALLOC_BACKGROUND_H */
|
||||
|
|
|
@ -45,7 +45,7 @@
|
|||
|
||||
void __bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob)
|
||||
{
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
|
||||
|
||||
if (ob->ec) {
|
||||
bch2_ec_bucket_written(c, ob);
|
||||
|
@ -55,9 +55,9 @@ void __bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob)
|
|||
percpu_down_read(&c->mark_lock);
|
||||
spin_lock(&ob->lock);
|
||||
|
||||
bch2_mark_alloc_bucket(c, ca, PTR_BUCKET_NR(ca, &ob->ptr), false);
|
||||
bch2_mark_alloc_bucket(c, ca, ob->bucket, false);
|
||||
ob->valid = false;
|
||||
ob->type = 0;
|
||||
ob->data_type = 0;
|
||||
|
||||
spin_unlock(&ob->lock);
|
||||
percpu_up_read(&c->mark_lock);
|
||||
|
@ -81,8 +81,7 @@ void bch2_open_bucket_write_error(struct bch_fs *c,
|
|||
unsigned i;
|
||||
|
||||
open_bucket_for_each(c, obs, ob, i)
|
||||
if (ob->ptr.dev == dev &&
|
||||
ob->ec)
|
||||
if (ob->dev == dev && ob->ec)
|
||||
bch2_ec_bucket_cancel(c, ob);
|
||||
}
|
||||
|
||||
|
@ -95,18 +94,19 @@ static struct open_bucket *bch2_open_bucket_alloc(struct bch_fs *c)
|
|||
ob = c->open_buckets + c->open_buckets_freelist;
|
||||
c->open_buckets_freelist = ob->freelist;
|
||||
atomic_set(&ob->pin, 1);
|
||||
ob->type = 0;
|
||||
ob->data_type = 0;
|
||||
|
||||
c->open_buckets_nr_free--;
|
||||
return ob;
|
||||
}
|
||||
|
||||
|
||||
static void open_bucket_free_unused(struct bch_fs *c,
|
||||
struct write_point *wp,
|
||||
struct open_bucket *ob)
|
||||
{
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
|
||||
bool may_realloc = wp->type == BCH_DATA_user;
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
|
||||
bool may_realloc = wp->data_type == BCH_DATA_user;
|
||||
|
||||
BUG_ON(ca->open_buckets_partial_nr >
|
||||
ARRAY_SIZE(ca->open_buckets_partial));
|
||||
|
@ -133,11 +133,13 @@ static void verify_not_stale(struct bch_fs *c, const struct open_buckets *obs)
|
|||
struct open_bucket *ob;
|
||||
unsigned i;
|
||||
|
||||
rcu_read_lock();
|
||||
open_bucket_for_each(c, obs, ob, i) {
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
|
||||
|
||||
BUG_ON(ptr_stale(ca, &ob->ptr));
|
||||
BUG_ON(bucket(ca, ob->bucket)->mark.gen != ob->gen);
|
||||
}
|
||||
rcu_read_unlock();
|
||||
#endif
|
||||
}
|
||||
|
||||
|
@ -246,13 +248,9 @@ struct open_bucket *bch2_bucket_alloc(struct bch_fs *c, struct bch_dev *ca,
|
|||
ob->valid = true;
|
||||
ob->sectors_free = ca->mi.bucket_size;
|
||||
ob->alloc_reserve = reserve;
|
||||
ob->ptr = (struct bch_extent_ptr) {
|
||||
.type = 1 << BCH_EXTENT_ENTRY_ptr,
|
||||
.gen = bucket(ca, b)->mark.gen,
|
||||
.offset = bucket_to_sector(ca, b),
|
||||
.dev = ca->dev_idx,
|
||||
};
|
||||
|
||||
ob->dev = ca->dev_idx;
|
||||
ob->gen = bucket(ca, b)->mark.gen;
|
||||
ob->bucket = b;
|
||||
spin_unlock(&ob->lock);
|
||||
|
||||
if (c->blocked_allocate_open_bucket) {
|
||||
|
@ -333,9 +331,9 @@ static void add_new_bucket(struct bch_fs *c,
|
|||
struct open_bucket *ob)
|
||||
{
|
||||
unsigned durability =
|
||||
bch_dev_bkey_exists(c, ob->ptr.dev)->mi.durability;
|
||||
bch_dev_bkey_exists(c, ob->dev)->mi.durability;
|
||||
|
||||
__clear_bit(ob->ptr.dev, devs_may_alloc->d);
|
||||
__clear_bit(ob->dev, devs_may_alloc->d);
|
||||
*nr_effective += (flags & BUCKET_ALLOC_USE_DURABILITY)
|
||||
? durability : 1;
|
||||
*have_cache |= !durability;
|
||||
|
@ -445,13 +443,13 @@ static int bucket_alloc_from_stripe(struct bch_fs *c,
|
|||
continue;
|
||||
|
||||
ob = c->open_buckets + h->s->blocks[ec_idx];
|
||||
if (ob->ptr.dev == devs_sorted.devs[i] &&
|
||||
if (ob->dev == devs_sorted.devs[i] &&
|
||||
!test_and_set_bit(ec_idx, h->s->blocks_allocated))
|
||||
goto got_bucket;
|
||||
}
|
||||
goto out_put_head;
|
||||
got_bucket:
|
||||
ca = bch_dev_bkey_exists(c, ob->ptr.dev);
|
||||
ca = bch_dev_bkey_exists(c, ob->dev);
|
||||
|
||||
ob->ec_idx = ec_idx;
|
||||
ob->ec = h->s;
|
||||
|
@ -481,12 +479,12 @@ static void get_buckets_from_writepoint(struct bch_fs *c,
|
|||
unsigned i;
|
||||
|
||||
open_bucket_for_each(c, &wp->ptrs, ob, i) {
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
|
||||
|
||||
if (*nr_effective < nr_replicas &&
|
||||
test_bit(ob->ptr.dev, devs_may_alloc->d) &&
|
||||
test_bit(ob->dev, devs_may_alloc->d) &&
|
||||
(ca->mi.durability ||
|
||||
(wp->type == BCH_DATA_user && !*have_cache)) &&
|
||||
(wp->data_type == BCH_DATA_user && !*have_cache)) &&
|
||||
(ob->ec || !need_ec)) {
|
||||
add_new_bucket(c, ptrs, devs_may_alloc,
|
||||
nr_effective, have_cache,
|
||||
|
@ -518,7 +516,7 @@ static int open_bucket_add_buckets(struct bch_fs *c,
|
|||
unsigned i;
|
||||
|
||||
rcu_read_lock();
|
||||
devs = target_rw_devs(c, wp->type, target);
|
||||
devs = target_rw_devs(c, wp->data_type, target);
|
||||
rcu_read_unlock();
|
||||
|
||||
/* Don't allocate from devices we already have pointers to: */
|
||||
|
@ -526,7 +524,7 @@ static int open_bucket_add_buckets(struct bch_fs *c,
|
|||
__clear_bit(devs_have->devs[i], devs.d);
|
||||
|
||||
open_bucket_for_each(c, ptrs, ob, i)
|
||||
__clear_bit(ob->ptr.dev, devs.d);
|
||||
__clear_bit(ob->dev, devs.d);
|
||||
|
||||
if (erasure_code) {
|
||||
if (!ec_open_bucket(c, ptrs)) {
|
||||
|
@ -586,7 +584,7 @@ void bch2_open_buckets_stop_dev(struct bch_fs *c, struct bch_dev *ca,
|
|||
unsigned i, j;
|
||||
|
||||
open_bucket_for_each(c, obs, ob, i) {
|
||||
bool drop = !ca || ob->ptr.dev == ca->dev_idx;
|
||||
bool drop = !ca || ob->dev == ca->dev_idx;
|
||||
|
||||
if (!drop && ob->ec) {
|
||||
mutex_lock(&ob->ec->lock);
|
||||
|
@ -595,7 +593,7 @@ void bch2_open_buckets_stop_dev(struct bch_fs *c, struct bch_dev *ca,
|
|||
continue;
|
||||
|
||||
ob2 = c->open_buckets + ob->ec->blocks[j];
|
||||
drop |= ob2->ptr.dev == ca->dev_idx;
|
||||
drop |= ob2->dev == ca->dev_idx;
|
||||
}
|
||||
mutex_unlock(&ob->ec->lock);
|
||||
}
|
||||
|
@ -779,11 +777,11 @@ struct write_point *bch2_alloc_sectors_start(struct bch_fs *c,
|
|||
|
||||
wp = writepoint_find(c, write_point.v);
|
||||
|
||||
if (wp->type == BCH_DATA_user)
|
||||
if (wp->data_type == BCH_DATA_user)
|
||||
ob_flags |= BUCKET_MAY_ALLOC_PARTIAL;
|
||||
|
||||
/* metadata may not allocate on cache devices: */
|
||||
if (wp->type != BCH_DATA_user)
|
||||
if (wp->data_type != BCH_DATA_user)
|
||||
have_cache = true;
|
||||
|
||||
if (!target || (flags & BCH_WRITE_ONLY_SPECIFIED_DEVS)) {
|
||||
|
@ -861,6 +859,20 @@ struct write_point *bch2_alloc_sectors_start(struct bch_fs *c,
|
|||
}
|
||||
}
|
||||
|
||||
struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *c, struct open_bucket *ob)
|
||||
{
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
|
||||
|
||||
return (struct bch_extent_ptr) {
|
||||
.type = 1 << BCH_EXTENT_ENTRY_ptr,
|
||||
.gen = ob->gen,
|
||||
.dev = ob->dev,
|
||||
.offset = bucket_to_sector(ca, ob->bucket) +
|
||||
ca->mi.bucket_size -
|
||||
ob->sectors_free,
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
* Append pointers to the space we just allocated to @k, and mark @sectors space
|
||||
* as allocated out of @ob
|
||||
|
@ -877,14 +889,13 @@ void bch2_alloc_sectors_append_ptrs(struct bch_fs *c, struct write_point *wp,
|
|||
wp->sectors_free -= sectors;
|
||||
|
||||
open_bucket_for_each(c, &wp->ptrs, ob, i) {
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->ptr.dev);
|
||||
struct bch_extent_ptr ptr = ob->ptr;
|
||||
struct bch_dev *ca = bch_dev_bkey_exists(c, ob->dev);
|
||||
struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob);
|
||||
|
||||
ptr.cached = cached ||
|
||||
(!ca->mi.durability &&
|
||||
wp->type == BCH_DATA_user);
|
||||
wp->data_type == BCH_DATA_user);
|
||||
|
||||
ptr.offset += ca->mi.bucket_size - ob->sectors_free;
|
||||
bch2_bkey_append_ptr(k, ptr);
|
||||
|
||||
BUG_ON(sectors > ob->sectors_free);
|
||||
|
@ -915,7 +926,7 @@ static inline void writepoint_init(struct write_point *wp,
|
|||
enum bch_data_type type)
|
||||
{
|
||||
mutex_init(&wp->lock);
|
||||
wp->type = type;
|
||||
wp->data_type = type;
|
||||
}
|
||||
|
||||
void bch2_fs_allocator_foreground_init(struct bch_fs *c)
|
||||
|
@ -952,3 +963,22 @@ void bch2_fs_allocator_foreground_init(struct bch_fs *c)
|
|||
writepoint_hash(c, wp->write_point));
|
||||
}
|
||||
}
|
||||
|
||||
void bch2_open_buckets_to_text(struct printbuf *out, struct bch_fs *c)
|
||||
{
|
||||
struct open_bucket *ob;
|
||||
|
||||
for (ob = c->open_buckets;
|
||||
ob < c->open_buckets + ARRAY_SIZE(c->open_buckets);
|
||||
ob++) {
|
||||
spin_lock(&ob->lock);
|
||||
if (ob->valid && !ob->on_partial_list) {
|
||||
pr_buf(out, "%zu ref %u type %s\n",
|
||||
ob - c->open_buckets,
|
||||
atomic_read(&ob->pin),
|
||||
bch2_data_types[ob->data_type]);
|
||||
}
|
||||
spin_unlock(&ob->lock);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -85,7 +85,7 @@ static inline void bch2_open_bucket_get(struct bch_fs *c,
|
|||
unsigned i;
|
||||
|
||||
open_bucket_for_each(c, &wp->ptrs, ob, i) {
|
||||
ob->type = wp->type;
|
||||
ob->data_type = wp->data_type;
|
||||
atomic_inc(&ob->pin);
|
||||
ob_push(c, ptrs, ob);
|
||||
}
|
||||
|
@ -105,6 +105,7 @@ struct write_point *bch2_alloc_sectors_start(struct bch_fs *,
|
|||
unsigned,
|
||||
struct closure *);
|
||||
|
||||
struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *, struct open_bucket *);
|
||||
void bch2_alloc_sectors_append_ptrs(struct bch_fs *, struct write_point *,
|
||||
struct bkey_i *, unsigned, bool);
|
||||
void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *);
|
||||
|
@ -127,4 +128,6 @@ static inline struct write_point_specifier writepoint_ptr(struct write_point *wp
|
|||
|
||||
void bch2_fs_allocator_foreground_init(struct bch_fs *);
|
||||
|
||||
void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *);
|
||||
|
||||
#endif /* _BCACHEFS_ALLOC_FOREGROUND_H */
|
||||
|
|
|
@ -49,12 +49,15 @@ struct open_bucket {
|
|||
* the block in the stripe this open_bucket corresponds to:
|
||||
*/
|
||||
u8 ec_idx;
|
||||
u8 type;
|
||||
enum bch_data_type data_type:3;
|
||||
unsigned valid:1;
|
||||
unsigned on_partial_list:1;
|
||||
int alloc_reserve:3;
|
||||
|
||||
unsigned sectors_free;
|
||||
struct bch_extent_ptr ptr;
|
||||
u8 dev;
|
||||
u8 gen;
|
||||
u64 bucket;
|
||||
struct ec_stripe_new *ec;
|
||||
};
|
||||
|
||||
|
@ -74,7 +77,7 @@ struct write_point {
|
|||
struct mutex lock;
|
||||
u64 last_used;
|
||||
unsigned long write_point;
|
||||
enum bch_data_type type;
|
||||
enum bch_data_type data_type;
|
||||
|
||||
/* calculated based on how many pointers we're actually going to use: */
|
||||
unsigned sectors_free;
|
||||
|
|
|
@ -1063,7 +1063,7 @@ void *bch2_writepoint_ec_buf(struct bch_fs *c, struct write_point *wp)
|
|||
if (!ob)
|
||||
return NULL;
|
||||
|
||||
ca = bch_dev_bkey_exists(c, ob->ptr.dev);
|
||||
ca = bch_dev_bkey_exists(c, ob->dev);
|
||||
offset = ca->mi.bucket_size - ob->sectors_free;
|
||||
|
||||
return ob->ec->new_stripe.data[ob->ec_idx] + (offset << 9);
|
||||
|
@ -1318,7 +1318,7 @@ static int new_stripe_alloc_buckets(struct bch_fs *c, struct ec_stripe_head *h,
|
|||
BUG_ON(j >= h->s->nr_data + h->s->nr_parity);
|
||||
|
||||
h->s->blocks[j] = buckets.v[i];
|
||||
h->s->new_stripe.key.v.ptrs[j] = ob->ptr;
|
||||
h->s->new_stripe.key.v.ptrs[j] = bch2_ob_ptr(c, ob);
|
||||
__set_bit(j, h->s->blocks_gotten);
|
||||
}
|
||||
|
||||
|
@ -1346,7 +1346,7 @@ static int new_stripe_alloc_buckets(struct bch_fs *c, struct ec_stripe_head *h,
|
|||
BUG_ON(j >= h->s->nr_data);
|
||||
|
||||
h->s->blocks[j] = buckets.v[i];
|
||||
h->s->new_stripe.key.v.ptrs[j] = ob->ptr;
|
||||
h->s->new_stripe.key.v.ptrs[j] = bch2_ob_ptr(c, ob);
|
||||
__set_bit(j, h->s->blocks_gotten);
|
||||
}
|
||||
|
||||
|
@ -1535,7 +1535,7 @@ void bch2_ec_stop_dev(struct bch_fs *c, struct bch_dev *ca)
|
|||
continue;
|
||||
|
||||
ob = c->open_buckets + h->s->blocks[i];
|
||||
if (ob->ptr.dev == ca->dev_idx)
|
||||
if (ob->dev == ca->dev_idx)
|
||||
goto found;
|
||||
}
|
||||
goto unlock;
|
||||
|
|
|
@ -789,7 +789,7 @@ static int __bch2_set_nr_journal_buckets(struct bch_dev *ca, unsigned nr,
|
|||
goto err;
|
||||
}
|
||||
|
||||
b = sector_to_bucket(ca, ob->ptr.offset);
|
||||
b = ob->bucket;
|
||||
}
|
||||
|
||||
if (c)
|
||||
|
|
|
@ -10,6 +10,7 @@
|
|||
|
||||
#include "bcachefs.h"
|
||||
#include "alloc_background.h"
|
||||
#include "alloc_foreground.h"
|
||||
#include "sysfs.h"
|
||||
#include "btree_cache.h"
|
||||
#include "btree_io.h"
|
||||
|
@ -723,7 +724,7 @@ static void dev_alloc_debug_to_text(struct printbuf *out, struct bch_dev *ca)
|
|||
memset(nr, 0, sizeof(nr));
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(c->open_buckets); i++)
|
||||
nr[c->open_buckets[i].type]++;
|
||||
nr[c->open_buckets[i].data_type]++;
|
||||
|
||||
pr_buf(out,
|
||||
"\t\t buckets\t sectors fragmented\n"
|
||||
|
|
Loading…
Reference in a new issue