diff options
Diffstat (limited to 'fs/bcachefs/buckets.c')
| -rw-r--r-- | fs/bcachefs/buckets.c | 975 |
1 files changed, 975 insertions, 0 deletions
diff --git a/fs/bcachefs/buckets.c b/fs/bcachefs/buckets.c new file mode 100644 index 000000000000..f347c93e0c6e --- /dev/null +++ b/fs/bcachefs/buckets.c @@ -0,0 +1,975 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Code for manipulating bucket marks for garbage collection. + * + * Copyright 2014 Datera, Inc. + * + * Bucket states: + * - free bucket: mark == 0 + * The bucket contains no data and will not be read + * + * - allocator bucket: owned_by_allocator == 1 + * The bucket is on a free list, or it is an open bucket + * + * - cached bucket: owned_by_allocator == 0 && + * dirty_sectors == 0 && + * cached_sectors > 0 + * The bucket contains data but may be safely discarded as there are + * enough replicas of the data on other cache devices, or it has been + * written back to the backing device + * + * - dirty bucket: owned_by_allocator == 0 && + * dirty_sectors > 0 + * The bucket contains data that we must not discard (either only copy, + * or one of the 'main copies' for data requiring multiple replicas) + * + * - metadata bucket: owned_by_allocator == 0 && is_metadata == 1 + * This is a btree node, journal or gen/prio bucket + * + * Lifecycle: + * + * bucket invalidated => bucket on freelist => open bucket => + * [dirty bucket =>] cached bucket => bucket invalidated => ... + * + * Note that cache promotion can skip the dirty bucket step, as data + * is copied from a deeper tier to a shallower tier, onto a cached + * bucket. + * Note also that a cached bucket can spontaneously become dirty -- + * see below. + * + * Only a traversal of the key space can determine whether a bucket is + * truly dirty or cached. + * + * Transitions: + * + * - free => allocator: bucket was invalidated + * - cached => allocator: bucket was invalidated + * + * - allocator => dirty: open bucket was filled up + * - allocator => cached: open bucket was filled up + * - allocator => metadata: metadata was allocated + * + * - dirty => cached: dirty sectors were copied to a deeper tier + * - dirty => free: dirty sectors were overwritten or moved (copy gc) + * - cached => free: cached sectors were overwritten + * + * - metadata => free: metadata was freed + * + * Oddities: + * - cached => dirty: a device was removed so formerly replicated data + * is no longer sufficiently replicated + * - free => cached: cannot happen + * - free => dirty: cannot happen + * - free => metadata: cannot happen + */ + +#include "bcachefs.h" +#include "alloc.h" +#include "btree_gc.h" +#include "buckets.h" +#include "error.h" +#include "movinggc.h" +#include "trace.h" + +#include <linux/preempt.h> + +#ifdef DEBUG_BUCKETS + +#define lg_local_lock lg_global_lock +#define lg_local_unlock lg_global_unlock + +static void bch2_fs_stats_verify(struct bch_fs *c) +{ + struct bch_fs_usage stats = + __bch2_fs_usage_read(c); + unsigned i; + + for (i = 0; i < ARRAY_SIZE(stats.s); i++) { + if ((s64) stats.s[i].data[S_META] < 0) + panic("replicas %u meta underflow: %lli\n", + i + 1, stats.s[i].data[S_META]); + + if ((s64) stats.s[i].data[S_DIRTY] < 0) + panic("replicas %u dirty underflow: %lli\n", + i + 1, stats.s[i].data[S_DIRTY]); + + if ((s64) stats.s[i].persistent_reserved < 0) + panic("replicas %u reserved underflow: %lli\n", + i + 1, stats.s[i].persistent_reserved); + } + + if ((s64) stats.online_reserved < 0) + panic("sectors_online_reserved underflow: %lli\n", + stats.online_reserved); +} + +static void bch2_dev_stats_verify(struct bch_dev *ca) +{ + struct bch_dev_usage stats = + __bch2_dev_usage_read(ca); + u64 n = ca->mi.nbuckets - ca->mi.first_bucket; + unsigned i; + + for (i = 0; i < ARRAY_SIZE(stats.buckets); i++) + BUG_ON(stats.buckets[i] > n); + BUG_ON(stats.buckets_alloc > n); + BUG_ON(stats.buckets_unavailable > n); +} + +static void bch2_disk_reservations_verify(struct bch_fs *c, int flags) +{ + if (!(flags & BCH_DISK_RESERVATION_NOFAIL)) { + u64 used = __bch2_fs_sectors_used(c); + u64 cached = 0; + u64 avail = atomic64_read(&c->sectors_available); + int cpu; + + for_each_possible_cpu(cpu) + cached += per_cpu_ptr(c->usage_percpu, cpu)->available_cache; + + if (used + avail + cached > c->capacity) + panic("used %llu avail %llu cached %llu capacity %llu\n", + used, avail, cached, c->capacity); + } +} + +#else + +static void bch2_fs_stats_verify(struct bch_fs *c) {} +static void bch2_dev_stats_verify(struct bch_dev *ca) {} +static void bch2_disk_reservations_verify(struct bch_fs *c, int flags) {} + +#endif + +/* + * Clear journal_seq_valid for buckets for which it's not needed, to prevent + * wraparound: + */ +void bch2_bucket_seq_cleanup(struct bch_fs *c) +{ + u16 last_seq_ondisk = c->journal.last_seq_ondisk; + struct bch_dev *ca; + struct bucket_array *buckets; + struct bucket *g; + struct bucket_mark m; + unsigned i; + + for_each_member_device(ca, c, i) { + down_read(&ca->bucket_lock); + buckets = bucket_array(ca); + + for_each_bucket(g, buckets) { + bucket_cmpxchg(g, m, ({ + if (!m.journal_seq_valid || + bucket_needs_journal_commit(m, last_seq_ondisk)) + break; + + m.journal_seq_valid = 0; + })); + } + up_read(&ca->bucket_lock); + } +} + +#define bch2_usage_add(_acc, _stats) \ +do { \ + typeof(_acc) _a = (_acc), _s = (_stats); \ + unsigned i; \ + \ + for (i = 0; i < sizeof(*_a) / sizeof(u64); i++) \ + ((u64 *) (_a))[i] += ((u64 *) (_s))[i]; \ +} while (0) + +#define bch2_usage_read_raw(_stats) \ +({ \ + typeof(*this_cpu_ptr(_stats)) _acc; \ + int cpu; \ + \ + memset(&_acc, 0, sizeof(_acc)); \ + \ + for_each_possible_cpu(cpu) \ + bch2_usage_add(&_acc, per_cpu_ptr((_stats), cpu)); \ + \ + _acc; \ +}) + +#define bch2_usage_read_cached(_c, _cached, _uncached) \ +({ \ + typeof(_cached) _ret; \ + unsigned _seq; \ + \ + do { \ + _seq = read_seqcount_begin(&(_c)->gc_pos_lock); \ + _ret = (_c)->gc_pos.phase == GC_PHASE_DONE \ + ? bch2_usage_read_raw(_uncached) \ + : (_cached); \ + } while (read_seqcount_retry(&(_c)->gc_pos_lock, _seq)); \ + \ + _ret; \ +}) + +struct bch_dev_usage __bch2_dev_usage_read(struct bch_dev *ca) +{ + return bch2_usage_read_raw(ca->usage_percpu); +} + +struct bch_dev_usage bch2_dev_usage_read(struct bch_fs *c, struct bch_dev *ca) +{ + return bch2_usage_read_cached(c, ca->usage_cached, ca->usage_percpu); +} + +struct bch_fs_usage +__bch2_fs_usage_read(struct bch_fs *c) +{ + return bch2_usage_read_raw(c->usage_percpu); +} + +struct bch_fs_usage +bch2_fs_usage_read(struct bch_fs *c) +{ + return bch2_usage_read_cached(c, + c->usage_cached, + c->usage_percpu); +} + +struct fs_usage_sum { + u64 data; + u64 reserved; +}; + +static inline struct fs_usage_sum __fs_usage_sum(struct bch_fs_usage stats) +{ + struct fs_usage_sum sum = { 0 }; + unsigned i; + + for (i = 0; i < ARRAY_SIZE(stats.s); i++) { + sum.data += (stats.s[i].data[S_META] + + stats.s[i].data[S_DIRTY]) * (i + 1); + sum.reserved += stats.s[i].persistent_reserved * (i + 1); + } + + sum.reserved += stats.online_reserved; + return sum; +} + +#define RESERVE_FACTOR 6 + +static u64 reserve_factor(u64 r) +{ + return r + (round_up(r, (1 << RESERVE_FACTOR)) >> RESERVE_FACTOR); +} + +static u64 avail_factor(u64 r) +{ + return (r << RESERVE_FACTOR) / (1 << RESERVE_FACTOR) + 1; +} + +u64 __bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage stats) +{ + struct fs_usage_sum sum = __fs_usage_sum(stats); + + return sum.data + reserve_factor(sum.reserved); +} + +u64 bch2_fs_sectors_used(struct bch_fs *c, struct bch_fs_usage stats) +{ + return min(c->capacity, __bch2_fs_sectors_used(c, stats)); +} + +u64 bch2_fs_sectors_free(struct bch_fs *c, struct bch_fs_usage stats) +{ + return avail_factor(c->capacity - bch2_fs_sectors_used(c, stats)); +} + +static inline int is_unavailable_bucket(struct bucket_mark m) +{ + return !is_available_bucket(m); +} + +static inline int is_fragmented_bucket(struct bucket_mark m, + struct bch_dev *ca) +{ + if (!m.owned_by_allocator && + m.data_type == BCH_DATA_USER && + bucket_sectors_used(m)) + return max_t(int, 0, (int) ca->mi.bucket_size - + bucket_sectors_used(m)); + return 0; +} + +static inline enum bch_data_type bucket_type(struct bucket_mark m) +{ + return m.cached_sectors && !m.dirty_sectors + ? BCH_DATA_CACHED + : m.data_type; +} + +static bool bucket_became_unavailable(struct bch_fs *c, + struct bucket_mark old, + struct bucket_mark new) +{ + return is_available_bucket(old) && + !is_available_bucket(new) && + (!c || c->gc_pos.phase == GC_PHASE_DONE); +} + +void bch2_fs_usage_apply(struct bch_fs *c, + struct bch_fs_usage *stats, + struct disk_reservation *disk_res, + struct gc_pos gc_pos) +{ + struct fs_usage_sum sum = __fs_usage_sum(*stats); + s64 added = sum.data + sum.reserved; + + /* + * Not allowed to reduce sectors_available except by getting a + * reservation: + */ + BUG_ON(added > (s64) (disk_res ? disk_res->sectors : 0)); + + if (added > 0) { + disk_res->sectors -= added; + stats->online_reserved -= added; + } + + percpu_down_read(&c->usage_lock); + preempt_disable(); + /* online_reserved not subject to gc: */ + this_cpu_add(c->usage_percpu->online_reserved, stats->online_reserved); + stats->online_reserved = 0; + + if (!gc_will_visit(c, gc_pos)) + bch2_usage_add(this_cpu_ptr(c->usage_percpu), stats); + + bch2_fs_stats_verify(c); + preempt_enable(); + percpu_up_read(&c->usage_lock); + + memset(stats, 0, sizeof(*stats)); +} + +static void bch2_dev_usage_update(struct bch_fs *c, struct bch_dev *ca, + struct bucket_mark old, struct bucket_mark new) +{ + struct bch_dev_usage *dev_usage; + + if (c) + percpu_rwsem_assert_held(&c->usage_lock); + + if (old.data_type && new.data_type && + old.data_type != new.data_type) { + BUG_ON(!c); + bch2_fs_inconsistent(c, + "different types of data in same bucket: %s, %s", + bch2_data_types[old.data_type], + bch2_data_types[new.data_type]); + } + + preempt_disable(); + dev_usage = this_cpu_ptr(ca->usage_percpu); + + dev_usage->buckets[bucket_type(old)]--; + dev_usage->buckets[bucket_type(new)]++; + + dev_usage->buckets_alloc += + (int) new.owned_by_allocator - (int) old.owned_by_allocator; + dev_usage->buckets_unavailable += + is_unavailable_bucket(new) - is_unavailable_bucket(old); + + dev_usage->sectors[old.data_type] -= old.dirty_sectors; + dev_usage->sectors[new.data_type] += new.dirty_sectors; + dev_usage->sectors[BCH_DATA_CACHED] += + (int) new.cached_sectors - (int) old.cached_sectors; + dev_usage->sectors_fragmented += + is_fragmented_bucket(new, ca) - is_fragmented_bucket(old, ca); + preempt_enable(); + + if (!is_available_bucket(old) && is_available_bucket(new)) + bch2_wake_allocator(ca); + + bch2_dev_stats_verify(ca); +} + +#define bucket_data_cmpxchg(c, ca, g, new, expr) \ +({ \ + struct bucket_mark _old = bucket_cmpxchg(g, new, expr); \ + \ + bch2_dev_usage_update(c, ca, _old, new); \ + _old; \ +}) + +bool bch2_invalidate_bucket(struct bch_fs *c, struct bch_dev *ca, + size_t b, struct bucket_mark *old) +{ + struct bucket *g; + struct bucket_mark new; + + percpu_rwsem_assert_held(&c->usage_lock); + + g = bucket(ca, b); + + *old = bucket_data_cmpxchg(c, ca, g, new, ({ + if (!is_available_bucket(new)) + return false; + + new.owned_by_allocator = 1; + new.data_type = 0; + new.cached_sectors = 0; + new.dirty_sectors = 0; + new.gen++; + })); + + if (!old->owned_by_allocator && old->cached_sectors) + trace_invalidate(ca, bucket_to_sector(ca, b), + old->cached_sectors); + return true; +} + +void bch2_mark_alloc_bucket(struct bch_fs *c, struct bch_dev *ca, + size_t b, bool owned_by_allocator, + struct gc_pos pos, unsigned flags) +{ + struct bucket *g; + struct bucket_mark old, new; + + percpu_rwsem_assert_held(&c->usage_lock); + g = bucket(ca, b); + + if (!(flags & BCH_BUCKET_MARK_GC_LOCK_HELD) && + gc_will_visit(c, pos)) + return; + + old = bucket_data_cmpxchg(c, ca, g, new, ({ + new.owned_by_allocator = owned_by_allocator; + })); + + BUG_ON(!owned_by_allocator && !old.owned_by_allocator && + c->gc_pos.phase == GC_PHASE_DONE); +} + +#define saturated_add(ca, dst, src, max) \ +do { \ + BUG_ON((int) (dst) + (src) < 0); \ + if ((dst) == (max)) \ + ; \ + else if ((dst) + (src) <= (max)) \ + dst += (src); \ + else { \ + dst = (max); \ + trace_sectors_saturated(ca); \ + } \ +} while (0) + +void bch2_mark_metadata_bucket(struct bch_fs *c, struct bch_dev *ca, + size_t b, enum bch_data_type type, + unsigned sectors, struct gc_pos pos, + unsigned flags) +{ + struct bucket *g; + struct bucket_mark old, new; + + BUG_ON(!type); + + if (likely(c)) { + percpu_rwsem_assert_held(&c->usage_lock); + + if (!(flags & BCH_BUCKET_MARK_GC_LOCK_HELD) && + gc_will_visit(c, pos)) + return; + } + + rcu_read_lock(); + + g = bucket(ca, b); + old = bucket_data_cmpxchg(c, ca, g, new, ({ + saturated_add(ca, new.dirty_sectors, sectors, + GC_MAX_SECTORS_USED); + new.data_type = type; + })); + + rcu_read_unlock(); + + BUG_ON(!(flags & BCH_BUCKET_MARK_MAY_MAKE_UNAVAILABLE) && + bucket_became_unavailable(c, old, new)); +} + +/* Reverting this until the copygc + compression issue is fixed: */ + +static int __disk_sectors(struct bch_extent_crc_unpacked crc, unsigned sectors) +{ + if (!sectors) + return 0; + + return max(1U, DIV_ROUND_UP(sectors * crc.compressed_size, + crc.uncompressed_size)); +} + +/* + * Checking against gc's position has to be done here, inside the cmpxchg() + * loop, to avoid racing with the start of gc clearing all the marks - GC does + * that with the gc pos seqlock held. + */ +static void bch2_mark_pointer(struct bch_fs *c, + struct bkey_s_c_extent e, + const struct bch_extent_ptr *ptr, + struct bch_extent_crc_unpacked crc, + s64 sectors, enum s_alloc type, + struct bch_fs_usage *stats, + u64 journal_seq, unsigned flags) +{ + struct bucket_mark old, new; + unsigned saturated; + struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev); + struct bucket *g = PTR_BUCKET(ca, ptr); + enum bch_data_type data_type = type == S_META + ? BCH_DATA_BTREE : BCH_DATA_USER; + u64 v; + + if (crc.compression_type) { + unsigned old_sectors, new_sectors; + + if (sectors > 0) { + old_sectors = 0; + new_sectors = sectors; + } else { + old_sectors = e.k->size; + new_sectors = e.k->size + sectors; + } + + sectors = -__disk_sectors(crc, old_sectors) + +__disk_sectors(crc, new_sectors); + } + + if (flags & BCH_BUCKET_MARK_GC_WILL_VISIT) { + if (journal_seq) + bucket_cmpxchg(g, new, ({ + new.journal_seq_valid = 1; + new.journal_seq = journal_seq; + })); + + return; + } + + v = atomic64_read(&g->_mark.v); + do { + new.v.counter = old.v.counter = v; + saturated = 0; + + /* + * Check this after reading bucket mark to guard against + * the allocator invalidating a bucket after we've already + * checked the gen + */ + if (gen_after(new.gen, ptr->gen)) { + BUG_ON(!test_bit(BCH_FS_ALLOC_READ_DONE, &c->flags)); + EBUG_ON(!ptr->cached && + test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags)); + return; + } + + if (!ptr->cached && + new.dirty_sectors == GC_MAX_SECTORS_USED && + sectors < 0) + saturated = -sectors; + + if (ptr->cached) + saturated_add(ca, new.cached_sectors, sectors, + GC_MAX_SECTORS_USED); + else + saturated_add(ca, new.dirty_sectors, sectors, + GC_MAX_SECTORS_USED); + + if (!new.dirty_sectors && + !new.cached_sectors) { + new.data_type = 0; + + if (journal_seq) { + new.journal_seq_valid = 1; + new.journal_seq = journal_seq; + } + } else { + new.data_type = data_type; + } + + if (flags & BCH_BUCKET_MARK_NOATOMIC) { + g->_mark = new; + break; + } + } while ((v = atomic64_cmpxchg(&g->_mark.v, + old.v.counter, + new.v.counter)) != old.v.counter); + + bch2_dev_usage_update(c, ca, old, new); + + BUG_ON(!(flags & BCH_BUCKET_MARK_MAY_MAKE_UNAVAILABLE) && + bucket_became_unavailable(c, old, new)); + + if (saturated && + atomic_long_add_return(saturated, + &ca->saturated_count) >= + bucket_to_sector(ca, ca->free_inc.size)) { + if (c->gc_thread) { + trace_gc_sectors_saturated(c); + wake_up_process(c->gc_thread); + } + } +} + +void bch2_mark_key(struct bch_fs *c, struct bkey_s_c k, + s64 sectors, bool metadata, + struct gc_pos pos, + struct bch_fs_usage *stats, + u64 journal_seq, unsigned flags) +{ + /* + * synchronization w.r.t. GC: + * + * Normally, bucket sector counts/marks are updated on the fly, as + * references are added/removed from the btree, the lists of buckets the + * allocator owns, other metadata buckets, etc. + * + * When GC is in progress and going to mark this reference, we do _not_ + * mark this reference here, to avoid double counting - GC will count it + * when it gets to it. + * + * To know whether we should mark a given reference (GC either isn't + * running, or has already marked references at this position) we + * construct a total order for everything GC walks. Then, we can simply + * compare the position of the reference we're marking - @pos - with + * GC's current position. If GC is going to mark this reference, GC's + * current position will be less than @pos; if GC's current position is + * greater than @pos GC has either already walked this position, or + * isn't running. + * + * To avoid racing with GC's position changing, we have to deal with + * - GC's position being set to GC_POS_MIN when GC starts: + * usage_lock guards against this + * - GC's position overtaking @pos: we guard against this with + * whatever lock protects the data structure the reference lives in + * (e.g. the btree node lock, or the relevant allocator lock). + */ + + percpu_down_read(&c->usage_lock); + if (!(flags & BCH_BUCKET_MARK_GC_LOCK_HELD) && + gc_will_visit(c, pos)) + flags |= BCH_BUCKET_MARK_GC_WILL_VISIT; + + if (!stats) + stats = this_cpu_ptr(c->usage_percpu); + + switch (k.k->type) { + case BCH_EXTENT: + case BCH_EXTENT_CACHED: { + struct bkey_s_c_extent e = bkey_s_c_to_extent(k); + const struct bch_extent_ptr *ptr; + struct bch_extent_crc_unpacked crc; + enum s_alloc type = metadata ? S_META : S_DIRTY; + unsigned replicas = 0; + + BUG_ON(metadata && bkey_extent_is_cached(e.k)); + BUG_ON(!sectors); + + extent_for_each_ptr_crc(e, ptr, crc) { + bch2_mark_pointer(c, e, ptr, crc, sectors, type, + stats, journal_seq, flags); + replicas += !ptr->cached; + } + + if (replicas) { + BUG_ON(replicas - 1 > ARRAY_SIZE(stats->s)); + stats->s[replicas - 1].data[type] += sectors; + } + break; + } + case BCH_RESERVATION: { + struct bkey_s_c_reservation r = bkey_s_c_to_reservation(k); + + if (r.v->nr_replicas) { + BUG_ON(r.v->nr_replicas - 1 > ARRAY_SIZE(stats->s)); + stats->s[r.v->nr_replicas - 1].persistent_reserved += sectors; + } + break; + } + } + percpu_up_read(&c->usage_lock); +} + +/* Disk reservations: */ + +static u64 __recalc_sectors_available(struct bch_fs *c) +{ + int cpu; + + for_each_possible_cpu(cpu) + per_cpu_ptr(c->usage_percpu, cpu)->available_cache = 0; + + return bch2_fs_sectors_free(c, bch2_fs_usage_read(c)); +} + +/* Used by gc when it's starting: */ +void bch2_recalc_sectors_available(struct bch_fs *c) +{ + percpu_down_write(&c->usage_lock); + atomic64_set(&c->sectors_available, __recalc_sectors_available(c)); + percpu_up_write(&c->usage_lock); +} + +void __bch2_disk_reservation_put(struct bch_fs *c, struct disk_reservation *res) +{ + percpu_down_read(&c->usage_lock); + this_cpu_sub(c->usage_percpu->online_reserved, + res->sectors); + + bch2_fs_stats_verify(c); + percpu_up_read(&c->usage_lock); + + res->sectors = 0; +} + +#define SECTORS_CACHE 1024 + +int bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res, + unsigned sectors, int flags) +{ + struct bch_fs_usage *stats; + u64 old, v, get; + s64 sectors_available; + int ret; + + percpu_down_read(&c->usage_lock); + preempt_disable(); + stats = this_cpu_ptr(c->usage_percpu); + + if (sectors <= stats->available_cache) + goto out; + + v = atomic64_read(&c->sectors_available); + do { + old = v; + get = min((u64) sectors + SECTORS_CACHE, old); + + if (get < sectors) { + preempt_enable(); + percpu_up_read(&c->usage_lock); + goto recalculate; + } + } while ((v = atomic64_cmpxchg(&c->sectors_available, + old, old - get)) != old); + + stats->available_cache += get; + +out: + stats->available_cache -= sectors; + stats->online_reserved += sectors; + res->sectors += sectors; + + bch2_disk_reservations_verify(c, flags); + bch2_fs_stats_verify(c); + preempt_enable(); + percpu_up_read(&c->usage_lock); + return 0; + +recalculate: + /* + * GC recalculates sectors_available when it starts, so that hopefully + * we don't normally end up blocking here: + */ + + /* + * Piss fuck, we can be called from extent_insert_fixup() with btree + * locks held: + */ + + if (!(flags & BCH_DISK_RESERVATION_GC_LOCK_HELD)) { + if (!(flags & BCH_DISK_RESERVATION_BTREE_LOCKS_HELD)) + down_read(&c->gc_lock); + else if (!down_read_trylock(&c->gc_lock)) + return -EINTR; + } + + percpu_down_write(&c->usage_lock); + sectors_available = __recalc_sectors_available(c); + + if (sectors <= sectors_available || + (flags & BCH_DISK_RESERVATION_NOFAIL)) { + atomic64_set(&c->sectors_available, + max_t(s64, 0, sectors_available - sectors)); + stats->online_reserved += sectors; + res->sectors += sectors; + ret = 0; + + bch2_disk_reservations_verify(c, flags); + } else { + atomic64_set(&c->sectors_available, sectors_available); + ret = -ENOSPC; + } + + bch2_fs_stats_verify(c); + percpu_up_write(&c->usage_lock); + + if (!(flags & BCH_DISK_RESERVATION_GC_LOCK_HELD)) + up_read(&c->gc_lock); + + return ret; +} + +/* Startup/shutdown: */ + +static void buckets_free_rcu(struct rcu_head *rcu) +{ + struct bucket_array *buckets = + container_of(rcu, struct bucket_array, rcu); + + kvpfree(buckets, + sizeof(struct bucket_array) + + buckets->nbuckets * sizeof(struct bucket)); +} + +int bch2_dev_buckets_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) +{ + struct bucket_array *buckets = NULL, *old_buckets = NULL; + unsigned long *buckets_dirty = NULL; + u8 *oldest_gens = NULL; + alloc_fifo free[RESERVE_NR]; + alloc_fifo free_inc; + alloc_heap alloc_heap; + copygc_heap copygc_heap; + + size_t btree_reserve = DIV_ROUND_UP(BTREE_NODE_RESERVE, + ca->mi.bucket_size / c->opts.btree_node_size); + /* XXX: these should be tunable */ + size_t reserve_none = max_t(size_t, 4, ca->mi.nbuckets >> 9); + size_t copygc_reserve = max_t(size_t, 16, ca->mi.nbuckets >> 7); + size_t free_inc_reserve = copygc_reserve / 2; + bool resize = ca->buckets != NULL, + start_copygc = ca->copygc_thread != NULL; + int ret = -ENOMEM; + unsigned i; + + memset(&free, 0, sizeof(free)); + memset(&free_inc, 0, sizeof(free_inc)); + memset(&alloc_heap, 0, sizeof(alloc_heap)); + memset(©gc_heap, 0, sizeof(copygc_heap)); + + if (!(buckets = kvpmalloc(sizeof(struct bucket_array) + + nbuckets * sizeof(struct bucket), + GFP_KERNEL|__GFP_ZERO)) || + !(oldest_gens = kvpmalloc(nbuckets * sizeof(u8), + GFP_KERNEL|__GFP_ZERO)) || + !(buckets_dirty = kvpmalloc(BITS_TO_LONGS(nbuckets) * + sizeof(unsigned long), + GFP_KERNEL|__GFP_ZERO)) || + !init_fifo(&free[RESERVE_BTREE], btree_reserve, GFP_KERNEL) || + !init_fifo(&free[RESERVE_MOVINGGC], + copygc_reserve, GFP_KERNEL) || + !init_fifo(&free[RESERVE_NONE], reserve_none, GFP_KERNEL) || + !init_fifo(&free_inc, free_inc_reserve, GFP_KERNEL) || + !init_heap(&alloc_heap, free_inc_reserve, GFP_KERNEL) || + !init_heap(©gc_heap, copygc_reserve, GFP_KERNEL)) + goto err; + + buckets->first_bucket = ca->mi.first_bucket; + buckets->nbuckets = nbuckets; + + bch2_copygc_stop(ca); + + if (resize) { + down_write(&c->gc_lock); + down_write(&ca->bucket_lock); + percpu_down_write(&c->usage_lock); + } + + old_buckets = bucket_array(ca); + + if (resize) { + size_t n = min(buckets->nbuckets, old_buckets->nbuckets); + + memcpy(buckets->b, + old_buckets->b, + n * sizeof(struct bucket)); + memcpy(oldest_gens, + ca->oldest_gens, + n * sizeof(u8)); + memcpy(buckets_dirty, + ca->buckets_dirty, + BITS_TO_LONGS(n) * sizeof(unsigned long)); + } + + rcu_assign_pointer(ca->buckets, buckets); + buckets = old_buckets; + + swap(ca->oldest_gens, oldest_gens); + swap(ca->buckets_dirty, buckets_dirty); + + if (resize) + percpu_up_write(&c->usage_lock); + + spin_lock(&c->freelist_lock); + for (i = 0; i < RESERVE_NR; i++) { + fifo_move(&free[i], &ca->free[i]); + swap(ca->free[i], free[i]); + } + fifo_move(&free_inc, &ca->free_inc); + swap(ca->free_inc, free_inc); + spin_unlock(&c->freelist_lock); + + /* with gc lock held, alloc_heap can't be in use: */ + swap(ca->alloc_heap, alloc_heap); + + /* and we shut down copygc: */ + swap(ca->copygc_heap, copygc_heap); + + nbuckets = ca->mi.nbuckets; + + if (resize) { + up_write(&ca->bucket_lock); + up_write(&c->gc_lock); + } + + if (start_copygc && + bch2_copygc_start(c, ca)) + bch_err(ca, "error restarting copygc thread"); + + ret = 0; +err: + free_heap(©gc_heap); + free_heap(&alloc_heap); + free_fifo(&free_inc); + for (i = 0; i < RESERVE_NR; i++) + free_fifo(&free[i]); + kvpfree(buckets_dirty, + BITS_TO_LONGS(nbuckets) * sizeof(unsigned long)); + kvpfree(oldest_gens, + nbuckets * sizeof(u8)); + if (buckets) + call_rcu(&old_buckets->rcu, buckets_free_rcu); + + return ret; +} + +void bch2_dev_buckets_free(struct bch_dev *ca) +{ + unsigned i; + + free_heap(&ca->copygc_heap); + free_heap(&ca->alloc_heap); + free_fifo(&ca->free_inc); + for (i = 0; i < RESERVE_NR; i++) + free_fifo(&ca->free[i]); + kvpfree(ca->buckets_dirty, + BITS_TO_LONGS(ca->mi.nbuckets) * sizeof(unsigned long)); + kvpfree(ca->oldest_gens, ca->mi.nbuckets * sizeof(u8)); + kvpfree(rcu_dereference_protected(ca->buckets, 1), + sizeof(struct bucket_array) + + ca->mi.nbuckets * sizeof(struct bucket)); + + free_percpu(ca->usage_percpu); +} + +int bch2_dev_buckets_alloc(struct bch_fs *c, struct bch_dev *ca) +{ + if (!(ca->usage_percpu = alloc_percpu(struct bch_dev_usage))) + return -ENOMEM; + + return bch2_dev_buckets_resize(c, ca, ca->mi.nbuckets);; +} |