diff --git a/flake.lock b/flake.lock index c4852971d7..6e2b3f8823 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "crane": { "locked": { - "lastModified": 1780099841, - "narHash": "sha256-EVZd2RsbpreRUDSi9rBwPY+ZxoyMaiEBbZxxhljbaS4=", + "lastModified": 1785782307, + "narHash": "sha256-MPaRdVkf6zZP5fCPxYCi8Dr4pZzgmXzg8T9nVEbp3Mw=", "owner": "ipetkov", "repo": "crane", - "rev": "0532eb17955225173906d671fb36306bdeb1e2dc", + "rev": "2c71e194474d13de031d729b729c968ddbe3507f", "type": "github" }, "original": { @@ -36,11 +36,11 @@ "nixpkgs-lib": "nixpkgs-lib" }, "locked": { - "lastModified": 1778716662, - "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=", + "lastModified": 1785627969, + "narHash": "sha256-4dtXQk/NMePegK/nWp5NSeuZKLATItOq61lpEvmXqGw=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb", + "rev": "427bf4bd9435fdf21321c8cc628c24efc14c0f7a", "type": "github" }, "original": { @@ -71,11 +71,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1782467914, - "narHash": "sha256-pGvFkM8N0xEkIIXDe5YYfbEAvHrk4IxBrjB/x8OomhE=", + "lastModified": 1785967620, + "narHash": "sha256-IItrdb7Puk05RqOBWZYFC5X6Wl1sJmCfh5MWVHw5iMM=", "owner": "nixos", "repo": "nixpkgs", - "rev": "e73de5be04e0eff4190a1432b946d469c794e7b4", + "rev": "b7c2ada94fe99c15b0dbcf4d11fd7850b957a436", "type": "github" }, "original": { @@ -87,11 +87,11 @@ }, "nixpkgs-lib": { "locked": { - "lastModified": 1777168982, - "narHash": "sha256-GOkGPcboWE9BmGCRMLX3worL4EMnsnG8MyKmXNeYuhQ=", + "lastModified": 1785031560, + "narHash": "sha256-OmshNvn2vupOFpYinLUu+1Dnpu4n7Q5N3ggGVNHpkUI=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "f5901329dade4a6ea039af1433fb087bd9c1fe14", + "rev": "0e79af5e3d4dcfcd676ab5ba3f95d2e3352e078c", "type": "github" }, "original": { @@ -118,11 +118,11 @@ ] }, "locked": { - "lastModified": 1780110990, - "narHash": "sha256-6QBThUi7SuK+dgA+DCaEkQGZN4kYx6DpXmK45+MG9zI=", + "lastModified": 1786076960, + "narHash": "sha256-jfR6OhwurCKn1tREyfOcK/Omxf1Q/DzDDFbnEr1mBLs=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "85570ef134d92a8702de6afd1f6f0209c863fa91", + "rev": "57a23bfaf4f7017267294b161175db1e32eb1c85", "type": "github" }, "original": { @@ -138,11 +138,11 @@ ] }, "locked": { - "lastModified": 1775636079, - "narHash": "sha256-pc20NRoMdiar8oPQceQT47UUZMBTiMdUuWrYu2obUP0=", + "lastModified": 1785945821, + "narHash": "sha256-NLSyTCW4K4ofhNBllt3omPasm6QpralXH1DBZOc91Dw=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "790751ff7fd3801feeaf96d7dc416a8d581265ba", + "rev": "ae7910970dddc408fe6ab1c8e4b277bb21d72dc0", "type": "github" }, "original": { diff --git a/fs/alloc/accounting.c b/fs/alloc/accounting.c index 363ca5b8aa..5d54239c91 100644 --- a/fs/alloc/accounting.c +++ b/fs/alloc/accounting.c @@ -1,14 +1,21 @@ // SPDX-License-Identifier: GPL-2.0 +#include "alloc/background.h" +#include "alloc/format.h" #include "bcachefs.h" +#include "bcachefs_format.h" #include "bcachefs_ioctl.h" +#include "alloc/accounting_format.h" #include "alloc/accounting.h" #include "alloc/buckets.h" #include "alloc/replicas.h" +#include "btree/bkey_types.h" #include "btree/cache.h" +#include "btree/iter.h" #include "btree/journal_overlay.h" +#include "btree/types.h" #include "btree/update.h" #include "btree/write_buffer.h" @@ -1260,6 +1267,55 @@ int bch2_accounting_read(struct bch_fs *c) return accounting_read_mem_fixups(trans); } +int bch2_dev_truncate_accounting(struct bch_fs *c, struct bch_dev *ca, u64 old_nbuckets, u64 cutoff) { + struct bpos start = POS(ca->dev_idx, cutoff); + struct bpos end = POS(ca->dev_idx, old_nbuckets); + + CLASS(btree_trans, trans)(c); + + /* accumulate deltas per data type, then apply them */ + return commit_do(trans, NULL, NULL, 0, ({ + s64 delta[BCH_DATA_NR][3] = {}; + s64 keys = 0; + struct bkey_s_c k; + int ret = 0; + + for_each_btree_key_max_norestart(trans, iter, BTREE_ID_alloc, start, end, BTREE_ITER_prefetch, k, ret) { + if (!k.k->type) + continue; + + struct bch_alloc_v4 _alloc; + const struct bch_alloc_v4 *alloc = bch2_alloc_to_v4(k, &_alloc); + + enum bch_data_type data_type = alloc->data_type; + + delta[data_type][0] -= 1; + delta[data_type][1] -= bch2_bucket_sectors(*alloc); + delta[data_type][2] -= bch2_bucket_sectors_fragmented(ca, *alloc); + + s64 unstriped = bch2_bucket_sectors_unstriped(*alloc); + if (unstriped) { + delta[BCH_DATA_unstriped][0] -= 1; + delta[BCH_DATA_unstriped][1] -= unstriped; + } + + keys++; + } + if (!ret) { + delta[BCH_DATA_free][0] -= (old_nbuckets - cutoff) - keys; + + for (unsigned type = 0; type < BCH_DATA_NR; type++) { + ret = bch2_disk_accounting_mod2(trans, false, delta[type], dev_data_type, .dev = ca->dev_idx, .data_type = type); + + if (ret) + break; + } + } + + ret; + })); +} + int bch2_dev_usage_remove(struct bch_fs *c, struct bch_dev *ca) { CLASS(btree_trans, trans)(c); diff --git a/fs/alloc/accounting.h b/fs/alloc/accounting.h index df358411fe..9cd500f1a4 100644 --- a/fs/alloc/accounting.h +++ b/fs/alloc/accounting.h @@ -345,6 +345,9 @@ int bch2_gc_accounting_done(struct bch_fs *); int bch2_accounting_read(struct bch_fs *); + +int bch2_dev_truncate_accounting(struct bch_fs *, struct bch_dev *, u64, u64); + int bch2_dev_usage_remove(struct bch_fs *, struct bch_dev *); int bch2_dev_usage_init(struct bch_dev *, bool); diff --git a/fs/alloc/accounting_format.h b/fs/alloc/accounting_format.h index 81df920a0a..e8b9bbf7f8 100644 --- a/fs/alloc/accounting_format.h +++ b/fs/alloc/accounting_format.h @@ -10,7 +10,7 @@ * Here, the key has considerably more structure than a typical key (bpos); an * accounting key is 'struct disk_accounting_pos', which is a union of bpos. * - * More specifically: a key is just a muliword integer (where word endianness + * More specifically: a key is just a multiword integer (where word endianness * matches native byte order), so we're treating bpos as an opaque 20 byte * integer and mapping bch_accounting_key to that. * diff --git a/fs/alloc/background.c b/fs/alloc/background.c index cccd24c9a6..7b8bf2e589 100644 --- a/fs/alloc/background.c +++ b/fs/alloc/background.c @@ -1100,6 +1100,52 @@ int bch2_alloc_read(struct bch_fs *c) return ret; } +int bch2_dev_remove_bucket_gens(struct bch_fs *c, struct bch_dev *ca, u64 cutoff) +{ + unsigned offset; + struct bpos pos = alloc_gens_pos(POS(ca->dev_idx, cutoff), &offset); + + /* cutoff on key boundary */ + if (!offset) + return bch2_btree_delete_range(c, BTREE_ID_bucket_gens, + pos, + POS(ca->dev_idx, U64_MAX), + BTREE_TRIGGER_norun); + + /* Cutoff mid-key + * + * Scope the zero-tail-gens transaction so it is cleared + * before bch2_btree_delete_range creates its own transaction. + */ + { + CLASS(btree_trans, trans)(c); + + /* zero tail gens */ + try(commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc, ({ + CLASS(btree_iter, iter)(trans, BTREE_ID_bucket_gens, pos, BTREE_ITER_intent); + struct bkey_s_c k = bkey_try(bch2_btree_iter_peek_slot(&iter)); + + try(bkey_err(k)); + + if (k.k->type == KEY_TYPE_bucket_gens) { + struct bkey_i_bucket_gens *g = + errptr_try(bch2_trans_kmalloc(trans, sizeof(*g))); + + bkey_reassemble(&g->k_i, k); + memset(&g->v.gens[offset], 0, sizeof(g->v.gens) - offset); + try(bch2_trans_update(trans, &iter, &g->k_i, 0)); + } + 0; + }))); + } + + /* delete remaining keys */ + return bch2_btree_delete_range(c, BTREE_ID_bucket_gens, + bpos_nosnap_successor(pos), + POS(ca->dev_idx, U64_MAX), + BTREE_TRIGGER_norun); +} + /* Free space/discard btree: */ int bch2_bucket_do_freespace_index(struct btree_trans *trans, @@ -1489,9 +1535,9 @@ int bch2_trigger_alloc(struct btree_trans *trans, struct btree_trigger_op op) return ret; } -/* device removal */ +/* device removal / shrinking */ -static int bch2_dev_remove_need_discard(struct bch_fs *c, struct bch_dev *ca) +static int bch2_dev_remove_need_discard(struct bch_fs *c, struct bch_dev *ca, u64 cutoff) { CLASS(btree_trans, trans)(c); unsigned dev_idx = ca->dev_idx; @@ -1499,37 +1545,39 @@ static int bch2_dev_remove_need_discard(struct bch_fs *c, struct bch_dev *ca) return for_each_btree_key_commit(trans, iter, BTREE_ID_need_discard, POS_MIN, BTREE_ITER_intent|BTREE_ITER_prefetch, k, - NULL, NULL, BCH_TRANS_COMMIT_no_enospc, ({ + NULL, NULL, BCH_WATERMARK_reclaim| + BCH_TRANS_COMMIT_no_enospc, ({ struct bpos bucket = u64_to_bucket(k.k->p.offset); - (bucket.inode == dev_idx) + (bucket.inode == dev_idx && bucket.offset >= cutoff) ? bch2_btree_delete_at(trans, &iter, BTREE_TRIGGER_norun) : 0; })); } -int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) +int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca, u64 cutoff) { - struct bpos start = POS(ca->dev_idx, 0); + struct bpos start = POS(ca->dev_idx, cutoff); struct bpos end = POS(ca->dev_idx, U64_MAX); + struct bpos bp_start = bucket_pos_to_bp_start(ca, start); + struct bpos bp_end = POS(ca->dev_idx + 1, 0); int ret; /* * We clear the LRU and need_discard btrees first so that we don't race - * with bch2_do_invalidates() and bch2_do_discards_async() + * with bch2_do_invalidates() and bch2_do_discards() */ - ret = bch2_dev_remove_lrus(c, ca) ?: - bch2_dev_remove_need_discard(c, ca) ?: + ret = bch2_dev_remove_lrus(c, ca, cutoff) ?: + bch2_dev_remove_need_discard(c, ca, cutoff) ?: bch2_btree_delete_range(c, BTREE_ID_freespace, start, end, BTREE_TRIGGER_norun) ?: - bch2_btree_delete_range(c, BTREE_ID_backpointers, start, end, - BTREE_TRIGGER_norun) ?: - bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end, + bch2_btree_delete_range(c, BTREE_ID_backpointers, bp_start, bp_end, BTREE_TRIGGER_norun) ?: + bch2_dev_remove_bucket_gens(c, ca, cutoff) ?: bch2_btree_delete_range(c, BTREE_ID_alloc, start, end, - BTREE_TRIGGER_norun) ?: - bch2_dev_usage_remove(c, ca); - bch_err_msg_dev(ca, ret, "removing dev alloc info"); + BTREE_TRIGGER_norun); + bch_err_msg_dev(ca, ret, "%s dev alloc info", + cutoff ? "truncating" : "removing"); return ret; } @@ -1712,7 +1760,7 @@ void bch2_dev_allocator_remove(struct bch_fs *c, struct bch_dev *ca) */ bch2_recalc_capacity(c); - bch2_open_buckets_stop(c, ca, false); + bch2_open_buckets_stop(c, ca, false, 0); /* * Wake up threads that were blocked on allocation, so they can notice diff --git a/fs/alloc/background.h b/fs/alloc/background.h index b60b586f9c..bb454cfac8 100644 --- a/fs/alloc/background.h +++ b/fs/alloc/background.h @@ -297,6 +297,7 @@ void bch2_bucket_gens_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_ }) int bch2_bucket_gens_init(struct bch_fs *); +int bch2_dev_remove_bucket_gens(struct bch_fs *, struct bch_dev *, u64); int bch2_alloc_read(struct bch_fs *); @@ -308,7 +309,7 @@ int bch2_alloc_key_to_dev_counters(struct btree_trans *, struct bch_dev *, const struct bch_alloc_v4 *, unsigned); int bch2_trigger_alloc(struct btree_trans *, struct btree_trigger_op); -int bch2_dev_remove_alloc(struct bch_fs *, struct bch_dev *); +int bch2_dev_remove_alloc(struct bch_fs *, struct bch_dev *, u64); void bch2_recalc_capacity(struct bch_fs *); unsigned long bch2_fs_ra_pages(struct bch_fs *); diff --git a/fs/alloc/buckets.c b/fs/alloc/buckets.c index 8dd4728a23..72327e28c2 100644 --- a/fs/alloc/buckets.c +++ b/fs/alloc/buckets.c @@ -605,6 +605,80 @@ static int __trigger_extent(struct btree_trans *trans, return 0; } +/* in-place update RECONCILE_PHYS if it is the only thing that changed */ +static int bch2_trigger_extent_reconcile_phys_update(struct btree_trans *trans, + enum btree_id btree, + unsigned level, + struct bkey_s_c old, + struct bkey_s new, + bool *handled) +{ + struct bch_fs *c = trans->c; + struct bkey_i_backpointer old_bp[BCH_REPLICAS_MAX * 2]; + struct bkey_i_backpointer new_bp[BCH_REPLICAS_MAX * 2]; + struct bkey_ptrs_c ptrs; + const union bch_extent_entry *entry; + struct extent_ptr_decoded p; + unsigned nr_old = 0, nr_new = 0; + unsigned i; + + *handled = false; + if (level) + return 0; + + ptrs = bch2_bkey_ptrs_c(old); + bkey_for_each_ptr_decode(old.k, ptrs, p, entry) { + BUG_ON(nr_old == ARRAY_SIZE(old_bp)); + bch2_extent_ptr_to_bp(c, btree, level, old, p, entry, &old_bp[nr_old++]); + } + + ptrs = bch2_bkey_ptrs_c(new.s_c); + bkey_for_each_ptr_decode(new.k, ptrs, p, entry) { + BUG_ON(nr_new == ARRAY_SIZE(new_bp)); + bch2_extent_ptr_to_bp(c, btree, level, new.s_c, p, entry, &new_bp[nr_new++]); + } + + if (nr_old != nr_new) + return 0; + + for (i = 0; i < nr_old; i++) { + struct bch_backpointer old_v = old_bp[i].v; + struct bch_backpointer new_v = new_bp[i].v; + + SET_BACKPOINTER_RECONCILE_PHYS(&old_v, 0); + SET_BACKPOINTER_RECONCILE_PHYS(&new_v, 0); + + if (!bpos_eq(old_bp[i].k.p, new_bp[i].k.p) || + memcmp(&old_v, &new_v, sizeof(old_v))) + return 0; + } + + for (i = 0; i < nr_old; i++) { + unsigned old_phys = BACKPOINTER_RECONCILE_PHYS(&old_bp[i].v); + unsigned new_phys = BACKPOINTER_RECONCILE_PHYS(&new_bp[i].v); + + if (!old_phys && !new_phys) + continue; + + if (old_phys) + try(bch2_btree_bit_mod_buffered(trans, + reconcile_work_phys_btree[old_phys], + old_bp[i].k.p, false)); + + if (new_phys) + try(bch2_btree_bit_mod_buffered(trans, + reconcile_work_phys_btree[new_phys], + new_bp[i].k.p, true)); + + try(bch2_trans_update_buffered(trans, + backpointer_btree(&new_bp[i].v), + &new_bp[i].k_i)); + } + + *handled = true; + return 0; +} + int bch2_trigger_extent(struct btree_trans *trans, struct btree_trigger_op op) { struct bkey_ptrs_c new_ptrs = bch2_bkey_ptrs_c(op.new.s_c); @@ -612,7 +686,15 @@ int bch2_trigger_extent(struct btree_trans *trans, struct btree_trigger_op op) unsigned new_ptrs_bytes = (void *) new_ptrs.end - (void *) new_ptrs.start; unsigned old_ptrs_bytes = (void *) old_ptrs.end - (void *) old_ptrs.start; - /* if pointers aren't changing - nothing to do: */ + /* optimization for in-place updates to reconcile_phys to avoid delete-insert churn */ + if (op.level == 0 && + op.new.k->u64s == op.old.k->u64s) { + bool handled; + + try(bch2_trigger_extent_reconcile_phys_update(trans, op.btree, op.level, op.old, op.new, &handled)); + if (handled) + return bch2_trigger_extent_reconcile(trans, op); + } if (new_ptrs_bytes == old_ptrs_bytes && !memcmp(new_ptrs.start, old_ptrs.start, diff --git a/fs/alloc/buckets.h b/fs/alloc/buckets.h index 00feb3ec2a..b8f5594d90 100644 --- a/fs/alloc/buckets.h +++ b/fs/alloc/buckets.h @@ -246,7 +246,8 @@ static inline struct bch_dev_usage_full bch2_dev_usage_full_read(struct bch_dev void bch2_dev_usage_to_text(struct printbuf *, struct bch_dev *, struct bch_dev_usage_full *); -static inline u64 bch2_dev_buckets_reserved(struct bch_dev *ca, enum bch_watermark watermark) +static inline u64 __bch2_dev_buckets_reserved(u64 nbuckets, unsigned nr_btree_reserve, + enum bch_watermark watermark) { s64 reserved = 0; @@ -254,16 +255,16 @@ static inline u64 bch2_dev_buckets_reserved(struct bch_dev *ca, enum bch_waterma case BCH_WATERMARK_NR: BUG(); case BCH_WATERMARK_stripe: - reserved += ca->mi.nbuckets >> 6; + reserved += nbuckets >> 6; fallthrough; case BCH_WATERMARK_normal: - reserved += ca->mi.nbuckets >> 6; + reserved += nbuckets >> 6; fallthrough; case BCH_WATERMARK_copygc: - reserved += ca->nr_btree_reserve; + reserved += nr_btree_reserve; fallthrough; case BCH_WATERMARK_btree: - reserved += ca->nr_btree_reserve; + reserved += nr_btree_reserve; fallthrough; case BCH_WATERMARK_btree_copygc: case BCH_WATERMARK_reclaim: @@ -274,14 +275,29 @@ static inline u64 bch2_dev_buckets_reserved(struct bch_dev *ca, enum bch_waterma return reserved; } +static inline u64 bch2_dev_buckets_reserved(struct bch_dev *ca, enum bch_watermark watermark) +{ + return __bch2_dev_buckets_reserved(ca->mi.nbuckets, ca->nr_btree_reserve, watermark); +} + static inline u64 __dev_buckets_free(struct bch_dev *ca, struct bch_dev_usage usage, enum bch_watermark watermark) { + u64 free = usage.buckets[BCH_DATA_free]; + u64 nbuckets = READ_ONCE(ca->mi.nbuckets); + + /* buckets in shrink tail are not allocatable and thus shouldn't count as free */ + u64 tail_free = atomic64_read(&ca->shrinking_tail_free); + if (unlikely(tail_free)) + free -= min(free, tail_free); + if (unlikely(bch2_dev_is_shrinking(ca))) + nbuckets = bch2_dev_resize_target(ca); + return max_t(s64, 0, - usage.buckets[BCH_DATA_free]- + free - ca->nr_open_buckets - - bch2_dev_buckets_reserved(ca, watermark)); + __bch2_dev_buckets_reserved(nbuckets, ca->nr_btree_reserve, watermark)); } static inline u64 dev_buckets_free(struct bch_dev *ca, diff --git a/fs/alloc/discard.c b/fs/alloc/discard.c index 8616d3f214..f587192c96 100644 --- a/fs/alloc/discard.c +++ b/fs/alloc/discard.c @@ -454,6 +454,16 @@ static void calculate_discard_sectors_to_release(struct btree_trans *trans) s->r.flush_journal = true; } +static bool bch2_discard_blocked_by_resize(struct bch_fs *c, struct bpos bucket) +{ + guard(rcu)(); + struct bch_dev *ca = bch2_dev_rcu_noerror(c, bucket.inode); + + return ca && + bch2_dev_is_shrinking(ca) && + bucket.offset >= bch2_dev_resize_target(ca); +} + static void bch2_do_discards(struct bch_fs *c) { struct bch_fs_discards *d = &c->discards; @@ -472,6 +482,10 @@ static void bch2_do_discards(struct bch_fs *c) /* * Iterate need_discard btree (sorted by journal_seq). * Stop when we hit a seq beyond rewind_seq_ondisk. + * + * Drop need_discard iterator before we update alloc and + * commit to avoid deadlock against alloc/freespace updates + * when removing the corresponding index entry */ ret = for_each_btree_key(trans, iter, BTREE_ID_need_discard, POS_MIN, 0, k, ({ @@ -483,6 +497,15 @@ static void bch2_do_discards(struct bch_fs *c) c->journal.flushed_seq_ondisk + 1)) break; + /* + * Leave buckets in the shrink tail queued, as the + * alloc update can deadlock with reconcile. + */ + if (bch2_discard_blocked_by_resize(c, bucket)) { + s->pos = iter.pos; + continue; + } + if (!bpos_eq(s->pos, iter.pos)) s->seen += bucket_size; s->pos = iter.pos; diff --git a/fs/alloc/disk_groups.h b/fs/alloc/disk_groups.h index 225ea65e31..02917218c7 100644 --- a/fs/alloc/disk_groups.h +++ b/fs/alloc/disk_groups.h @@ -3,6 +3,7 @@ #define _BCACHEFS_DISK_GROUPS_H #include "disk_groups_types.h" +#include "sb/members.h" extern const struct bch_sb_field_ops bch_sb_field_ops_disk_groups; @@ -80,6 +81,22 @@ static inline bool bch2_target_accepts_data(struct bch_fs *c, return !bitmap_empty(rw_devs.d, BCH_SB_MEMBERS_MAX); } +static inline bool bch2_target_has_non_shrinking_dev(struct bch_fs *c, + enum bch_data_type data_type, + u16 target) +{ + struct bch_devs_mask devs = target_rw_devs(c, data_type, target); + + guard(rcu)(); + unsigned i; + for_each_set_bit(i, devs.d, BCH_SB_MEMBERS_MAX) { + struct bch_dev *ca = bch2_dev_rcu_noerror(c, i); + if (ca && !bch2_dev_is_shrinking(ca)) + return true; + } + return false; +} + bool bch2_dev_in_target_rcu(struct bch_fs *, unsigned, unsigned); static inline bool bch2_dev_in_target(struct bch_fs *c, unsigned dev, unsigned target) diff --git a/fs/alloc/foreground.c b/fs/alloc/foreground.c index 5e4f6634e0..26f39dccf8 100644 --- a/fs/alloc/foreground.c +++ b/fs/alloc/foreground.c @@ -279,7 +279,9 @@ static struct open_bucket *__try_alloc_bucket(struct bch_fs *c, if (unlikely(is_superblock_bucket(c, ca, bucket))) return NULL; - if (unlikely(bch2_bucket_nouse(ca, bucket))) { + if (unlikely(bch2_bucket_nouse(ca, bucket) || + (bch2_dev_is_shrinking(ca) && + bucket >= bch2_dev_resize_target(ca)))) { req->counters.skipped_nouse++; return NULL; } @@ -1364,20 +1366,21 @@ static int bucket_alloc_cached(struct btree_trans *trans, struct alloc_request * * @ec: if true, we're shutting down erasure coding and killing all ec * open_buckets * otherwise, return true + * @tail_cutoff: if != 0 only drop buckets after the cutoff * Returns: true if we should kill this open_bucket * * We're killing open_buckets because we're shutting down a device, erasure * coding, or the entire filesystem - check if this open_bucket matches: */ static bool should_drop_bucket(struct open_bucket *ob, struct bch_fs *c, - struct bch_dev *ca, bool ec) + struct bch_dev *ca, bool ec, u64 tail_cutoff) { struct bch_fs_allocator *a = &c->allocator; if (ec) { return ob->ec != NULL; } else if (ca) { - bool drop = ob->dev == ca->dev_idx; + bool drop = dev_and_region_matches(ob, ca, tail_cutoff); if (!drop && ob->ec) { guard(mutex)(&ob->ec->lock); @@ -1388,8 +1391,12 @@ static bool should_drop_bucket(struct open_bucket *ob, struct bch_fs *c, continue; struct open_bucket *ob2 = a->open_buckets + ob->ec->blocks[i]; - drop |= ob2->dev == ca->dev_idx; + drop |= dev_and_region_matches(ob2, ca, tail_cutoff); } + + if (!drop && ob->ec->have_old_stripe) + drop = stripe_dev_and_region_matches(c, ca, + &ob->ec->old_stripe.key.v, tail_cutoff); } return drop; @@ -1399,7 +1406,7 @@ static bool should_drop_bucket(struct open_bucket *ob, struct bch_fs *c, } static void bch2_writepoint_stop(struct bch_fs *c, struct bch_dev *ca, - bool ec, struct write_point *wp) + bool ec, struct write_point *wp, u64 tail_cutoff) { struct open_buckets ptrs = { .nr = 0 }; struct open_bucket *ob; @@ -1407,26 +1414,27 @@ static void bch2_writepoint_stop(struct bch_fs *c, struct bch_dev *ca, guard(mutex)(&wp->lock); open_bucket_for_each(c, &wp->ptrs, ob, i) - if (should_drop_bucket(ob, c, ca, ec)) + if (should_drop_bucket(ob, c, ca, ec, tail_cutoff)) bch2_open_bucket_put(c, ob); else ob_push(c, &ptrs, ob); wp->ptrs = ptrs; } +/* stop open buckets on @ca, if tail_cutoff isn't zero, only after the cutoff */ void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *ca, - bool ec) + bool ec, u64 tail_cutoff) { struct bch_fs_allocator *a = &c->allocator; unsigned i; - /* Next, close write points that point to this device... */ + /* Next, close write points that point to this device or the to-be-shrunk region... */ for (i = 0; i < ARRAY_SIZE(a->write_points); i++) - bch2_writepoint_stop(c, ca, ec, &a->write_points[i]); + bch2_writepoint_stop(c, ca, ec, &a->write_points[i], tail_cutoff); - bch2_writepoint_stop(c, ca, ec, &c->copygc.write_point); - bch2_writepoint_stop(c, ca, ec, &a->reconcile_write_point); - bch2_writepoint_stop(c, ca, ec, &a->btree_write_point); + bch2_writepoint_stop(c, ca, ec, &c->copygc.write_point, tail_cutoff); + bch2_writepoint_stop(c, ca, ec, &a->reconcile_write_point, tail_cutoff); + bch2_writepoint_stop(c, ca, ec, &a->btree_write_point, tail_cutoff); scoped_guard(mutex, &c->btree.reserve_cache.lock) while (c->btree.reserve_cache.nr) { @@ -1442,7 +1450,7 @@ void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *ca, struct open_bucket *ob = a->open_buckets + a->open_buckets_partial[i]; - if (should_drop_bucket(ob, c, ca, ec)) { + if (should_drop_bucket(ob, c, ca, ec, tail_cutoff)) { --a->open_buckets_partial_nr; swap(a->open_buckets_partial[i], a->open_buckets_partial[a->open_buckets_partial_nr]); @@ -1460,7 +1468,7 @@ void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *ca, } } - bch2_ec_stop_dev(c, ca); + bch2_ec_stop_dev_cutoff(c, ca, tail_cutoff); } static inline struct hlist_head *writepoint_hash(struct bch_fs_allocator *a, diff --git a/fs/alloc/foreground.h b/fs/alloc/foreground.h index 0e4710eeb3..59bf9a35cf 100644 --- a/fs/alloc/foreground.h +++ b/fs/alloc/foreground.h @@ -381,6 +381,25 @@ enum bch_write_flags; int bch2_bucket_alloc_set_trans(struct btree_trans *, struct alloc_request *, struct dev_stripe_state *); +/* Returns whether the @open_buckets device matches @ca and whether the @open_bucket falls behind the cutoff */ +static inline bool dev_and_region_matches(struct open_bucket *ob, struct bch_dev *ca, u64 tail_cutoff) { + return ob->dev == ca->dev_idx && ob->bucket >= tail_cutoff; +} + +/* Whether any block of @v references @ca's region at/after @tail_cutoff */ +static inline bool stripe_dev_and_region_matches(struct bch_fs *c, struct bch_dev *ca, + struct bch_stripe *v, u64 tail_cutoff) +{ + guard(rcu)(); + for (unsigned i = 0; i < v->nr_blocks; i++) { + struct bch_dev *oca = bch2_dev_rcu_noerror(c, v->ptrs[i].dev); + if (oca && oca->dev_idx == ca->dev_idx && + PTR_BUCKET_NR(oca, &v->ptrs[i]) >= tail_cutoff) + return true; + } + return false; +} + int bch2_alloc_sectors_req(struct btree_trans *, struct alloc_request *, struct write_point_specifier, struct write_point **); @@ -503,7 +522,7 @@ void bch2_alloc_sectors_append_ptrs(struct bch_fs *, struct write_point *, struct bkey_i *, unsigned, bool); void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *); -void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *, bool); +void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *, bool, u64 tail_cutoff); static inline struct write_point_specifier writepoint_hashed(unsigned long v) { diff --git a/fs/alloc/lru.c b/fs/alloc/lru.c index 08bf0977af..c9eca0c3cb 100644 --- a/fs/alloc/lru.c +++ b/fs/alloc/lru.c @@ -2,6 +2,7 @@ #include "bcachefs.h" +#include "alloc/backpointers.h" #include "alloc/background.h" #include "alloc/lru.h" @@ -118,7 +119,8 @@ static struct bbpos lru_pos_to_bp(struct bkey_s_c lru_k) } } -int bch2_dev_remove_lrus(struct bch_fs *c, struct bch_dev *ca) + +int bch2_dev_remove_lrus(struct bch_fs *c, struct bch_dev *ca, u64 cutoff) { CLASS(btree_trans, trans)(c); int ret = bch2_btree_write_buffer_flush_sync(trans) ?: @@ -126,7 +128,7 @@ int bch2_dev_remove_lrus(struct bch_fs *c, struct bch_dev *ca) BTREE_ID_lru, POS_MIN, BTREE_ITER_prefetch, k, ({ struct bbpos bp = lru_pos_to_bp(k); - bp.btree == BTREE_ID_alloc && bp.pos.inode == ca->dev_idx + bp.btree == BTREE_ID_alloc && bp.pos.inode == ca->dev_idx && bp.pos.offset >= cutoff ? (bch2_btree_delete_at(trans, &iter, 0) ?: bch2_trans_commit(trans, NULL, NULL, 0)) : 0; diff --git a/fs/alloc/lru.h b/fs/alloc/lru.h index e3074fde3d..7f19b43914 100644 --- a/fs/alloc/lru.h +++ b/fs/alloc/lru.h @@ -81,7 +81,7 @@ static inline int bch2_lru_change(struct btree_trans *trans, : 0; } -int bch2_dev_remove_lrus(struct bch_fs *, struct bch_dev *); +int bch2_dev_remove_lrus(struct bch_fs *, struct bch_dev *, u64); struct wb_maybe_flush; int bch2_lru_check_set(struct btree_trans *, u16, u64, u64, struct bkey_s_c, diff --git a/fs/bcachefs.h b/fs/bcachefs.h index 5ad2d4526f..cd1d6e1ce8 100644 --- a/fs/bcachefs.h +++ b/fs/bcachefs.h @@ -33,6 +33,8 @@ #define race_fault(...) dynamic_fault("bcachefs:race") +struct task_struct; + #include #include #include @@ -577,7 +579,14 @@ struct bch_dev { unsigned nr_partial_buckets; unsigned nr_btree_reserve; + atomic64_t shrinking_tail_free; + struct work_struct invalidate_work; + spinlock_t resize_lock; + u64 resize_seq; + wait_queue_head_t resize_wait; + int resize_status; + struct task_struct *resize_thread; struct work_struct discard_fast_work; darray_u64 discard_fast; diff --git a/fs/bcachefs_format.h b/fs/bcachefs_format.h index 78a4db705c..5685cdb184 100644 --- a/fs/bcachefs_format.h +++ b/fs/bcachefs_format.h @@ -1129,7 +1129,10 @@ LE64_BITMASK(BCH_SB_EXT_BTREE_CACHE_SHRINKER_SEEKS, "discard eligibility checks", "2026-03") \ x(per_dev_fragmentation_lru, BCH_VERSION(1, 39), \ "Per-device bucket fragmentation LRUs, so copygc can reason " \ - "about fragmentation per device", "2026-07") + "about fragmentation per device", "2026-07") \ + x(snapshot_nr_keys, BCH_VERSION(1, 40), \ + "Per-snapshot key-count accounting, so snapshot deletion can tell " \ + "an empty node from one whose keys would be stranded", "2026-07") enum bcachefs_metadata_version { bcachefs_metadata_version_min = 9, diff --git a/fs/btree/interior.c b/fs/btree/interior.c index 55ed70bc6d..a956906a3f 100644 --- a/fs/btree/interior.c +++ b/fs/btree/interior.c @@ -183,6 +183,14 @@ static const char * const bch2_btree_update_modes[] = { NULL }; +static unsigned btree_alloc_target(struct bch_fs *c, unsigned target) +{ + target = target ?: c->opts.metadata_target ?: c->opts.foreground_target; + return target && bch2_target_has_non_shrinking_dev(c, BCH_DATA_btree, target) + ? target + : 0; +} + static __cold void bch2_btree_update_to_text(struct printbuf *, struct btree_update *); static int bch2_btree_insert_node(struct btree_update *, struct btree_trans *, @@ -1558,17 +1566,16 @@ bch2_btree_update_start(struct btree_trans *trans, btree_path_idx_t path_idx, goto err; struct bch_devs_list devs_have = (struct bch_devs_list) { 0 }; + unsigned alloc_target = btree_alloc_target(c, target); req = alloc_request_get(trans, - target ?: - c->opts.metadata_target ?: - c->opts.foreground_target, - false, - &devs_have, - as->disk_res.nr_replicas, - as->disk_res.nr_replicas, - watermark, - write_flags, - NULL); + alloc_target, + false, + &devs_have, + as->disk_res.nr_replicas, + as->disk_res.nr_replicas, + watermark, + write_flags, + NULL); ret = PTR_ERR_OR_ZERO(req); if (ret) goto err; diff --git a/fs/data/copygc.c b/fs/data/copygc.c index 04f1dbe983..67305b08d3 100644 --- a/fs/data/copygc.c +++ b/fs/data/copygc.c @@ -571,6 +571,10 @@ bool bch2_copygc_can_make_progress(struct bch_dev *ca) * Caller must hold mark_lock (read), for the dev_leaving accounting read - * and must take it outside any rcu read section, mark_lock can block. * + * Free buckets in the shrink tail aren't allocatable + * and don't count, and while a shrink is in progress the device counts as + * target-sized. + * * The allowance at the limit - when the device is full - is the space we * reserved in bch2_recalc_capacity; we can't have more than that amount of * disk space stranded due to fragmentation and store everything we have @@ -600,9 +604,18 @@ s64 bch2_copygc_dev_wait_amount(struct bch_dev *ca) bch2_accounting_mem_read_locked(c, disk_accounting_pos_to_bpos(&pos), &leaving, 1); leaving = max(0LL, leaving); + /* + * Free buckets in the shrink tail aren't allocatable + */ + u64 free_buckets = usage.buckets[BCH_DATA_free]; + u64 nbuckets = bch2_dev_resize_target(ca); + u64 tail_free = atomic64_read(&ca->shrinking_tail_free); + if (unlikely(tail_free)) + free_buckets -= min(free_buckets, tail_free); + /* Don't start until less than 20% of the device is free: */ - s64 free = usage.buckets[BCH_DATA_free] * ca->mi.bucket_size + leaving; - s64 wait = free * 5 - ca->mi.nbuckets * ca->mi.bucket_size; + s64 free = free_buckets * ca->mi.bucket_size + leaving; + s64 wait = free * 5 - nbuckets * ca->mi.bucket_size; if (wait > 0) return wait; diff --git a/fs/data/ec/create.c b/fs/data/ec/create.c index 14a9f13ff8..eea957386e 100644 --- a/fs/data/ec/create.c +++ b/fs/data/ec/create.c @@ -295,13 +295,19 @@ static int stripe_update_extent(struct btree_trans *trans, if (p.ec.idx == new_stripe->k.p.offset) return 0; - if (old_stripe == new_stripe || - p.ec.idx != old_stripe->k.p.offset) { - CLASS(printbuf, buf)(); - ret_log_fsck_err(trans, stripe_update_stale_stripe_ptr, - "dropping stale stripe pointer (idx %llu) while updating extent\n%s", - (u64) p.ec.idx, - (bch2_bkey_val_to_text(&buf, c, k), buf.buf)); + if (p.ec.idx != old_stripe->k.p.offset) { + /* + * The extent references a stripe that no longer owns this block. + * Let the migration below handle it, and just log it as an fsck + * error for check_allocations to verify. + */ + CLASS(bch_log_msg_ratelimited, msg)(c); + prt_printf(&msg.m, "dropping stale stripe pointer (idx %llu) while updating extent\n", + (u64) p.ec.idx); + bch2_bkey_val_to_text(&msg.m, c, k); + prt_newline(&msg.m); + + bch2_count_fsck_err(c, stripe_update_stale_stripe_ptr, &msg.m); } } @@ -436,6 +442,56 @@ static int stripe_update_bucket(struct btree_trans *trans, stats.nr_done, stats.sectors_done); })); + /* + * count deltas are unreliable under concurrency, leading to phantom counts + * that are never drained. No extent references (old_stripe, old_blocknr) + * any more, so just zero the count. + * + * Safe because the scan covers the whole bucket, and an extent referencing + * the old stripe block always has a backpointer in this bucket, and will + * thus be removed. + * + * TODO: a cleaner / possibly more robust solution would be solving the + * concurrency unrealiability directly + */ + if (old_stripe != new_stripe) { + lockrestart_do(trans, ({ + struct bkey_i_stripe *s = bch2_bkey_get_mut_typed(trans, + BTREE_ID_stripes, POS(0, old_stripe->k.p.offset), + BTREE_ITER_cached, stripe); + int ret = PTR_ERR_OR_ZERO(s); + + if (ret) { + if (bch2_err_matches(ret, ENOENT)) + ret = 0; /* stripe already deleted */ + } else { + stripe_blockcount_set(&s->v, old_blocknr, 0); + ret = bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); + } + ret; + })); + + /* + * The block count also has the same concurrency / key-cached RMW unreliability, + * so set it manually here as well. + */ + lockrestart_do(trans, ({ + struct bkey_i_stripe *s = bch2_bkey_get_mut_typed(trans, + BTREE_ID_stripes, POS(0, new_stripe->k.p.offset), + BTREE_ITER_cached, stripe); + int ret = PTR_ERR_OR_ZERO(s); + + if (ret) { + if (bch2_err_matches(ret, ENOENT)) + ret = 0; /* stripe already deleted - nothing to fix */ + } else { + stripe_blockcount_set(&s->v, new_blocknr, stats.sectors_done); + ret = bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc); + } + ret; + })); + } + return 0; } @@ -543,9 +599,22 @@ static void zero_out_rest_of_ec_bucket(struct bch_fs *c, void bch2_ec_stripe_new_free(struct bch_fs *c, struct ec_stripe_new *s) { + bool had_old_stripe = s->old_stripe_handle.idx != 0; + bch2_stripe_new_buckets_del(c, s); bch2_stripe_handle_put(c, &s->new_stripe_handle); bch2_stripe_handle_put(c, &s->old_stripe_handle); + + /* + * If this create had an old stripe, it may have become empty while we + * held it open: the trigger defers the delete to the LRU work, which + * skips open stripes, and nothing re-queues it once we release the + * handle. Re-trigger it now that the stripe is closed so the deferred + * delete actually runs. + */ + if (had_old_stripe) + bch2_do_stripe_deletes(c); + kfree(s); } @@ -1453,13 +1522,17 @@ static bool may_reuse_stripe(struct bch_fs *c, old->nr_redundant != new->new_stripe.key.v.nr_redundant) return false; + for (unsigned i = 0; i < old->nr_blocks; i++) + if (bch2_ptr_bad_or_evacuating(c, &old->ptrs[i])) + return false; + struct bch_devs_mask devs_may_alloc = new->devs; unsigned nr_data = old->nr_blocks - old->nr_redundant; unsigned live_data = 0; for_each_data_block(i, nr_data) if (stripe_blockcount_get(old, i)) { - if (!bch2_dev_bad_or_evacuating(c, old->ptrs[i].dev)) + if (!bch2_ptr_bad_or_evacuating(c, &old->ptrs[i])) __clear_bit(old->ptrs[i].dev, devs_may_alloc.d); live_data++; } @@ -1563,7 +1636,7 @@ static void init_new_stripe_from_old(struct bch_fs *c, struct ec_stripe_new *s, for_each_data_block(i, old_nr_data) { if (stripe_blockcount_get(old_v, i)) { - if (!bch2_dev_bad_or_evacuating(c, old_v->ptrs[i].dev)) + if (!bch2_ptr_bad_or_evacuating(c, &old_v->ptrs[i])) __set_bit(s->old_blocks_nr, s->blocks_gotten); else __set_bit(s->old_blocks_nr, s->blocks_moving); @@ -1571,6 +1644,14 @@ static void init_new_stripe_from_old(struct bch_fs *c, struct ec_stripe_new *s, new_v->ptrs[s->old_blocks_nr] = old_v->ptrs[i]; + /* + * Per-extent key-cache RMW deltas aren't reliable under + * concurrency, so just set the new stripe's blockcount to + * the old stripe's value directly. + */ + stripe_blockcount_set(new_v, s->old_blocks_nr, + stripe_blockcount_get(old_v, i)); + s->old_block_map[s->old_blocks_nr++] = i; BUG_ON(s->old_blocks_nr + !repair > new_nr_data); } @@ -2044,7 +2125,7 @@ struct ec_stripe_head *bch2_ec_stripe_head_get(struct btree_trans *trans, static bool stripe_degraded(struct bch_fs *c, const struct bch_stripe *s) { for (unsigned i = 0; i < s->nr_blocks; i++) - if (bch2_dev_bad_or_evacuating(c, s->ptrs[i].dev)) + if (bch2_ptr_bad_or_evacuating(c, &s->ptrs[i])) return true; return false; } @@ -2076,8 +2157,21 @@ int bch2_stripe_repair(struct moving_context *ctxt, for_each_data_block(i, nr_data) nr_live_data_blocks += stripe_blockcount_get(old_s, i) != 0; - if (!nr_live_data_blocks) + if (!nr_live_data_blocks) { + /* + * Nothing to repair - the stripe is empty, so it should just be + * deleted. Clear needs_reconcile so reconcile stops re-queueing this work. + */ + if (old_s->needs_reconcile) { + struct bkey_i_stripe *n = + errptr_try(bch2_bkey_make_mut_typed(trans, iter, &s.s_c, 0, stripe)); + n->v.needs_reconcile = 0; + + try(bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc)); + } + return 0; + } struct bch_devs_mask devs; bch2_disk_label_ec_devs(c, old_s->disk_label, &devs, le16_to_cpu(old_s->sectors)); diff --git a/fs/data/ec/init.c b/fs/data/ec/init.c index 9234998267..5d783e9602 100644 --- a/fs/data/ec/init.c +++ b/fs/data/ec/init.c @@ -169,7 +169,7 @@ int bch2_dev_remove_stripes(struct bch_fs *c, unsigned dev_idx, /* startup/shutdown */ -static bool should_cancel_stripe(struct bch_fs *c, struct ec_stripe_new *s, struct bch_dev *ca) +static bool should_cancel_stripe(struct bch_fs *c, struct ec_stripe_new *s, struct bch_dev *ca, u64 tail_cutoff) { if (!ca) return true; @@ -177,10 +177,10 @@ static bool should_cancel_stripe(struct bch_fs *c, struct ec_stripe_new *s, stru struct bch_stripe *v = &s->new_stripe.key.v; for (unsigned i = 0; i < v->nr_blocks; i++) { - /* Freshly allocated block - check the open_bucket's device: */ + /* Freshly allocated block - check the open_bucket's device and region: */ if (s->blocks[i]) { struct open_bucket *ob = c->allocator.open_buckets + s->blocks[i]; - if (ob->dev == ca->dev_idx) + if (dev_and_region_matches(ob, ca, tail_cutoff)) return true; } @@ -195,29 +195,38 @@ static bool should_cancel_stripe(struct bch_fs *c, struct ec_stripe_new *s, stru return true; } + if (s->have_old_stripe && + stripe_dev_and_region_matches(c, ca, &s->old_stripe.key.v, tail_cutoff)) + return true; + return false; } -static void __bch2_ec_stop(struct bch_fs *c, struct bch_dev *ca) +static void __bch2_ec_stop(struct bch_fs *c, struct bch_dev *ca, u64 tail_cutoff) { struct ec_stripe_head *h; guard(mutex)(&c->ec.stripe_head_lock); list_for_each_entry(h, &c->ec.stripe_head_list, list) { guard(mutex)(&h->lock); - if (h->s && should_cancel_stripe(c, h->s, ca)) + if (h->s && should_cancel_stripe(c, h->s, ca, tail_cutoff)) bch2_ec_stripe_new_cancel(c, h, -BCH_ERR_erofs_no_writes); } } +void bch2_ec_stop_dev_cutoff(struct bch_fs *c, struct bch_dev *ca, u64 tail_cutoff) +{ + __bch2_ec_stop(c, ca, tail_cutoff); +} + void bch2_ec_stop_dev(struct bch_fs *c, struct bch_dev *ca) { - __bch2_ec_stop(c, ca); + __bch2_ec_stop(c, ca, 0); } void bch2_fs_ec_stop(struct bch_fs *c) { - __bch2_ec_stop(c, NULL); + __bch2_ec_stop(c, NULL, 0); } static bool bch2_fs_ec_flush_done(struct bch_fs *c) diff --git a/fs/data/ec/init.h b/fs/data/ec/init.h index 0e7c3ba000..d2bf3f4687 100644 --- a/fs/data/ec/init.h +++ b/fs/data/ec/init.h @@ -7,6 +7,7 @@ int bch2_invalidate_stripe_to_dev(struct btree_trans *, struct btree_iter *, unsigned, struct printbuf *, bool *); int bch2_dev_remove_stripes(struct bch_fs *, unsigned, unsigned, struct printbuf *); +void bch2_ec_stop_dev_cutoff(struct bch_fs *, struct bch_dev *, u64); void bch2_ec_stop_dev(struct bch_fs *, struct bch_dev *); void bch2_fs_ec_stop(struct bch_fs *); void bch2_fs_ec_flush(struct bch_fs *); @@ -23,4 +24,3 @@ int bch2_bucket_nr_stripes(struct btree_trans *, struct bpos); int bch2_check_stripe_refs(struct btree_trans *); #endif /* _BCACHEFS_DATA_EC_INIT_H */ - diff --git a/fs/data/extents.c b/fs/data/extents.c index a2d7da1aae..4f9baf5ff0 100644 --- a/fs/data/extents.c +++ b/fs/data/extents.c @@ -1038,7 +1038,7 @@ int bch2_bkey_durability(struct btree_trans *trans, struct bkey_s_c k, struct bk * evacuating device reads as durability 0 for the minimum. */ ret->acct += desired; - unsigned d_min = !p.has_ec && bch2_dev_bad_or_evacuating(c, p.ptr.dev) + unsigned d_min = !p.has_ec && bch2_ptr_bad_or_evacuating(c, &p.ptr) ? 0 : desired; ret->min_durability = min(ret->min_durability, d_min); } @@ -1369,12 +1369,12 @@ bool bch2_bkey_in_target(struct bch_fs *c, struct bkey_s_c k, unsigned target) return true; } -bool bch2_bkey_has_dev_bad_or_evacuating(struct bch_fs *c, struct bkey_s_c k) +bool bch2_bkey_has_ptr_bad_or_evacuating(struct bch_fs *c, struct bkey_s_c k) { guard(rcu)(); struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); bkey_for_each_ptr(ptrs, ptr) - if (bch2_dev_bad_or_evacuating_rcu(c, ptr->dev)) + if (bch2_ptr_bad_or_evacuating_rcu(c, ptr)) return true; return false; } @@ -1519,6 +1519,9 @@ static bool maybe_drop_cached_ptr(struct bch_fs *c, struct bch_inode_opts *opts, return drop_cached_pointer_trace(c, k, ptr, "pointer is stale"); if (!ca || ca->mi.state == BCH_MEMBER_STATE_evacuating) return drop_cached_pointer_trace(c, k, ptr, "device bad or evacuating"); + if (bch2_dev_is_shrinking(ca) && + bch2_dev_resize_target(ca) <= PTR_BUCKET_NR(ca, ptr)) + return drop_cached_pointer_trace(c, k, ptr, "past shrink cutoff"); unsigned target = opts->promote_target ?: opts->foreground_target; if (target && !bch2_dev_in_target_rcu(c, ptr->dev, target)) diff --git a/fs/data/extents.h b/fs/data/extents.h index 5ffef8cd2d..f2e38ce8a2 100644 --- a/fs/data/extents.h +++ b/fs/data/extents.h @@ -688,7 +688,7 @@ bool bch2_bkey_devs_rw(struct bch_fs *, struct bkey_s_c); bool bch2_bkey_has_target(struct bch_fs *, struct bkey_s_c, unsigned); bool bch2_bkey_in_target(struct bch_fs *, struct bkey_s_c, unsigned); -bool bch2_bkey_has_dev_bad_or_evacuating(struct bch_fs *, struct bkey_s_c); +bool bch2_bkey_has_ptr_bad_or_evacuating(struct bch_fs *, struct bkey_s_c); void bch2_bkey_extent_entry_drop_s(const struct bch_fs *, struct bkey_s, union bch_extent_entry *); void bch2_bkey_extent_entry_drop(const struct bch_fs *, struct bkey_i *, union bch_extent_entry *); diff --git a/fs/data/move.c b/fs/data/move.c index dd3c1d98e0..5e7d250aab 100644 --- a/fs/data/move.c +++ b/fs/data/move.c @@ -389,7 +389,7 @@ int bch2_move_extent(struct moving_context *ctxt, ret = bch2_btree_node_rewrite_pos(trans, iter->btree_id, level, k.k->p, data_opts->target, - data_opts->commit_flags, + commit_flags, data_opts->write_flags); /* ENOMEM becomes a restart below and gets retried - not an outcome */ diff --git a/fs/data/reconcile/trigger.c b/fs/data/reconcile/trigger.c index f7fedde42d..96aa95ae19 100644 --- a/fs/data/reconcile/trigger.c +++ b/fs/data/reconcile/trigger.c @@ -498,7 +498,7 @@ static int bch2_bkey_needs_reconcile(struct btree_trans *trans, struct bkey_s_c incompressible |= p.crc.compression_type == BCH_COMPRESSION_TYPE_incompressible; unwritten |= p.ptr.unwritten; - bool evacuating = bch2_dev_bad_or_evacuating(c, p.ptr.dev) && !p.has_ec; + bool evacuating = bch2_ptr_bad_or_evacuating(c, &p.ptr) && !p.has_ec; if (!poisoned && !btree && @@ -539,8 +539,14 @@ static int bch2_bkey_needs_reconcile(struct btree_trans *trans, struct bkey_s_c if (k.k->type == KEY_TYPE_stripe) { *ret = r; - return (r.need_rb & BIT(BCH_RECONCILE_data_replicas)) && - !bkey_s_c_to_stripe(k).v->needs_reconcile; + /* + * A repair pass can lose the stripe's work item (e.g. a stale + * needs_reconcile read racing the scan's flag update), and unless + * the scan re-queues it the stripe stays flagged-but-never-repaired + * forever. + * TODO: a cleaner fix would be avoiding the race in the first place + */ + return r.need_rb & BIT(BCH_RECONCILE_data_replicas); } if (unwritten || incompressible) @@ -729,6 +735,16 @@ static int new_needs_rb_allowed(struct btree_trans *trans, return 0; } + /* + * Shrink/evacuation can mark an extent under-replicated as soon as one + * pointer falls onto a bad/evacuating location, before the device scan + * has rewritten the reconcile entry. That's a transient physical-state + * change, not a missing io-opts propagation cookie. + */ + if ((new_need_rb & BIT(BCH_RECONCILE_data_replicas)) && + bch2_bkey_has_ptr_bad_or_evacuating(c, k)) + return 0; + /* * Either the extent data or the extent io options (from * bch_extent_reconcile) should match the io_opts from the @@ -751,7 +767,14 @@ static int new_needs_rb_allowed(struct btree_trans *trans, if (ret) return min(ret, 0); - if (new_need_rb == BIT(BCH_RECONCILE_data_replicas)) { + /* + * Device scans can add evacuation/rereplicate work on top of existing + * reconcile state, e.g. an extent that was already waiting on + * erasure-code conversion. While that scan cookie is pending, tolerate + * missing data_replicas/hipri state even if other reconcile bits are + * also set; the scan will rewrite the full reconcile entry. + */ + if (new_need_rb & BIT(BCH_RECONCILE_data_replicas)) { ret = check_dev_reconcile_scan_cookie(trans, k, s ? &s->dev_cookie : NULL); if (ret) return min(ret, 0); @@ -805,6 +828,17 @@ static int set_needs_reconcile_stripe(struct btree_trans *trans, } s.v->needs_reconcile = new_needs_reconcile; + + /* + * A repair pass can lose the stripe's work item (e.g. a stale + * needs_reconcile read racing the scan's flag update), and unless + * the scan re-queues it the stripe stays flagged-but-never-repaired + * forever. + * TODO: a cleaner fix would be avoiding the race in the first place + */ + if (new_needs_reconcile) + try(reconcile_work_mod(trans, k.s_c, RECONCILE_WORK_hipri, + data_to_rb_work_pos(BTREE_ID_stripes, k.k->p), true)); return 0; } diff --git a/fs/data/reconcile/types.h b/fs/data/reconcile/types.h index 127b6abcc8..434bc8d130 100644 --- a/fs/data/reconcile/types.h +++ b/fs/data/reconcile/types.h @@ -2,6 +2,8 @@ #ifndef _BCACHEFS_REBALANCE_TYPES_H #define _BCACHEFS_REBALANCE_TYPES_H +#include + #include "btree/bbpos_types.h" #include "data/move_types.h" #include "init/progress.h" @@ -11,7 +13,9 @@ struct bch_fs_reconcile { struct task_struct __rcu *thread; - u32 kick; + atomic_t kick; + wait_queue_head_t wait; + atomic_t completed_kick; bool running; u64 wait_iotime_start; diff --git a/fs/data/reconcile/work.c b/fs/data/reconcile/work.c index 041eddfac5..5e79cb3055 100644 --- a/fs/data/reconcile/work.c +++ b/fs/data/reconcile/work.c @@ -349,14 +349,23 @@ static struct bkey_s_c next_reconcile_entry(struct btree_trans *trans, if (work_pos->btree == BTREE_ID_reconcile_scan) { buf->nr = 0; - int ret = for_each_btree_key_max(trans, iter, work_pos->btree, work_pos->pos, end, - flags, k, ({ - bkey_reassemble(&darray_top(*buf), k); - return bkey_i_to_s_c(&darray_top(*buf)); - 0; - })); + /* + * Do not return out of a for_each_btree_key_* body here: we want + * the iterator fully dropped before reconcile starts mutating the + * extent that this work item points at. Keeping the reconcile_scan + * path pinned across the later move/rewrite can deadlock against + * alloc/freespace updates from the same transaction. + */ + CLASS(btree_iter, iter)(trans, work_pos->btree, work_pos->pos, flags); + struct bkey_s_c k = bch2_btree_iter_peek(&iter); + + if (bkey_err(k)) + return bkey_s_c_err(bkey_err(k)); + if (!k.k || bpos_gt(k.k->p, end)) + return bkey_s_c_null; - return ret ? bkey_s_c_err(ret) : bkey_s_c_null; + bkey_reassemble(&darray_top(*buf), k); + return bkey_i_to_s_c(&darray_top(*buf)); } if (unlikely(!buf->nr)) { @@ -442,7 +451,8 @@ static int reconcile_set_data_opts(struct btree_trans *trans, * (data_update_useless_write_fail). Better to fail with freelist_empty * and retry from a fresh read. */ - data_opts->write_flags |= BCH_WRITE_alloc_nowait; + if (!bkey_is_btree_ptr(k.k)) + data_opts->write_flags |= BCH_WRITE_alloc_nowait; /* * we can't add/drop replicas from btree nodes incrementally, we always @@ -467,7 +477,7 @@ static int reconcile_set_data_opts(struct btree_trans *trans, guard(rcu)(); bkey_for_each_ptr(ptrs, ptr) { - if (bch2_dev_bad_or_evacuating(c, ptr->dev)) + if (bch2_ptr_bad_or_evacuating_rcu(c, ptr)) data_opts->ptrs_kill |= ptr_bit; ptr_bit <<= 1; } @@ -499,7 +509,7 @@ static int reconcile_set_data_opts(struct btree_trans *trans, !test_bit(ptr->dev, c->devs_online.d); if (!ptr->cached && (online_floor || offline)) { - bool force = bch2_dev_bad_or_evacuating(c, ptr->dev); + bool force = bch2_ptr_bad_or_evacuating(c, ptr); ptr->cached = true; struct bkey_durability d; @@ -518,7 +528,7 @@ static int reconcile_set_data_opts(struct btree_trans *trans, * reads as durability=0, but we drop it once the * required durability is held by other devices. */ - if (have >= r->data_replicas && + if ((force || have >= r->data_replicas) && (force || have < was)) { data_opts->ptrs_kill |= ptr_bit; cur = d; @@ -706,10 +716,17 @@ static int check_reconcile_pending_err(struct btree_trans *trans, { struct bch_fs *c = trans->c; - if (!bch2_err_matches(err, BCH_ERR_data_update_fail_no_rw_devs) && - !bch2_err_matches(err, BCH_ERR_insufficient_devices) && - !bch2_err_matches(err, ENOSPC)) - return err; + /* + * Reconcile can park transient allocation failures on the pending list + * and retry after space becomes available. The allocator's internal + * no_buckets_found code is the same class of "try again later" failure + * as ENOSPC, but it bypasses the errno-based match. + */ + if (!bch2_err_matches(err, BCH_ERR_data_update_fail_no_rw_devs) && + !bch2_err_matches(err, BCH_ERR_insufficient_devices) && + !bch2_err_matches(err, BCH_ERR_freelist_empty) && + !bch2_err_matches(err, ENOSPC)) + return err; s64 sectors = bkey_is_btree_ptr(k.k) ? btree_sectors(c) : k.k->size; @@ -909,13 +926,17 @@ static int do_reconcile_extent(struct moving_context *ctxt, struct bch_fs *c = trans->c; struct bbpos data_pos = rb_work_to_data_pos(work.pos); - /* We require holding an intent lock when calling - * bch2_stripe_handle_tryget(), to avoid racing with the stripe trigger - * deleting the stripe */ - enum btree_iter_update_trigger_flags flags = data_pos.btree == BTREE_ID_stripes - ? BTREE_ITER_intent : 0; + /* + * Reconcile always mutates the key it is looking at, either by updating + * reconcile state or by queueing a move. Start with an intent iterator + * so we do not pay one transaction restart_upgrade per extent just to + * upgrade the lock during commit. Stripe keys also require intent before + * bch2_stripe_handle_tryget() can safely race with stripe deletion. + */ + enum btree_iter_update_trigger_flags flags = + BTREE_ITER_all_snapshots|BTREE_ITER_intent; - CLASS(btree_iter, iter)(trans, data_pos.btree, data_pos.pos, BTREE_ITER_all_snapshots|flags); + CLASS(btree_iter, iter)(trans, data_pos.btree, data_pos.pos, flags); struct bkey_s_c k = bkey_try(bch2_btree_iter_peek_slot(&iter)); if (!k.k) return 0; @@ -959,11 +980,13 @@ static int do_reconcile_extent_phys(struct moving_context *ctxt, if (bch2_data_update_in_flight(c, &pos, BCH_DATA_UPDATE_reconcile)) return 0; - /* We require holding an intent lock when calling - * bch2_stripe_handle_tryget(), to avoid racing with the stripe trigger - * deleting the stripe */ - enum btree_iter_update_trigger_flags flags = bp.v->btree_id == BTREE_ID_stripes - ? BTREE_ITER_intent : 0; + /* + * Phys reconcile reaches the owning extent through a backpointer and + * then updates that extent in the same transaction. Take the extent + * iterator in intent mode up front to avoid restart_upgrade churn when + * many adjacent extents are being reprocessed after shrink. + */ + enum btree_iter_update_trigger_flags flags = BTREE_ITER_intent; CLASS(btree_iter_uninit, iter)(trans); struct bkey_s_c k = bkey_try(bch2_backpointer_get_key(trans, bp, &iter, flags, last_flushed)); @@ -1099,12 +1122,27 @@ static int do_reconcile_scan_bps(struct moving_context *ctxt, struct btree_trans *trans = ctxt->trans; struct bch_fs *c = trans->c; struct bch_fs_reconcile *r = &c->reconcile; + struct bpos start = POS(s.dev, 0); + struct bch_dev *ca; - r->scan_start = BBPOS(BTREE_ID_backpointers, POS(s.dev, 0)); + /* + * Shrink only needs reconcile work for the tail that will be truncated. + * Scanning the whole device requeues metadata below the retained region, + * which can churn btree rewrites for tens of seconds before the resize + * worker ever gets to commit the smaller size. + */ + scoped_guard(rcu) { + ca = bch2_dev_rcu_noerror(c, s.dev); + if (ca && bch2_dev_is_shrinking(ca)) + start = bucket_pos_to_bp_start(ca, + POS(s.dev, bch2_dev_resize_target(ca))); + } + + r->scan_start = BBPOS(BTREE_ID_backpointers, start); r->scan_end = BBPOS(BTREE_ID_backpointers, POS(s.dev, U64_MAX)); return backpointer_scan_for_each(trans, iter, BTREE_ID_backpointers, - POS(s.dev, 0), POS(s.dev, U64_MAX), + start, POS(s.dev, U64_MAX), last_flushed, NULL, bp, ({ ctxt->stats->pos = BBPOS(BTREE_ID_backpointers, iter.pos); @@ -1356,10 +1394,10 @@ static void reconcile_wait(struct bch_fs *c, u32 kick) r->wait_iotime_end = now + (min_member_capacity >> 6); - if (r->running) { + if (READ_ONCE(r->running)) { r->wait_iotime_start = now; r->wait_wallclock_start = ktime_get_real_ns(); - r->running = false; + WRITE_ONCE(r->running, false); } /* @@ -1368,7 +1406,7 @@ static void reconcile_wait(struct bch_fs *c, u32 kick) * wakeups: */ set_current_state(TASK_INTERRUPTIBLE); - if (kick == READ_ONCE(r->kick)) + if (kick == atomic_read(&r->kick)) bch2_kthread_io_clock_wait_once(clock, r->wait_iotime_end, MAX_SCHEDULE_TIMEOUT); __set_current_state(TASK_RUNNING); } @@ -1490,6 +1528,15 @@ static CLOSURE_CALLBACK(do_reconcile_phys_thread) k.k->p.inode != thr->dev) break; + /* + * Drop any cached search paths from next_reconcile_entry() before + * we start the actual move work: otherwise reconcile_scan or + * backpointer paths looked up to choose the work item can stay + * pinned in this transaction and deadlock against alloc/freespace + * updates during btree node rewrites. + */ + bch2_trans_begin(trans); + int ret = lockrestart_do(trans, do_reconcile_extent_phys(&ctxt, &snapshot_io_opts, BBPOS(work_pos.btree, k.k->p), @@ -1636,7 +1683,7 @@ static int do_reconcile_phase_iter(struct reconcile_pass *p, u32 kick, while (!bch2_move_ratelimit(ctxt) && !test_bit(BCH_FS_going_ro, &c->flags) && bch2_reconcile_enabled(c) && - kick == r->kick) { + kick == atomic_read(&r->kick)) { bch2_trans_begin(trans); struct bkey_s_c k = next_reconcile_entry(trans, p->work, &r->work_pos, @@ -1725,8 +1772,9 @@ static int do_reconcile(struct moving_context *ctxt) struct bch_fs *c = trans->c; struct bch_fs_reconcile *r = &c->reconcile; u64 sectors_scanned = 0; - u32 kick = r->kick; + u32 kick = atomic_read(&r->kick); u32 copygc_run_count = c->copygc.run_count; + bool pass_complete = false; int ret = 0; CLASS(darray_reconcile_work, work)(); @@ -1777,7 +1825,7 @@ static int do_reconcile(struct moving_context *ctxt) * uses this latest value to decide whether anyone is still * asking for more work. */ - kick = r->kick; + kick = atomic_read(&r->kick); for (r->phase = 0; r->phase < ARRAY_SIZE(reconcile_phases); r->phase++) { reconcile_phase_start(c); @@ -1798,7 +1846,7 @@ static int do_reconcile(struct moving_context *ctxt) work.nr = 0; - if (kick != r->kick || + if (kick != atomic_read(&r->kick) || test_bit(BCH_FS_going_ro, &c->flags) || bch2_move_ratelimit(ctxt)) break; @@ -1808,21 +1856,37 @@ static int do_reconcile(struct moving_context *ctxt) } /* Completed a clean pass through all phases — we're done. */ - if (r->phase == ARRAY_SIZE(reconcile_phases)) + if (r->phase == ARRAY_SIZE(reconcile_phases)) { + pass_complete = true; break; + } } out: if (!ret && !bkey_deleted(&pending_cookie.k)) try(bch2_clear_reconcile_needs_scan(trans, pending_cookie.k.p, pending_cookie.v.cookie)); + /* + * A completed kick means reconcile drained every phase for the current + * request generation without being superseded by a newer wakeup. + * Shrink uses this as the point where all scan-generated downstream + * work has been attempted before it decides a tail is still impossible + * to evacuate. + */ + if (!ret && + pass_complete && + kick == atomic_read(&r->kick)) { + atomic_set(&r->completed_kick, kick); + wake_up_all(&r->wait); + } + bch2_move_stats_exit(&r->work_stats, c); if (!ret && !kthread_should_stop() && !atomic64_read(&r->work_stats.sectors_seen) && !sectors_scanned && - kick == r->kick) { + kick == atomic_read(&r->kick)) { bch2_moving_ctxt_flush_all(ctxt); bch2_trans_unlock_long(trans); reconcile_wait(c, kick); @@ -2037,6 +2101,10 @@ int bch2_fs_reconcile_init(struct bch_fs *c) { struct bch_fs_reconcile *r = &c->reconcile; + atomic_set(&r->kick, 0); + init_waitqueue_head(&r->wait); + atomic_set(&r->completed_kick, 0); + mutex_init(&r->scans_in_flight_lock); try(rhashtable_init(&r->scans_in_flight, &reconcile_scan_in_flight_params)); r->scans_in_flight_init_done = true; diff --git a/fs/data/reconcile/work.h b/fs/data/reconcile/work.h index f945cfa212..e84cacace2 100644 --- a/fs/data/reconcile/work.h +++ b/fs/data/reconcile/work.h @@ -64,13 +64,24 @@ int bch2_set_fs_needs_reconcile(struct bch_fs *); int bch2_reconcile_scan_cookie_is_set(struct btree_trans *, u64); -static inline void bch2_reconcile_wakeup(struct bch_fs *c) +static inline u32 bch2_reconcile_kick(struct bch_fs *c) { - c->reconcile.kick++; + u32 kick = atomic_inc_return(&c->reconcile.kick); guard(rcu)(); struct task_struct *p = rcu_dereference(c->reconcile.thread); if (p) wake_up_process(p); + return kick; +} + +static inline void bch2_reconcile_wakeup(struct bch_fs *c) +{ + bch2_reconcile_kick(c); +} + +static inline u32 bch2_reconcile_completed_kick(struct bch_fs *c) +{ + return atomic_read(&c->reconcile.completed_kick); } static inline int bch2_reconcile_pending_wakeup(struct bch_fs *c) diff --git a/fs/data/update.c b/fs/data/update.c index 9ee2f01491..829e07f45c 100644 --- a/fs/data/update.c +++ b/fs/data/update.c @@ -744,7 +744,20 @@ bool bch2_data_update_fail_should_trace(enum bch_data_update_types type, int ret bch2_err_matches(ret, BCH_ERR_operation_blocked) || ((type == BCH_DATA_UPDATE_reconcile || type == BCH_DATA_UPDATE_promote) && - bch2_err_matches(ret, BCH_ERR_data_update_fail_no_rw_devs))) + (bch2_err_matches(ret, BCH_ERR_data_update_fail_no_rw_devs) || + /* + * The allocator reports a fully-exhausted retry sequence as + * BCH_ERR_freelist_empty/no_buckets_found. Reconcile demotes + * that to pending work just like ENOSPC, so don't count it as a + * hard data-update failure. + */ + bch2_err_matches(ret, BCH_ERR_freelist_empty) || + /* + * Reconcile promotes ENOSPC-class write failures to pending work + * and retries after space/stripe availability changes, so don't + * count those transient allocation misses as data update failures. + */ + bch2_err_matches(ret, ENOSPC)))) return false; return true; @@ -1088,11 +1101,9 @@ static bool bch2_btree_ptr_has_dev_evacuating(struct bch_fs *c, struct bkey_s_c struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); guard(rcu)(); - bkey_for_each_ptr(ptrs, ptr) { - struct bch_dev *ca = bch2_dev_rcu_noerror(c, ptr->dev); - if (ca && ca->mi.state == BCH_MEMBER_STATE_evacuating) + bkey_for_each_ptr(ptrs, ptr) + if (bch2_ptr_bad_or_evacuating_rcu(c, ptr)) return true; - } return false; } @@ -1249,9 +1260,13 @@ int bch2_can_do_data_update(struct btree_trans *trans, .ec_replicas = opts->data_replicas + data_opts->extra_replicas, .watermark = BCH_WATERMARK_normal, }; + darray_init(&req.trace); struct ec_stripe_head *h = bch2_ec_stripe_head_get(trans, &req, 0); + + darray_exit(&req.trace); + if (IS_ERR_OR_NULL(h)) return bch_err_throw(c, ec_alloc_failed); bch2_ec_stripe_head_put(c, h); @@ -1473,8 +1488,24 @@ int bch2_data_update_init(struct btree_trans *trans, goto out; } - if (!rhltable_insert_key(&c->update_table, &m->pos, &m->hash, bch_update_params)) - m->on_hashtable = true; + /* + * Phys reconcile can queue the same logical extent from multiple + * source devices. The lookup above is only a fast path; another + * mover can still win the race and claim @m->pos before we insert. + * Use the non-list insert helper so only one update owns the key. + */ + ret = rhashtable_lookup_insert_fast(&c->update_table.ht, + &m->hash.rhead, + bch_update_params); + if (ret == -EEXIST) { + event_inc(c, data_update_in_flight); + ret = bch_err_throw(c, data_update_fail_in_flight); + goto out; + } + if (ret) + goto out; + + m->on_hashtable = true; } else { if (unwritten) { ret = bch_err_throw(c, data_update_done_unwritten); diff --git a/fs/errcode.h b/fs/errcode.h index 15488d4a54..6c8fb65077 100644 --- a/fs/errcode.h +++ b/fs/errcode.h @@ -289,6 +289,7 @@ x(EINVAL, bucket_size_too_small) \ x(EINVAL, device_size_too_small) \ x(EINVAL, device_size_too_big) \ + x(EINVAL, device_already_resizing) \ x(EINVAL, device_not_a_member_of_filesystem) \ x(EINVAL, device_has_been_removed) \ x(EINVAL, device_splitbrain) \ diff --git a/fs/init/dev.c b/fs/init/dev.c index 03b1a114f3..0c0262735f 100644 --- a/fs/init/dev.c +++ b/fs/init/dev.c @@ -167,14 +167,39 @@ * * \subsubsection{Resize} * - * \texttt{bcachefs device resize} grows a device to use additional space - * (shrinking is not yet supported). If no size is specified, the device grows to + * \texttt{bcachefs device resize} resizes a device to use more or less space. + * If no size is specified, the device grows to * fill its underlying block device. Resize works online---no unmount required. + * Resize is executed by a per-device kthread, which is restarted when a + * newer request overwrites an in-progress resize, automatically cancelling the old resize. + * Resizes are persisted as \texttt{target\_nbuckets} and will be automatically resumed across restarts. + * + * \subsubsubsection{Growing} * The new size is subject to a maximum bucket count * (\texttt{BCH\_MEMBER\_NBUCKETS\_MAX}); resize will fail if the requested size * would exceed this limit. After resize, the reconcile subsystem is notified to * account for the newly available space. * + * \subsubsubsection{Shrinking} + * Shrinking removes the tail region of a device and evacuates any data + * located there. If this is not possible, the operation will fail with \texttt{-ENOSPC}. + * + * Once \texttt{target\_nbuckets < nbuckets} is persisted, the allocator + * refuses new allocations in the tail, cached pointers past the cutoff are + * treated as stale, and metadata allocation spills to non-shrinking devices + * so shrink does not deadlock on its own journal or btree-rewrite needs. + * Reconcile is then used to discover and evacuate the remaining data. + * + * Before draining the tail, the worker relocates any journal buckets in the truncating region explicitly + * (\texttt{move\_journal\_past\_cutoff()}) so journal activity does not keep + * reintroducing references into the region being evacuated. + * + * Once the tail is empty, the worker finalises under \texttt{state\_lock}: + * it flushes journal pins, clears \texttt{NEED\_DISCARD} bookkeeping for + * the removed buckets, drops superblock copies and alloc metadata past the + * cutoff, then commits \texttt{nbuckets = target\_nbuckets} to the + * superblock. + * * \texttt{bcachefs device resize-journal} adjusts the per-device journal size * independently of the data area. * @@ -204,28 +229,51 @@ #include "bcachefs.h" +#include "alloc/buckets.h" #include "alloc/accounting.h" #include "alloc/background.h" #include "alloc/backpointers.h" #include "alloc/check.h" #include "alloc/discard.h" #include "alloc/replicas.h" +#include "alloc/foreground.h" + +#include "asm/byteorder.h" +#include "bcachefs_format.h" +#include "btree/bkey_methods.h" +#include "btree/bkey_types.h" #include "btree/interior.h" +#include "btree/iter.h" +#include "btree/update.h" +#include "btree/types.h" +#include "btree/write_buffer.h" #include "data/ec/init.h" +#include "data/extents.h" #include "data/migrate.h" #include "data/reconcile/work.h" #include "debug/sysfs.h" #include "journal/init.h" +#include "journal/journal.h" #include "journal/reclaim.h" #include "init/dev.h" #include "init/fs.h" +#include "init/passes.h" +#include "linux/bitmap.h" +#include "linux/kthread.h" +#include "linux/sched.h" +#include "linux/sched/signal.h" +#include "sb/io.h" #include "sb/members.h" +#include "sb/members_format.h" +#include "util/util.h" + +static void bch2_dev_resize_thread_stop(struct bch_dev *); #define x(n) #n, const char * const bch2_dev_read_refs[] = { @@ -422,7 +470,7 @@ static void __bch2_dev_read_only(struct bch_fs *c, struct bch_dev *ca) /* * may_reuse_stripe() / init_new_stripe_from_old() check - * bch2_dev_bad_or_evacuating at stripe allocation time, but state can + * bch2_ptr_bad_or_evacuating at stripe allocation time, but state can * flip to non-rw between that check and the stripe commit. Stripes * still on stripe_head_list were cancelled by bch2_ec_stop_dev() via * bch2_dev_allocator_remove() above; stripes that had moved to @@ -497,6 +545,8 @@ void bch2_dev_free(struct bch_dev *ca) WARN_ON(!enumerated_ref_is_zero(&ca->io_ref[WRITE])); WARN_ON(!enumerated_ref_is_zero(&ca->io_ref[READ])); + bch2_dev_resize_thread_stop(ca); + bch2_dev_unlink(ca); if (ca->kobj.state_in_sysfs) @@ -584,6 +634,9 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c, init_completion(&ca->ref_completion); refcount_set(&ca->ref_outer, 1); init_completion(&ca->ref_outer_completion); + spin_lock_init(&ca->resize_lock); + init_waitqueue_head(&ca->resize_wait); + ca->resize_status = 0; INIT_WORK(&ca->io_error_work, bch2_io_error_work); @@ -871,8 +924,6 @@ int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca, enum bch_member_state new_state, int flags, struct printbuf *err) { - int ret = 0; - bool was_rw = ca->mi.state == BCH_MEMBER_STATE_rw; bool do_reconcile_scan = new_state == BCH_MEMBER_STATE_rw || @@ -926,7 +977,7 @@ int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca, if (do_reconcile_scan) try(bch2_set_reconcile_needs_scan(c, s, true)); - return ret; + return 0; } int bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca, @@ -1032,11 +1083,16 @@ static int __bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, */ __bch2_dev_offline(c, ca); - ret = bch2_dev_remove_alloc(c, ca); + ret = bch2_dev_remove_alloc(c, ca, 0); if (ret) { prt_printf(err, "bch2_dev_remove_alloc() error: %s\n", bch2_err_str(ret)); goto err; } + ret = bch2_dev_usage_remove(c, ca); + if (ret) { + prt_printf(err, "bch2_dev_usage_remove() error: %s\n", bch2_err_str(ret)); + goto err; + } /* * dev_remove_alloc issued btree deletes that need to be durable before @@ -1162,11 +1218,9 @@ int bch2_dev_add_initialize(struct bch_fs *c, struct bch_dev *ca) /* Add new device to running filesystem: */ int bch2_dev_add(struct bch_fs *c, const char *path, struct printbuf *err) { - int ret = 0; - struct bch_opts opts = bch2_opts_empty(); struct bch_sb_handle sb __cleanup(bch2_free_super) = {}; - ret = bch2_read_super(path, &opts, &sb); + int ret = bch2_read_super(path, &opts, &sb); if (ret) { prt_printf(err, "error reading superblock: %s\n", bch2_err_str(ret)); return ret; @@ -1215,6 +1269,13 @@ int bch2_dev_add(struct bch_fs *c, const char *path, struct printbuf *err) * only thing that adds a device to a not-started filesystem: */ try(bch2_set_reconcile_needs_scan(c, s, false)); + + /* + * Adding a device grows the EC widening target (RW members of + * the disk_label) for existing stripes. + */ + try(bch2_set_reconcile_needs_scan(c, + (struct reconcile_scan) { .type = RECONCILE_SCAN_stripes }, false)); } scoped_guard(rwsem_write, &c->state_lock) { @@ -1435,44 +1496,198 @@ int bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca, int flags, struct pri return 0; } -int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets, struct printbuf *err) +static u64 bch2_dev_resize_seq(struct bch_dev *ca) { - u64 old_nbuckets; - int ret = 0; + scoped_guard(spinlock, &ca->resize_lock) + return ca->resize_seq; +} + +static bool bch2_dev_resize_wait_done(struct bch_dev *ca, u64 seq, int *status) +{ + scoped_guard(spinlock, &ca->resize_lock) { + if (ca->resize_seq != seq) { + *status = -ECANCELED; + return true; + } + if (ca->resize_status != -EINPROGRESS) { + *status = ca->resize_status; + return true; + } + } + + return false; +} + +static int bch2_dev_resize_wait(struct bch_dev *ca, u64 seq) +{ + int status = -EINPROGRESS; + int ret = wait_event_killable(ca->resize_wait, + bch2_dev_resize_wait_done(ca, seq, &status)); + + return ret ? -EINTR : status; +} + +static bool bch2_dev_resize_finish(struct bch_dev *ca, u64 seq, int status) +{ + bool is_current; + + scoped_guard(spinlock, &ca->resize_lock) { + is_current = ca->resize_seq == seq; + if (is_current) + ca->resize_status = status; + } + + if (is_current) { + wake_up_all(&ca->resize_wait); + + /* Discards are deferred during resize to avoid allocator/journal deadlocks, restart them now that we are done */ + bch2_do_discards_async(ca->fs); + } + + return is_current; +} + +/* checks for kthread interruption, and resize seq having changed */ +static int bch2_dev_resize_restart_check(struct bch_dev *ca, u64 seq) +{ + if (kthread_should_stop()) + return -EINTR; + + return bch2_dev_resize_seq(ca) != seq ? -EAGAIN : 0; +} + +static int bch2_dev_resize_thread(void *arg); + +static int bch2_dev_resize_thread_start(struct bch_dev *ca) +{ + struct bch_fs *c = ca->fs; + + lockdep_assert_held(&c->state_lock); + + if (ca->resize_thread) + return 0; + + struct task_struct *p = + kthread_create(bch2_dev_resize_thread, ca, + "bch-resize/%s:%u", c->name, ca->dev_idx); + try(PTR_ERR_OR_ZERO(p)); + + get_task_struct(p); + ca->resize_thread = p; + wake_up_process(p); + return 0; +} + +static void bch2_dev_resize_thread_stop(struct bch_dev *ca) +{ + scoped_guard(spinlock, &ca->resize_lock) { + ca->resize_seq++; + ca->resize_status = -EINTR; + } + + if (ca->resize_thread) { + kthread_stop(ca->resize_thread); + put_task_struct(ca->resize_thread); + ca->resize_thread = NULL; + } + + wake_up_all(&ca->resize_wait); +} + +void bch2_dev_resize_threads_stop(struct bch_fs *c) +{ + for_each_member_device(c, ca) + bch2_dev_resize_thread_stop(ca); +} + +static int bch2_dev_resize_validate_target(struct bch_fs *c, struct bch_dev *ca, + u64 target_nbuckets, struct printbuf *err) +{ + lockdep_assert_held(&c->state_lock); + + /* validate target_nbuckets */ + u64 old_nbuckets = ca->mi.nbuckets; + + if (target_nbuckets > BCH_MEMBER_NBUCKETS_MAX) { + prt_printf(err, "New device size too big (%llu greater than max %u)\n", + target_nbuckets, BCH_MEMBER_NBUCKETS_MAX); + return bch_err_throw(c, device_size_too_big); + } + + if (target_nbuckets < old_nbuckets && + target_nbuckets < ca->mi.first_bucket + BCH_MIN_NR_NBUCKETS) { + prt_printf(err, "New device size too small (%llu smaller than min %llu)\n", + target_nbuckets, + (u64) ca->mi.first_bucket + BCH_MIN_NR_NBUCKETS); + return bch_err_throw(c, device_size_too_small); + } + + if (target_nbuckets > old_nbuckets && + bch2_dev_is_online(ca) && + get_capacity(ca->disk_sb.bdev->bd_disk) < + ca->mi.bucket_size * target_nbuckets) { + prt_printf(err, "New size %llu larger than device size %llu\n", + ca->mi.bucket_size * target_nbuckets, + get_capacity(ca->disk_sb.bdev->bd_disk)); + return bch_err_throw(c, device_size_too_small); + } + + return 0; +} +static int bch2_dev_resize_set_target(struct bch_fs *c, struct bch_dev *ca, u64 target_nbuckets) +{ + lockdep_assert_held(&c->state_lock); + + /* commit target_nbuckets */ + scoped_guard(mutex_noio, &c->sb_lock) { + struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); + + m->target_nbuckets = cpu_to_le64(target_nbuckets); + try(bch2_write_super(c)); + } + + return 0; +} + +static int __bch2_dev_grow(struct bch_fs *c, struct bch_dev *ca, + u64 new_nbuckets, struct printbuf *err) +{ guard(rwsem_write)(&c->state_lock); + int ret = 0; + if (READ_ONCE(ca->removing)) return bch_err_throw(c, device_has_been_removed); - old_nbuckets = ca->mi.nbuckets; + u64 old_nbuckets = ca->mi.nbuckets; - if (nbuckets < ca->mi.nbuckets) { - prt_printf(err, "Cannot shrink yet\n"); - return bch_err_throw(c, EINVAL_dev_resize_shrink); + if (new_nbuckets <= old_nbuckets) { + return 0; } - bool wakeup_reconcile_pending = nbuckets > ca->mi.nbuckets; + /* we have more space -> wake up pending */ + bool wakeup_reconcile_pending = new_nbuckets > old_nbuckets; struct reconcile_scan s = { .type = RECONCILE_SCAN_pending }; if (wakeup_reconcile_pending) try(bch2_set_reconcile_needs_scan(c, s, false)); - if (nbuckets > BCH_MEMBER_NBUCKETS_MAX) { + if (new_nbuckets > BCH_MEMBER_NBUCKETS_MAX) { prt_printf(err, "New device size too big (%llu greater than max %u)\n", - nbuckets, BCH_MEMBER_NBUCKETS_MAX); + new_nbuckets, BCH_MEMBER_NBUCKETS_MAX); return bch_err_throw(c, device_size_too_big); } if (bch2_dev_is_online(ca) && get_capacity(ca->disk_sb.bdev->bd_disk) < - ca->mi.bucket_size * nbuckets) { + ca->mi.bucket_size * new_nbuckets) { prt_printf(err, "New size %llu larger than device size %llu\n", - ca->mi.bucket_size * nbuckets, + ca->mi.bucket_size * new_nbuckets, get_capacity(ca->disk_sb.bdev->bd_disk)); return bch_err_throw(c, device_size_too_small); } - ret = bch2_dev_buckets_resize(c, ca, nbuckets); + ret = bch2_dev_buckets_resize(c, ca, new_nbuckets); if (ret) { prt_printf(err, "bch2_dev_buckets_resize() error: %s\n", bch2_err_str(ret)); return ret; @@ -1486,13 +1701,15 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets, struct p scoped_guard(mutex_noio, &c->sb_lock) { struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); - m->nbuckets = cpu_to_le64(nbuckets); + m->nbuckets = cpu_to_le64(new_nbuckets); + if (bch2_dev_resize_target(ca) == new_nbuckets) + m->target_nbuckets = 0; bch2_write_super(c); } if (ca->mi.freespace_initialized) { - ret = __bch2_dev_resize_alloc(ca, old_nbuckets, nbuckets); + ret = __bch2_dev_resize_alloc(ca, old_nbuckets, new_nbuckets); if (ret) { prt_printf(err, "__bch2_dev_resize_alloc() error: %s\n", bch2_err_str(ret)); return ret; @@ -1506,12 +1723,680 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets, struct p return 0; } +static int drop_sbs_after_cutoff(struct bch_fs *c, struct bch_dev *ca, u64 cutoff) { + u64 cutoff_sector = bucket_to_sector(ca, cutoff); + + guard(mutex_noio)(&c->sb_lock); + + struct bch_sb_layout *layout = &ca->disk_sb.sb->layout; + + u64 max_sectors = 1 << layout->sb_max_size_bits; + + u8 i; + /* offsets are sorted in ascending order, see validate_sb_layout() overlapping checks for evidence */ + for (i = 0; i < layout->nr_superblocks; i++) { + u64 offset = le64_to_cpu(layout->sb_offset[i]); + if (offset + max_sectors > cutoff_sector) { + break; + } + } + + /* this should never happen, as we only call to this function after checking the cutoff against the minimum fs size, + * which includes at least the first sb copy */ + BUG_ON(i == 0); + + layout->nr_superblocks = i; + + return bch2_write_super(c); +} + +static int move_journal_past_cutoff(struct bch_fs *c, struct bch_dev *ca, + u64 cutoff, struct printbuf *err) +{ + bool grew = false; + + while (true) { + u64 bucket_to_delete = 0; + unsigned nr = 0, nr_past_cutoff = 0; + bool cur_bucket_past_cutoff = false; + int ret; + + scoped_guard(spinlock, &c->journal.lock) { + struct journal_device *ja = &ca->journal; + + nr = ja->nr; + if (!nr) + break; + + cur_bucket_past_cutoff = ja->buckets[ja->cur_idx] >= cutoff; + + for (unsigned i = 0; i < ja->nr; i++) { + if (ja->buckets[i] < cutoff) + continue; + + nr_past_cutoff++; + if (i != ja->cur_idx && !bucket_to_delete) + bucket_to_delete = ja->buckets[i]; + } + } + + if (!nr_past_cutoff) + return 0; + + if (!grew) { + ret = bch2_set_nr_journal_buckets(c, ca, nr + nr_past_cutoff); + if (ret) { + prt_printf(err, "Failed to relocate journal buckets: %s\n", + bch2_err_str(ret)); + return ret; + } + grew = true; + continue; + } + + if (!bucket_to_delete && cur_bucket_past_cutoff) { + scoped_guard(spinlock, &c->journal.lock) { + struct journal_device *ja = &ca->journal; + + if (ja->nr && + ja->buckets[ja->cur_idx] >= cutoff) + ja->sectors_free = 0; + } + + scoped_guard(mutex_noio, &c->sb_lock) { + ret = bch2_write_super(c); + } + if (ret) { + prt_printf(err, "Failed to advance journal off shrink tail: %s\n", + bch2_err_str(ret)); + return ret; + } + + ret = bch2_journal_flush(&c->journal); + if (ret) { + prt_printf(err, "Failed to flush relocated journal: %s\n", + bch2_err_str(ret)); + return ret; + } + continue; + } + + ret = bch2_dev_journal_bucket_delete(ca, bucket_to_delete); + if (ret) { + prt_printf(err, "Failed to drop journal bucket %llu from shrink tail: %s\n", + bucket_to_delete, bch2_err_str(ret)); + return ret; + } + } +} + +struct shrink_tail_head { + struct bpos bucket; + struct bpos first_bp; + unsigned nr_backpointers; +}; + + + +static inline bool shrink_tail_head_empty(const struct shrink_tail_head *head) +{ + return bpos_eq(head->first_bp, SPOS_MAX); +} + +static bool shrink_tail_head_progressed(const struct shrink_tail_head *old, + const struct shrink_tail_head *new) +{ + if (shrink_tail_head_empty(new)) + return true; + + if (shrink_tail_head_empty(old)) + return false; + + if (!bpos_eq(new->bucket, old->bucket)) + return bpos_gt(new->bucket, old->bucket); + + if (!bpos_eq(new->first_bp, old->first_bp)) + return bpos_gt(new->first_bp, old->first_bp); + + return new->nr_backpointers < old->nr_backpointers; +} + +/* + * Make sure everything is caught here: this snapshots backpointer-visible tail + * data. Journal buckets and superblock copies in the shrink tail are handled by + * move_journal_past_cutoff() and drop_sbs_after_cutoff(). + */ +static int tail_head_snapshot(struct bch_fs *c, struct bch_dev *ca, + u64 new_nbuckets, struct shrink_tail_head *head) +{ + struct bpos bp_start = bucket_pos_to_bp_start(ca, POS(ca->dev_idx, new_nbuckets)); + struct bpos bp_end = bucket_pos_to_bp_start(ca, POS(ca->dev_idx, ca->mi.nbuckets)); + + CLASS(btree_trans, trans)(c); + CLASS(backpointer_scan_iter, iter)(BTREE_ID_backpointers, bp_start, NULL); + + struct wb_maybe_flush last_flushed __cleanup(wb_maybe_flush_exit); + wb_maybe_flush_init(&last_flushed); + + struct bkey_s_c_backpointer bp = bch2_bp_scan_iter_peek(trans, &iter, bp_end, &last_flushed); + + try(bkey_err(bp)); + + *head = (struct shrink_tail_head) { + .bucket = SPOS_MAX, + .first_bp = SPOS_MAX, + }; + + if (!bp.k) + return 0; + + head->bucket = bp_pos_to_bucket(ca, bp.k->p); + head->first_bp = bp.k->p; + + do { + head->nr_backpointers++; + bch2_bp_scan_iter_advance(&iter); + bp = bch2_bp_scan_iter_peek(trans, &iter, bp_end, &last_flushed); + try(bkey_err(bp)); + } while (bp.k && bpos_eq(bp_pos_to_bucket(ca, bp.k->p), head->bucket)); + + return 0; +} + + + +static int tail_is_empty(struct bch_fs *c, struct bch_dev *ca, u64 new_nbuckets, + bool *empty) +{ + struct shrink_tail_head head; + + try(tail_head_snapshot(c, ca, new_nbuckets, &head)); + *empty = shrink_tail_head_empty(&head); + return 0; +} + + + + + + +static int bch2_dev_shrink_queue_reconcile(struct bch_fs *c, struct bch_dev *ca, + bool scan_device, u32 *kick, + struct printbuf *err) +{ + if (scan_device) { + struct reconcile_scan s = { + .type = RECONCILE_SCAN_device, // TODO(performance): make this range-based + .dev = ca->dev_idx, + }; + + int ret = bch2_set_reconcile_needs_scan(c, s, false); + if (ret) { + prt_printf(err, "Failed to queue device reconcile scan: %s\n", + bch2_err_str(ret)); + return ret; + } + } + + /* + * Shrink waits for a completed reconcile pass, not just for the device + * scan cookie to disappear. Queue the pending phase alongside the scan + * so the same pass also retries any work that evacuation demoted to the + * pending list. + */ + int ret = bch2_set_reconcile_needs_scan(c, + (struct reconcile_scan) { .type = RECONCILE_SCAN_pending }, false); + if (ret) { + prt_printf(err, "Failed to queue pending reconcile scan: %s\n", + bch2_err_str(ret)); + return ret; + } + + *kick = bch2_reconcile_kick(c); + return 0; +} + +static int bch2_dev_shrink_wait_reconcile(struct bch_dev *ca, u64 new_nbuckets, + u64 seq, u32 kick, + const struct shrink_tail_head *head_before, + bool *kick_complete, + struct printbuf *err) +{ + struct bch_fs *c = ca->fs; + + while (true) { + bool completed; + try(bch2_dev_resize_restart_check(ca, seq)); + + /* + * We only need reconcile to keep running until the shrink tail is + * actually empty. A kick can continue chewing through unrelated + * global reconcile work for minutes after the truncating region is + * already evacuated, especially with fragmented variable-bucket + * workloads. Poll the head of the shrink tail and bound each wait + * to a one-second slice so shrink can rescan its own state instead + * of sitting behind one long reconcile kick. + */ + struct shrink_tail_head head_after; + + try(tail_head_snapshot(c, ca, new_nbuckets, &head_after)); + completed = bch2_reconcile_completed_kick(c) >= kick; + + if (shrink_tail_head_empty(&head_after) || + shrink_tail_head_progressed(head_before, &head_after) || + completed) { + *kick_complete = completed; + return 0; + } + + int ret = wait_event_killable_timeout(c->reconcile.wait, + bch2_reconcile_completed_kick(c) >= kick || + bch2_dev_resize_seq(ca) != seq || + kthread_should_stop(), + HZ); + if (ret < 0) + return -EINTR; + if (!ret) { + *kick_complete = false; + return 0; + } + } +} + +static int bch2_dev_shrink_clear_target(struct bch_fs *c, struct bch_dev *ca, + u64 new_nbuckets, u64 seq, + struct printbuf *err) +{ + scoped_guard(rwsem_write, &c->state_lock) { + try(bch2_dev_resize_restart_check(ca, seq)); + + if (bch2_dev_resize_target(ca) != new_nbuckets || + !bch2_dev_is_shrinking(ca)) + return -EAGAIN; + + try(bch2_dev_resize_set_target(c, ca, 0)); + } + + atomic64_set(&ca->shrinking_tail_free, 0); + + /* allocations are now no longer blocked after the cutoff, so there may now be more usable space */ + int ret = bch2_reconcile_pending_wakeup(c); + if (ret) + bch_err_fn(c, ret); + + return 0; +} + +static int bch2_dev_count_tail_free(struct bch_fs *c, struct bch_dev *ca, + u64 new_nbuckets) +{ + struct bpos start = POS(ca->dev_idx, new_nbuckets); + struct bpos end = POS(ca->dev_idx, ca->mi.nbuckets); + u64 count = 0; + + CLASS(btree_trans, trans)(c); + try(for_each_btree_key_max(trans, iter, BTREE_ID_freespace, + start, end, 0, k, ({ + u64 first = max_t(u64, new_nbuckets, + bkey_start_offset(k.k)); + u64 last = min_t(u64, ca->mi.nbuckets, + k.k->p.offset); + if (last > first) + count += last - first; + 0; + }))); + + atomic64_set(&ca->shrinking_tail_free, count); + return 0; +} + +static int bch2_dev_shrink_finalize(struct bch_fs *c, struct bch_dev *ca, + u64 old_nbuckets, u64 new_nbuckets, + u64 seq, struct printbuf *err) +{ + scoped_guard(rwsem_write, &c->state_lock) { + bool empty = false; + + try(bch2_dev_resize_restart_check(ca, seq)); + + if (bch2_dev_resize_target(ca) != new_nbuckets || + !bch2_dev_is_shrinking(ca)) + return -EAGAIN; + + /* flush interior updates - mirroring dev remove path */ + bch2_btree_interior_updates_flush(c); + + /* + * Only flush pins that were already outstanding when shrink + * entered the final commit path. Reconcile can continue + * generating unrelated key-cache journal pins on newer + * sequences while the tail is already empty; waiting for every + * future pin here can turn shrink into an unbounded global + * journal drain. + */ + bch2_journal_flush_outstanding_pins(&c->journal); + + int ret = bch2_journal_flush_device_pins(&c->journal, ca->dev_idx); + if (ret) { + prt_printf(err, "bch2_journal_flush_device_pins() error: %s\n", + bch2_err_str(ret)); + return ret; + } + + ret = bch2_journal_flush(&c->journal); + if (ret) { + prt_printf(err, "bch2_journal_flush() error: %s\n", + bch2_err_str(ret)); + return ret; + } + + /* re-check that tail is really empty */ + try(tail_is_empty(c, ca, new_nbuckets, &empty)); + + if (!empty) { + prt_printf(err, "Shrink failed: still has data\n"); + return -EBUSY; + } + + /* drop references to now-truncated superblock copies */ + ret = drop_sbs_after_cutoff(c, ca, new_nbuckets); + if (ret) { + prt_printf(err, "Error dropping superblocks after cutoff: %s\n", + bch2_err_str(ret)); + return ret; + } + + /* update accounting info - has to happen before truncating alloc info */ + ret = bch2_dev_truncate_accounting(c, ca, old_nbuckets, new_nbuckets); + if (ret) { + prt_printf(err, "error updating accounting info: %s\n", + bch2_err_str(ret)); + return ret; + } + + /* truncate alloc info */ + ret = bch2_dev_remove_alloc(c, ca, new_nbuckets); + if (ret) { + prt_printf(err, "error truncating alloc info: %s\n", + bch2_err_str(ret)); + return ret; + } + + /* + * Commit the shrink only after the truncated tail has been + * removed from alloc metadata, so later transactions can't see + * stale tail buckets after the new size is visible. + */ + scoped_guard(mutex_noio, &c->sb_lock) { + struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); + m->nbuckets = cpu_to_le64(new_nbuckets); + if (bch2_dev_resize_target(ca) == new_nbuckets) + m->target_nbuckets = 0; + + try(bch2_write_super(c)); + } + + /* resize buckets */ + ret = bch2_dev_buckets_resize(c, ca, new_nbuckets); + if (ret) { + prt_printf(err, "bch2_dev_buckets_resize() error: %s\n", + bch2_err_str(ret)); + return ret; + } + + bch2_recalc_capacity(c); + atomic64_set(&ca->shrinking_tail_free, 0); + } + + return 0; +} + + +static int __bch2_dev_shrink(struct bch_fs *c, struct bch_dev *ca, + u64 new_nbuckets, u64 seq, struct printbuf *err) +{ + u64 old_nbuckets = ca->mi.nbuckets; + + scoped_guard(rwsem_write, &c->state_lock) { + /* validate shrink size */ + if (new_nbuckets >= old_nbuckets) { + return -EAGAIN; + } + + try(bch2_dev_resize_restart_check(ca, seq)); + + if (bch2_dev_resize_target(ca) != new_nbuckets || + !bch2_dev_is_shrinking(ca)) + return -EAGAIN; + + /* close open buckets in the to-be-shrunk region */ + bch2_open_buckets_stop(c, ca, false, new_nbuckets); + bch2_reset_alloc_cursors(c); // avoid churn + }; + + try(bch2_dev_resize_restart_check(ca, seq)); + + /* + * Shrink can start while a discard worker from earlier freespace + * churn is still in flight. Drain that work before we begin the + * evacuation/journal-flush path: once shrink has started, later + * discard passes skip the shrinking device, but an already-running + * discard can still race in and deadlock against resize/reconcile's + * allocator and btree rewrite work. + */ + flush_work(&c->discards.work); + flush_work(&ca->discard_fast_work); + + try(bch2_dev_resize_restart_check(ca, seq)); + + /* + * Move journal buckets out of the tail up front: otherwise the journal + * can keep reintroducing metadata references in the region we're trying + * to evacuate while reconcile is draining backpointers from it. + */ + try(move_journal_past_cutoff(c, ca, new_nbuckets, err)); + + try(bch2_dev_resize_restart_check(ca, seq)); + + /* wait for to-be-shrunk region to be empty */ + const unsigned stalled_kicks_limit = 32; + struct shrink_tail_head best_head = { + .bucket = SPOS_MAX, + .first_bp = SPOS_MAX, + }; + bool scan_device = true; + unsigned stalled_kicks = 0; + + /* initialize, will be updated in the loop */ + try(bch2_dev_count_tail_free(c, ca, new_nbuckets)); + + /* + * AI output below for detailed reference. TLDR: EC accounting is currently slightly off, so just + * schedule an accounting pass for the next mount. + * + * The stripe repair re-points extents from a stripe with a block in the + * shrink tail to the rebuilt stripe on the surviving devices, and the + * re-pointed extent is committed without a stripe_ptr the insert trigger + * can see - so the new stripe's replicas entry is never incremented and + * the old stripe's entry is left overcounted. + * + * Proper fix (TODO): find why the re-pointed extent's insert loses its + * stripe_ptr. The old-key overwrite triggers but no insert for the new + * stripes ever does; the stripe_ptr must be dropped or rewritten in + * stripe_update_extent()'s key construction or the extent update/merge + * path. Once the insert trigger sees the new stripe_ptr, the accounting + * follows the migration online and this scheduling can go away. + */ + CLASS(bch_log_msg, msg)(c); + int ret = bch2_run_explicit_recovery_pass(c, &msg.m, BCH_RECOVERY_PASS_check_allocations, 0); + if (bch2_err_matches(ret, BCH_ERR_recovery_will_run)) + ret = 0; /* pass will run on next mount */ + try(ret); + + for (unsigned pass = 0; ; pass++) { + bool kick_complete; + struct shrink_tail_head head; + u32 kick; + bool did_scan = pass == 0 || scan_device; + + try(bch2_dev_resize_restart_check(ca, seq)); + + try(tail_head_snapshot(c, ca, new_nbuckets, &head)); + + /* do a definitive check */ + if (shrink_tail_head_empty(&head)) { + { + CLASS(btree_trans, trans)(c); + try(bch2_btree_write_buffer_flush_sync(trans)); + } + + try(tail_head_snapshot(c, ca, new_nbuckets, &head)); + if (shrink_tail_head_empty(&head)) + break; + } + + try(bch2_dev_shrink_queue_reconcile(c, ca, did_scan, &kick, err)); + + try(bch2_dev_shrink_wait_reconcile(ca, new_nbuckets, seq, kick, + &head, &kick_complete, err)); + + bch2_fs_ec_flush_outstanding(c); + + /* Free buckets may have been changed during reconcile; refresh the count */ + try(bch2_dev_count_tail_free(c, ca, new_nbuckets)); + + try(tail_head_snapshot(c, ca, new_nbuckets, &head)); + if (shrink_tail_head_empty(&head)) + break; + + if (shrink_tail_head_progressed(&best_head, &head)) { + best_head = head; + scan_device = false; + stalled_kicks = 0; + } else if (kick_complete) { + /* + * Reconcile drained a full pass but the head didn't + * advance. If we haven't yet tried a full device rescan, + * do one. If we have and it also made no progress, this + * tail is genuinely impossible to evacuate. + */ + if (!did_scan) { + scan_device = true; + } else if (++stalled_kicks >= stalled_kicks_limit) { + prt_printf(err, + "Shrink failed: evacuating all data from the shrink tail not possible\n"); + try(bch2_dev_shrink_clear_target(c, ca, new_nbuckets, seq, err)); + return -ENOSPC; + } + } + } + + return bch2_dev_shrink_finalize(c, ca, old_nbuckets, new_nbuckets, seq, err); +} + +static int bch2_dev_resize_thread(void *arg) +{ + struct bch_dev *ca = arg; + struct bch_fs *c = ca->fs; + u64 seen_seq = 0; + + set_freezable(); + + while (!kthread_should_stop()) { + kthread_wait_freezable(kthread_should_stop() || + bch2_dev_resize_seq(ca) != seen_seq); + if (kthread_should_stop()) + break; + + while (!kthread_should_stop()) { + u64 seq = bch2_dev_resize_seq(ca); + u64 target = bch2_dev_resize_target(ca); + int ret; + CLASS(printbuf, err)(); + + if (target == ca->mi.nbuckets) { + ret = 0; + } else if (target > ca->mi.nbuckets) { + ret = __bch2_dev_grow(c, ca, target, &err); + } else { + ret = __bch2_dev_shrink(c, ca, target, seq, &err); + } + + if (ret == -EAGAIN) + continue; + + if (ret && err.pos) + bch_err_dev(ca, "%s", err.buf); + else if (ret && ret != -EINTR) + bch_err_fn_dev(ca, ret); + + seen_seq = bch2_dev_resize_seq(ca); + if (ret == -EINTR) + break; + if (!bch2_dev_resize_finish(ca, seq, ret)) + continue; + break; + } + } + + return 0; +} + +static int bch2_dev_resize_kick(struct bch_dev *ca) +{ + u64 seq; + + scoped_guard(spinlock, &ca->resize_lock) { + seq = ++ca->resize_seq; + ca->resize_status = -EINPROGRESS; + } + + wake_up_process(ca->resize_thread); + return bch2_dev_resize_wait(ca, seq); +} + +int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 target_nbuckets, struct printbuf *err) +{ + scoped_guard(rwsem_write, &c->state_lock) { + try(bch2_dev_resize_validate_target(c, ca, target_nbuckets, err)); + + // normalize + target_nbuckets = target_nbuckets == ca->mi.nbuckets ? 0 : target_nbuckets; + + try(bch2_dev_resize_thread_start(ca)); + try(bch2_dev_resize_set_target(c, ca, target_nbuckets)); + } + + int ret = bch2_dev_resize_kick(ca); + if (ret == -ECANCELED) + prt_printf(err, "Resize request superseded by a newer target\n"); + else if (ret && ret != -EINTR && !err->pos) + prt_printf(err, "Resize worker failed; see kernel log for details\n"); + return ret; +} + +int bch2_dev_resize_resume(struct bch_fs *c, struct bch_dev *ca, + struct printbuf *err) +{ + if (!bch2_dev_resize_pending(ca)) + return 0; + + scoped_guard(rwsem_write, &c->state_lock) { + try(bch2_dev_resize_thread_start(ca)); + } + + int ret = bch2_dev_resize_kick(ca); + if (ret && ret != -ECANCELED && ret != -EINTR && !err->pos) + prt_printf(err, "Resize resume failed; see kernel log for details\n"); + return ret; +} + /* Resize on mount */ int __bch2_dev_resize_alloc(struct bch_dev *ca, u64 old_nbuckets, u64 new_nbuckets) { struct bch_fs *c = ca->fs; - u64 v[3] = { new_nbuckets - old_nbuckets, 0, 0 }; + s64 v[3] = { (s64) new_nbuckets - (s64) old_nbuckets, 0, 0 }; return bch2_trans_commit_do(ca->fs, NULL, NULL, 0, bch2_disk_accounting_mod2(trans, false, v, dev_data_type, diff --git a/fs/init/dev.h b/fs/init/dev.h index b87fc4b2ed..c680a4733b 100644 --- a/fs/init/dev.h +++ b/fs/init/dev.h @@ -38,6 +38,8 @@ int bch2_dev_add(struct bch_fs *, const char *, struct printbuf *); int bch2_dev_online(struct bch_fs *, const char *, struct printbuf *); int bch2_dev_offline(struct bch_fs *, struct bch_dev *, int, struct printbuf *); int bch2_dev_resize(struct bch_fs *, struct bch_dev *, u64, struct printbuf *); +int bch2_dev_resize_resume(struct bch_fs *, struct bch_dev *, struct printbuf *); +void bch2_dev_resize_threads_stop(struct bch_fs *); int __bch2_dev_resize_alloc(struct bch_dev *, u64, u64); diff --git a/fs/init/fs.c b/fs/init/fs.c index f2d71de705..bd0b93c440 100644 --- a/fs/init/fs.c +++ b/fs/init/fs.c @@ -320,8 +320,7 @@ static void __bch2_fs_read_only(struct bch_fs *c) u64 seq = 0; bch2_maybe_schedule_btree_bitmap_gc_stop(c); - bch2_fs_ec_stop(c); - bch2_open_buckets_stop(c, NULL, true); + bch2_open_buckets_stop(c, NULL, true, 0); bch2_copygc_stop(c); bch2_btree_write_buffer_stop(c); bch2_fs_ec_flush(c); @@ -409,6 +408,34 @@ void bch2_fs_read_only(struct bch_fs *c) bch_verbose(c, "going read-only"); + /* + * Close write points before stopping background data movers. Shrink can + * leave copygc waiting on allocator space from its dedicated + * write_point; dropping those open buckets first breaks that dependency + * so kthread_stop() does not hang behind a move write that's blocked in + * the allocator. + */ + bch2_open_buckets_stop(c, NULL, true, 0); + + /* + * Stop per-device resize workers while writes are still available. + * A pending resize may be in the middle of transactional alloc/accounting + * updates; if it survives into clean shutdown it can trip write-path + * assertions while the filesystem is already going read-only. + */ + bch2_dev_resize_threads_stop(c); + + /* + * Stop background data movers before disabling writes globally: + * reconcile/copygc move writes don't hold c->writes refs, but they do + * still need journal/btree write access to finish their final index + * updates. If we shut off writes first they'll trip -EROFS during + * shutdown and spuriously account data_update failures. + */ + bch2_fs_ec_stop(c); + bch2_reconcile_stop(c); + bch2_copygc_stop(c); + /* * Block new foreground-end write operations from starting - any new * writes will return -EROFS. Set before stopping reconcile so the @@ -1548,41 +1575,79 @@ static bool bch2_fs_will_resize_on_mount(struct bch_fs *c) int bch2_fs_resize_on_mount(struct bch_fs *c) { - for_each_online_member(c, ca, BCH_DEV_READ_REF_fs_resize_on_mount) { - if (bch2_dev_will_resize_on_mount(ca)) { - u64 old_nbuckets = ca->mi.nbuckets; - u64 new_nbuckets = div64_u64(get_capacity(ca->disk_sb.bdev->bd_disk), - ca->mi.bucket_size); - - bch_info_dev(ca, "resizing to size %llu", new_nbuckets * ca->mi.bucket_size); - int ret = bch2_dev_buckets_resize(c, ca, new_nbuckets); - bch_err_fn_dev(ca, ret); - if (ret) { - enumerated_ref_put(&ca->io_ref[READ], - BCH_DEV_READ_REF_fs_resize_on_mount); - return ret; - } - - scoped_guard(mutex_noio, &c->sb_lock) { - struct bch_member *m = - bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); - m->nbuckets = cpu_to_le64(new_nbuckets); - SET_BCH_MEMBER_RESIZE_ON_MOUNT(m, false); - - c->disk_sb.sb->features[0] &= ~cpu_to_le64(BIT_ULL(BCH_FEATURE_small_image)); - bch2_write_super(c); - } - - if (ca->mi.freespace_initialized) { - ret = __bch2_dev_resize_alloc(ca, old_nbuckets, new_nbuckets); + scoped_guard(rwsem_write, &c->state_lock) { + for_each_online_member(c, ca, BCH_DEV_READ_REF_fs_resize_on_mount) { + if (bch2_dev_will_resize_on_mount(ca)) { + u64 old_nbuckets = ca->mi.nbuckets; + u64 new_nbuckets = div64_u64(get_capacity(ca->disk_sb.bdev->bd_disk), + ca->mi.bucket_size); + + bch_info_dev(ca, "resizing to size %llu", new_nbuckets * ca->mi.bucket_size); + int ret = bch2_dev_buckets_resize(c, ca, new_nbuckets); + bch_err_fn_dev(ca, ret); if (ret) { enumerated_ref_put(&ca->io_ref[READ], - BCH_DEV_READ_REF_fs_resize_on_mount); + BCH_DEV_READ_REF_fs_resize_on_mount); return ret; } + + scoped_guard(mutex_noio, &c->sb_lock) { + struct bch_member *m = + bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx); + m->nbuckets = cpu_to_le64(new_nbuckets); + SET_BCH_MEMBER_RESIZE_ON_MOUNT(m, false); + + c->disk_sb.sb->features[0] &= ~cpu_to_le64(BIT_ULL(BCH_FEATURE_small_image)); + bch2_write_super(c); + } + + if (ca->mi.freespace_initialized) { + ret = __bch2_dev_resize_alloc(ca, old_nbuckets, new_nbuckets); + if (ret) { + enumerated_ref_put(&ca->io_ref[READ], + BCH_DEV_READ_REF_fs_resize_on_mount); + return ret; + } + } } } } + + /* + * A pending shrink needs reconcile_scan and backpointer btree access. + * The early resize-on-mount hook runs before btree roots are read so it + * can handle grow-on-mount image expansion; defer shrink resume until a + * later call once BCH_FS_btree_running is set. + */ + if (!test_bit(BCH_FS_btree_running, &c->flags)) + return 0; + + if (c->opts.read_only || + (c->sb.features & (BIT_ULL(BCH_FEATURE_small_image) | + BIT_ULL(BCH_FEATURE_no_default_sb)))) + return 0; + + for_each_online_member(c, ca, BCH_DEV_READ_REF_fs_resize_on_mount) { + if (!bch2_dev_resize_pending(ca)) + continue; + + CLASS(printbuf, err)(); + int ret; + + bch_info_dev(ca, "resuming resize to size %llu", + bch2_dev_resize_target(ca) * ca->mi.bucket_size); + ret = bch2_dev_resize_resume(c, ca, &err); + if (ret) { + if (err.pos) + bch_err_dev(ca, "%s", err.buf); + else + bch_err_fn_dev(ca, ret); + + enumerated_ref_put(&ca->io_ref[READ], + BCH_DEV_READ_REF_fs_resize_on_mount); + return ret; + } + } return 0; } diff --git a/fs/init/recovery.c b/fs/init/recovery.c index abe65d931a..a402a1737f 100644 --- a/fs/init/recovery.c +++ b/fs/init/recovery.c @@ -832,8 +832,7 @@ static int __bch2_fs_recovery(struct bch_fs *c) try(journal_replay_early(c, clean)); - scoped_guard(rwsem_write, &c->state_lock) - try(bch2_fs_resize_on_mount(c)); + try(bch2_fs_resize_on_mount(c)); if (c->sb.features & BIT_ULL(BCH_FEATURE_small_image)) { bch_info(c, "filesystem is an unresized image file, mounting ro"); @@ -969,6 +968,8 @@ static int __bch2_fs_recovery(struct bch_fs *c) set_bit(BCH_FS_may_go_rw, &c->flags); clear_bit(BCH_FS_in_fsck, &c->flags); + try(bch2_fs_resize_on_mount(c)); + /* in case we don't run journal replay, i.e. norecovery mode */ set_bit(BCH_FS_accounting_replay_done, &c->flags); diff --git a/fs/journal/init.c b/fs/journal/init.c index 69c8be00a7..3c60846453 100644 --- a/fs/journal/init.c +++ b/fs/journal/init.c @@ -217,37 +217,53 @@ int bch2_dev_journal_bucket_delete(struct bch_dev *ca, u64 b) &new_buckets[pos + 1], (ja->nr - 1 - pos) * sizeof(new_buckets[0])); - int ret = bch2_journal_buckets_to_sb(c, ca, ja->buckets, ja->nr - 1) ?: - bch2_write_super(c); + int ret; + + scoped_guard(journal_block, &c->journal) { + ret = bch2_journal_buckets_to_sb(c, ca, new_buckets, ja->nr - 1) ?: + bch2_write_super(c); + if (ret) + break; + + scoped_guard(spinlock, &j->lock) { + if (pos < ja->discard_idx) + --ja->discard_idx; + if (pos < ja->dirty_idx_ondisk) + --ja->dirty_idx_ondisk; + if (pos < ja->dirty_idx) + --ja->dirty_idx; + if (pos < ja->cur_idx) + --ja->cur_idx; + + ja->nr--; + + memmove(&ja->buckets[pos], + &ja->buckets[pos + 1], + (ja->nr - pos) * sizeof(ja->buckets[0])); + + memmove(&ja->bucket_seq[pos], + &ja->bucket_seq[pos + 1], + (ja->nr - pos) * sizeof(ja->bucket_seq[0])); + + bch2_journal_space_available(j); + } + } + if (ret) { kfree(new_buckets); return ret; } - scoped_guard(spinlock, &j->lock) { - if (pos < ja->discard_idx) - --ja->discard_idx; - if (pos < ja->dirty_idx_ondisk) - --ja->dirty_idx_ondisk; - if (pos < ja->dirty_idx) - --ja->dirty_idx; - if (pos < ja->cur_idx) - --ja->cur_idx; - - ja->nr--; - - memmove(&ja->buckets[pos], - &ja->buckets[pos + 1], - (ja->nr - pos) * sizeof(ja->buckets[0])); - - memmove(&ja->bucket_seq[pos], - &ja->bucket_seq[pos + 1], - (ja->nr - pos) * sizeof(ja->bucket_seq[0])); - - bch2_journal_space_available(j); + kfree(new_buckets); + + { + CLASS(btree_trans, trans)(c); + ret = commit_do(trans, NULL, NULL, 0, + bch2_trans_mark_metadata_bucket(trans, ca, b, + BCH_DATA_free, 0, + BTREE_TRIGGER_transactional)); } - kfree(new_buckets); return 0; } diff --git a/fs/journal/write.c b/fs/journal/write.c index a90a57988b..0e0d913318 100644 --- a/fs/journal/write.c +++ b/fs/journal/write.c @@ -23,9 +23,18 @@ #include "sb/clean.h" #include "sb/counters.h" +#include "sb/members.h" #include +static unsigned journal_alloc_target(struct bch_fs *c) +{ + unsigned target = c->opts.metadata_target ?: c->opts.foreground_target; + return target && bch2_target_has_non_shrinking_dev(c, BCH_DATA_journal, target) + ? target + : 0; +} + static void journal_advance_devs_to_next_bucket(struct journal *j, struct bch_devs_mask *devs, unsigned sectors, __le64 seq) @@ -138,8 +147,7 @@ static int journal_write_alloc(struct journal *j, struct journal_buf *w, struct bch_fs *c = container_of(j, struct bch_fs, journal); struct bch_devs_mask devs; unsigned sectors = vstruct_sectors(w->data, c->block_bits); - unsigned target = c->opts.metadata_target ?: - c->opts.foreground_target; + unsigned target = journal_alloc_target(c); unsigned replicas_want = READ_ONCE(c->opts.metadata_replicas); bool advance_done = false; diff --git a/fs/sb/downgrade.c b/fs/sb/downgrade.c index 08f6effe1d..26136b8546 100644 --- a/fs/sb/downgrade.c +++ b/fs/sb/downgrade.c @@ -213,6 +213,9 @@ BIT_ULL(BCH_RECOVERY_PASS_check_alloc_to_lru_refs), \ BCH_FSCK_ERR_lru_entry_bad, \ BCH_FSCK_ERR_alloc_key_to_missing_lru_entry, \ + BCH_FSCK_ERR_accounting_mismatch) \ + x(snapshot_nr_keys, \ + BIT_ULL(BCH_RECOVERY_PASS_check_allocations), \ BCH_FSCK_ERR_accounting_mismatch) struct upgrade_downgrade_entry { diff --git a/fs/sb/members.c b/fs/sb/members.c index 14751e9855..18a48ce27e 100644 --- a/fs/sb/members.c +++ b/fs/sb/members.c @@ -216,6 +216,29 @@ static int validate_member(struct printbuf *err, return -BCH_ERR_invalid_sb_members; } + u64 target_nbuckets = le64_to_cpu(m.target_nbuckets); + + if (target_nbuckets) { + if (target_nbuckets < first_bucket) { + prt_printf(err, "device %u: target buckets starts before first bucket (got %llu, first %u)", + i, target_nbuckets, first_bucket); + return -BCH_ERR_invalid_sb_members; + } + + if (target_nbuckets < nbuckets && + target_nbuckets - first_bucket < BCH_MIN_NR_NBUCKETS) { + prt_printf(err, "device %u: not enough target buckets (got %llu, min %u)", + i, target_nbuckets - first_bucket, BCH_MIN_NR_NBUCKETS); + return -BCH_ERR_invalid_sb_members; + } + + if (target_nbuckets > BCH_MEMBER_NBUCKETS_MAX) { + prt_printf(err, "device %u: target buckets too big (got %llu, max %u)", + i, target_nbuckets, BCH_MEMBER_NBUCKETS_MAX); + return -BCH_ERR_invalid_sb_members; + } + } + return 0; } @@ -227,6 +250,7 @@ __cold void bch2_member_to_text(struct printbuf *out, { u64 bucket_size = le16_to_cpu(m->bucket_size); u64 device_size = le64_to_cpu(m->nbuckets) * bucket_size; + u64 target_device_size = le64_to_cpu(m->target_nbuckets) * bucket_size; prt_printf(out, "Label:\t"); if (BCH_MEMBER_GROUP(m)) @@ -248,6 +272,14 @@ __cold void bch2_member_to_text(struct printbuf *out, prt_units_u64(out, device_size << 9); prt_newline(out); + prt_printf(out, "Target size:\t"); + if (target_device_size) { + prt_units_u64(out, target_device_size << 9); + } else { + prt_printf(out, "Inactive"); + } + prt_newline(out); + for (unsigned i = 0; i < BCH_MEMBER_ERROR_NR; i++) prt_printf(out, "%s errors:\t%llu\n", bch2_member_error_strs[i], le64_to_cpu(m->errors[i])); @@ -261,6 +293,14 @@ __cold void bch2_member_to_text(struct printbuf *out, prt_printf(out, "First bucket:\t%u\n", le16_to_cpu(m->first_bucket)); prt_printf(out, "Buckets:\t%llu\n", le64_to_cpu(m->nbuckets)); + prt_printf(out, "Target buckets:\t"); + if (target_device_size) { + prt_printf(out, "%llu", le64_to_cpu(m->target_nbuckets)); + } else { + prt_printf(out, "Inactive"); + } + prt_newline(out); + prt_printf(out, "Last mount:\t"); if (m->last_mount) bch2_prt_datetime(out, le64_to_cpu(m->last_mount)); diff --git a/fs/sb/members.h b/fs/sb/members.h index 824a3c6cd9..f4463097e5 100644 --- a/fs/sb/members.h +++ b/fs/sb/members.h @@ -269,18 +269,49 @@ static inline struct bch_dev *bch2_dev_rcu_noerror(const struct bch_fs *c, unsig : NULL; } -static inline bool bch2_dev_bad_or_evacuating_rcu(struct bch_fs *c, unsigned dev) +static inline u64 bch2_dev_resize_target(const struct bch_dev *ca) { - struct bch_dev *ca = bch2_dev_rcu_noerror(c, dev); - return !ca || ca->mi.state == BCH_MEMBER_STATE_evacuating; + u64 target = READ_ONCE(ca->mi.target_nbuckets); + + return target ?: READ_ONCE(ca->mi.nbuckets); +} + +static inline bool bch2_dev_resize_pending(const struct bch_dev *ca) +{ + return bch2_dev_resize_target(ca) != READ_ONCE(ca->mi.nbuckets); +} + +static inline bool bch2_dev_is_shrinking(const struct bch_dev *ca) +{ + return bch2_dev_resize_target(ca) < READ_ONCE(ca->mi.nbuckets); +} + +static inline bool bch2_dev_is_growing(const struct bch_dev *ca) +{ + return bch2_dev_resize_target(ca) > READ_ONCE(ca->mi.nbuckets); } -static inline bool bch2_dev_bad_or_evacuating(struct bch_fs *c, unsigned dev) +static inline bool bch2_ptr_bad_or_evacuating_rcu(struct bch_fs *c, const struct bch_extent_ptr *ptr) +{ + struct bch_dev *ca = bch2_dev_rcu_noerror(c, ptr->dev); + u64 resize_target; + + if (!ca || ca->mi.state == BCH_MEMBER_STATE_evacuating) + return true; + + resize_target = bch2_dev_resize_target(ca); + + return resize_target < READ_ONCE(ca->mi.nbuckets) && + ptr->offset >= resize_target * ca->mi.bucket_size; +} + +static inline bool bch2_ptr_bad_or_evacuating(struct bch_fs *c, const struct bch_extent_ptr *ptr) { guard(rcu)(); - return bch2_dev_bad_or_evacuating_rcu(c, dev); + return bch2_ptr_bad_or_evacuating_rcu(c, ptr); } + int bch2_dev_missing_bkey_msg(struct bch_fs *, struct bkey_s_c, unsigned, struct printbuf *out); int bch2_dev_missing_bkey(struct bch_fs *, struct bkey_s_c, unsigned); @@ -429,13 +460,14 @@ static inline struct bch_member_cpu bch2_mi_to_cpu(struct bch_member *mi) .durability = BCH_MEMBER_DURABILITY(mi) ? BCH_MEMBER_DURABILITY(mi) - 1 : 1, - .freespace_initialized = BCH_MEMBER_FREESPACE_INITIALIZED(mi), + .freespace_initialized = BCH_MEMBER_FREESPACE_INITIALIZED(mi), .initialized = BCH_MEMBER_INITIALIZED(mi), .resize_on_mount = BCH_MEMBER_RESIZE_ON_MOUNT(mi), .rotational = BCH_MEMBER_ROTATIONAL(mi), .valid = bch2_member_alive(mi), .btree_bitmap_shift = mi->btree_bitmap_shift, .btree_allocated_bitmap = le64_to_cpu(mi->btree_allocated_bitmap), + .target_nbuckets = le64_to_cpu(mi->target_nbuckets), }; } diff --git a/fs/sb/members_format.h b/fs/sb/members_format.h index 7768576bc6..fd4147eb12 100644 --- a/fs/sb/members_format.h +++ b/fs/sb/members_format.h @@ -85,6 +85,7 @@ struct bch_member { * the allocation path. */ __u8 failure_domain[32] __nonstring; + __le64 target_nbuckets; /* 0 => idle, nbuckets => idle, < nbuckets => shrink target, > nbuckets => grow target */ }; /* diff --git a/fs/sb/members_types.h b/fs/sb/members_types.h index 46b26e49db..d48ce65182 100644 --- a/fs/sb/members_types.h +++ b/fs/sb/members_types.h @@ -20,6 +20,7 @@ struct bch_member_cpu { u8 valid; u8 btree_bitmap_shift; u64 btree_allocated_bitmap; + u64 target_nbuckets; /* 0 => idle, nbuckets => idle, < nbuckets => shrink target, > nbuckets => grow target */ }; #endif /* _BCACHEFS_SB_MEMBERS_H */ diff --git a/include/linux/wait.h b/include/linux/wait.h index bfd40a5f45..ec4fdb2227 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -113,6 +113,8 @@ do { \ __ret; \ }) +#define wait_event_killable_timeout(wq, condition, timeout) ({wait_event_timeout(wq, condition, timeout); 0; }) + void wake_up_bit(void *, int); void __wait_on_bit(void *, int, unsigned); void __wait_on_bit_lock(void *, int, unsigned); diff --git a/src/commands/device.rs b/src/commands/device.rs index 00a6714e1d..e8dc34b3c3 100644 --- a/src/commands/device.rs +++ b/src/commands/device.rs @@ -389,25 +389,38 @@ pub struct ResizeCli { /// Device path device: String, - /// New size (human-readable, e.g. 1G); defaults to device size + /// New size (human-readable, e.g. 1G) or "cancel" (resizes to the currently persisted size); defaults to device size size: Option, } +enum SizeOrCancel { + Size(u64), + Cancel, +} fn cmd_device_resize(cli: ResizeCli) -> Result<()> { + use SizeOrCancel::*; let size_bytes = match cli.size { - Some(ref s) => parse_human_size(s)?, - None => device_size(&cli.device)?, + Some(ref s) => match s.as_str() { + "cancel" | "Cancel" => Cancel, + other => Size(parse_human_size(other)?), + }, + None => Size(device_size(&cli.device)?), + }; + let size_sectors = match size_bytes { + Size(bytes) => Size(bytes >> 9), + Cancel => Cancel, }; - let size_sectors = size_bytes >> 9; match open_dev(&cli.device) { Ok((handle, dev_idx)) => { println!("Doing online resize of {}", cli.device); - let usage = handle.dev_usage(dev_idx) - .context("querying device usage")?; - let nbuckets = size_sectors / usage.bucket_size as u64; + let usage = handle.dev_usage(dev_idx).context("querying device usage")?; + let nbuckets = match size_sectors { + Size(sectors) => sectors / usage.bucket_size as u64, + Cancel => usage.nr_buckets, + }; let shrinking = nbuckets < usage.nr_buckets; println!("resizing {} to {} buckets", cli.device, nbuckets); @@ -423,7 +436,7 @@ fn cmd_device_resize(cli: ResizeCli) -> Result<()> { Err(_) if Path::new(&cli.device).exists() => { println!("Doing offline resize of {}", cli.device); resize_offline(&cli.device, size_sectors)?; - } + }, Err(e) => return Err(e), } @@ -454,8 +467,10 @@ fn find_single_online_dev(fs: &Fs) -> Result { .ok_or_else(|| anyhow!("could not get reference to device {}", found_idx)) } -fn resize_offline(device: &str, size_sectors: u64) -> Result<()> { +fn resize_offline(device: &str, size_sectors: SizeOrCancel) -> Result<()> { use bcachefs_kernel::util::printbuf::Printbuf; + use SizeOrCancel::*; + let opts: c::bch_opts = Default::default(); let fs = crate::device_scan::open_scan(&[PathBuf::from(device)], opts) @@ -463,7 +478,10 @@ fn resize_offline(device: &str, size_sectors: u64) -> Result<()> { let ca = find_single_online_dev(&fs)?; - let nbuckets = size_sectors / ca.mi.bucket_size as u64; + let nbuckets = match size_sectors { + Size(sectors) => sectors / ca.mi.bucket_size as u64, + Cancel => ca.mi.nbuckets, + }; if nbuckets < ca.mi.nbuckets { bail!("shrinking not supported (requested {} buckets, have {})",