From 6ac414963a7e862ee6e075da6183807ea4537fb9 Mon Sep 17 00:00:00 2001 From: Kenta Akagi Date: Fri, 29 Aug 2025 01:32:14 +0900 Subject: [PATCH 1/3] md/raid1,raid10: Do not set MD_BROKEN on failfast io failure This commit ensures that an MD_FAILFAST IO failure does not put the array into a broken state. When failfast is enabled on rdev in RAID1 or RAID10, the array may be flagged MD_BROKEN in the following cases. - If MD_FAILFAST IOs to multiple rdevs fail simultaneously - If an MD_FAILFAST metadata write to the 'last' rdev fails The MD_FAILFAST bio error handler always calls md_error on IO failure, under the assumption that raid{1,10}_error will neither fail the last rdev nor break the array. After commit 9631abdbf406 ("md: Set MD_BROKEN for RAID1 and RAID10"), calling md_error on the 'last' rdev in RAID1/10 always sets the MD_BROKEN flag on the array. As a result, when failfast IO fails on the last rdev, the array immediately becomes failed. Normally, MD_FAILFAST IOs are not issued to the 'last' rdev, so this is an edge case; however, it can occur if rdevs in a non-degraded array share the same path and that path is lost, or if a metadata write is triggered in a degraded array and fails due to failfast. When a failfast metadata write fails, it is retried through the following sequence. If a metadata write without failfast fails, the array will be marked with MD_BROKEN. 1. MD_SB_NEED_REWRITE is set in sb_flags by super_written. 2. md_super_wait, called by the function executing md_super_write, returns -EAGAIN due to MD_SB_NEED_REWRITE. 3. The caller of md_super_wait (e.g., md_update_sb) receives the negative return value and retries md_super_write. 4. md_super_write issues the metadata write again, this time without MD_FAILFAST. Fixes: 9631abdbf406 ("md: Set MD_BROKEN for RAID1 and RAID10") Signed-off-by: Kenta Akagi --- drivers/md/md.c | 14 +++++++++----- drivers/md/md.h | 13 +++++++------ drivers/md/raid1.c | 18 ++++++++++++++++-- drivers/md/raid10.c | 21 ++++++++++++++++++--- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/drivers/md/md.c b/drivers/md/md.c index 09042b060086..2cf4db007aeb 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -986,14 +986,18 @@ static void super_written(struct bio *bio) if (bio->bi_status) { pr_err("md: %s gets error=%d\n", __func__, blk_status_to_errno(bio->bi_status)); + if (bio->bi_opf & MD_FAILFAST) + set_bit(FailfastIOFailure, &rdev->flags); md_error(mddev, rdev); if (!test_bit(Faulty, &rdev->flags) && (bio->bi_opf & MD_FAILFAST)) { + pr_warn("md: %s: Metadata write will be repeated to %pg\n", + mdname(mddev), rdev->bdev); set_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags); - set_bit(LastDev, &rdev->flags); } - } else - clear_bit(LastDev, &rdev->flags); + } else { + clear_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags); + } bio_put(bio); @@ -1035,7 +1039,7 @@ void md_super_write(struct mddev *mddev, struct md_rdev *rdev, if (test_bit(MD_FAILFAST_SUPPORTED, &mddev->flags) && test_bit(FailFast, &rdev->flags) && - !test_bit(LastDev, &rdev->flags)) + !test_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags)) bio->bi_opf |= MD_FAILFAST; atomic_inc(&mddev->pending_writes); @@ -1046,7 +1050,7 @@ int md_super_wait(struct mddev *mddev) { /* wait for all superblock writes that were scheduled to complete */ wait_event(mddev->sb_wait, atomic_read(&mddev->pending_writes)==0); - if (test_and_clear_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags)) + if (test_bit(MD_SB_NEED_REWRITE, &mddev->sb_flags)) return -EAGAIN; return 0; } diff --git a/drivers/md/md.h b/drivers/md/md.h index d45a9e6ead80..10e13bf6a2e6 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -281,9 +281,10 @@ enum flag_bits { * It is expects that no bad block log * is present. */ - LastDev, /* Seems to be the last working dev as - * it didn't fail, so don't use FailFast - * any more for metadata + FailfastIOFailure, /* rdev with failfast IO failure + * but md_error not yet completed. + * If the last rdev has this flag, + * error_handler must not fail the array */ CollisionCheck, /* * check if there is collision between raid1 @@ -331,8 +332,8 @@ struct md_cluster_operations; * @MD_CLUSTER_RESYNC_LOCKED: cluster raid only, which means node, already took * resync lock, need to release the lock. * @MD_FAILFAST_SUPPORTED: Using MD_FAILFAST on metadata writes is supported as - * calls to md_error() will never cause the array to - * become failed. + * calls to md_error() with FailfastIOFailure will + * never cause the array to become failed. * @MD_HAS_PPL: The raid array has PPL feature set. * @MD_HAS_MULTIPLE_PPLS: The raid array has multiple PPLs feature set. * @MD_NOT_READY: do_md_run() is active, so 'array_state', ust not report that @@ -360,7 +361,7 @@ enum mddev_sb_flags { MD_SB_CHANGE_DEVS, /* Some device status has changed */ MD_SB_CHANGE_CLEAN, /* transition to or from 'clean' */ MD_SB_CHANGE_PENDING, /* switch from 'clean' to 'active' in progress */ - MD_SB_NEED_REWRITE, /* metadata write needs to be repeated */ + MD_SB_NEED_REWRITE, /* metadata write needs to be repeated, do not use failfast */ }; #define NR_SERIAL_INFOS 8 diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 64b8176907a9..f8e710360880 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -471,6 +471,7 @@ static void raid1_end_write_request(struct bio *bio) (bio->bi_opf & MD_FAILFAST) && /* We never try FailFast to WriteMostly devices */ !test_bit(WriteMostly, &rdev->flags)) { + set_bit(FailfastIOFailure, &rdev->flags); md_error(r1_bio->mddev, rdev); } @@ -1748,8 +1749,12 @@ static void raid1_status(struct seq_file *seq, struct mddev *mddev) * - recovery is interrupted. * - &mddev->degraded is bumped. * - * @rdev is marked as &Faulty excluding case when array is failed and - * &mddev->fail_last_dev is off. + * If @rdev has &FailfastIOFailure and it is the 'last' rdev, + * then @mddev and @rdev will not be marked as failed. + * + * @rdev is marked as &Faulty excluding any cases: + * - when @mddev is failed and &mddev->fail_last_dev is off + * - when @rdev is last device and &FailfastIOFailure flag is set */ static void raid1_error(struct mddev *mddev, struct md_rdev *rdev) { @@ -1760,6 +1765,13 @@ static void raid1_error(struct mddev *mddev, struct md_rdev *rdev) if (test_bit(In_sync, &rdev->flags) && (conf->raid_disks - mddev->degraded) == 1) { + if (test_and_clear_bit(FailfastIOFailure, &rdev->flags)) { + spin_unlock_irqrestore(&conf->device_lock, flags); + pr_warn_ratelimited("md/raid1:%s: Failfast IO failure on %pg, " + "last device but ignoring it\n", + mdname(mddev), rdev->bdev); + return; + } set_bit(MD_BROKEN, &mddev->flags); if (!mddev->fail_last_dev) { @@ -2150,6 +2162,7 @@ static int fix_sync_read_error(struct r1bio *r1_bio) if (test_bit(FailFast, &rdev->flags)) { /* Don't try recovering from here - just fail it * ... unless it is the last working device of course */ + set_bit(FailfastIOFailure, &rdev->flags); md_error(mddev, rdev); if (test_bit(Faulty, &rdev->flags)) /* Don't try to read from here, but make sure @@ -2654,6 +2667,7 @@ static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio) fix_read_error(conf, r1_bio); unfreeze_array(conf); } else if (mddev->ro == 0 && test_bit(FailFast, &rdev->flags)) { + set_bit(FailfastIOFailure, &rdev->flags); md_error(mddev, rdev); } else { r1_bio->bios[r1_bio->read_disk] = IO_BLOCKED; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index c9bd2005bfd0..3223945fa995 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -488,6 +488,7 @@ static void raid10_end_write_request(struct bio *bio) dec_rdev = 0; if (test_bit(FailFast, &rdev->flags) && (bio->bi_opf & MD_FAILFAST)) { + set_bit(FailfastIOFailure, &rdev->flags); md_error(rdev->mddev, rdev); } @@ -1995,8 +1996,12 @@ static int enough(struct r10conf *conf, int ignore) * - recovery is interrupted. * - &mddev->degraded is bumped. * - * @rdev is marked as &Faulty excluding case when array is failed and - * &mddev->fail_last_dev is off. + * If @rdev has &FailfastIOFailure and it is the 'last' rdev, + * then @mddev and @rdev will not be marked as failed. + * + * @rdev is marked as &Faulty excluding any cases: + * - when @mddev is failed and &mddev->fail_last_dev is off + * - when @rdev is last device and &FailfastIOFailure flag is set */ static void raid10_error(struct mddev *mddev, struct md_rdev *rdev) { @@ -2006,6 +2011,13 @@ static void raid10_error(struct mddev *mddev, struct md_rdev *rdev) spin_lock_irqsave(&conf->device_lock, flags); if (test_bit(In_sync, &rdev->flags) && !enough(conf, rdev->raid_disk)) { + if (test_and_clear_bit(FailfastIOFailure, &rdev->flags)) { + spin_unlock_irqrestore(&conf->device_lock, flags); + pr_warn_ratelimited("md/raid10:%s: Failfast IO failure on %pg, " + "last device but ignoring it\n", + mdname(mddev), rdev->bdev); + return; + } set_bit(MD_BROKEN, &mddev->flags); if (!mddev->fail_last_dev) { @@ -2413,6 +2425,7 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio) continue; } else if (test_bit(FailFast, &rdev->flags)) { /* Just give up on this device */ + set_bit(FailfastIOFailure, &rdev->flags); md_error(rdev->mddev, rdev); continue; } @@ -2868,8 +2881,10 @@ static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio) freeze_array(conf, 1); fix_read_error(conf, mddev, r10_bio); unfreeze_array(conf); - } else + } else { + set_bit(FailfastIOFailure, &rdev->flags); md_error(mddev, rdev); + } rdev_dec_pending(rdev, mddev); r10_bio->state = 0; From 1c2bdf39f322a94b1e5542f95d618525aab244ec Mon Sep 17 00:00:00 2001 From: Kenta Akagi Date: Fri, 29 Aug 2025 01:32:15 +0900 Subject: [PATCH 2/3] md/raid1,raid10: Add error message when setting MD_BROKEN Once MD_BROKEN is set on an array, no further writes can be performed to it. The user must be informed that the array cannot continue operation. Signed-off-by: Kenta Akagi --- drivers/md/raid1.c | 5 +++++ drivers/md/raid10.c | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index f8e710360880..419e19b80160 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1772,7 +1772,12 @@ static void raid1_error(struct mddev *mddev, struct md_rdev *rdev) mdname(mddev), rdev->bdev); return; } + set_bit(MD_BROKEN, &mddev->flags); + pr_crit("md/raid1:%s: Disk failure on %pg, this is the last device.\n" + "md/raid1:%s: Cannot continue operation (%d/%d failed).\n", + mdname(mddev), rdev->bdev, + mdname(mddev), mddev->degraded + 1, conf->raid_disks); if (!mddev->fail_last_dev) { conf->recovery_disabled = mddev->recovery_disabled; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 3223945fa995..8046972a20ef 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -2018,7 +2018,12 @@ static void raid10_error(struct mddev *mddev, struct md_rdev *rdev) mdname(mddev), rdev->bdev); return; } + set_bit(MD_BROKEN, &mddev->flags); + pr_crit("md/raid10:%s: Disk failure on %pg, this is the last device.\n" + "md/raid10:%s: Cannot continue operation (%d/%d failed).\n", + mdname(mddev), rdev->bdev, + mdname(mddev), mddev->degraded + 1, conf->geo.raid_disks); if (!mddev->fail_last_dev) { spin_unlock_irqrestore(&conf->device_lock, flags); From d14d29c0d25a16fdb4ebc1d079d43ab9908b81ba Mon Sep 17 00:00:00 2001 From: Kenta Akagi Date: Fri, 29 Aug 2025 01:32:16 +0900 Subject: [PATCH 3/3] md/raid1,raid10: Fix: Operation continuing on 0 devices. Since commit 9a567843f7ce ("md: allow last device to be forcibly removed from RAID1/RAID10."), RAID1/10 arrays can now lose all rdevs. Before that commit, losing the array last rdev or reaching the end of the function without early return in raid{1,10}_error never occurred. However, both situations can occur in the current implementation. As a result, when mddev->fail_last_dev is set, a spurious pr_crit message can be printed. This patch prevents "Operation continuing" printed if the array is not operational. root@fedora:~# mdadm --create --verbose /dev/md0 --level=1 \ --raid-devices=2 /dev/loop0 /dev/loop1 mdadm: Note: this array has metadata at the start and may not be suitable as a boot device. If you plan to store '/boot' on this device please ensure that your boot-loader understands md/v1.x metadata, or use --metadata=0.90 mdadm: size set to 1046528K Continue creating array? y mdadm: Defaulting to version 1.2 metadata mdadm: array /dev/md0 started. root@fedora:~# echo 1 > /sys/block/md0/md/fail_last_dev root@fedora:~# mdadm --fail /dev/md0 loop0 mdadm: set loop0 faulty in /dev/md0 root@fedora:~# mdadm --fail /dev/md0 loop1 mdadm: set device faulty failed for loop1: Device or resource busy root@fedora:~# dmesg | tail -n 4 [ 1314.359674] md/raid1:md0: Disk failure on loop0, disabling device. md/raid1:md0: Operation continuing on 1 devices. [ 1315.506633] md/raid1:md0: Disk failure on loop1, disabling device. md/raid1:md0: Operation continuing on 0 devices. root@fedora:~# Fixes: 9a567843f7ce ("md: allow last device to be forcibly removed from RAID1/RAID10.") Signed-off-by: Kenta Akagi --- drivers/md/raid1.c | 9 +++++---- drivers/md/raid10.c | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 419e19b80160..face493c8903 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -1789,6 +1789,11 @@ static void raid1_error(struct mddev *mddev, struct md_rdev *rdev) if (test_and_clear_bit(In_sync, &rdev->flags)) mddev->degraded++; set_bit(Faulty, &rdev->flags); + if ((conf->raid_disks - mddev->degraded) > 0) + pr_crit("md/raid1:%s: Disk failure on %pg, disabling device.\n" + "md/raid1:%s: Operation continuing on %d devices.\n", + mdname(mddev), rdev->bdev, + mdname(mddev), conf->raid_disks - mddev->degraded); spin_unlock_irqrestore(&conf->device_lock, flags); /* * if recovery is running, make sure it aborts. @@ -1796,10 +1801,6 @@ static void raid1_error(struct mddev *mddev, struct md_rdev *rdev) set_bit(MD_RECOVERY_INTR, &mddev->recovery); set_mask_bits(&mddev->sb_flags, 0, BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); - pr_crit("md/raid1:%s: Disk failure on %pg, disabling device.\n" - "md/raid1:%s: Operation continuing on %d devices.\n", - mdname(mddev), rdev->bdev, - mdname(mddev), conf->raid_disks - mddev->degraded); } static void print_conf(struct r1conf *conf) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 8046972a20ef..5a5d09206607 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -2038,11 +2038,12 @@ static void raid10_error(struct mddev *mddev, struct md_rdev *rdev) set_bit(Faulty, &rdev->flags); set_mask_bits(&mddev->sb_flags, 0, BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING)); + if (enough(conf, -1)) + pr_crit("md/raid10:%s: Disk failure on %pg, disabling device.\n" + "md/raid10:%s: Operation continuing on %d devices.\n", + mdname(mddev), rdev->bdev, + mdname(mddev), conf->geo.raid_disks - mddev->degraded); spin_unlock_irqrestore(&conf->device_lock, flags); - pr_crit("md/raid10:%s: Disk failure on %pg, disabling device.\n" - "md/raid10:%s: Operation continuing on %d devices.\n", - mdname(mddev), rdev->bdev, - mdname(mddev), conf->geo.raid_disks - mddev->degraded); } static void print_conf(struct r10conf *conf)