diff --git a/drivers/md/md.c b/drivers/md/md.c index 9a0a1e0934d5..fd4af4de03b4 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -2147,14 +2147,12 @@ EXPORT_SYMBOL(md_integrity_register); */ int md_integrity_add_rdev(struct md_rdev *rdev, struct mddev *mddev) { - struct blk_integrity *bi_rdev; struct blk_integrity *bi_mddev; char name[BDEVNAME_SIZE]; if (!mddev->gendisk) return 0; - bi_rdev = bdev_get_integrity(rdev->bdev); bi_mddev = blk_get_integrity(mddev->gendisk); if (!bi_mddev) /* nothing to do */ @@ -5693,14 +5691,10 @@ int md_run(struct mddev *mddev) return 0; abort: - if (mddev->flush_bio_pool) { - mempool_destroy(mddev->flush_bio_pool); - mddev->flush_bio_pool = NULL; - } - if (mddev->flush_pool){ - mempool_destroy(mddev->flush_pool); - mddev->flush_pool = NULL; - } + mempool_destroy(mddev->flush_bio_pool); + mddev->flush_bio_pool = NULL; + mempool_destroy(mddev->flush_pool); + mddev->flush_pool = NULL; return err; } diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index b98e746e7fc4..abb5d382f64d 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -1124,6 +1124,29 @@ static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule) kfree(plug); } +/* + * 1. Register the new request and wait if the reconstruction thread has put + * up a bar for new requests. Continue immediately if no resync is active + * currently. + * 2. If IO spans the reshape position. Need to wait for reshape to pass. + */ +static void regular_request_wait(struct mddev *mddev, struct r10conf *conf, + struct bio *bio, sector_t sectors) +{ + wait_barrier(conf); + while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && + bio->bi_iter.bi_sector < conf->reshape_progress && + bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { + raid10_log(conf->mddev, "wait reshape"); + allow_barrier(conf); + wait_event(conf->wait_barrier, + conf->reshape_progress <= bio->bi_iter.bi_sector || + conf->reshape_progress >= bio->bi_iter.bi_sector + + sectors); + wait_barrier(conf); + } +} + static void raid10_read_request(struct mddev *mddev, struct bio *bio, struct r10bio *r10_bio) { @@ -1132,7 +1155,6 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, const int op = bio_op(bio); const unsigned long do_sync = (bio->bi_opf & REQ_SYNC); int max_sectors; - sector_t sectors; struct md_rdev *rdev; char b[BDEVNAME_SIZE]; int slot = r10_bio->read_slot; @@ -1166,30 +1188,8 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, } rcu_read_unlock(); } - /* - * Register the new request and wait if the reconstruction - * thread has put up a bar for new requests. - * Continue immediately if no resync is active currently. - */ - wait_barrier(conf); - - sectors = r10_bio->sectors; - while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && - bio->bi_iter.bi_sector < conf->reshape_progress && - bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { - /* - * IO spans the reshape position. Need to wait for reshape to - * pass - */ - raid10_log(conf->mddev, "wait reshape"); - allow_barrier(conf); - wait_event(conf->wait_barrier, - conf->reshape_progress <= bio->bi_iter.bi_sector || - conf->reshape_progress >= bio->bi_iter.bi_sector + - sectors); - wait_barrier(conf); - } + regular_request_wait(mddev, conf, bio, r10_bio->sectors); rdev = read_balance(conf, r10_bio, &max_sectors); if (!rdev) { if (err_rdev) { @@ -1209,7 +1209,9 @@ static void raid10_read_request(struct mddev *mddev, struct bio *bio, struct bio *split = bio_split(bio, max_sectors, gfp, &conf->bio_split); bio_chain(split, bio); + allow_barrier(conf); generic_make_request(bio); + wait_barrier(conf); bio = split; r10_bio->master_bio = bio; r10_bio->sectors = max_sectors; @@ -1332,30 +1334,8 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio, finish_wait(&conf->wait_barrier, &w); } - /* - * Register the new request and wait if the reconstruction - * thread has put up a bar for new requests. - * Continue immediately if no resync is active currently. - */ - wait_barrier(conf); - sectors = r10_bio->sectors; - while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && - bio->bi_iter.bi_sector < conf->reshape_progress && - bio->bi_iter.bi_sector + sectors > conf->reshape_progress) { - /* - * IO spans the reshape position. Need to wait for reshape to - * pass - */ - raid10_log(conf->mddev, "wait reshape"); - allow_barrier(conf); - wait_event(conf->wait_barrier, - conf->reshape_progress <= bio->bi_iter.bi_sector || - conf->reshape_progress >= bio->bi_iter.bi_sector + - sectors); - wait_barrier(conf); - } - + regular_request_wait(mddev, conf, bio, sectors); if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) && (mddev->reshape_backwards ? (bio->bi_iter.bi_sector < conf->reshape_safe && @@ -1514,7 +1494,9 @@ retry_write: struct bio *split = bio_split(bio, r10_bio->sectors, GFP_NOIO, &conf->bio_split); bio_chain(split, bio); + allow_barrier(conf); generic_make_request(bio); + wait_barrier(conf); bio = split; r10_bio->master_bio = bio; } diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index ea8505204fdf..605cf46c17bd 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -35,6 +35,7 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; #include #include #include +#include #include /* Not standard, but glibc defines it */ @@ -52,7 +53,9 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; #define __init #define __exit -#define __attribute_const__ __attribute__((const)) +#ifndef __attribute_const__ +# define __attribute_const__ __attribute__((const)) +#endif #define noinline __attribute__((noinline)) #define preempt_enable() @@ -67,6 +70,9 @@ extern const char raid6_empty_zero_page[PAGE_SIZE]; #define MODULE_DESCRIPTION(desc) #define subsys_initcall(x) #define module_exit(x) + +#define IS_ENABLED(x) (x) +#define CONFIG_RAID6_PQ_BENCHMARK 1 #endif /* __KERNEL__ */ /* Routine choices */ diff --git a/lib/Kconfig b/lib/Kconfig index 79bc2eef9c14..a9e56539bd11 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -10,6 +10,14 @@ menu "Library routines" config RAID6_PQ tristate +config RAID6_PQ_BENCHMARK + bool "Automatically choose fastest RAID6 PQ functions" + depends on RAID6_PQ + default y + help + Benchmark all available RAID6 PQ functions on init and choose the + fastest one. + config BITREVERSE tristate diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c index 5065b1e7e327..7e4f7a8ffa8e 100644 --- a/lib/raid6/algos.c +++ b/lib/raid6/algos.c @@ -34,64 +34,64 @@ struct raid6_calls raid6_call; EXPORT_SYMBOL_GPL(raid6_call); const struct raid6_calls * const raid6_algos[] = { -#if defined(__ia64__) - &raid6_intx16, - &raid6_intx32, -#endif #if defined(__i386__) && !defined(__arch_um__) - &raid6_mmxx1, - &raid6_mmxx2, - &raid6_sse1x1, - &raid6_sse1x2, - &raid6_sse2x1, - &raid6_sse2x2, -#ifdef CONFIG_AS_AVX2 - &raid6_avx2x1, - &raid6_avx2x2, -#endif #ifdef CONFIG_AS_AVX512 - &raid6_avx512x1, &raid6_avx512x2, + &raid6_avx512x1, #endif +#ifdef CONFIG_AS_AVX2 + &raid6_avx2x2, + &raid6_avx2x1, +#endif + &raid6_sse2x2, + &raid6_sse2x1, + &raid6_sse1x2, + &raid6_sse1x1, + &raid6_mmxx2, + &raid6_mmxx1, #endif #if defined(__x86_64__) && !defined(__arch_um__) - &raid6_sse2x1, - &raid6_sse2x2, - &raid6_sse2x4, -#ifdef CONFIG_AS_AVX2 - &raid6_avx2x1, - &raid6_avx2x2, - &raid6_avx2x4, -#endif #ifdef CONFIG_AS_AVX512 - &raid6_avx512x1, - &raid6_avx512x2, &raid6_avx512x4, + &raid6_avx512x2, + &raid6_avx512x1, #endif +#ifdef CONFIG_AS_AVX2 + &raid6_avx2x4, + &raid6_avx2x2, + &raid6_avx2x1, +#endif + &raid6_sse2x4, + &raid6_sse2x2, + &raid6_sse2x1, #endif #ifdef CONFIG_ALTIVEC - &raid6_altivec1, - &raid6_altivec2, - &raid6_altivec4, - &raid6_altivec8, - &raid6_vpermxor1, - &raid6_vpermxor2, - &raid6_vpermxor4, &raid6_vpermxor8, + &raid6_vpermxor4, + &raid6_vpermxor2, + &raid6_vpermxor1, + &raid6_altivec8, + &raid6_altivec4, + &raid6_altivec2, + &raid6_altivec1, #endif #if defined(CONFIG_S390) &raid6_s390vx8, #endif - &raid6_intx1, - &raid6_intx2, - &raid6_intx4, - &raid6_intx8, #ifdef CONFIG_KERNEL_MODE_NEON - &raid6_neonx1, - &raid6_neonx2, - &raid6_neonx4, &raid6_neonx8, + &raid6_neonx4, + &raid6_neonx2, + &raid6_neonx1, #endif +#if defined(__ia64__) + &raid6_intx32, + &raid6_intx16, +#endif + &raid6_intx8, + &raid6_intx4, + &raid6_intx2, + &raid6_intx1, NULL }; @@ -163,6 +163,11 @@ static inline const struct raid6_calls *raid6_choose_gen( if ((*algo)->valid && !(*algo)->valid()) continue; + if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { + best = *algo; + break; + } + perf = 0; preempt_disable(); diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile index 79777645cac9..3ab8720aa2f8 100644 --- a/lib/raid6/test/Makefile +++ b/lib/raid6/test/Makefile @@ -34,6 +34,9 @@ endif ifeq ($(IS_X86),yes) OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o + CFLAGS += $(shell echo "pshufb %xmm0, %xmm0" | \ + gcc -c -x assembler - >&/dev/null && \ + rm ./-.o && echo -DCONFIG_AS_SSSE3=1) CFLAGS += $(shell echo "vpbroadcastb %xmm0, %ymm1" | \ gcc -c -x assembler - >&/dev/null && \ rm ./-.o && echo -DCONFIG_AS_AVX2=1)