From e9d338a0b1799c988b678e8ccb66a442272e6aa3 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 24 Jun 2020 15:59:59 -0700 Subject: scftorture: Add smp_call_function() torture test This commit adds an smp_call_function() torture test that repeatedly invokes this function and complains if things go badly awry. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 350 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 kernel/scftorture.c (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c new file mode 100644 index 000000000000..44f1e49ba6e9 --- /dev/null +++ b/kernel/scftorture.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0+ +// +// Torture test for smp_call_function() and friends. +// +// Copyright (C) Facebook, 2020. +// +// Author: Paul E. McKenney + +#define pr_fmt(fmt) fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SCFTORT_STRING "scftorture" +#define SCFTORT_FLAG SCFTORT_STRING ": " + +#define SCFTORTOUT(s, x...) \ + pr_alert(SCFTORT_FLAG s, ## x) + +#define VERBOSE_SCFTORTOUT(s, x...) \ + do { if (verbose) pr_alert(SCFTORT_FLAG s, ## x); } while (0) + +#define VERBOSE_SCFTORTOUT_ERRSTRING(s, x...) \ + do { if (verbose) pr_alert(SCFTORT_FLAG "!!! " s, ## x); } while (0) + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Paul E. McKenney "); + +// Wait until there are multiple CPUs before starting test. +torture_param(int, holdoff, IS_BUILTIN(CONFIG_SCF_TORTURE_TEST) ? 10 : 0, + "Holdoff time before test start (s)"); +torture_param(int, longwait, 0, "Include ridiculously long waits? (seconds)"); +torture_param(int, nthreads, -1, "# threads, defaults to -1 for all CPUs."); +torture_param(int, onoff_holdoff, 0, "Time after boot before CPU hotplugs (s)"); +torture_param(int, onoff_interval, 0, "Time between CPU hotplugs (s), 0=disable"); +torture_param(int, shutdown_secs, 0, "Shutdown time (ms), <= zero to disable."); +torture_param(int, stat_interval, 60, "Number of seconds between stats printk()s."); +torture_param(int, stutter_cpus, 5, "Number of jiffies to change CPUs under test, 0=disable"); +torture_param(bool, use_cpus_read_lock, 0, "Use cpus_read_lock() to exclude CPU hotplug."); +torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); +torture_param(int, weight_single, -1, "Testing weight for single-CPU no-wait operations."); +torture_param(int, weight_single_wait, -1, "Testing weight for single-CPU operations."); +torture_param(int, weight_mult, -1, "Testing weight for multi-CPU no-wait operations."); +torture_param(int, weight_mult_wait, -1, "Testing weight for multi-CPU operations."); +torture_param(int, weight_all, -1, "Testing weight for all-CPU no-wait operations."); +torture_param(int, weight_all_wait, -1, "Testing weight for all-CPU operations."); + +char *torture_type = ""; + +#ifdef MODULE +# define SCFTORT_SHUTDOWN 0 +#else +# define SCFTORT_SHUTDOWN 1 +#endif + +torture_param(bool, shutdown, SCFTORT_SHUTDOWN, "Shutdown at end of torture test."); + +struct scf_statistics { + struct task_struct *task; + int cpu; + long long n_single; + long long n_single_wait; + long long n_multi; + long long n_multi_wait; + long long n_all; + long long n_all_wait; +}; + +static struct scf_statistics *scf_stats_p; +static struct task_struct *scf_torture_stats_task; +static DEFINE_PER_CPU(long long, scf_invoked_count); + +// Use to wait for all threads to start. +static atomic_t n_started; +static atomic_t n_errs; +static bool scfdone; + +DEFINE_TORTURE_RANDOM_PERCPU(scf_torture_rand); + +// Print torture statistics. Caller must ensure serialization. +static void scf_torture_stats_print(void) +{ + int cpu; + long long invoked_count = 0; + bool isdone = READ_ONCE(scfdone); + + for_each_possible_cpu(cpu) + invoked_count += data_race(per_cpu(scf_invoked_count, cpu)); + pr_alert("%s scf_invoked_count %s: %lld ", + SCFTORT_FLAG, isdone ? "VER" : "ver", invoked_count); + torture_onoff_stats(); + pr_cont("\n"); +} + +// Periodically prints torture statistics, if periodic statistics printing +// was specified via the stat_interval module parameter. +static int +scf_torture_stats(void *arg) +{ + VERBOSE_TOROUT_STRING("scf_torture_stats task started"); + do { + schedule_timeout_interruptible(stat_interval * HZ); + scf_torture_stats_print(); + torture_shutdown_absorb("scf_torture_stats"); + } while (!torture_must_stop()); + torture_kthread_stopping("scf_torture_stats"); + return 0; +} + +// Update statistics and occasionally burn up mass quantities of CPU time, +// if told to do so via scftorture.longwait. Otherwise, occasionally burn +// a little bit. +static void scf_handler(void *unused) +{ + int i; + int j; + unsigned long r = torture_random(this_cpu_ptr(&scf_torture_rand)); + + this_cpu_inc(scf_invoked_count); + if (longwait <= 0) { + if (!(r & 0xffc0)) + udelay(r & 0x3f); + return; + } + if (r & 0xfff) + return; + r = (r >> 12); + if (longwait <= 0) { + udelay((r & 0xff) + 1); + return; + } + r = r % longwait + 1; + for (i = 0; i < r; i++) { + for (j = 0; j < 1000; j++) { + udelay(1000); + cpu_relax(); + } + } +} + +// Randomly do an smp_call_function*() invocation. +static void scftorture_invoke_one(struct scf_statistics *scfp,struct torture_random_state *trsp) +{ + if (use_cpus_read_lock) + cpus_read_lock(); + else + preempt_disable(); + scfp->n_all++; + smp_call_function(scf_handler, NULL, 0); + if (use_cpus_read_lock) + cpus_read_unlock(); + else + preempt_enable(); + if (!(torture_random(trsp) & 0xfff)) + schedule_timeout_uninterruptible(1); +} + +// SCF test kthread. Repeatedly does calls to members of the +// smp_call_function() family of functions. +static int scftorture_invoker(void *arg) +{ + DEFINE_TORTURE_RANDOM(rand); + struct scf_statistics *scfp = (struct scf_statistics *)arg; + + VERBOSE_SCFTORTOUT("scftorture_invoker %d: task started", scfp->cpu); + set_cpus_allowed_ptr(current, cpumask_of(scfp->cpu % nr_cpu_ids)); + set_user_nice(current, MAX_NICE); + if (holdoff) + schedule_timeout_interruptible(holdoff * HZ); + + VERBOSE_SCFTORTOUT("scftorture_invoker %d: Waiting for all SCF torturers from cpu %d", scfp->cpu, smp_processor_id()); + + // Make sure that the CPU is affinitized appropriately during testing. + WARN_ON_ONCE(smp_processor_id() != scfp->cpu); + + if (!atomic_dec_return(&n_started)) + while (atomic_read_acquire(&n_started)) { + if (torture_must_stop()) { + VERBOSE_SCFTORTOUT("scftorture_invoker %d ended before starting", scfp->cpu); + goto end; + } + schedule_timeout_uninterruptible(1); + } + + VERBOSE_SCFTORTOUT("scftorture_invoker %d started", scfp->cpu); + + do { + scftorture_invoke_one(scfp, &rand); + } while (!torture_must_stop()); + + VERBOSE_SCFTORTOUT("scftorture_invoker %d ended", scfp->cpu); +end: + torture_kthread_stopping("scftorture_invoker"); + return 0; +} + +static void +scftorture_print_module_parms(const char *tag) +{ + pr_alert(SCFTORT_FLAG + "--- %s: verbose=%d holdoff=%d longwait=%d nthreads=%d onoff_holdoff=%d onoff_interval=%d shutdown_secs=%d stat_interval=%d stutter_cpus=%d use_cpus_read_lock=%d, weight_single=%d, weight_single_wait=%d, weight_mult=%d, weight_mult_wait=%d, weight_all=%d, weight_all_wait=%d\n", tag, + verbose, holdoff, longwait, nthreads, onoff_holdoff, onoff_interval, shutdown, stat_interval, stutter_cpus, use_cpus_read_lock, weight_single, weight_single_wait, weight_mult, weight_mult_wait, weight_all, weight_all_wait); +} + +static void scf_cleanup_handler(void *unused) +{ +} + +static void scf_torture_cleanup(void) +{ + int i; + + if (torture_cleanup_begin()) + return; + + WRITE_ONCE(scfdone, true); + if (nthreads) + for (i = 0; i < nthreads; i++) + torture_stop_kthread("scftorture_invoker", scf_stats_p[i].task); + else + goto end; + kfree(scf_stats_p); + scf_stats_p = NULL; + smp_call_function(scf_cleanup_handler, NULL, 0); + torture_stop_kthread(scf_torture_stats, scf_torture_stats_task); + scf_torture_stats_print(); // -After- the stats thread is stopped! + + if (atomic_read(&n_errs)) + scftorture_print_module_parms("End of test: FAILURE"); + else if (torture_onoff_failures()) + scftorture_print_module_parms("End of test: LOCK_HOTPLUG"); + else + scftorture_print_module_parms("End of test: SUCCESS"); + +end: + torture_cleanup_end(); +} + +static int __init scf_torture_init(void) +{ + long i; + int firsterr = 0; + + if (!torture_init_begin(SCFTORT_STRING, verbose)) + return -EBUSY; + + scftorture_print_module_parms("Start of test"); + + if (weight_single == -1 && weight_single_wait == -1 && + weight_mult == -1 && weight_mult_wait == -1 && + weight_all == -1 && weight_all_wait == -1) { + weight_single = 1; + weight_single_wait = 1; + weight_mult = 1; + weight_mult_wait = 1; + weight_all = 1; + weight_all_wait = 1; + } else { + if (weight_single == -1) + weight_single = 0; + if (weight_single_wait == -1) + weight_single_wait = 0; + if (weight_mult == -1) + weight_mult = 0; + if (weight_mult_wait == -1) + weight_mult_wait = 0; + if (weight_all == -1) + weight_all = 0; + if (weight_all_wait == -1) + weight_all_wait = 0; + } + if (weight_single == 0 && weight_single_wait == 0 && + weight_mult == 0 && weight_mult_wait == 0 && + weight_all == 0 && weight_all_wait == 0) { + firsterr = -EINVAL; + goto unwind; + } + + if (onoff_interval > 0) { + firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL); + if (firsterr) + goto unwind; + } + if (shutdown_secs > 0) { + firsterr = torture_shutdown_init(shutdown_secs, scf_torture_cleanup); + if (firsterr) + goto unwind; + } + + // Worker tasks invoking smp_call_function(). + if (nthreads < 0) + nthreads = num_online_cpus(); + scf_stats_p = kcalloc(nthreads, sizeof(scf_stats_p[0]), GFP_KERNEL); + if (!scf_stats_p) { + VERBOSE_SCFTORTOUT_ERRSTRING("out of memory"); + firsterr = -ENOMEM; + goto unwind; + } + + VERBOSE_SCFTORTOUT("Starting %d smp_call_function() threads\n", nthreads); + + atomic_set(&n_started, nthreads); + for (i = 0; i < nthreads; i++) { + scf_stats_p[i].cpu = i; + firsterr = torture_create_kthread(scftorture_invoker, (void *)&scf_stats_p[i], + scf_stats_p[i].task); + if (firsterr) + goto unwind; + } + if (stat_interval > 0) { + firsterr = torture_create_kthread(scf_torture_stats, NULL, scf_torture_stats_task); + if (firsterr) + goto unwind; + } + + torture_init_end(); + return 0; + +unwind: + torture_init_end(); + scf_torture_cleanup(); + return firsterr; +} + +module_init(scf_torture_init); +module_exit(scf_torture_cleanup); -- cgit v1.2.3 From 5022b8ac608f8b80b042a8041fe2738c4b9ea8cf Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 25 Jun 2020 17:05:58 -0700 Subject: scftorture: Implement weighted primitive selection This commit uses the scftorture.weight* kernel parameters to randomly chooses between smp_call_function_single(), smp_call_function_many(), and smp_call_function(). For each variant, it also randomly chooses whether to invoke it synchronously (wait=1) or asynchronously (wait=0). The percentage weighting for each option are dumped to the console log (search for "scf_sel_dump"). This accumulates statistics, which a later commit will dump out at the end of the run. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 182 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 155 insertions(+), 27 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 44f1e49ba6e9..5f1984522ae4 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -64,8 +64,8 @@ torture_param(bool, use_cpus_read_lock, 0, "Use cpus_read_lock() to exclude CPU torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); torture_param(int, weight_single, -1, "Testing weight for single-CPU no-wait operations."); torture_param(int, weight_single_wait, -1, "Testing weight for single-CPU operations."); -torture_param(int, weight_mult, -1, "Testing weight for multi-CPU no-wait operations."); -torture_param(int, weight_mult_wait, -1, "Testing weight for multi-CPU operations."); +torture_param(int, weight_many, -1, "Testing weight for multi-CPU no-wait operations."); +torture_param(int, weight_many_wait, -1, "Testing weight for multi-CPU operations."); torture_param(int, weight_all, -1, "Testing weight for all-CPU no-wait operations."); torture_param(int, weight_all_wait, -1, "Testing weight for all-CPU operations."); @@ -83,9 +83,11 @@ struct scf_statistics { struct task_struct *task; int cpu; long long n_single; + long long n_single_ofl; long long n_single_wait; - long long n_multi; - long long n_multi_wait; + long long n_single_wait_ofl; + long long n_many; + long long n_many_wait; long long n_all; long long n_all_wait; }; @@ -94,6 +96,27 @@ static struct scf_statistics *scf_stats_p; static struct task_struct *scf_torture_stats_task; static DEFINE_PER_CPU(long long, scf_invoked_count); +// Data for random primitive selection +#define SCF_PRIM_SINGLE 0 +#define SCF_PRIM_MANY 1 +#define SCF_PRIM_ALL 2 +#define SCF_NPRIMS (2 * 3) // Need wait and no-wait versions of each. + +static char *scf_prim_name[] = { + "smp_call_function_single", + "smp_call_function_many", + "smp_call_function", +}; + +struct scf_selector { + unsigned long scfs_weight; + int scfs_prim; + bool scfs_wait; +}; +static struct scf_selector scf_sel_array[SCF_NPRIMS]; +static int scf_sel_array_len; +static unsigned long scf_sel_totweight; + // Use to wait for all threads to start. static atomic_t n_started; static atomic_t n_errs; @@ -131,6 +154,57 @@ scf_torture_stats(void *arg) return 0; } +// Add a primitive to the scf_sel_array[]. +static void scf_sel_add(unsigned long weight, int prim, bool wait) +{ + struct scf_selector *scfsp = &scf_sel_array[scf_sel_array_len]; + + // If no weight, if array would overflow, if computing three-place + // percentages would overflow, or if the scf_prim_name[] array would + // overflow, don't bother. In the last three two cases, complain. + if (!weight || + WARN_ON_ONCE(scf_sel_array_len >= ARRAY_SIZE(scf_sel_array)) || + WARN_ON_ONCE(0 - 100000 * weight <= 100000 * scf_sel_totweight) || + WARN_ON_ONCE(prim >= ARRAY_SIZE(scf_prim_name))) + return; + scf_sel_totweight += weight; + scfsp->scfs_weight = scf_sel_totweight; + scfsp->scfs_prim = prim; + scfsp->scfs_wait = wait; + scf_sel_array_len++; +} + +// Dump out weighting percentages for scf_prim_name[] array. +static void scf_sel_dump(void) +{ + int i; + unsigned long oldw = 0; + struct scf_selector *scfsp; + unsigned long w; + + for (i = 0; i < scf_sel_array_len; i++) { + scfsp = &scf_sel_array[i]; + w = (scfsp->scfs_weight - oldw) * 100000 / scf_sel_totweight; + pr_info("%s: %3lu.%03lu %s(%s)\n", __func__, w / 1000, w % 1000, + scf_prim_name[scfsp->scfs_prim], + scfsp->scfs_wait ? "wait" : "nowait"); + oldw = scfsp->scfs_weight; + } +} + +// Randomly pick a primitive and wait/nowait, based on weightings. +static struct scf_selector *scf_sel_rand(struct torture_random_state *trsp) +{ + int i; + unsigned long w = torture_random(trsp) % (scf_sel_totweight + 1); + + for (i = 0; i < scf_sel_array_len; i++) + if (scf_sel_array[i].scfs_weight >= w) + return &scf_sel_array[i]; + WARN_ON_ONCE(1); + return &scf_sel_array[0]; +} + // Update statistics and occasionally burn up mass quantities of CPU time, // if told to do so via scftorture.longwait. Otherwise, occasionally burn // a little bit. @@ -162,15 +236,55 @@ static void scf_handler(void *unused) } } +// As above, but check for correct CPU. +static void scf_handler_1(void *me) +{ + if (WARN_ON_ONCE(smp_processor_id() != (uintptr_t)me)) + atomic_inc(&n_errs); + scf_handler(NULL); +} + // Randomly do an smp_call_function*() invocation. -static void scftorture_invoke_one(struct scf_statistics *scfp,struct torture_random_state *trsp) +static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_random_state *trsp) { + uintptr_t cpu; + int ret; + struct scf_selector *scfsp = scf_sel_rand(trsp); + if (use_cpus_read_lock) cpus_read_lock(); else preempt_disable(); - scfp->n_all++; - smp_call_function(scf_handler, NULL, 0); + switch (scfsp->scfs_prim) { + case SCF_PRIM_SINGLE: + cpu = torture_random(trsp) % nr_cpu_ids; + if (scfsp->scfs_wait) + scfp->n_single_wait++; + else + scfp->n_single++; + ret = smp_call_function_single(cpu, scf_handler_1, (void *)cpu, scfsp->scfs_wait); + if (ret) { + if (scfsp->scfs_wait) + scfp->n_single_wait_ofl++; + else + scfp->n_single_ofl++; + } + break; + case SCF_PRIM_MANY: + if (scfsp->scfs_wait) + scfp->n_many_wait++; + else + scfp->n_many++; + smp_call_function_many(cpu_online_mask, scf_handler, NULL, scfsp->scfs_wait); + break; + case SCF_PRIM_ALL: + if (scfsp->scfs_wait) + scfp->n_all_wait++; + else + scfp->n_all++; + smp_call_function(scf_handler, NULL, scfsp->scfs_wait); + break; + } if (use_cpus_read_lock) cpus_read_unlock(); else @@ -222,8 +336,8 @@ static void scftorture_print_module_parms(const char *tag) { pr_alert(SCFTORT_FLAG - "--- %s: verbose=%d holdoff=%d longwait=%d nthreads=%d onoff_holdoff=%d onoff_interval=%d shutdown_secs=%d stat_interval=%d stutter_cpus=%d use_cpus_read_lock=%d, weight_single=%d, weight_single_wait=%d, weight_mult=%d, weight_mult_wait=%d, weight_all=%d, weight_all_wait=%d\n", tag, - verbose, holdoff, longwait, nthreads, onoff_holdoff, onoff_interval, shutdown, stat_interval, stutter_cpus, use_cpus_read_lock, weight_single, weight_single_wait, weight_mult, weight_mult_wait, weight_all, weight_all_wait); + "--- %s: verbose=%d holdoff=%d longwait=%d nthreads=%d onoff_holdoff=%d onoff_interval=%d shutdown_secs=%d stat_interval=%d stutter_cpus=%d use_cpus_read_lock=%d, weight_single=%d, weight_single_wait=%d, weight_many=%d, weight_many_wait=%d, weight_all=%d, weight_all_wait=%d\n", tag, + verbose, holdoff, longwait, nthreads, onoff_holdoff, onoff_interval, shutdown, stat_interval, stutter_cpus, use_cpus_read_lock, weight_single, weight_single_wait, weight_many, weight_many_wait, weight_all, weight_all_wait); } static void scf_cleanup_handler(void *unused) @@ -264,6 +378,12 @@ static int __init scf_torture_init(void) { long i; int firsterr = 0; + unsigned long weight_single1 = weight_single; + unsigned long weight_single_wait1 = weight_single_wait; + unsigned long weight_many1 = weight_many; + unsigned long weight_many_wait1 = weight_many_wait; + unsigned long weight_all1 = weight_all; + unsigned long weight_all_wait1 = weight_all_wait; if (!torture_init_begin(SCFTORT_STRING, verbose)) return -EBUSY; @@ -271,34 +391,42 @@ static int __init scf_torture_init(void) scftorture_print_module_parms("Start of test"); if (weight_single == -1 && weight_single_wait == -1 && - weight_mult == -1 && weight_mult_wait == -1 && + weight_many == -1 && weight_many_wait == -1 && weight_all == -1 && weight_all_wait == -1) { - weight_single = 1; - weight_single_wait = 1; - weight_mult = 1; - weight_mult_wait = 1; - weight_all = 1; - weight_all_wait = 1; + weight_single1 = 2 * nr_cpu_ids; + weight_single_wait1 = 2 * nr_cpu_ids; + weight_many1 = 2; + weight_many_wait1 = 2; + weight_all1 = 1; + weight_all_wait1 = 1; } else { if (weight_single == -1) - weight_single = 0; + weight_single1 = 0; if (weight_single_wait == -1) - weight_single_wait = 0; - if (weight_mult == -1) - weight_mult = 0; - if (weight_mult_wait == -1) - weight_mult_wait = 0; + weight_single_wait1 = 0; + if (weight_many == -1) + weight_many1 = 0; + if (weight_many_wait == -1) + weight_many_wait1 = 0; if (weight_all == -1) - weight_all = 0; + weight_all1 = 0; if (weight_all_wait == -1) - weight_all_wait = 0; + weight_all_wait1 = 0; } - if (weight_single == 0 && weight_single_wait == 0 && - weight_mult == 0 && weight_mult_wait == 0 && - weight_all == 0 && weight_all_wait == 0) { + if (weight_single1 == 0 && weight_single_wait1 == 0 && + weight_many1 == 0 && weight_many_wait1 == 0 && + weight_all1 == 0 && weight_all_wait1 == 0) { + VERBOSE_SCFTORTOUT_ERRSTRING("all zero weights makes no sense"); firsterr = -EINVAL; goto unwind; } + scf_sel_add(weight_single1, SCF_PRIM_SINGLE, false); + scf_sel_add(weight_single_wait1, SCF_PRIM_SINGLE, true); + scf_sel_add(weight_many1, SCF_PRIM_MANY, false); + scf_sel_add(weight_many_wait1, SCF_PRIM_MANY, true); + scf_sel_add(weight_all1, SCF_PRIM_ALL, false); + scf_sel_add(weight_all_wait1, SCF_PRIM_ALL, true); + scf_sel_dump(); if (onoff_interval > 0) { firsterr = torture_onoff_init(onoff_holdoff * HZ, onoff_interval, NULL); -- cgit v1.2.3 From dba3142b37f343734bf61dbce2914acb76e69fb6 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 30 Jun 2020 16:13:37 -0700 Subject: scftorture: Summarize per-thread statistics This commit summarizes the per-thread statistics, providing counts of the number of single, many, and all calls, both no-wait and wait, and, for the single case, the number where the target CPU was offline. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 5f1984522ae4..09a62424bb8c 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -128,13 +128,27 @@ DEFINE_TORTURE_RANDOM_PERCPU(scf_torture_rand); static void scf_torture_stats_print(void) { int cpu; + int i; long long invoked_count = 0; bool isdone = READ_ONCE(scfdone); + struct scf_statistics scfs = {}; for_each_possible_cpu(cpu) invoked_count += data_race(per_cpu(scf_invoked_count, cpu)); - pr_alert("%s scf_invoked_count %s: %lld ", - SCFTORT_FLAG, isdone ? "VER" : "ver", invoked_count); + for (i = 0; i < nthreads; i++) { + scfs.n_single += scf_stats_p[i].n_single; + scfs.n_single_ofl += scf_stats_p[i].n_single_ofl; + scfs.n_single_wait += scf_stats_p[i].n_single_wait; + scfs.n_single_wait_ofl += scf_stats_p[i].n_single_wait_ofl; + scfs.n_many += scf_stats_p[i].n_many; + scfs.n_many_wait += scf_stats_p[i].n_many_wait; + scfs.n_all += scf_stats_p[i].n_all; + scfs.n_all_wait += scf_stats_p[i].n_all_wait; + } + pr_alert("%s scf_invoked_count %s: %lld single: %lld/%lld single_ofl: %lld/%lld many: %lld/%lld all: %lld/%lld ", + SCFTORT_FLAG, isdone ? "VER" : "ver", invoked_count, + scfs.n_single, scfs.n_single_wait, scfs.n_single_ofl, scfs.n_single_wait_ofl, + scfs.n_many, scfs.n_many_wait, scfs.n_all, scfs.n_all_wait); torture_onoff_stats(); pr_cont("\n"); } @@ -357,11 +371,11 @@ static void scf_torture_cleanup(void) torture_stop_kthread("scftorture_invoker", scf_stats_p[i].task); else goto end; - kfree(scf_stats_p); - scf_stats_p = NULL; smp_call_function(scf_cleanup_handler, NULL, 0); torture_stop_kthread(scf_torture_stats, scf_torture_stats_task); scf_torture_stats_print(); // -After- the stats thread is stopped! + kfree(scf_stats_p); // -After- the last stats print has completed! + scf_stats_p = NULL; if (atomic_read(&n_errs)) scftorture_print_module_parms("End of test: FAILURE"); -- cgit v1.2.3 From b93e21a51e1c8ed3816da888d34f88193ad1b917 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 30 Jun 2020 20:49:50 -0700 Subject: scftorture: Add smp_call_function_single() memory-ordering checks This commit adds checks for memory misordering across calls to smp_call_function_single() and also across returns in the case where the caller waits. Misordering results in a splat. [ paulmck: s/GFP_KERNEL/GFP_ATOMIC/ per kernel test robot feedback. ] Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 56 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 09a62424bb8c..9b42271d64f1 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -117,9 +117,20 @@ static struct scf_selector scf_sel_array[SCF_NPRIMS]; static int scf_sel_array_len; static unsigned long scf_sel_totweight; +// Communicate between caller and handler. +struct scf_check { + bool scfc_in; + bool scfc_out; + int scfc_cpu; // -1 for not _single(). + bool scfc_wait; +}; + // Use to wait for all threads to start. static atomic_t n_started; static atomic_t n_errs; +static atomic_t n_mb_in_errs; +static atomic_t n_mb_out_errs; +static atomic_t n_alloc_errs; static bool scfdone; DEFINE_TORTURE_RANDOM_PERCPU(scf_torture_rand); @@ -222,24 +233,27 @@ static struct scf_selector *scf_sel_rand(struct torture_random_state *trsp) // Update statistics and occasionally burn up mass quantities of CPU time, // if told to do so via scftorture.longwait. Otherwise, occasionally burn // a little bit. -static void scf_handler(void *unused) +static void scf_handler(void *scfc_in) { int i; int j; unsigned long r = torture_random(this_cpu_ptr(&scf_torture_rand)); + struct scf_check *scfcp = scfc_in; + if (likely(scfcp) && WARN_ON_ONCE(unlikely(!READ_ONCE(scfcp->scfc_in)))) + atomic_inc(&n_mb_in_errs); this_cpu_inc(scf_invoked_count); if (longwait <= 0) { if (!(r & 0xffc0)) udelay(r & 0x3f); - return; + goto out; } if (r & 0xfff) - return; + goto out; r = (r >> 12); if (longwait <= 0) { udelay((r & 0xff) + 1); - return; + goto out; } r = r % longwait + 1; for (i = 0; i < r; i++) { @@ -248,14 +262,24 @@ static void scf_handler(void *unused) cpu_relax(); } } +out: + if (unlikely(!scfcp)) + return; + if (scfcp->scfc_wait) + WRITE_ONCE(scfcp->scfc_out, true); + else + kfree(scfcp); } // As above, but check for correct CPU. -static void scf_handler_1(void *me) +static void scf_handler_1(void *scfc_in) { - if (WARN_ON_ONCE(smp_processor_id() != (uintptr_t)me)) + struct scf_check *scfcp = scfc_in; + + if (likely(scfcp) && WARN_ONCE(smp_processor_id() != scfcp->scfc_cpu, "%s: Wanted CPU %d got CPU %d\n", __func__, scfcp->scfc_cpu, smp_processor_id())) { atomic_inc(&n_errs); - scf_handler(NULL); + } + scf_handler(scfcp); } // Randomly do an smp_call_function*() invocation. @@ -263,6 +287,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra { uintptr_t cpu; int ret; + struct scf_check *scfcp = NULL; struct scf_selector *scfsp = scf_sel_rand(trsp); if (use_cpus_read_lock) @@ -271,17 +296,32 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra preempt_disable(); switch (scfsp->scfs_prim) { case SCF_PRIM_SINGLE: + scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC); + if (WARN_ON_ONCE(!scfcp)) + atomic_inc(&n_alloc_errs); cpu = torture_random(trsp) % nr_cpu_ids; if (scfsp->scfs_wait) scfp->n_single_wait++; else scfp->n_single++; - ret = smp_call_function_single(cpu, scf_handler_1, (void *)cpu, scfsp->scfs_wait); + if (scfcp) { + scfcp->scfc_cpu = cpu; + scfcp->scfc_wait = scfsp->scfs_wait; + scfcp->scfc_out = false; + scfcp->scfc_in = true; + } + ret = smp_call_function_single(cpu, scf_handler_1, (void *)scfcp, scfsp->scfs_wait); if (ret) { if (scfsp->scfs_wait) scfp->n_single_wait_ofl++; else scfp->n_single_ofl++; + kfree(scfcp); + } else if (scfcp && scfsp->scfs_wait) { + if (WARN_ON_ONCE(!scfcp->scfc_out)) + atomic_inc(&n_mb_out_errs); // Leak rather than trash! + else + kfree(scfcp); } break; case SCF_PRIM_MANY: -- cgit v1.2.3 From 980205ee8489d53c4380f7762debac87312b0fb3 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 1 Jul 2020 12:30:02 -0700 Subject: scftorture: Add smp_call_function_many() memory-ordering checks This commit adds checks for memory misordering across calls to and returns from smp_call_function_many() in the case where the caller waits. Misordering results in a splat. Note that in contrast to smp_call_function_single(), this code does not test memory ordering into the handler in the no-wait case because none of the handlers would be able to free the scf_check structure without introducing heavy synchronization to work out which was last. [ paulmck: s/GFP_KERNEL/GFP_ATOMIC/ per kernel test robot feedback. ] Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 9b42271d64f1..3519ad1b3278 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -240,8 +240,11 @@ static void scf_handler(void *scfc_in) unsigned long r = torture_random(this_cpu_ptr(&scf_torture_rand)); struct scf_check *scfcp = scfc_in; - if (likely(scfcp) && WARN_ON_ONCE(unlikely(!READ_ONCE(scfcp->scfc_in)))) - atomic_inc(&n_mb_in_errs); + if (likely(scfcp)) { + WRITE_ONCE(scfcp->scfc_out, false); // For multiple receivers. + if (WARN_ON_ONCE(unlikely(!READ_ONCE(scfcp->scfc_in)))) + atomic_inc(&n_mb_in_errs); + } this_cpu_inc(scf_invoked_count); if (longwait <= 0) { if (!(r & 0xffc0)) @@ -325,11 +328,28 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra } break; case SCF_PRIM_MANY: + if (scfsp->scfs_wait) { + scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC); + if (WARN_ON_ONCE(!scfcp)) + atomic_inc(&n_alloc_errs); + } if (scfsp->scfs_wait) scfp->n_many_wait++; else scfp->n_many++; - smp_call_function_many(cpu_online_mask, scf_handler, NULL, scfsp->scfs_wait); + if (scfcp) { + scfcp->scfc_cpu = -1; + scfcp->scfc_wait = true; + scfcp->scfc_out = false; + scfcp->scfc_in = true; + } + smp_call_function_many(cpu_online_mask, scf_handler, scfcp, scfsp->scfs_wait); + if (scfcp) { + if (WARN_ON_ONCE(!scfcp->scfc_out)) + atomic_inc(&n_mb_out_errs); // Leak rather than trash! + else + kfree(scfcp); + } break; case SCF_PRIM_ALL: if (scfsp->scfs_wait) -- cgit v1.2.3 From 34e8c4837adb579962e528a4f7dd1f75cb120be4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 1 Jul 2020 13:49:06 -0700 Subject: scftorture: Add smp_call_function() memory-ordering checks This commit adds checks for memory misordering across calls to and returns from smp_call_function() in the case where the caller waits. Misordering results in a splat. Note that in contrast to smp_call_function_single(), this code does not test memory ordering into the handler in the no-wait case because none of the handlers would be able to free the scf_check structure without introducing heavy synchronization to work out which was last. [ paulmck: s/GFP_KERNEL/GFP_ATOMIC/ per kernel test robot feedback. ] Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 3519ad1b3278..0d7299d32dd0 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -297,11 +297,13 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra cpus_read_lock(); else preempt_disable(); - switch (scfsp->scfs_prim) { - case SCF_PRIM_SINGLE: + if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) { scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC); if (WARN_ON_ONCE(!scfcp)) atomic_inc(&n_alloc_errs); + } + switch (scfsp->scfs_prim) { + case SCF_PRIM_SINGLE: cpu = torture_random(trsp) % nr_cpu_ids; if (scfsp->scfs_wait) scfp->n_single_wait++; @@ -328,11 +330,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra } break; case SCF_PRIM_MANY: - if (scfsp->scfs_wait) { - scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC); - if (WARN_ON_ONCE(!scfcp)) - atomic_inc(&n_alloc_errs); - } if (scfsp->scfs_wait) scfp->n_many_wait++; else @@ -356,7 +353,19 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfp->n_all_wait++; else scfp->n_all++; - smp_call_function(scf_handler, NULL, scfsp->scfs_wait); + if (scfcp) { + scfcp->scfc_cpu = -1; + scfcp->scfc_wait = true; + scfcp->scfc_out = false; + scfcp->scfc_in = true; + } + smp_call_function(scf_handler, scfcp, scfsp->scfs_wait); + if (scfcp) { + if (WARN_ON_ONCE(!scfcp->scfc_out)) + atomic_inc(&n_mb_out_errs); // Leak rather than trash! + else + kfree(scfcp); + } break; } if (use_cpus_read_lock) -- cgit v1.2.3 From 676e5469643e716df7f39ef77ba8f09c85b0c4f8 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 1 Jul 2020 14:13:02 -0700 Subject: scftorture: Consolidate scftorture_invoke_one() check and kfree() This commit moves checking of the ->scfc_out field and the freeing of the scf_check structure down below the end of switch statement, thus saving a few lines of code. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 0d7299d32dd0..f220cd364e23 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -289,7 +289,7 @@ static void scf_handler_1(void *scfc_in) static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_random_state *trsp) { uintptr_t cpu; - int ret; + int ret = 0; struct scf_check *scfcp = NULL; struct scf_selector *scfsp = scf_sel_rand(trsp); @@ -322,11 +322,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra else scfp->n_single_ofl++; kfree(scfcp); - } else if (scfcp && scfsp->scfs_wait) { - if (WARN_ON_ONCE(!scfcp->scfc_out)) - atomic_inc(&n_mb_out_errs); // Leak rather than trash! - else - kfree(scfcp); + scfcp = NULL; } break; case SCF_PRIM_MANY: @@ -341,12 +337,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfcp->scfc_in = true; } smp_call_function_many(cpu_online_mask, scf_handler, scfcp, scfsp->scfs_wait); - if (scfcp) { - if (WARN_ON_ONCE(!scfcp->scfc_out)) - atomic_inc(&n_mb_out_errs); // Leak rather than trash! - else - kfree(scfcp); - } break; case SCF_PRIM_ALL: if (scfsp->scfs_wait) @@ -360,14 +350,14 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfcp->scfc_in = true; } smp_call_function(scf_handler, scfcp, scfsp->scfs_wait); - if (scfcp) { - if (WARN_ON_ONCE(!scfcp->scfc_out)) - atomic_inc(&n_mb_out_errs); // Leak rather than trash! - else - kfree(scfcp); - } break; } + if (scfcp && scfsp->scfs_wait) { + if (WARN_ON_ONCE(!scfcp->scfc_out)) + atomic_inc(&n_mb_out_errs); // Leak rather than trash! + else + kfree(scfcp); + } if (use_cpus_read_lock) cpus_read_unlock(); else -- cgit v1.2.3 From 4df55bddc1a360e94c86e227fe417ac9422cb615 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 9 Jul 2020 13:58:32 -0700 Subject: scftorture: Consolidate scftorture_invoke_one() scf_check initialization This commit hoists much of the initialization of the scf_check structure out of the switch statement, thus saving a few lines of code. The initialization of the ->scfc_in field remains in each leg of the switch statement in order to more heavily stress memory ordering. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index f220cd364e23..8ab72e545a61 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -299,8 +299,13 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra preempt_disable(); if (scfsp->scfs_prim == SCF_PRIM_SINGLE || scfsp->scfs_wait) { scfcp = kmalloc(sizeof(*scfcp), GFP_ATOMIC); - if (WARN_ON_ONCE(!scfcp)) + if (WARN_ON_ONCE(!scfcp)) { atomic_inc(&n_alloc_errs); + } else { + scfcp->scfc_cpu = -1; + scfcp->scfc_wait = scfsp->scfs_wait; + scfcp->scfc_out = false; + } } switch (scfsp->scfs_prim) { case SCF_PRIM_SINGLE: @@ -311,8 +316,6 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfp->n_single++; if (scfcp) { scfcp->scfc_cpu = cpu; - scfcp->scfc_wait = scfsp->scfs_wait; - scfcp->scfc_out = false; scfcp->scfc_in = true; } ret = smp_call_function_single(cpu, scf_handler_1, (void *)scfcp, scfsp->scfs_wait); @@ -330,12 +333,8 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfp->n_many_wait++; else scfp->n_many++; - if (scfcp) { - scfcp->scfc_cpu = -1; - scfcp->scfc_wait = true; - scfcp->scfc_out = false; + if (scfcp) scfcp->scfc_in = true; - } smp_call_function_many(cpu_online_mask, scf_handler, scfcp, scfsp->scfs_wait); break; case SCF_PRIM_ALL: @@ -343,12 +342,8 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfp->n_all_wait++; else scfp->n_all++; - if (scfcp) { - scfcp->scfc_cpu = -1; - scfcp->scfc_wait = true; - scfcp->scfc_out = false; + if (scfcp) scfcp->scfc_in = true; - } smp_call_function(scf_handler, scfcp, scfsp->scfs_wait); break; } -- cgit v1.2.3 From dbf83b655a7853bc430af10e9a3e7eb1f4c90f86 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 1 Jul 2020 16:06:22 -0700 Subject: scftorture: Flag errors in torture-compatible manner This commit prints error counts on the statistics line and also adds a "!!!" if any of the counters are non-zero. Allocation failures are (somewhat) forgiven, but all other errors result in a "FAILURE" print at the end of the test. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 8ab72e545a61..880c2cef13e7 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -132,6 +132,7 @@ static atomic_t n_mb_in_errs; static atomic_t n_mb_out_errs; static atomic_t n_alloc_errs; static bool scfdone; +static char *bangstr = ""; DEFINE_TORTURE_RANDOM_PERCPU(scf_torture_rand); @@ -156,12 +157,17 @@ static void scf_torture_stats_print(void) scfs.n_all += scf_stats_p[i].n_all; scfs.n_all_wait += scf_stats_p[i].n_all_wait; } - pr_alert("%s scf_invoked_count %s: %lld single: %lld/%lld single_ofl: %lld/%lld many: %lld/%lld all: %lld/%lld ", - SCFTORT_FLAG, isdone ? "VER" : "ver", invoked_count, + if (atomic_read(&n_errs) || atomic_read(&n_mb_in_errs) || + atomic_read(&n_mb_out_errs) || atomic_read(&n_alloc_errs)) + bangstr = "!!! "; + pr_alert("%s %sscf_invoked_count %s: %lld single: %lld/%lld single_ofl: %lld/%lld many: %lld/%lld all: %lld/%lld ", + SCFTORT_FLAG, bangstr, isdone ? "VER" : "ver", invoked_count, scfs.n_single, scfs.n_single_wait, scfs.n_single_ofl, scfs.n_single_wait_ofl, scfs.n_many, scfs.n_many_wait, scfs.n_all, scfs.n_all_wait); torture_onoff_stats(); - pr_cont("\n"); + pr_cont("ste: %d stnmie: %d stnmoe: %d staf: %d\n", atomic_read(&n_errs), + atomic_read(&n_mb_in_errs), atomic_read(&n_mb_out_errs), + atomic_read(&n_alloc_errs)); } // Periodically prints torture statistics, if periodic statistics printing @@ -431,7 +437,7 @@ static void scf_torture_cleanup(void) kfree(scf_stats_p); // -After- the last stats print has completed! scf_stats_p = NULL; - if (atomic_read(&n_errs)) + if (atomic_read(&n_errs) || atomic_read(&n_mb_in_errs) || atomic_read(&n_mb_out_errs)) scftorture_print_module_parms("End of test: FAILURE"); else if (torture_onoff_failures()) scftorture_print_module_parms("End of test: LOCK_HOTPLUG"); -- cgit v1.2.3 From ee7035d29576dcb59b1191e5f609517cacab1e56 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 1 Jul 2020 16:38:16 -0700 Subject: scftorture: Prevent compiler from reducing race probabilities Detecting smp_call_function() memory misordering requires close timing, so it is necessary to have the checks immediately before and after the call to the smp_call_function*() function under test. This commit therefore inserts barrier() calls to prevent the compiler from optimizing memory-misordering detection down into the zone of extreme improbability. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 880c2cef13e7..83496810fc48 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -322,6 +322,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfp->n_single++; if (scfcp) { scfcp->scfc_cpu = cpu; + barrier(); // Prevent race-reduction compiler optimizations. scfcp->scfc_in = true; } ret = smp_call_function_single(cpu, scf_handler_1, (void *)scfcp, scfsp->scfs_wait); @@ -339,8 +340,10 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfp->n_many_wait++; else scfp->n_many++; - if (scfcp) + if (scfcp) { + barrier(); // Prevent race-reduction compiler optimizations. scfcp->scfc_in = true; + } smp_call_function_many(cpu_online_mask, scf_handler, scfcp, scfsp->scfs_wait); break; case SCF_PRIM_ALL: @@ -348,8 +351,10 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfp->n_all_wait++; else scfp->n_all++; - if (scfcp) + if (scfcp) { + barrier(); // Prevent race-reduction compiler optimizations. scfcp->scfc_in = true; + } smp_call_function(scf_handler, scfcp, scfsp->scfs_wait); break; } @@ -358,6 +363,7 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra atomic_inc(&n_mb_out_errs); // Leak rather than trash! else kfree(scfcp); + barrier(); // Prevent race-reduction compiler optimizations. } if (use_cpus_read_lock) cpus_read_unlock(); -- cgit v1.2.3 From 9a52a574676f8d4aa55f69319231ce6c343b00bb Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Thu, 2 Jul 2020 09:56:50 -0700 Subject: scftorture: Make symbol 'scf_torture_rand' static The sparse tool complains as follows kernel/scftorture.c:124:1: warning: symbol '__pcpu_scope_scf_torture_rand' was not declared. Should it be static? And this per-CPU variable is not used outside of scftorture.c, so this commit marks it static. Reported-by: Hulk Robot Signed-off-by: Wei Yongjun Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 83496810fc48..9180de73e4e8 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -134,7 +134,7 @@ static atomic_t n_alloc_errs; static bool scfdone; static char *bangstr = ""; -DEFINE_TORTURE_RANDOM_PERCPU(scf_torture_rand); +static DEFINE_TORTURE_RANDOM_PERCPU(scf_torture_rand); // Print torture statistics. Caller must ensure serialization. static void scf_torture_stats_print(void) -- cgit v1.2.3 From de77d4da54d10df97d265e7e99112bfc2fef7d4a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 2 Jul 2020 12:15:37 -0700 Subject: scftorture: Check unexpected "switch" statement value This commit adds a "default" case to the switch statement in scftorture_invoke_one() which contains a WARN_ON_ONCE() and an assignment to ->scfc_out to suppress knock-on warnings. These knock-on warnings could otherwise cause the user to think that there was a memory-ordering problem in smp_call_function() instead of a bug in scftorture.c itself. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 9180de73e4e8..d9c01c722e2a 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -357,6 +357,10 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra } smp_call_function(scf_handler, scfcp, scfsp->scfs_wait); break; + default: + WARN_ON_ONCE(1); + if (scfcp) + scfcp->scfc_out = true; } if (scfcp && scfsp->scfs_wait) { if (WARN_ON_ONCE(!scfcp->scfc_out)) -- cgit v1.2.3 From a7c072ef26644b632241d549869f10f8d2dd3b5c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 2 Jul 2020 14:15:33 -0700 Subject: scftorture: Block scftorture_invoker() kthreads for offline CPUs Currently, CPU-hotplug operations might result in all but two of (say) 100 CPUs being offline, which in turn might result in false-positive diagnostics due to overload. This commit therefore causes scftorture_invoker() kthreads for offline CPUs to loop blocking for 200 milliseconds at a time, thus continuously adjusting the number of threads to match the number of online CPUs. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index d9c01c722e2a..04d3a4279413 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -381,11 +381,14 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra // smp_call_function() family of functions. static int scftorture_invoker(void *arg) { + int cpu; DEFINE_TORTURE_RANDOM(rand); struct scf_statistics *scfp = (struct scf_statistics *)arg; + bool was_offline = false; VERBOSE_SCFTORTOUT("scftorture_invoker %d: task started", scfp->cpu); - set_cpus_allowed_ptr(current, cpumask_of(scfp->cpu % nr_cpu_ids)); + cpu = scfp->cpu % nr_cpu_ids; + set_cpus_allowed_ptr(current, cpumask_of(cpu)); set_user_nice(current, MAX_NICE); if (holdoff) schedule_timeout_interruptible(holdoff * HZ); @@ -408,6 +411,14 @@ static int scftorture_invoker(void *arg) do { scftorture_invoke_one(scfp, &rand); + while (cpu_is_offline(cpu) && !torture_must_stop()) { + schedule_timeout_interruptible(HZ / 5); + was_offline = true; + } + if (was_offline) { + set_cpus_allowed_ptr(current, cpumask_of(cpu)); + was_offline = false; + } } while (!torture_must_stop()); VERBOSE_SCFTORTOUT("scftorture_invoker %d ended", scfp->cpu); -- cgit v1.2.3 From 9e66bf03f9c538863e614a72c5799bcd9579630e Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 3 Jul 2020 15:23:19 -0700 Subject: scftorture: Adapt memory-ordering test to UP operation On uniprocessor systems, smp_call_function() does nothing. This commit therefore avoids complaining about the lack of handler accesses in the single-CPU case where there is no handler. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index 04d3a4279413..fc22bcc9a589 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -363,7 +363,8 @@ static void scftorture_invoke_one(struct scf_statistics *scfp, struct torture_ra scfcp->scfc_out = true; } if (scfcp && scfsp->scfs_wait) { - if (WARN_ON_ONCE(!scfcp->scfc_out)) + if (WARN_ON_ONCE((num_online_cpus() > 1 || scfsp->scfs_prim == SCF_PRIM_SINGLE) && + !scfcp->scfc_out)) atomic_inc(&n_mb_out_errs); // Leak rather than trash! else kfree(scfcp); -- cgit v1.2.3 From 65bd77f554336407f5fd7ced7a6df686767fba21 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 23 Jul 2020 15:53:02 -0700 Subject: scftorture: Add cond_resched() to test loop Although the test loop does randomly delay, which would provide quiescent states and so forth, it is possible for there to be a series of long smp_call_function*() handler runtimes with no delays, which results in softlockup and RCU CPU stall warning messages. This commit therefore inserts a cond_resched() into the main test loop. Signed-off-by: Paul E. McKenney --- kernel/scftorture.c | 1 + 1 file changed, 1 insertion(+) (limited to 'kernel/scftorture.c') diff --git a/kernel/scftorture.c b/kernel/scftorture.c index fc22bcc9a589..554a521ee235 100644 --- a/kernel/scftorture.c +++ b/kernel/scftorture.c @@ -420,6 +420,7 @@ static int scftorture_invoker(void *arg) set_cpus_allowed_ptr(current, cpumask_of(cpu)); was_offline = false; } + cond_resched(); } while (!torture_must_stop()); VERBOSE_SCFTORTOUT("scftorture_invoker %d ended", scfp->cpu); -- cgit v1.2.3