[Devel,RH7,v2] sysctl: panic only after softlockup_panic seconds of softlockups in raw

Submitted by Pavel Tikhomirov on Sept. 29, 2016, 7:47 a.m.

Details

Message ID 1475135269-14731-1-git-send-email-ptikhomirov@virtuozzo.com
State New
Series "sysctl: panic only after softlockup_panic seconds of softlockups in raw"
Headers show

Commit Message

Pavel Tikhomirov Sept. 29, 2016, 7:47 a.m.
sysctl kernel.softlockup_panic (integer >= 0)

  0(default) - do not generate panics on softlockup

  n > 0 - panic if cpu is in softlockup state for a period
  of n seconds

For instance: If n == 1 sysctl will work same as mainstream one,
kernel will panic imediately on first softlockup, as kernel
should lock at least for (2*kernel.watchdog_thresh) seconds,
where kernel.watchdog_thresh >= 1, to be softlockuped.

If kernel.watchdog_thresh is 10(default) real softlockups will
come every 20-22 seconds as sample_period is (kernel.watchdog_thresh
/ 5.0) == 2.0. Assume softlockup_panic == 60, there can be situation
when first softlockup will be printed at time 0s, second at time 22s,
third at 44s, forth will come at 66s and finaly as 66 > 60 kernel will
panic.

Actually softlockups are not required to be realy consecuent(processor
can be unlocked during period) as we allow 4 * kernel.watchdog_thresh
between the begining of softlockup and begining of next consequent one.

So it mean that: panic can really happen in intervall between
kernel.softlockup_panic and (kernel.softlockup_panic +
4 * kernel.watchdog_thresh + sample_period). So in case of example
in question between 60 and 102 seconds. Bigger watchdog_thresh means
bigger error. To get locked time grep in dmesg: "BUG: continious
softlockup for".

https://jira.sw.ru/browse/PSBM-52199
Signed-off-by: Pavel Tikhomirov <ptikhomirov@virtuozzo.com>
---
 kernel/sysctl.c   |  3 ++-
 kernel/watchdog.c | 27 ++++++++++++++++++++++++++-
 2 files changed, 28 insertions(+), 2 deletions(-)

Patch hide | download patch | download mbox

diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index c8f7bc3..40dfc98 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -123,6 +123,7 @@  EXPORT_SYMBOL(ve_allow_module_load);
 #ifdef CONFIG_LOCKUP_DETECTOR
 static int sixty = 60;
 static int neg_one = -1;
+static int int_max = INT_MAX;
 #endif
 
 static int zero;
@@ -910,7 +911,7 @@  static struct ctl_table kern_table[] = {
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec_minmax,
 		.extra1		= &zero,
-		.extra2		= &one,
+		.extra2		= &int_max,
 	},
 #ifdef CONFIG_SMP
 	{
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index ba61141..9addd5b 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -54,6 +54,8 @@  static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
 static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
 #endif
 static unsigned long soft_lockup_nmi_warn;
+static DEFINE_PER_CPU(unsigned long, first_softlockup_ts);
+static DEFINE_PER_CPU(unsigned long, prev_softlockup_ts);
 
 /* boot commands */
 /*
@@ -304,6 +306,23 @@  static void watchdog_interrupt_count(void)
 static int watchdog_nmi_enable(unsigned int cpu);
 static void watchdog_nmi_disable(unsigned int cpu);
 
+static int need_panic_on_softlockup(void)
+{
+	unsigned long first_ts = __this_cpu_read(first_softlockup_ts);
+	unsigned long now = get_timestamp();
+	int slp = softlockup_panic;
+
+	if (!slp)
+		return 0;
+
+	if (time_after(now, first_ts + slp)) {
+		printk(KERN_EMERG "BUG: continious softlockup for %lus!\n",
+		       now - first_ts);
+		return 1;
+	}
+	return 0;
+}
+
 /* watchdog kicker functions */
 static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 {
@@ -345,6 +364,12 @@  static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 	 */
 	duration = is_softlockup(touch_ts);
 	if (unlikely(duration)) {
+		unsigned long prev_ts = __this_cpu_read(prev_softlockup_ts);
+
+		if (!prev_ts || time_after(touch_ts, prev_ts + 2 * get_softlockup_thresh()))
+			__this_cpu_write(first_softlockup_ts, touch_ts);
+		__this_cpu_write(prev_softlockup_ts, touch_ts);
+
 		/*
 		 * If a virtual machine is stopped by the host it can look to
 		 * the watchdog like a soft lockup, check to see if the host
@@ -390,7 +415,7 @@  static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
 		}
 
 		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
-		if (softlockup_panic)
+		if (need_panic_on_softlockup())
 			panic("softlockup: hung tasks");
 		__this_cpu_write(soft_watchdog_warn, true);
 	} else

Comments

Konstantin Khorenko March 17, 2017, 1:51 p.m.
ack,

but as we've beaten softlockup in VM issues already (for now :)),
i won't apply the patch right now.

Let it wait until we need it again.

--
Best regards,

Konstantin Khorenko,
Virtuozzo Linux Kernel Team

On 09/29/2016 10:47 AM, Pavel Tikhomirov wrote:
> sysctl kernel.softlockup_panic (integer >= 0)
>
>   0(default) - do not generate panics on softlockup
>
>   n > 0 - panic if cpu is in softlockup state for a period
>   of n seconds
>
> For instance: If n == 1 sysctl will work same as mainstream one,
> kernel will panic imediately on first softlockup, as kernel
> should lock at least for (2*kernel.watchdog_thresh) seconds,
> where kernel.watchdog_thresh >= 1, to be softlockuped.
>
> If kernel.watchdog_thresh is 10(default) real softlockups will
> come every 20-22 seconds as sample_period is (kernel.watchdog_thresh
> / 5.0) == 2.0. Assume softlockup_panic == 60, there can be situation
> when first softlockup will be printed at time 0s, second at time 22s,
> third at 44s, forth will come at 66s and finaly as 66 > 60 kernel will
> panic.
>
> Actually softlockups are not required to be realy consecuent(processor
> can be unlocked during period) as we allow 4 * kernel.watchdog_thresh
> between the begining of softlockup and begining of next consequent one.
>
> So it mean that: panic can really happen in intervall between
> kernel.softlockup_panic and (kernel.softlockup_panic +
> 4 * kernel.watchdog_thresh + sample_period). So in case of example
> in question between 60 and 102 seconds. Bigger watchdog_thresh means
> bigger error. To get locked time grep in dmesg: "BUG: continious
> softlockup for".
>
> https://jira.sw.ru/browse/PSBM-52199
> Signed-off-by: Pavel Tikhomirov <ptikhomirov@virtuozzo.com>
> ---
>  kernel/sysctl.c   |  3 ++-
>  kernel/watchdog.c | 27 ++++++++++++++++++++++++++-
>  2 files changed, 28 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/sysctl.c b/kernel/sysctl.c
> index c8f7bc3..40dfc98 100644
> --- a/kernel/sysctl.c
> +++ b/kernel/sysctl.c
> @@ -123,6 +123,7 @@ EXPORT_SYMBOL(ve_allow_module_load);
>  #ifdef CONFIG_LOCKUP_DETECTOR
>  static int sixty = 60;
>  static int neg_one = -1;
> +static int int_max = INT_MAX;
>  #endif
>
>  static int zero;
> @@ -910,7 +911,7 @@ static struct ctl_table kern_table[] = {
>  		.mode		= 0644,
>  		.proc_handler	= proc_dointvec_minmax,
>  		.extra1		= &zero,
> -		.extra2		= &one,
> +		.extra2		= &int_max,
>  	},
>  #ifdef CONFIG_SMP
>  	{
> diff --git a/kernel/watchdog.c b/kernel/watchdog.c
> index ba61141..9addd5b 100644
> --- a/kernel/watchdog.c
> +++ b/kernel/watchdog.c
> @@ -54,6 +54,8 @@ static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
>  static DEFINE_PER_CPU(struct perf_event *, watchdog_ev);
>  #endif
>  static unsigned long soft_lockup_nmi_warn;
> +static DEFINE_PER_CPU(unsigned long, first_softlockup_ts);
> +static DEFINE_PER_CPU(unsigned long, prev_softlockup_ts);
>
>  /* boot commands */
>  /*
> @@ -304,6 +306,23 @@ static void watchdog_interrupt_count(void)
>  static int watchdog_nmi_enable(unsigned int cpu);
>  static void watchdog_nmi_disable(unsigned int cpu);
>
> +static int need_panic_on_softlockup(void)
> +{
> +	unsigned long first_ts = __this_cpu_read(first_softlockup_ts);
> +	unsigned long now = get_timestamp();
> +	int slp = softlockup_panic;
> +
> +	if (!slp)
> +		return 0;
> +
> +	if (time_after(now, first_ts + slp)) {
> +		printk(KERN_EMERG "BUG: continious softlockup for %lus!\n",
> +		       now - first_ts);
> +		return 1;
> +	}
> +	return 0;
> +}
> +
>  /* watchdog kicker functions */
>  static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
>  {
> @@ -345,6 +364,12 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
>  	 */
>  	duration = is_softlockup(touch_ts);
>  	if (unlikely(duration)) {
> +		unsigned long prev_ts = __this_cpu_read(prev_softlockup_ts);
> +
> +		if (!prev_ts || time_after(touch_ts, prev_ts + 2 * get_softlockup_thresh()))
> +			__this_cpu_write(first_softlockup_ts, touch_ts);
> +		__this_cpu_write(prev_softlockup_ts, touch_ts);
> +
>  		/*
>  		 * If a virtual machine is stopped by the host it can look to
>  		 * the watchdog like a soft lockup, check to see if the host
> @@ -390,7 +415,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
>  		}
>
>  		add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
> -		if (softlockup_panic)
> +		if (need_panic_on_softlockup())
>  			panic("softlockup: hung tasks");
>  		__this_cpu_write(soft_watchdog_warn, true);
>  	} else
>