diff --git a/pyperf/__main__.py b/pyperf/__main__.py index aee19d1..639ffeb 100644 --- a/pyperf/__main__.py +++ b/pyperf/__main__.py @@ -491,10 +491,13 @@ def display_benchmarks(args, show_metadata=False, hist=False, stats=False, empty_line(output) output.extend(lines) + contains_warning = False for line in output: + if line.startswith("WARNING:"): + contains_warning = True print(line) - if not output and only_checks: + if not contains_warning and only_checks: if len(data) == 1: print("The benchmark seems to be stable") else: diff --git a/pyperf/_bench.py b/pyperf/_bench.py index a0cc80c..6278e8a 100644 --- a/pyperf/_bench.py +++ b/pyperf/_bench.py @@ -437,7 +437,11 @@ def required_nsamples(self): # Get the means of the values per run values = [] for run in self._runs: - values.append(statistics.mean(run.values)) + if len(run.values): + values.append(statistics.mean(run.values)) + + if len(values) < 2: + return None total = math.fsum(values) mean = total / len(values) diff --git a/pyperf/_cli.py b/pyperf/_cli.py index db27061..266031e 100644 --- a/pyperf/_cli.py +++ b/pyperf/_cli.py @@ -425,7 +425,10 @@ def format_checks(bench, lines=None): % (bench.format_value(stdev), percent, bench.format_value(mean))) else: # display a warning if the number of samples isn't enough to get a stable result - if required_nsamples > len(bench._runs): + if ( + required_nsamples is not None and + required_nsamples > len(bench._runs) + ): warn("Not enough samples to get a stable result (95% certainly of less than 1% variation)") # Minimum and maximum, detect obvious outliers @@ -463,7 +466,10 @@ def format_checks(bench, lines=None): lines.append("Use pyperf stats, pyperf dump and pyperf hist to analyze results.") lines.append("Use --quiet option to hide these warnings.") - if required_nsamples < len(bench._runs) * 0.75: + if ( + required_nsamples is not None and + required_nsamples < len(bench._runs) * 0.75 + ): lines.append("Benchmark was run more times than necessary to get a stable result.") lines.append( "Consider passing processes=%d to the Runner constructor to save time." % diff --git a/pyperf/tests/test_perf_cli.py b/pyperf/tests/test_perf_cli.py index 6423c52..8ce62cc 100644 --- a/pyperf/tests/test_perf_cli.py +++ b/pyperf/tests/test_perf_cli.py @@ -478,11 +478,16 @@ def test_hist(self): 22.8 ms: 3 ############## 22.9 ms: 4 ################### 22.9 ms: 4 ################### + Benchmark was run more times than necessary to get a stable result. + Consider passing processes=7 to the Runner constructor to save time. """) self.check_command(expected, 'hist', TELCO, env=env) def test_show(self): expected = (""" + Benchmark was run more times than necessary to get a stable result. + Consider passing processes=7 to the Runner constructor to save time. + Mean +- std dev: 22.5 ms +- 0.2 ms """) self.check_command(expected, 'show', TELCO) @@ -518,6 +523,8 @@ def test_stats(self): 100th percentile: 22.9 ms (+2% of the mean) -- maximum Number of outlier (out of 22.0 ms..23.0 ms): 0 + Benchmark was run more times than necessary to get a stable result. + Consider passing processes=7 to the Runner constructor to save time. """) self.check_command(expected, 'stats', TELCO) @@ -628,8 +635,10 @@ def test_slowest(self): def test_check_stable(self): stdout = self.run_command('check', TELCO) - self.assertEqual(stdout.rstrip(), - 'The benchmark seems to be stable') + self.assertTrue( + 'The benchmark seems to be stable' in + stdout.rstrip() + ) def test_command(self): command = [sys.executable, '-c', 'pass'] @@ -689,7 +698,7 @@ def _check_track_memory(self, track_option): '[1,2]*1000', '-o', tmp_name) bench = pyperf.Benchmark.load(tmp_name) - + self._check_track_memory_bench(bench, loops=5) def test_track_memory(self):