Merge #19710: bench: Prevent thread oversubscription and decreases the variance of result values

3edc4e34fe bench: Prevent thread oversubscription (Hennadii Stepanov)
ce3e6a7cb2 bench: Allow skip benchmark (Hennadii Stepanov)

Pull request description:

  Split out from #18710.

  Some results (borrowed from #18710):
  ![89121718-a3329800-d4c1-11ea-8bd1-66da20619696](https://user-images.githubusercontent.com/32963518/90146614-ecb89800-dd89-11ea-80fe-bac0e46e735e.png)

ACKs for top commit:
  fjahr:
    Code review ACK 3edc4e34fe

Tree-SHA512: df7413ec9ea326564a8e8de54752c9d1444ff7de34edb03e1e0c2120fc333e4640767fdbe3e87eab6a7b389a4863c02e22ad2ae0dbf139fad6a9b85e00f563b4

src/bench/bench.cpp

@@ -70,8 +70,11 @@ void benchmark::BenchRunner::RunAll(const Args& args)
             }
             std::cout << bench.complexityBigO() << std::endl;
         }
+
+        if (!bench.results().empty()) {
             benchmarkResults.push_back(bench.results().back());
         }
+    }
 
     GenerateTemplateResults(benchmarkResults, args.output_csv, "# Benchmark, evals, iterations, total, min, max, median\n"
                                                                "{{#result}}{{name}}, {{epochs}}, {{average(iterations)}}, {{sumProduct(iterations, elapsed)}}, {{minimum(elapsed)}}, {{maximum(elapsed)}}, {{median(elapsed)}}\n"

src/bench/checkqueue.cpp

@@ -14,8 +14,6 @@
 
 #include <vector>
 
-
-static const int MIN_CORES = 2;
 static const size_t BATCHES = 101;
 static const size_t BATCH_SIZE = 30;
 static const int PREVECTOR_SIZE = 28;
@@ -26,6 +24,9 @@ static const unsigned int QUEUE_BATCH_SIZE = 128;
 // and there is a little bit of work done between calls to Add.
 static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench)
 {
+    // We shouldn't ever be running with the checkqueue on a single core machine.
+    if (GetNumCores() <= 1) return;
+
     const ECCVerifyHandle verify_handle;
     ECC_Start();
 
@@ -44,7 +45,9 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench)
     };
     CCheckQueue<PrevectorJob> queue {QUEUE_BATCH_SIZE};
     boost::thread_group tg;
-    for (auto x = 0; x < std::max(MIN_CORES, GetNumCores()); ++x) {
+    // The main thread should be counted to prevent thread oversubscription, and
+    // to decrease the variance of benchmark results.
+    for (auto x = 0; x < GetNumCores() - 1; ++x) {
        tg.create_thread([&]{queue.Thread();});
     }