bench covering both NNUE and classical, PGO, CI

vondele · vondele · commit 21ff09286b64 · 2020-08-10T23:31:55.000+02:00
Add additional options to the bench command classical, NNUE, mixed. requires the default net to be present, can be obtained with ``` make net ``` Default bench is mixed, further examples: ``` ./stockfish bench 16 1 13 default depth mixed ./stockfish bench 16 1 13 default depth classical ./stockfish bench 16 1 13 default depth NNUE ``` Requires the default net to be present, download automatically if needed for profile-build. PGO gives a nice speedup on fishtest, passed STC: LLR: 2.93 (-2.94,2.94) {-0.50,1.50} Total: 3360 W: 469 L: 343 D: 2548 Ptnml(0-2): 20, 246, 1030, 356, 28 https://tests.stockfishchess.org/tests/view/5f31b5499081672066537569 requires fishtest work, and travis work, before commit. Bench: 4290577
diff --git a/src/Makefile b/src/Makefile
@@ -564,7 +564,7 @@ help:
 build: config-sanity
 	$(MAKE) ARCH=$(ARCH) COMP=$(COMP) all
 
-profile-build: config-sanity objclean profileclean
+profile-build: config-sanity objclean profileclean net
 	@echo ""
 	@echo "Step 1/4. Building instrumented executable ..."
 	$(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make)
diff --git a/src/benchmark.cpp b/src/benchmark.cpp
@@ -95,8 +95,9 @@ const vector<string> Defaults = {
 /// setup_bench() builds a list of UCI commands to be run by bench. There
 /// are five parameters: TT size in MB, number of search threads that
 /// should be used, the limit value spent for each position, a file name
-/// where to look for positions in FEN format and the type of the limit:
-/// depth, perft, nodes and movetime (in millisecs).
+/// where to look for positions in FEN format, the type of the limit:
+/// depth, perft, nodes and movetime (in millisecs), and evaluation type
+/// mixed (default), classical, NNUE.
 ///
 /// bench -> search default positions up to depth 13
 /// bench 64 1 15 -> search default positions up to depth 15 (TT = 64MB)
@@ -115,6 +116,7 @@ vector<string> setup_bench(const Position& current, istream& is) {
   string limit     = (is >> token) ? token : "13";
   string fenFile   = (is >> token) ? token : "default";
   string limitType = (is >> token) ? token : "depth";
+  string evalType  = (is >> token) ? token : "mixed";
 
   go = limitType == "eval" ? "eval" : "go " + limitType + " " + limit;
 
@@ -146,13 +148,20 @@ vector<string> setup_bench(const Position& current, istream& is) {
   list.emplace_back("setoption name Hash value " + ttSize);
   list.emplace_back("ucinewgame");
 
+  size_t posCounter = 0;
+
   for (const string& fen : fens)
       if (fen.find("setoption") != string::npos)
           list.emplace_back(fen);
       else
       {
+          if (evalType == "classical" || (evalType == "mixed" && posCounter % 2 == 0))
+              list.emplace_back("setoption name Use NNUE value false");
+          else if (evalType == "NNUE" || (evalType == "mixed" && posCounter % 2 != 0))
+              list.emplace_back("setoption name Use NNUE value true");
           list.emplace_back("position fen " + fen);
           list.emplace_back(go);
+          ++posCounter;
       }
 
   return list;
