Skip to content

Commit e1044c1

Browse files
linrockmstembera
andcommitted
Dual NNUE with L1-128 smallnet
Credit goes to @mstembera for: - writing the code enabling dual NNUE: #4898 - the idea of trying L1-128 trained exclusively on high simple eval positions The L1-128 smallnet is: - epoch 399 of a single-stage training from scratch - trained only on positions from filtered data with high material difference - defined by abs(simple_eval) > 1000 ```yaml experiment-name: 128--S1-only-hse-v2 training-dataset: - /data/hse/S3/dfrc99-16tb7p-eval-filt-v2.min.high-simple-eval-1k.binpack - /data/hse/S3/leela96-filt-v2.min.high-simple-eval-1k.binpack - /data/hse/S3/test80-apr2022-16tb7p.min.high-simple-eval-1k.binpack - /data/hse/S7/test60-2020-2tb7p.v6-3072.high-simple-eval-1k.binpack - /data/hse/S7/test60-novdec2021-12tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test77-nov2021-2tb7p.v6-3072.min.high-simple-eval-1k.binpack - /data/hse/S7/test77-dec2021-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test77-jan2022-2tb7p.high-simple-eval-1k.binpack - /data/hse/S7/test78-jantomay2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test78-juntosep2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test79-apr2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test79-may2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack # T80 2022 - /data/hse/S7/test80-may2022-16tb7p.high-simple-eval-1k.binpack - /data/hse/S7/test80-jun2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-jul2022-16tb7p.v6-dd.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-aug2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-sep2022-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-oct2022-16tb7p.v6-dd.high-simple-eval-1k.binpack - /data/hse/S7/test80-nov2022-16tb7p-v6-dd.min.high-simple-eval-1k.binpack # T80 2023 - /data/hse/S7/test80-jan2023-3of3-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-feb2023-16tb7p-filter-v6-dd.min-mar2023.unmin.high-simple-eval-1k.binpack - /data/hse/S7/test80-mar2023-2tb7p.v6-sk16.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-apr2023-2tb7p-filter-v6-sk16.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-may2023-2tb7p.v6.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-jun2023-2tb7p.v6-3072.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-jul2023-2tb7p.v6-3072.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-aug2023-2tb7p.v6.min.high-simple-eval-1k.binpack - /data/hse/S7/test80-sep2023-2tb7p.high-simple-eval-1k.binpack - /data/hse/S7/test80-oct2023-2tb7p.high-simple-eval-1k.binpack start-from-engine-test-net: False nnue-pytorch-branch: linrock/nnue-pytorch/L1-128 engine-test-branch: linrock/Stockfish/L1-128-nolazy engine-base-branch: linrock/Stockfish/L1-128 num-epochs: 500 lambda: 1.0 ``` Training data can be found at: https://robotmoon.com/nnue-training-data/ Local elo at 25k nodes per move of L1-128 smallnet (nnue-only eval) vs. L1-128 trained on standard S1 data: nn-epoch399.nnue : -318.1 +/- 2.1 Passed STC: https://tests.stockfishchess.org/tests/view/6574cb9d95ea6ba1fcd49e3b LLR: 2.93 (-2.94,2.94) <0.00,2.00> Total: 62432 W: 15875 L: 15521 D: 31036 Ptnml(0-2): 177, 7331, 15872, 7633, 203 Passed LTC: https://tests.stockfishchess.org/tests/view/6575da2d4d789acf40aaac6e LLR: 2.94 (-2.94,2.94) <0.50,2.50> Total: 64830 W: 16118 L: 15738 D: 32974 Ptnml(0-2): 43, 7129, 17697, 7497, 49 bench 1485861 Co-authored-by: mstembera <m_stembera@yahoo.com>
1 parent 36db936 commit e1044c1

File tree

13 files changed

+375
-220
lines changed

13 files changed

+375
-220
lines changed

src/Makefile

Lines changed: 52 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -791,6 +791,7 @@ help:
791791
@echo "profile-build > standard build with profile-guided optimization"
792792
@echo "build > skip profile-guided optimization"
793793
@echo "net > Download the default nnue net"
794+
@echo "net2 > Download the smaller nnue net"
794795
@echo "strip > Strip executable"
795796
@echo "install > Install executable"
796797
@echo "clean > Clean up"
@@ -857,13 +858,13 @@ endif
857858
clang-profile-use clang-profile-make FORCE \
858859
format analyze
859860

860-
analyze: net config-sanity objclean
861+
analyze: net net2 config-sanity objclean
861862
$(MAKE) -k ARCH=$(ARCH) COMP=$(COMP) $(OBJS)
862863

863-
build: net config-sanity
864+
build: net net2 config-sanity
864865
$(MAKE) ARCH=$(ARCH) COMP=$(COMP) all
865866

866-
profile-build: net config-sanity objclean profileclean
867+
profile-build: net net2 config-sanity objclean profileclean
867868
@echo ""
868869
@echo "Step 1/4. Building instrumented executable ..."
869870
$(MAKE) ARCH=$(ARCH) COMP=$(COMP) $(profile_make)
@@ -907,7 +908,7 @@ profileclean:
907908

908909
# set up shell variables for the net stuff
909910
netvariables:
910-
$(eval nnuenet := $(shell grep EvalFileDefaultName evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/'))
911+
$(eval nnuenet := $(shell grep EvalFileDefaultNameBig evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/'))
911912
$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet))
912913
$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet))
913914
$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi))
@@ -951,6 +952,52 @@ net: netvariables
951952
fi; \
952953
fi; \
953954

955+
netvariables2:
956+
$(eval nnuenet := $(shell grep EvalFileDefaultNameSmall evaluate.h | grep define | sed 's/.*\(nn-[a-z0-9]\{12\}.nnue\).*/\1/'))
957+
$(eval nnuedownloadurl1 := https://tests.stockfishchess.org/api/nn/$(nnuenet))
958+
$(eval nnuedownloadurl2 := https://github.com/official-stockfish/networks/raw/master/$(nnuenet))
959+
$(eval curl_or_wget := $(shell if hash curl 2>/dev/null; then echo "curl -skL"; elif hash wget 2>/dev/null; then echo "wget -qO-"; fi))
960+
$(eval shasum_command := $(shell if hash shasum 2>/dev/null; then echo "shasum -a 256 "; elif hash sha256sum 2>/dev/null; then echo "sha256sum "; fi))
961+
962+
# evaluation network (nnue)
963+
net2: netvariables2
964+
@echo "Default net: $(nnuenet)"
965+
@if [ "x$(curl_or_wget)" = "x" ]; then \
966+
echo "Neither curl nor wget is installed. Install one of these tools unless the net has been downloaded manually"; \
967+
fi
968+
@if [ "x$(shasum_command)" = "x" ]; then \
969+
echo "shasum / sha256sum not found, skipping net validation"; \
970+
elif test -f "$(nnuenet)"; then \
971+
if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
972+
echo "Removing invalid network"; rm -f $(nnuenet); \
973+
fi; \
974+
fi;
975+
@for nnuedownloadurl in "$(nnuedownloadurl1)" "$(nnuedownloadurl2)"; do \
976+
if test -f "$(nnuenet)"; then \
977+
echo "$(nnuenet) available : OK"; break; \
978+
else \
979+
if [ "x$(curl_or_wget)" != "x" ]; then \
980+
echo "Downloading $${nnuedownloadurl}"; $(curl_or_wget) $${nnuedownloadurl} > $(nnuenet);\
981+
else \
982+
echo "No net found and download not possible"; exit 1;\
983+
fi; \
984+
fi; \
985+
if [ "x$(shasum_command)" != "x" ]; then \
986+
if [ "$(nnuenet)" != "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
987+
echo "Removing failed download"; rm -f $(nnuenet); \
988+
fi; \
989+
fi; \
990+
done
991+
@if ! test -f "$(nnuenet)"; then \
992+
echo "Failed to download $(nnuenet)."; \
993+
fi;
994+
@if [ "x$(shasum_command)" != "x" ]; then \
995+
if [ "$(nnuenet)" = "nn-"`$(shasum_command) $(nnuenet) | cut -c1-12`".nnue" ]; then \
996+
echo "Network validated"; break; \
997+
fi; \
998+
fi; \
999+
1000+
9541001
format:
9551002
$(CLANG-FORMAT) -i $(SRCS) $(HEADERS) -style=file
9561003

@@ -1073,6 +1120,6 @@ icx-profile-use:
10731120
.depend: $(SRCS)
10741121
-@$(CXX) $(DEPENDFLAGS) -MM $(SRCS) > $@ 2> /dev/null
10751122

1076-
ifeq (, $(filter $(MAKECMDGOALS), help strip install clean net objclean profileclean config-sanity))
1123+
ifeq (, $(filter $(MAKECMDGOALS), help strip install clean net net2 objclean profileclean config-sanity))
10771124
-include .depend
10781125
endif

src/evaluate.cpp

Lines changed: 89 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <cassert>
2323
#include <cstdlib>
2424
#include <fstream>
25+
#include <initializer_list>
2526
#include <iomanip>
2627
#include <iostream>
2728
#include <sstream>
@@ -43,19 +44,25 @@
4344
// const unsigned int gEmbeddedNNUESize; // the size of the embedded file
4445
// Note that this does not work in Microsoft Visual Studio.
4546
#if !defined(_MSC_VER) && !defined(NNUE_EMBEDDING_OFF)
46-
INCBIN(EmbeddedNNUE, EvalFileDefaultName);
47+
INCBIN(EmbeddedNNUEBig, EvalFileDefaultNameBig);
48+
INCBIN(EmbeddedNNUESmall, EvalFileDefaultNameSmall);
4749
#else
48-
const unsigned char gEmbeddedNNUEData[1] = {0x0};
49-
const unsigned char* const gEmbeddedNNUEEnd = &gEmbeddedNNUEData[1];
50-
const unsigned int gEmbeddedNNUESize = 1;
50+
const unsigned char gEmbeddedNNUEBigData[1] = {0x0};
51+
const unsigned char* const gEmbeddedNNUEBigEnd = &gEmbeddedNNUEBigData[1];
52+
const unsigned int gEmbeddedNNUEBigSize = 1;
53+
const unsigned char gEmbeddedNNUESmallData[1] = {0x0};
54+
const unsigned char* const gEmbeddedNNUESmallEnd = &gEmbeddedNNUESmallData[1];
55+
const unsigned int gEmbeddedNNUESmallSize = 1;
5156
#endif
5257

5358

5459
namespace Stockfish {
5560

5661
namespace Eval {
5762

58-
std::string currentEvalFileName = "None";
63+
std::string currentEvalFileName[2] = {"None", "None"};
64+
const std::string EvFiles[2] = {"EvalFileBig", "EvalFile"};
65+
const std::string EvFileNames[2] = {EvalFileDefaultNameBig, EvalFileDefaultNameSmall};
5966

6067
// Tries to load a NNUE network at startup time, or when the engine
6168
// receives a UCI command "setoption name EvalFile value nn-[a-z0-9]{12}.nnue"
@@ -66,93 +73,92 @@ std::string currentEvalFileName = "None";
6673
// variable to have the engine search in a special directory in their distro.
6774
void NNUE::init() {
6875

69-
std::string eval_file = std::string(Options["EvalFile"]);
70-
if (eval_file.empty())
71-
eval_file = EvalFileDefaultName;
76+
for (bool small : {false, true})
77+
{
78+
std::string eval_file = std::string(Options[EvFiles[small]]);
79+
if (eval_file.empty())
80+
eval_file = EvFileNames[small];
7281

7382
#if defined(DEFAULT_NNUE_DIRECTORY)
74-
std::vector<std::string> dirs = {"<internal>", "", CommandLine::binaryDirectory,
75-
stringify(DEFAULT_NNUE_DIRECTORY)};
83+
std::vector<std::string> dirs = {"<internal>", "", CommandLine::binaryDirectory,
84+
stringify(DEFAULT_NNUE_DIRECTORY)};
7685
#else
77-
std::vector<std::string> dirs = {"<internal>", "", CommandLine::binaryDirectory};
86+
std::vector<std::string> dirs = {"<internal>", "", CommandLine::binaryDirectory};
7887
#endif
7988

80-
for (const std::string& directory : dirs)
81-
if (currentEvalFileName != eval_file)
89+
for (const std::string& directory : dirs)
8290
{
83-
if (directory != "<internal>")
84-
{
85-
std::ifstream stream(directory + eval_file, std::ios::binary);
86-
if (NNUE::load_eval(eval_file, stream))
87-
currentEvalFileName = eval_file;
88-
}
89-
90-
if (directory == "<internal>" && eval_file == EvalFileDefaultName)
91+
if (currentEvalFileName[small] != eval_file)
9192
{
92-
// C++ way to prepare a buffer for a memory stream
93-
class MemoryBuffer: public std::basic_streambuf<char> {
94-
public:
95-
MemoryBuffer(char* p, size_t n) {
96-
setg(p, p, p + n);
97-
setp(p, p + n);
98-
}
99-
};
100-
101-
MemoryBuffer buffer(
102-
const_cast<char*>(reinterpret_cast<const char*>(gEmbeddedNNUEData)),
103-
size_t(gEmbeddedNNUESize));
104-
(void) gEmbeddedNNUEEnd; // Silence warning on unused variable
105-
106-
std::istream stream(&buffer);
107-
if (NNUE::load_eval(eval_file, stream))
108-
currentEvalFileName = eval_file;
93+
if (directory != "<internal>")
94+
{
95+
std::ifstream stream(directory + eval_file, std::ios::binary);
96+
if (NNUE::load_eval(eval_file, stream, small))
97+
currentEvalFileName[small] = eval_file;
98+
}
99+
100+
if (directory == "<internal>" && eval_file == EvFileNames[small])
101+
{
102+
// C++ way to prepare a buffer for a memory stream
103+
class MemoryBuffer: public std::basic_streambuf<char> {
104+
public:
105+
MemoryBuffer(char* p, size_t n) {
106+
setg(p, p, p + n);
107+
setp(p, p + n);
108+
}
109+
};
110+
111+
MemoryBuffer buffer(
112+
const_cast<char*>(reinterpret_cast<const char*>(
113+
small ? gEmbeddedNNUESmallData : gEmbeddedNNUEBigData)),
114+
size_t(small ? gEmbeddedNNUESmallSize : gEmbeddedNNUEBigSize));
115+
(void) gEmbeddedNNUEBigEnd; // Silence warning on unused variable
116+
(void) gEmbeddedNNUESmallEnd;
117+
118+
std::istream stream(&buffer);
119+
if (NNUE::load_eval(eval_file, stream, small))
120+
currentEvalFileName[small] = eval_file;
121+
}
109122
}
110123
}
124+
}
111125
}
112126

113127
// Verifies that the last net used was loaded successfully
114128
void NNUE::verify() {
115129

116-
std::string eval_file = std::string(Options["EvalFile"]);
117-
if (eval_file.empty())
118-
eval_file = EvalFileDefaultName;
119-
120-
if (currentEvalFileName != eval_file)
130+
for (bool small : {false, true})
121131
{
132+
std::string eval_file = std::string(Options[EvFiles[small]]);
133+
if (eval_file.empty())
134+
eval_file = EvFileNames[small];
122135

123-
std::string msg1 =
124-
"Network evaluation parameters compatible with the engine must be available.";
125-
std::string msg2 = "The network file " + eval_file + " was not loaded successfully.";
126-
std::string msg3 = "The UCI option EvalFile might need to specify the full path, "
127-
"including the directory name, to the network file.";
128-
std::string msg4 = "The default net can be downloaded from: "
129-
"https://tests.stockfishchess.org/api/nn/"
130-
+ std::string(EvalFileDefaultName);
131-
std::string msg5 = "The engine will be terminated now.";
132-
133-
sync_cout << "info string ERROR: " << msg1 << sync_endl;
134-
sync_cout << "info string ERROR: " << msg2 << sync_endl;
135-
sync_cout << "info string ERROR: " << msg3 << sync_endl;
136-
sync_cout << "info string ERROR: " << msg4 << sync_endl;
137-
sync_cout << "info string ERROR: " << msg5 << sync_endl;
138-
139-
exit(EXIT_FAILURE);
140-
}
136+
if (currentEvalFileName[small] != eval_file)
137+
{
138+
std::string msg1 =
139+
"Network evaluation parameters compatible with the engine must be available.";
140+
std::string msg2 = "The network file " + eval_file + " was not loaded successfully.";
141+
std::string msg3 = "The UCI option EvalFile might need to specify the full path, "
142+
"including the directory name, to the network file.";
143+
std::string msg4 = "The default net can be downloaded from: "
144+
"https://tests.stockfishchess.org/api/nn/"
145+
+ std::string(EvFileNames[small]);
146+
std::string msg5 = "The engine will be terminated now.";
147+
148+
sync_cout << "info string ERROR: " << msg1 << sync_endl;
149+
sync_cout << "info string ERROR: " << msg2 << sync_endl;
150+
sync_cout << "info string ERROR: " << msg3 << sync_endl;
151+
sync_cout << "info string ERROR: " << msg4 << sync_endl;
152+
sync_cout << "info string ERROR: " << msg5 << sync_endl;
153+
154+
exit(EXIT_FAILURE);
155+
}
141156

142-
sync_cout << "info string NNUE evaluation using " << eval_file << sync_endl;
143-
}
157+
sync_cout << "info string NNUE evaluation using " << eval_file << sync_endl;
158+
}
144159
}
145-
146-
147-
// Returns a static, purely materialistic evaluation of the position from
148-
// the point of view of the given color. It can be divided by PawnValue to get
149-
// an approximation of the material advantage on the board in terms of pawns.
150-
Value Eval::simple_eval(const Position& pos, Color c) {
151-
return PawnValue * (pos.count<PAWN>(c) - pos.count<PAWN>(~c))
152-
+ (pos.non_pawn_material(c) - pos.non_pawn_material(~c));
153160
}
154161

155-
156162
// Evaluate is the evaluator for the outer world. It returns a static evaluation
157163
// of the position from the point of view of the side to move.
158164
Value Eval::evaluate(const Position& pos) {
@@ -162,18 +168,22 @@ Value Eval::evaluate(const Position& pos) {
162168
Value v;
163169
Color stm = pos.side_to_move();
164170
int shuffling = pos.rule50_count();
165-
int simpleEval = simple_eval(pos, stm) + (int(pos.key() & 7) - 3);
171+
int simpleEval = pos.simple_eval();
166172

167-
bool lazy = abs(simpleEval) >= RookValue + KnightValue + 16 * shuffling * shuffling
168-
+ abs(pos.this_thread()->bestValue)
169-
+ abs(pos.this_thread()->rootSimpleEval);
173+
int lazyThresholdSimpleEval = 2300;
174+
int lazyThresholdSmallNet = 1100;
170175

176+
bool lazy = abs(simpleEval) > lazyThresholdSimpleEval;
171177
if (lazy)
172178
v = Value(simpleEval);
173179
else
174180
{
175-
int nnueComplexity;
176-
Value nnue = NNUE::evaluate(pos, true, &nnueComplexity);
181+
bool smallNet = abs(simpleEval) > lazyThresholdSmallNet;
182+
183+
int nnueComplexity;
184+
185+
Value nnue = smallNet ? NNUE::evaluate<true>(pos, true, &nnueComplexity)
186+
: NNUE::evaluate<false>(pos, true, &nnueComplexity);
177187

178188
Value optimism = pos.this_thread()->optimism[stm];
179189

@@ -216,7 +226,7 @@ std::string Eval::trace(Position& pos) {
216226
ss << std::showpoint << std::showpos << std::fixed << std::setprecision(2) << std::setw(15);
217227

218228
Value v;
219-
v = NNUE::evaluate(pos, false);
229+
v = NNUE::evaluate<false>(pos, false);
220230
v = pos.side_to_move() == WHITE ? v : -v;
221231
ss << "NNUE evaluation " << 0.01 * UCI::to_cp(v) << " (white side)\n";
222232

src/evaluate.h

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,25 +21,24 @@
2121

2222
#include <string>
2323

24-
#include "types.h"
25-
2624
namespace Stockfish {
2725

2826
class Position;
27+
enum Value : int;
2928

3029
namespace Eval {
3130

3231
std::string trace(Position& pos);
3332

34-
Value simple_eval(const Position& pos, Color c);
3533
Value evaluate(const Position& pos);
3634

37-
extern std::string currentEvalFileName;
35+
extern std::string currentEvalFileName[2];
3836

3937
// The default net name MUST follow the format nn-[SHA256 first 12 digits].nnue
4038
// for the build process (profile-build and fishtest) to work. Do not change the
4139
// name of the macro, as it is used in the Makefile.
42-
#define EvalFileDefaultName "nn-0000000000a0.nnue"
40+
#define EvalFileDefaultNameBig "nn-0000000000a0.nnue"
41+
#define EvalFileDefaultNameSmall "nn-c01dc0ffeede.nnue"
4342

4443
namespace NNUE {
4544

0 commit comments

Comments
 (0)