Use smaller initial LR

vondele · vondele · commit f998880ad9e9 · 2021-12-13T07:24:24.000+01:00
lead to a new master net official-stockfish/Stockfish#3848 this is in combination with a restart from a good net (previous master). Visually, the net starts a bit better after the restart, probably logical as the initial perturbations are smaller.
diff --git a/model.py b/model.py
@@ -295,7 +295,7 @@ def test_step(self, batch, batch_idx):
 
   def configure_optimizers(self):
     # Train with a lower LR on the output layer
-    LR = 8.75e-4
+    LR = 4.375e-4
     train_params = [
       {'params' : get_parameters([self.input]), 'lr' : LR, 'gc_dim' : 0 },
       {'params' : [self.layer_stacks.l1_fact.weight], 'lr' : LR },
