Replace Ranger optimizer with RAdam

bmdanielsson · bmdanielsson · commit c2561b16d99d · 2024-01-28T17:52:10.000+01:00
diff --git a/README.md b/README.md
@@ -28,5 +28,3 @@ Then, go to http://localhost:6006/
 
 # Acknowledgements
 * Training code is based on https://github.com/glinscott/nnue-pytorch
-* Ranger optimizer is taken from https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer
-
diff --git a/ranger.py b/ranger.py
diff --git a/train.py b/train.py
@@ -2,7 +2,6 @@
 import model as M
 import nnue_dataset
 import torch
-import ranger
 import time
 import os.path
 from datetime import timedelta
@@ -149,8 +148,8 @@ def main(args):
     nnue = M.NNUE().to(main_device)
 
     # Configure optimizer
-    optimizer = ranger.Ranger(nnue.parameters(), lr=1e-3)
-    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.1, patience=1, verbose=True, min_lr=1e-6)
+    optimizer = torch.optim.RAdam(nnue.parameters(), lr=1e-3, betas=(.95, 0.999), eps=1e-5, weight_decay=0)
+    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.3, patience=1, verbose=True, min_lr=1e-6)
 
     # Main training loop
     start = time.monotonic()
