From 56f0ae2f06edbca6ebdf247584c0741f3fa75138 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Mon, 7 Feb 2022 16:24:27 -0800 Subject: [PATCH 01/25] Added overloads for addcdiv --- src/TorchSharp/Tensor/Tensor.Math.cs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/TorchSharp/Tensor/Tensor.Math.cs b/src/TorchSharp/Tensor/Tensor.Math.cs index 2ea83a092..03cd53fe2 100644 --- a/src/TorchSharp/Tensor/Tensor.Math.cs +++ b/src/TorchSharp/Tensor/Tensor.Math.cs @@ -229,6 +229,17 @@ public Tensor addcdiv(Tensor tensor1, Tensor tensor2, Scalar value) return new Tensor(res); } + /// + /// Perform the element-wise division of tensor1 by tensor2 and add it to input. + /// + /// First tensor + /// Second tensor + /// + public Tensor addcdiv(Tensor tensor1, Tensor tensor2) + { + return addcdiv(tensor1, tensor2, 1); + } + [DllImport("LibTorchSharp")] static extern IntPtr THSTensor_addcdiv_(IntPtr tensor, IntPtr tensor1, IntPtr tensor2, IntPtr value); @@ -247,6 +258,17 @@ public Tensor addcdiv_(Tensor tensor1, Tensor tensor2, Scalar value) return new Tensor(res); } + /// + /// Performs the in-place element-wise division of tensor1 by tensor2 and add it to input. + /// + /// First tensor + /// Second tensor + /// + public Tensor addcdiv_(Tensor tensor1, Tensor tensor2) + { + return addcdiv_(tensor1, tensor2, 1); + } + [DllImport("LibTorchSharp")] static extern IntPtr THSTensor_addcmul(IntPtr tensor, IntPtr tensor1, IntPtr tensor2, IntPtr value); From 1ab1e493497f11f5765868822cd2082f48c3661e Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Mon, 7 Feb 2022 16:25:15 -0800 Subject: [PATCH 02/25] Started migrating optimizer implementations to managed code. --- src/TorchSharp/NN/Optimizer.cs | 299 +++++++++++++++++++++++++-------- 1 file changed, 230 insertions(+), 69 deletions(-) diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index 15a08552a..8d1ee98e4 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -169,22 +169,17 @@ public static LBFGSOptimizer LBFGS(IEnumerable parameters, double lea /// /// Proposed by G.Hinton in his course. /// - /// Parameters to optimize + /// Parameters to optimize /// Learning rate (default: 1e-2) - /// Smoothing constant (default: 0.99) /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Smoothing constant (default: 0.99) /// Weight decay (L2 penalty) (default: 0) /// Momentum factor (default: 0) /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance /// - public static RMSPropOptimizer RMSProp(IEnumerable parameters, double learningRate = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) + public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 0.01, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0, bool centered = false) { - var parray = new PinnedArray(); - IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray()); - - var res = THSNN_RMSprop_ctor(paramsRef, parray.Array.Length, learningRate, alpha, eps, weight_decay, momentum, centered); - if (res == IntPtr.Zero) { torch.CheckForErrors(); } - return new RMSPropOptimizer(res, learningRate, momentum); + return new RMSPropOptimizer(named_parameters, learningRate, eps, alpha, weight_decay, momentum, centered); } [DllImport("LibTorchSharp")] @@ -364,27 +359,19 @@ public static RpropOptimizer Rprop(IEnumerable<(string name, Modules.Parameter p return new RpropOptimizer(named_parameters, lr, etaminus, etaplus, min_step, max_step); } - [DllImport("LibTorchSharp")] - private static extern IntPtr THSNN_SGD_ctor(IntPtr parameters, int len, double learningRate, double momentum, double dampening, double weight_decay, bool nesterov); - /// /// Implements stochastic gradient descent (optionally with momentum). /// - /// Parameters to optimize + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// Momentum factor (default: 0) /// Dampening for momentum (default: 0) /// Weight decay (L2 penalty) (default: 0) /// Enables Nesterov momentum (default: False) /// - public static SGDOptimizer SGD(IEnumerable parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false) + public static SGDOptimizer SGD(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false) { - var parray = new PinnedArray(); - IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray()); - - var res = THSNN_SGD_ctor(paramsRef, parray.Array.Length, learningRate, momentum, dampening, weight_decay, nesterov); - if (res == IntPtr.Zero) { torch.CheckForErrors(); } - return new SGDOptimizer(res, learningRate, momentum); + return new SGDOptimizer(named_parameters, learningRate, momentum, dampening, weight_decay, nesterov); } } } @@ -976,6 +963,121 @@ private class State private double _weight_decay; } + public class SGDOptimizer : OptimizerHelper, IMomentum + { + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public SGDOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) : base(named_parameters, lr) + { + if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + if (nesterov && (momentum <= 0 || dampening != 0)) throw new ArgumentException("Nesterov momentum requires a momentum and zero dampening"); + + LearningRate = lr; + InitialLearningRate = lr; + _momentum = momentum; + _dampening = dampening; + _weight_decay = weight_decay; + _nesterov = nesterov; + _maximize = maximize; + + foreach (var (name, p) in named_parameters) { + var state = new State(); + _state[name] = state; + state.momentum_buffer = null; + } + } + + public override Tensor step(Func closure = null) + { + Tensor loss = null; + + if (closure != null) { + using (var _ = torch.enable_grad()) + loss = closure(); + } + + using (var _ = torch.no_grad()) { + + using (var d = torch.NewDisposeScope()) { + + foreach (var (name, param) in _parameters) { + + var state = _state[name]; + + var grad = param.grad(); + + if (grad is null) continue; + + if (_weight_decay != 0) { + grad = grad.add(param, alpha: _weight_decay); + } + + if (_momentum != 0) { + var buf = state.momentum_buffer; + + if (buf is null) { + buf = grad.clone().detach().DetatchFromDisposeScope(); + state.momentum_buffer = buf; + } else { + buf.mul_(_momentum).add_(grad, alpha: (1 - _dampening)); + } + + if (_nesterov) { + grad = grad.add(buf, alpha: _momentum); + } else { + grad = buf; + } + + state.momentum_buffer = buf; + } + + var alpha = _maximize ? LearningRate : -LearningRate; + param.add_(grad, alpha: alpha); + + } + + d.DisposeEverything(); + } + } + + return loss; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + if (state.momentum_buffer is not null) { + state.momentum_buffer.Dispose(); + } + } + } + + private class State + { + public Tensor momentum_buffer; + } + + private Dictionary _state = new Dictionary(); + private double _momentum; + private double _dampening; + private double _weight_decay; + private bool _nesterov; + private bool _maximize; + + public double Momentum { get => _momentum; set => _momentum = value; } + } + public class RpropOptimizer : OptimizerHelper, ILearningRateController { /// @@ -1168,36 +1270,127 @@ public double LearningRate { } - public class RMSPropOptimizer : Optimizer, ILearningRateController, IMomentum + + public class RMSPropOptimizer : OptimizerHelper, IMomentum { - public RMSPropOptimizer(IntPtr handle, double lr, double momentum) : base(handle) + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Smoothing constant. + /// Momentum factor (default: 0) + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public RMSPropOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0.0, bool centered = false) : base(named_parameters, lr) { - _rate = lr; - _momentum = momentum; + if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (alpha < 0) throw new ArgumentException($"Invalid alpha: {alpha}"); + if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + LearningRate = lr; InitialLearningRate = lr; + _momentum = momentum; + _weight_decay = weight_decay; + _alpha = alpha; + _eps = eps; + _centered = centered; + + foreach (var (name, p) in named_parameters) { + var state = new State(); + _state[name] = state; + state.step = 0; + state.square_avg = torch.zeros_like(p); + state.grad_avg = torch.zeros_like(p); + state.momentum_buffer = torch.zeros_like(p); + } } - [DllImport("LibTorchSharp")] - private static extern void THSNN_RMSprop_set_lr(HType optimizer, double lr); + public override Tensor step(Func closure = null) + { + Tensor loss = null; - [DllImport("LibTorchSharp")] - private static extern void THSNN_RMSprop_set_momentum(HType optimizer, double momentum); + if (closure != null) { + using (var _ = torch.enable_grad()) + loss = closure(); + } - public double LearningRate { - get { return _rate; } - set { THSNN_RMSprop_set_lr(handle, value); torch.CheckForErrors(); _rate = value; } - } + using (var _ = torch.no_grad()) { - public double InitialLearningRate { get; set; } + using (var d = torch.NewDisposeScope()) { - private double _rate; + foreach (var (name, param) in _parameters) { + + var state = _state[name]; + + var grad = param.grad(); + + if (grad is null) continue; + + state.step += 1; + + if (_weight_decay != 0) { + grad = grad.add(param, alpha: _weight_decay); + } + + state.square_avg.mul_(_alpha).addcmul_(grad, grad, value: 1 - _alpha); + + Tensor avg = null; + + if (_centered) { + var grad_avg = state.grad_avg; + grad_avg.mul_(_alpha).add_(grad, alpha: 1 - _alpha); + avg = state.square_avg.addcmul(grad_avg, grad_avg, value: -1).sqrt_().add_(_eps); + } else { + avg = state.square_avg.sqrt().add_(_eps); + } + + if (_momentum > 0) { + var buf = state.momentum_buffer; + buf.mul_(_momentum).addcdiv_(grad, avg); + param.add_(buf, alpha: -LearningRate); + } else { + param.addcdiv_(grad, avg, -LearningRate); + } + } - public double Momentum { - get => _momentum; - set { THSNN_RMSprop_set_momentum(handle, value); torch.CheckForErrors(); _momentum = value; } + d.DisposeEverything(); + } + } + + return loss; } + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.momentum_buffer.Dispose(); + state.square_avg.Dispose(); + state.grad_avg.Dispose(); + } + } + + private class State + { + public int step; + public Tensor square_avg; + public Tensor momentum_buffer; + public Tensor grad_avg; + } + + private Dictionary _state = new Dictionary(); private double _momentum; + private double _alpha; + private double _eps; + private double _weight_decay; + private bool _centered; + + public double Momentum { get => _momentum; set => _momentum = value; } } public class LBFGSOptimizer : Optimizer, ILearningRateController @@ -1227,38 +1420,6 @@ public override Tensor step(Func closure = null) private double _rate; } - - public class SGDOptimizer : Optimizer, ILearningRateController, IMomentum - { - public SGDOptimizer(IntPtr handle, double lr, double momentum) : base(handle) - { - _rate = lr; - _momentum = momentum; - InitialLearningRate = lr; - } - - [DllImport("LibTorchSharp")] - private static extern void THSNN_SGD_set_lr(HType optimizer, double lr); - - [DllImport("LibTorchSharp")] - private static extern void THSNN_SGD_set_momentum(HType optimizer, double momentum); - - public double LearningRate { - get { return _rate; } - set { THSNN_SGD_set_lr(handle, value); torch.CheckForErrors(); _rate = value; } - } - - public double InitialLearningRate { get; set; } - - private double _rate; - - public double Momentum { - get => _momentum; - set { THSNN_SGD_set_momentum(handle, value); torch.CheckForErrors(); _momentum = value; } - } - - private double _momentum; - } } } From c5bfe57a1e0af6437d14bd16d1cac4725ca59269 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 8 Feb 2022 11:13:49 -0800 Subject: [PATCH 03/25] Migrated Adam and AdamW implementations to managed code. --- src/TorchSharp/NN/Optimizer.cs | 306 +++++++++++++++++++++++++++------ 1 file changed, 252 insertions(+), 54 deletions(-) diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index 8d1ee98e4..a81ac4393 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -161,9 +161,6 @@ public static LBFGSOptimizer LBFGS(IEnumerable parameters, double lea return new LBFGSOptimizer(res, learningRate); } - [DllImport("LibTorchSharp")] - private static extern IntPtr THSNN_RMSprop_ctor(IntPtr parameters, int len, double learningRate, double alpha, double eps, double weight_decay, double momemtum, bool centered); - /// /// Implements RMSprop algorithm. /// @@ -190,51 +187,43 @@ public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Modules.Paramet /// /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. /// - /// Parameters to optimize + /// Parameters to optimize /// learning rate (default: 1e-3) /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// /// - public static AdamOptimizer Adam(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false) + public static AdamOptimizer Adam(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { - var parray = new PinnedArray(); - IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray()); - - var res = THSNN_Adam_ctor(paramsRef, parray.Array.Length, learningRate, beta1, beta2, eps, weight_decay, amsgrad); - if (res == IntPtr.Zero) { torch.CheckForErrors(); } - return new AdamOptimizer(res, learningRate, beta1, beta2); + return new AdamOptimizer(named_parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); } [DllImport("LibTorchSharp")] private static extern IntPtr THSNN_AdamW_ctor(IntPtr parameters, int len, double learningRate, double beta1, double beta2, double eps, double weight_decay, bool amsgrad); /// - /// Implements Adam algorithm. + /// Implements AdamW algorithm. /// /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. /// - /// Parameters to optimize + /// Parameters to optimize /// learning rate (default: 1e-3) /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// /// - public static AdamWOptimizer AdamW(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false) + public static AdamWOptimizer AdamW(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { - var parray = new PinnedArray(); - IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray()); - - var res = THSNN_AdamW_ctor(paramsRef, parray.Array.Length, learningRate, beta1, beta2, eps, weight_decay, amsgrad); - if (res == IntPtr.Zero) { torch.CheckForErrors(); } - return new AdamWOptimizer(res, learningRate, beta1, beta2); + return new AdamWOptimizer(named_parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); } - [DllImport("LibTorchSharp")] + [DllImport("LibTorchSharp")] private static extern IntPtr THSNN_Adagrad_ctor(IntPtr parameters, int len, double learningRate, double lr_decay, double weight_decay, double initial_accumulator_value, double eps); /// @@ -1207,70 +1196,279 @@ public double LearningRate { private double _rate; } - public class AdamOptimizer : Optimizer, ILearningRateController, IBetas + public class AdamOptimizer : OptimizerHelper, IBetas { - public AdamOptimizer(IntPtr handle, double lr, double beta1, double beta2) : base(handle) + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// First coefficient used for computing running averages of gradient and its square + /// Second coefficient used for computing running averages of gradient and its square + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm + /// Maximize the params based on the objective, instead of minimizing + /// + public AdamOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) : base(named_parameters, lr) { - _betas = (beta1, beta2); - _rate = lr; + if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (beta1 < 0 || beta1 >= 1.0) throw new ArgumentException($"Invalid beta1: {beta1}"); + if (beta2 < 0 || beta2 >= 1.0) throw new ArgumentException($"Invalid beta2: {beta2}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + LearningRate = lr; InitialLearningRate = lr; + _beta1 = beta1; + _beta2 = beta2; + _weight_decay = weight_decay; + _eps = eps; + _amsgrad = amsgrad; + _maximize = maximize; + + foreach (var (name, p) in named_parameters) { + + if (p.dtype == ScalarType.ComplexFloat32 || p.dtype == ScalarType.ComplexFloat64) + throw new NotImplementedException("Adam optimizer with complex weights."); + + var state = new State(); + _state[name] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + if (_amsgrad) { + state.max_exp_avg_sq = torch.zeros_like(p); + } + } } - [DllImport("LibTorchSharp")] - private static extern void THSNN_Adam_set_lr(HType optimizer, double lr); + public override Tensor step(Func closure = null) + { + Tensor loss = null; - [DllImport("LibTorchSharp")] - private static extern void THSNN_Adam_set_betas(HType optimizer, double beta1, double beta2); + if (closure != null) { + using (var _ = torch.enable_grad()) + loss = closure(); + } - public double LearningRate { - get { return _rate; } - set { THSNN_Adam_set_lr(handle, value); torch.CheckForErrors(); _rate = value; } + using (var _ = torch.no_grad()) { + + using (var d = torch.NewDisposeScope()) { + + foreach (var (name, param) in _parameters) { + + var state = _state[name]; + + var grad = (_maximize) ? -param.grad() : param.grad(); + + if (grad is null) continue; + + state.step += 1; + + var bias_correction1 = 1 - Math.Pow(_beta1, state.step); + var bias_correction2 = 1 - Math.Pow(_beta2, state.step); + + if (_weight_decay != 0) { + grad = grad.add(param, alpha: _weight_decay); + } + + state.exp_avg.mul_(_beta1).add_(grad, alpha: 1 - _beta1); + // When complex types are supported: + //state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad.conj(), value: 1 - _beta2) + state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad, value: 1 - _beta2); + + Tensor denom = null; + if (_amsgrad) { + var t0 = state.max_exp_avg_sq; + state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); + t0.Dispose(); + denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); + } + else { + denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); + } + + var step_size = LearningRate / bias_correction1; + param.addcdiv_(state.exp_avg, denom, value: -step_size); + } + + d.DisposeEverything(); + } + } + + return loss; } - public double InitialLearningRate { get; set; } + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.exp_avg.Dispose(); + state.exp_avg_sq.Dispose(); + if (state.max_exp_avg_sq is not null) { + state.max_exp_avg_sq.Dispose(); + } + } + } + + private class State + { + public int step; + public Tensor exp_avg; + public Tensor exp_avg_sq; + public Tensor max_exp_avg_sq; + } public (double, double) Betas { - get => _betas; - set { THSNN_Adam_set_betas(handle, value.Item1, value.Item2); torch.CheckForErrors(); _betas = value; } + get => (_beta1, _beta2); + set { _beta1 = value.Item1; _beta2 = value.Item2; } } - private (double, double) _betas; - private double _rate; + + private Dictionary _state = new Dictionary(); + private double _beta1; + private double _beta2; + private double _eps; + private double _weight_decay; + private bool _amsgrad; + private bool _maximize; } - public class AdamWOptimizer : Optimizer, ILearningRateController, IBetas + public class AdamWOptimizer : OptimizerHelper, IBetas { - public AdamWOptimizer(IntPtr handle, double lr, double beta1, double beta2) : base(handle) + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// First coefficient used for computing running averages of gradient and its square + /// Second coefficient used for computing running averages of gradient and its square + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm + /// Maximize the params based on the objective, instead of minimizing + /// + public AdamWOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) : base(named_parameters, lr) { - _betas = (beta1, beta2); - _rate = lr; + if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (beta1 < 0 || beta1 >= 1.0) throw new ArgumentException($"Invalid beta1: {beta1}"); + if (beta2 < 0 || beta2 >= 1.0) throw new ArgumentException($"Invalid beta2: {beta2}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + LearningRate = lr; InitialLearningRate = lr; + _beta1 = beta1; + _beta2 = beta2; + _weight_decay = weight_decay; + _eps = eps; + _amsgrad = amsgrad; + _maximize = maximize; + + foreach (var (name, p) in named_parameters) { + + if (p.dtype == ScalarType.ComplexFloat32 || p.dtype == ScalarType.ComplexFloat64) + throw new NotImplementedException("Adam optimizer with complex weights."); + + var state = new State(); + _state[name] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + if (_amsgrad) { + state.max_exp_avg_sq = torch.zeros_like(p); + } + } } - [DllImport("LibTorchSharp")] - private static extern void THSNN_AdamW_set_lr(HType optimizer, double lr); + public override Tensor step(Func closure = null) + { + Tensor loss = null; - [DllImport("LibTorchSharp")] - private static extern void THSNN_AdamW_set_betas(HType optimizer, double beta1, double beta2); + if (closure != null) { + using (var _ = torch.enable_grad()) + loss = closure(); + } - public double LearningRate { - get { return _rate; } - set { THSNN_AdamW_set_lr(handle, value); torch.CheckForErrors(); _rate = value; } + using (var _ = torch.no_grad()) { + + using (var d = torch.NewDisposeScope()) { + + foreach (var (name, param) in _parameters) { + + var state = _state[name]; + + var grad = (_maximize) ? -param.grad() : param.grad(); + + if (grad is null) continue; + + state.step += 1; + + param.mul_(1 - LearningRate * _weight_decay); + + var bias_correction1 = 1 - Math.Pow(_beta1, state.step); + var bias_correction2 = 1 - Math.Pow(_beta2, state.step); + + state.exp_avg.mul_(_beta1).add_(grad, alpha: 1 - _beta1); + state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad, value: 1 - _beta2); + + Tensor denom = null; + if (_amsgrad) { + var t0 = state.max_exp_avg_sq; + state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); + t0.Dispose(); + denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); + } else { + denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); + } + + var step_size = LearningRate / bias_correction1; + param.addcdiv_(state.exp_avg, denom, value: -step_size); + } + + d.DisposeEverything(); + } + } + + return loss; } - public double InitialLearningRate { get; set; } + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.exp_avg.Dispose(); + state.exp_avg_sq.Dispose(); + if (state.max_exp_avg_sq is not null) { + state.max_exp_avg_sq.Dispose(); + } + } + } + + private class State + { + public int step; + public Tensor exp_avg; + public Tensor exp_avg_sq; + public Tensor max_exp_avg_sq; + } public (double, double) Betas { - get => _betas; - set { THSNN_AdamW_set_betas(handle, value.Item1, value.Item2); torch.CheckForErrors(); _betas = value; } + get => (_beta1, _beta2); + set { _beta1 = value.Item1; _beta2 = value.Item2; } } - private (double, double) _betas; - private double _rate; + private Dictionary _state = new Dictionary(); + private double _beta1; + private double _beta2; + private double _eps; + private double _weight_decay; + private bool _amsgrad; + private bool _maximize; } - public class RMSPropOptimizer : OptimizerHelper, IMomentum { /// From 875124bc98f3e1311c150dfd0458249308f5544a Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 8 Feb 2022 12:35:01 -0800 Subject: [PATCH 04/25] Migrated the Adagrad optimizer to managed code. --- src/TorchSharp/NN/Optimizer.cs | 153 ++++++++++++++++++++++++--------- 1 file changed, 114 insertions(+), 39 deletions(-) diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index a81ac4393..2cf3b93eb 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -179,9 +179,6 @@ public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Modules.Paramet return new RMSPropOptimizer(named_parameters, learningRate, eps, alpha, weight_decay, momentum, centered); } - [DllImport("LibTorchSharp")] - private static extern IntPtr THSNN_Adam_ctor(IntPtr parameters, int len, double learningRate, double beta1, double beta2, double eps, double weight_decay, bool amsgrad); - /// /// Implements Adam algorithm. /// @@ -201,9 +198,6 @@ public static AdamOptimizer Adam(IEnumerable<(string name, Modules.Parameter par return new AdamOptimizer(named_parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); } - [DllImport("LibTorchSharp")] - private static extern IntPtr THSNN_AdamW_ctor(IntPtr parameters, int len, double learningRate, double beta1, double beta2, double eps, double weight_decay, bool amsgrad); - /// /// Implements AdamW algorithm. /// @@ -223,29 +217,21 @@ public static AdamWOptimizer AdamW(IEnumerable<(string name, Modules.Parameter p return new AdamWOptimizer(named_parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); } - [DllImport("LibTorchSharp")] - private static extern IntPtr THSNN_Adagrad_ctor(IntPtr parameters, int len, double learningRate, double lr_decay, double weight_decay, double initial_accumulator_value, double eps); - /// /// Implements Adagrad algorithm. /// /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. /// - /// Parameters to optimize + /// Parameters to optimize /// learning rate (default: 1e-2) /// learning rate decay (default: 0) /// weight decay (L2 penalty) (default: 0) /// /// Term added to the denominator to improve numerical stability (default: 1e-10) /// - public static AdagradOptimizer Adagrad(IEnumerable parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + public static AdagradOptimizer Adagrad(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) { - var parray = new PinnedArray(); - IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray()); - - var res = THSNN_Adagrad_ctor(paramsRef, parray.Array.Length, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps); - if (res == IntPtr.Zero) { torch.CheckForErrors(); } - return new AdagradOptimizer(res, learningRate); + return new AdagradOptimizer(named_parameters, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps); } /// @@ -1173,33 +1159,120 @@ private class State private double _max_step; } - // The following optimizers are just wrappers for the native code implementations. - - public class AdagradOptimizer : Optimizer, ILearningRateController + public class AdagradOptimizer : OptimizerHelper { - public AdagradOptimizer(IntPtr handle, double lr) : base(handle) + /// + /// Implements Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public AdagradOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) : base(named_parameters, learningRate) { - _rate = lr; - InitialLearningRate = lr; + if (learningRate < 0) throw new ArgumentException($"Invalid learning rate: {learningRate}"); + if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + LearningRate = learningRate; + InitialLearningRate = learningRate; + _lr_decay = lr_decay; + _initial_accumulator_value = initial_accumulator_value; + _weight_decay = weight_decay; + _eps = eps; + + foreach (var (name, p) in named_parameters) { + + var state = new State(); + _state[name] = state; + state.step = 0; + var init_value = torch.is_complex(p.dtype) + ? (Scalar)new System.Numerics.Complex(initial_accumulator_value, initial_accumulator_value) + : (Scalar)initial_accumulator_value; + state.sum = torch.full_like(p, init_value); + } } - [DllImport("LibTorchSharp")] - private static extern void THSNN_Adagrad_set_lr(HType optimizer, double lr); + public override Tensor step(Func closure = null) + { + Tensor loss = null; - public double LearningRate { - get { return _rate; } - set { THSNN_Adagrad_set_lr(handle, value); torch.CheckForErrors(); _rate = value; } + if (closure != null) { + using (var _ = torch.enable_grad()) + loss = closure(); + } + + using (var _ = torch.no_grad()) { + + using (var d = torch.NewDisposeScope()) { + + foreach (var (name, param) in _parameters) { + + var state = _state[name]; + + var grad = param.grad(); + + if (grad is null) continue; + + state.step += 1; + + if (_weight_decay != 0) { + grad = grad.add(param, alpha: _weight_decay); + } + + var clr = LearningRate / (1 + (state.step - 1) * _lr_decay); + + if (grad.is_sparse) { + throw new NotImplementedException("Adagrad optimization over sparse parameters"); + } else if (torch.is_complex(param)) { + throw new NotImplementedException("Adagrad optimization over complex parameters"); + } else { + state.sum.addcmul_(grad, grad, value: 1); + var std = state.sum.sqrt().add_(_eps); + param.addcdiv_(grad, std, value: -clr); + } + + d.DisposeEverything(); + } + } + } + + return loss; } - public double InitialLearningRate { get; set; } + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.sum.Dispose(); + } + } - private double _rate; + private class State + { + public int step; + public Tensor sum; + } + + + private Dictionary _state = new Dictionary(); + private double _lr_decay; + private double _initial_accumulator_value; + private double _eps; + private double _weight_decay; } public class AdamOptimizer : OptimizerHelper, IBetas { /// - /// Implements stochastic gradient descent (optionally with momentum). + /// Implements Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. /// /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate @@ -1229,7 +1302,7 @@ public AdamOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> nam foreach (var (name, p) in named_parameters) { - if (p.dtype == ScalarType.ComplexFloat32 || p.dtype == ScalarType.ComplexFloat64) + if (torch.is_complex(p.dtype)) throw new NotImplementedException("Adam optimizer with complex weights."); var state = new State(); @@ -1284,8 +1357,7 @@ public override Tensor step(Func closure = null) state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); t0.Dispose(); denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); - } - else { + } else { denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); } @@ -1338,7 +1410,9 @@ private class State public class AdamWOptimizer : OptimizerHelper, IBetas { /// - /// Implements stochastic gradient descent (optionally with momentum). + /// Implements AdamW algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. /// /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate @@ -1368,9 +1442,6 @@ public AdamWOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> na foreach (var (name, p) in named_parameters) { - if (p.dtype == ScalarType.ComplexFloat32 || p.dtype == ScalarType.ComplexFloat64) - throw new NotImplementedException("Adam optimizer with complex weights."); - var state = new State(); _state[name] = state; state.step = 0; @@ -1472,7 +1543,9 @@ private class State public class RMSPropOptimizer : OptimizerHelper, IMomentum { /// - /// Implements stochastic gradient descent (optionally with momentum). + /// Implements RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. /// /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate @@ -1591,6 +1664,8 @@ private class State public double Momentum { get => _momentum; set => _momentum = value; } } + // The following optimizers are just wrappers for the native code implementations. + public class LBFGSOptimizer : Optimizer, ILearningRateController { public LBFGSOptimizer(IntPtr handle, double lr) : base(handle) From 35cde7822c5b573072804ca9ce88f4dbc5fc94f7 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 8 Feb 2022 12:36:51 -0800 Subject: [PATCH 05/25] The new optimizer construction APIs take named_parameters instead of parameters. --- src/Examples/CIFAR10.cs | 2 +- src/Examples/MNIST.cs | 2 +- src/Examples/SequenceToSequence.cs | 2 +- src/Examples/TextClassification.cs | 2 +- src/FSharp.Examples/AlexNet.fs | 2 +- test/TorchSharpTest/TestTraining.cs | 46 ++++++++++++++--------------- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/Examples/CIFAR10.cs b/src/Examples/CIFAR10.cs index 5aeb3de9b..ce56fb3a0 100644 --- a/src/Examples/CIFAR10.cs +++ b/src/Examples/CIFAR10.cs @@ -120,7 +120,7 @@ internal static void Main(string[] args) using (var train = new CIFARReader(targetDir, false, _trainBatchSize, shuffle: true, device: device, transforms: new torchvision.ITransform[] { })) using (var test = new CIFARReader(targetDir, true, _testBatchSize, device: device)) - using (var optimizer = torch.optim.Adam(model.parameters(), 0.001)) { + using (var optimizer = torch.optim.Adam(model.named_parameters(), 0.001)) { Stopwatch totalSW = new Stopwatch(); totalSW.Start(); diff --git a/src/Examples/MNIST.cs b/src/Examples/MNIST.cs index 0010d314a..e82ae9d96 100644 --- a/src/Examples/MNIST.cs +++ b/src/Examples/MNIST.cs @@ -82,7 +82,7 @@ internal static void TrainingLoop(string dataset, Device device, Model model, MN _epochs *= 4; } - var optimizer = torch.optim.Adam(model.parameters()); + var optimizer = torch.optim.Adam(model.named_parameters()); var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.75); diff --git a/src/Examples/SequenceToSequence.cs b/src/Examples/SequenceToSequence.cs index 10f2f8e67..baf2ca738 100644 --- a/src/Examples/SequenceToSequence.cs +++ b/src/Examples/SequenceToSequence.cs @@ -72,7 +72,7 @@ internal static void Main(string[] args) var model = new TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to((Device)device); var loss = cross_entropy_loss(); var lr = 2.50; - var optimizer = torch.optim.SGD(model.parameters(), lr); + var optimizer = torch.optim.SGD(model.named_parameters(), lr); var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.95, last_epoch: 15); var totalTime = new Stopwatch(); diff --git a/src/Examples/TextClassification.cs b/src/Examples/TextClassification.cs index 6e74b6173..f5a716fb3 100644 --- a/src/Examples/TextClassification.cs +++ b/src/Examples/TextClassification.cs @@ -63,7 +63,7 @@ internal static void Main(string[] args) var loss = cross_entropy_loss(); var lr = 5.0; - var optimizer = torch.optim.SGD(model.parameters(), lr); + var optimizer = torch.optim.SGD(model.named_parameters(), lr); var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.2, last_epoch: 5); // This data set is small enough that we can get away with diff --git a/src/FSharp.Examples/AlexNet.fs b/src/FSharp.Examples/AlexNet.fs index 4c410435f..7fff03eee 100644 --- a/src/FSharp.Examples/AlexNet.fs +++ b/src/FSharp.Examples/AlexNet.fs @@ -154,7 +154,7 @@ let test (model:Model) (dataLoader:CIFARReader) = let trainingLoop (model:Model) epochs trainData testData = - use optimizer = Adam(model.parameters(), 0.001) + use optimizer = Adam(model.named_parameters(), 0.001) let sw = Stopwatch() sw.Start() diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index ed92ac1a6..05744c88a 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -121,7 +121,7 @@ public void TestAdam() double learning_rate = 0.00001; - var optimizer = torch.optim.Adam(seq.parameters(), learning_rate); + var optimizer = torch.optim.Adam(seq.named_parameters(), learning_rate); Assert.NotNull(optimizer); } @@ -135,7 +135,7 @@ public void TestAdamBetas() double learning_rate = 0.00001; - var optimizer = torch.optim.Adam(seq.parameters(), learning_rate); + var optimizer = torch.optim.Adam(seq.named_parameters(), learning_rate); var (beta1, beta2) = optimizer.Betas; Assert.Equal(0.9, beta1); Assert.Equal(0.99, beta2); @@ -161,7 +161,7 @@ public void TestTrainingAdamDefaults() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.parameters()); + var optimizer = torch.optim.Adam(seq.named_parameters()); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -193,7 +193,7 @@ public void TestTrainingAdamAmsGrad() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.parameters(), amsgrad: true); + var optimizer = torch.optim.Adam(seq.named_parameters(), amsgrad: true); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -225,7 +225,7 @@ public void TestTrainingAdamOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.parameters(), amsgrad: true); + var optimizer = torch.optim.Adam(seq.named_parameters(), amsgrad: true); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -266,7 +266,7 @@ public void TestTrainingAdamWDefaults() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.AdamW(seq.parameters()); + var optimizer = torch.optim.AdamW(seq.named_parameters()); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -298,7 +298,7 @@ public void TestTrainingAdamWOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.AdamW(seq.parameters(), amsgrad: true); + var optimizer = torch.optim.AdamW(seq.named_parameters(), amsgrad: true); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -341,7 +341,7 @@ public void TestTrainingAdagrad() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.Adagrad(seq.parameters(), learning_rate); + var optimizer = torch.optim.Adagrad(seq.named_parameters(), learning_rate); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -702,7 +702,7 @@ public void TestTrainingRMSLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate); + var optimizer = torch.optim.RMSProp(seq.named_parameters(), learning_rate); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -739,7 +739,7 @@ public void TestTrainingRMSOneCycleLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate); + var optimizer = torch.optim.RMSProp(seq.named_parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -775,7 +775,7 @@ public void TestTrainingRMSAlpha() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, alpha: 0.75); + var optimizer = torch.optim.RMSProp(seq.named_parameters(), learning_rate, alpha: 0.75); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -808,7 +808,7 @@ public void TestTrainingRMSCentered() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, centered: true); + var optimizer = torch.optim.RMSProp(seq.named_parameters(), learning_rate, centered: true); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -845,7 +845,7 @@ public void TestTrainingSGDMomentum() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, momentum: 0.5); + var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate, momentum: 0.5); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -867,7 +867,7 @@ public void TestTrainingSGDMomentum() Assert.True(finalLoss < initialLoss); } - [Fact(Skip = "Fails with an exception in native code.")] + [Fact()]//(Skip = "Fails with an exception in native code.")] public void TestTrainingSGDNesterov() { var lin1 = Linear(1000, 100); @@ -878,7 +878,7 @@ public void TestTrainingSGDNesterov() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, nesterov: true); + var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate, momentum: 0.1, nesterov: true); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -911,7 +911,7 @@ public void TestTrainingSGDDefaults() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -944,7 +944,7 @@ public void TestTrainingSGDStepLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.97); var loss = mse_loss(Reduction.Sum); @@ -987,7 +987,7 @@ public void TestTrainingSGDMultiStepLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, new int[] { 3, 5, 7 }, 0.97); var loss = mse_loss(Reduction.Sum); @@ -1031,7 +1031,7 @@ public void TestTrainingSGDCosineAnnealingLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 10); var loss = mse_loss(Reduction.Sum); @@ -1075,7 +1075,7 @@ public void TestTrainingSGDCyclicLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, 0.0001, 0.0004, step_size_up: 5); var loss = mse_loss(Reduction.Sum); @@ -1113,7 +1113,7 @@ public void TestTrainingSGDOneCycleLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, 0.0004, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -1248,7 +1248,7 @@ public void TestTrainingConv2d() var x = torch.randn(new long[] { 64, 3, 28, 28 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.parameters()); + var optimizer = torch.optim.Adam(seq.named_parameters()); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -1292,7 +1292,7 @@ public void TestTrainingConv2dCUDA() seq.to((Device)device); - var optimizer = torch.optim.Adam(seq.parameters()); + var optimizer = torch.optim.Adam(seq.named_parameters()); var loss = mse_loss(Reduction.Sum); using (Tensor x = torch.randn(new long[] { 64, 3, 28, 28 }, device: (Device)device), From c7fe4877779cb348eb523f08e8707068c5940dea Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 8 Feb 2022 12:37:06 -0800 Subject: [PATCH 06/25] Migrating F# examples to new APIs. --- src/FSharp.Examples/MNIST.fs | 2 +- src/FSharp.Examples/SequenceToSequence.fs | 2 +- src/FSharp.Examples/TextClassification.fs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/FSharp.Examples/MNIST.fs b/src/FSharp.Examples/MNIST.fs index 297a25d4d..c6914640f 100644 --- a/src/FSharp.Examples/MNIST.fs +++ b/src/FSharp.Examples/MNIST.fs @@ -141,7 +141,7 @@ let trainingLoop (model:Model) epochs dataset trainData testData = let epochs = if device.``type`` = DeviceType.CUDA then epochs * 4 else epochs - let optimizer = Adam(model.parameters()) + let optimizer = Adam(model.named_parameters()) lr_scheduler.StepLR(optimizer, 1, 0.7, last_epoch=5) |> ignore let sw = Stopwatch() diff --git a/src/FSharp.Examples/SequenceToSequence.fs b/src/FSharp.Examples/SequenceToSequence.fs index ae6546c7c..421cd953d 100644 --- a/src/FSharp.Examples/SequenceToSequence.fs +++ b/src/FSharp.Examples/SequenceToSequence.fs @@ -244,7 +244,7 @@ let run epochs = use model = new TransformerModel(ntokens, device) let lr = 2.50 - let optimizer = SGD(model.parameters(), lr) + let optimizer = SGD(model.named_parameters(), lr) let scheduler = lr_scheduler.StepLR(optimizer, 1, 0.95, last_epoch=15) let totalTime = Stopwatch() diff --git a/src/FSharp.Examples/TextClassification.fs b/src/FSharp.Examples/TextClassification.fs index 0213b9708..fb17c0f20 100644 --- a/src/FSharp.Examples/TextClassification.fs +++ b/src/FSharp.Examples/TextClassification.fs @@ -137,7 +137,7 @@ let run epochs = let model = new TextClassificationModel((int64 vocab.Count), emsize, 4L, device) - let optimizer = torch.optim.SGD(model.parameters(), lr) + let optimizer = torch.optim.SGD(model.named_parameters(), lr) let scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.2, last_epoch=5) let sw = Stopwatch() From 956c3ea6fcdb59cd029f8741f9d555f7f17d6f3f Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 8 Feb 2022 12:37:17 -0800 Subject: [PATCH 07/25] Adding is_complex() --- .../Tensor/TensorExtensionMethods.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/TorchSharp/Tensor/TensorExtensionMethods.cs b/src/TorchSharp/Tensor/TensorExtensionMethods.cs index 398b97c57..249f93008 100644 --- a/src/TorchSharp/Tensor/TensorExtensionMethods.cs +++ b/src/TorchSharp/Tensor/TensorExtensionMethods.cs @@ -109,6 +109,25 @@ internal static bool IsFloatingPoint(this ScalarType type) } } + /// + /// Indicates whether a given element type is complex. + /// + /// The input type. + /// + /// + /// Complex numbers are not real, and thus will return 'false' + /// + internal static bool IsComplex(this ScalarType type) + { + switch (type) { + case ScalarType.ComplexFloat32: + case ScalarType.ComplexFloat64: + return true; + default: + return false; + } + } + /// /// Save the tensor in a .NET-specific format. /// From d051e4d054cc646cd7e2bd5cc8fecb2deb15292b Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 8 Feb 2022 16:25:04 -0800 Subject: [PATCH 08/25] Started support for parameter groups. --- RELEASENOTES.md | 10 + src/Examples/Program.cs | 6 +- src/Examples/SequenceToSequence.cs | 2 +- src/Examples/TextClassification.cs | 2 +- src/FSharp.Examples/SequenceToSequence.fs | 2 +- src/FSharp.Examples/TextClassification.fs | 2 +- src/TorchSharp/NN/Optimizer.cs | 385 ++++++++++++++-------- test/TorchSharpTest/TestTraining.cs | 55 +++- 8 files changed, 311 insertions(+), 153 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index d7f84cb87..e19faec5e 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,6 +2,16 @@ Releases, starting with 9/2/2021, are listed with the most recent release at the top. +## NuGet Version 0.96.1 + +__API Changes:__ + +__NOTE__: This release contains breaking changes. + +The APIs to create optimizers all take 'named_parameters' rather than 'parameters' now. + +__Fixed Bugs:__ + ## NuGet Version 0.96.0 __API Changes:__ diff --git a/src/Examples/Program.cs b/src/Examples/Program.cs index 032e42652..d7c30d373 100644 --- a/src/Examples/Program.cs +++ b/src/Examples/Program.cs @@ -11,9 +11,9 @@ public static class Program { public static void Main(string[] args) { - MNIST.Main(args); - AdversarialExampleGeneration.Main(args); - CIFAR10.Main(args); + //MNIST.Main(args); + //AdversarialExampleGeneration.Main(args); + //CIFAR10.Main(args); SequenceToSequence.Main(args); TextClassification.Main(args); //ImageTransforms.Main(args); diff --git a/src/Examples/SequenceToSequence.cs b/src/Examples/SequenceToSequence.cs index baf2ca738..10f2f8e67 100644 --- a/src/Examples/SequenceToSequence.cs +++ b/src/Examples/SequenceToSequence.cs @@ -72,7 +72,7 @@ internal static void Main(string[] args) var model = new TransformerModel(ntokens, emsize, nhead, nhid, nlayers, dropout).to((Device)device); var loss = cross_entropy_loss(); var lr = 2.50; - var optimizer = torch.optim.SGD(model.named_parameters(), lr); + var optimizer = torch.optim.SGD(model.parameters(), lr); var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.95, last_epoch: 15); var totalTime = new Stopwatch(); diff --git a/src/Examples/TextClassification.cs b/src/Examples/TextClassification.cs index f5a716fb3..6e74b6173 100644 --- a/src/Examples/TextClassification.cs +++ b/src/Examples/TextClassification.cs @@ -63,7 +63,7 @@ internal static void Main(string[] args) var loss = cross_entropy_loss(); var lr = 5.0; - var optimizer = torch.optim.SGD(model.named_parameters(), lr); + var optimizer = torch.optim.SGD(model.parameters(), lr); var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.2, last_epoch: 5); // This data set is small enough that we can get away with diff --git a/src/FSharp.Examples/SequenceToSequence.fs b/src/FSharp.Examples/SequenceToSequence.fs index 421cd953d..ae6546c7c 100644 --- a/src/FSharp.Examples/SequenceToSequence.fs +++ b/src/FSharp.Examples/SequenceToSequence.fs @@ -244,7 +244,7 @@ let run epochs = use model = new TransformerModel(ntokens, device) let lr = 2.50 - let optimizer = SGD(model.named_parameters(), lr) + let optimizer = SGD(model.parameters(), lr) let scheduler = lr_scheduler.StepLR(optimizer, 1, 0.95, last_epoch=15) let totalTime = Stopwatch() diff --git a/src/FSharp.Examples/TextClassification.fs b/src/FSharp.Examples/TextClassification.fs index fb17c0f20..0213b9708 100644 --- a/src/FSharp.Examples/TextClassification.fs +++ b/src/FSharp.Examples/TextClassification.fs @@ -137,7 +137,7 @@ let run epochs = let model = new TextClassificationModel((int64 vocab.Count), emsize, 4L, device) - let optimizer = torch.optim.SGD(model.named_parameters(), lr) + let optimizer = torch.optim.SGD(model.parameters(), lr) let scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.2, last_epoch=5) let sw = Stopwatch() diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index 2cf3b93eb..c4e42d326 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -174,7 +174,7 @@ public static LBFGSOptimizer LBFGS(IEnumerable parameters, double lea /// Momentum factor (default: 0) /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance /// - public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 0.01, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0, bool centered = false) + public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 0.01, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0, bool centered = false) { return new RMSPropOptimizer(named_parameters, learningRate, eps, alpha, weight_decay, momentum, centered); } @@ -193,7 +193,7 @@ public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Modules.Paramet /// Whether to use the AMSGrad variant of this algorithm. (default: False) /// /// - public static AdamOptimizer Adam(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + public static AdamOptimizer Adam(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { return new AdamOptimizer(named_parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); } @@ -212,7 +212,7 @@ public static AdamOptimizer Adam(IEnumerable<(string name, Modules.Parameter par /// Whether to use the AMSGrad variant of this algorithm. (default: False) /// /// - public static AdamWOptimizer AdamW(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + public static AdamWOptimizer AdamW(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { return new AdamWOptimizer(named_parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); } @@ -229,7 +229,7 @@ public static AdamWOptimizer AdamW(IEnumerable<(string name, Modules.Parameter p /// /// Term added to the denominator to improve numerical stability (default: 1e-10) /// - public static AdagradOptimizer Adagrad(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + public static AdagradOptimizer Adagrad(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) { return new AdagradOptimizer(named_parameters, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps); } @@ -245,7 +245,7 @@ public static AdagradOptimizer Adagrad(IEnumerable<(string name, Modules.Paramet /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) /// Weight decay (L2 penalty) (default: 0) /// - public static AdadeltaOptimizer Adadelta(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + public static AdadeltaOptimizer Adadelta(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) { return new AdadeltaOptimizer(named_parameters, lr, rho, eps, weight_decay); } @@ -263,7 +263,7 @@ public static AdadeltaOptimizer Adadelta(IEnumerable<(string name, Modules.Param /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// Momentum decay - public static NAdamOptimizer NAdam(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + public static NAdamOptimizer NAdam(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) { return new NAdamOptimizer(named_parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); } @@ -280,7 +280,7 @@ public static NAdamOptimizer NAdam(IEnumerable<(string name, Modules.Parameter p /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) - public static RAdamOptimizer RAdam(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + public static RAdamOptimizer RAdam(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) { return new RAdamOptimizer(named_parameters, lr, beta1, beta2, eps, weight_decay); } @@ -297,7 +297,7 @@ public static RAdamOptimizer RAdam(IEnumerable<(string name, Modules.Parameter p /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// - public static AdamaxOptimizer Adamax(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + public static AdamaxOptimizer Adamax(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) { return new AdamaxOptimizer(named_parameters, lr, beta1, beta2, eps, weight_decay); } @@ -314,7 +314,7 @@ public static AdamaxOptimizer Adamax(IEnumerable<(string name, Modules.Parameter /// Point at which to start averaging (default: 1e6) /// Weight decay (L2 penalty) (default: 0) /// - public static ASGDOptimizer ASGD(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + public static ASGDOptimizer ASGD(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) { return new ASGDOptimizer(named_parameters, lr, lambd, alpha, t0, weight_decay); } @@ -329,7 +329,7 @@ public static ASGDOptimizer ASGD(IEnumerable<(string name, Modules.Parameter par /// Minimum allowed step size. /// Maximum allowed step size. /// - public static RpropOptimizer Rprop(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + public static RpropOptimizer Rprop(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) { return new RpropOptimizer(named_parameters, lr, etaminus, etaplus, min_step, max_step); } @@ -337,16 +337,21 @@ public static RpropOptimizer Rprop(IEnumerable<(string name, Modules.Parameter p /// /// Implements stochastic gradient descent (optionally with momentum). /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// Momentum factor (default: 0) /// Dampening for momentum (default: 0) /// Weight decay (L2 penalty) (default: 0) /// Enables Nesterov momentum (default: False) /// - public static SGDOptimizer SGD(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false) + public static SGDOptimizer SGD(IEnumerable parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false) + { + return new SGDOptimizer(parameters, learningRate, momentum, dampening, weight_decay, nesterov); + } + + public static SGDOptimizer SGD(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) { - return new SGDOptimizer(named_parameters, learningRate, momentum, dampening, weight_decay, nesterov); + return new SGDOptimizer(parameters, lr, momentum, dampening, weight_decay, nesterov); } } } @@ -361,6 +366,7 @@ namespace Modules /// /// Base class to help with a couple of the things that managed-code implementations need. /// + public class OptimizerHelper : Optimizer, ILearningRateController { public OptimizerHelper(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate) : base(IntPtr.Zero) @@ -389,6 +395,224 @@ public override void zero_grad() protected IEnumerable<(string name, Modules.Parameter parameter)> _parameters; } + public class NewOptimizerHelper : Optimizer, ILearningRateController + { + public NewOptimizerHelper() : base(IntPtr.Zero) + { + } + + public override void zero_grad() + { + foreach (var g in _parameter_groups) { + + foreach (var p in g.Parameters) { + + using var grad = p.grad(); + + if (grad is null) continue; + + grad.zero_().Dispose(); + } + } + } + + public virtual void add_param_group(ParameterGroup param_group) + { + _parameter_groups.Add(param_group); + } + + public double LearningRate { get => _defaults.LearningRate.Value; set => _defaults.LearningRate = value; } + + public double InitialLearningRate { get => _defaults.InitialLearningRate; set => _defaults.InitialLearningRate = value; } + + protected OptimizerOptions _defaults; + protected IList _parameter_groups; + } + + public class OptimizerOptions + { + public double? LearningRate { get; set; } + public double InitialLearningRate { get; set; } + } + + public class ParameterGroup : ILearningRateController + { + public IEnumerable Parameters { get; set; } + + public OptimizerOptions Options { get; set; } + + public double LearningRate { get => Options.LearningRate.Value; set => Options.LearningRate = value; } + public double InitialLearningRate { get => Options.InitialLearningRate; set => Options.InitialLearningRate = value; } + } + + public class SGDOptimizer : NewOptimizerHelper, IMomentum + { + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public SGDOptimizer(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + : this(new ParameterGroup[] { new ParameterGroup { Parameters = parameters.ToList() } }, lr, momentum, dampening, weight_decay, nesterov, maximize) + { + } + + public SGDOptimizer(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + { + if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + if (nesterov && (momentum <= 0 || dampening != 0)) throw new ArgumentException("Nesterov momentum requires a momentum and zero dampening"); + + var options = new OptimizerOptions { + LearningRate = lr, + InitialLearningRate = lr, + dampening = dampening, + maximize = maximize, + momentum = momentum, + nesterov = nesterov, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + public override Tensor step(Func closure = null) + { + Tensor loss = null; + + if (closure != null) { + using (var _ = torch.enable_grad()) + loss = closure(); + } + + using (var _ = torch.no_grad()) { + + using (var d = torch.NewDisposeScope()) { + + foreach (var group in _parameter_groups) { + + var options = group.Options as OptimizerOptions; + var momentum = options.momentum.Value; + var dampening = options.dampening.Value; + var weight_decay = options.weight_decay.Value; + var nesterov = options.nesterov.Value; + var maximize = options.maximize.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var state = _state[param]; + + var grad = param.grad(); + + if (grad is null) continue; + + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } + + if (momentum != 0) { + var buf = state.momentum_buffer; + + if (buf is null) { + buf = grad.clone().detach().DetatchFromDisposeScope(); + state.momentum_buffer = buf; + } else { + buf.mul_(momentum).add_(grad, alpha: (1 - dampening)); + } + + if (nesterov) { + grad = grad.add(buf, alpha: momentum); + } else { + grad = buf; + } + + state.momentum_buffer = buf; + } + + var alpha = maximize ? lr : -lr; + param.add_(grad, alpha: alpha); + + } + } + + d.DisposeEverything(); + } + } + + return loss; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + if (state.momentum_buffer is not null) { + state.momentum_buffer.Dispose(); + } + } + _state.Clear(); + } + + private class State + { + public Tensor momentum_buffer; + } + + public override void add_param_group(ParameterGroup param_group) + { + var def = _defaults as OptimizerOptions; + if (param_group.Options is null) { + param_group.Options = new OptimizerOptions(); + } + + var opt = param_group.Options as OptimizerOptions; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.momentum.HasValue) opt.momentum = def.momentum; + if (!opt.dampening.HasValue) opt.dampening = def.dampening; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.nesterov.HasValue) opt.nesterov = def.nesterov; + if (!opt.maximize.HasValue) opt.maximize = def.maximize; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p] = state; + state.momentum_buffer = null; + } + } + + public class OptimizerOptions : Modules.OptimizerOptions + { + public double? momentum; + public double? dampening; + public double? weight_decay; + public bool? nesterov; + public bool? maximize; + } + + + private Dictionary _state = new Dictionary(); + + public double Momentum { get => (_defaults as OptimizerOptions).momentum.Value; set => (_defaults as OptimizerOptions).momentum = value; } + } + public class AdadeltaOptimizer : OptimizerHelper, ILearningRateController { /// @@ -399,7 +623,7 @@ public class AdadeltaOptimizer : OptimizerHelper, ILearningRateController /// Coefficient used for computing a running average of squared gradients (default: 0.9) /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) /// Weight decay (L2 penalty) (default: 0) - public AdadeltaOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) : base(named_parameters, lr) + public AdadeltaOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) : base(named_parameters, lr) { LearningRate = lr; InitialLearningRate = lr; @@ -497,7 +721,7 @@ public class AdamaxOptimizer : OptimizerHelper, ILearningRateController, IBetas /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// - public AdamaxOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) : base(named_parameters, lr) + public AdamaxOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) : base(named_parameters, lr) { LearningRate = lr; InitialLearningRate = lr; @@ -609,7 +833,7 @@ public class NAdamOptimizer : OptimizerHelper, ILearningRateController, IBetas /// Weight decay (L2 penalty) (default: 0) /// Momentum decay /// - public NAdamOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) : base(named_parameters, lr) + public NAdamOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) : base(named_parameters, lr) { LearningRate = lr; InitialLearningRate = lr; @@ -729,7 +953,7 @@ public class RAdamOptimizer : OptimizerHelper, ILearningRateController, IBetas /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// - public RAdamOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) : base(named_parameters, lr) + public RAdamOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) : base(named_parameters, lr) { LearningRate = lr; InitialLearningRate = lr; @@ -848,7 +1072,7 @@ public class ASGDOptimizer : OptimizerHelper, ILearningRateController /// Point at which to start averaging (default: 1e6) /// Weight decay (L2 penalty) (default: 0) /// - public ASGDOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) : base(named_parameters, lr) + public ASGDOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) : base(named_parameters, lr) { LearningRate = lr; InitialLearningRate = lr; @@ -938,121 +1162,6 @@ private class State private double _weight_decay; } - public class SGDOptimizer : OptimizerHelper, IMomentum - { - /// - /// Implements stochastic gradient descent (optionally with momentum). - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Momentum factor (default: 0) - /// Dampening for momentum (default: 0) - /// Weight decay (L2 penalty) (default: 0) - /// Enables Nesterov momentum (default: False) - /// - /// - public SGDOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) : base(named_parameters, lr) - { - if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - if (nesterov && (momentum <= 0 || dampening != 0)) throw new ArgumentException("Nesterov momentum requires a momentum and zero dampening"); - - LearningRate = lr; - InitialLearningRate = lr; - _momentum = momentum; - _dampening = dampening; - _weight_decay = weight_decay; - _nesterov = nesterov; - _maximize = maximize; - - foreach (var (name, p) in named_parameters) { - var state = new State(); - _state[name] = state; - state.momentum_buffer = null; - } - } - - public override Tensor step(Func closure = null) - { - Tensor loss = null; - - if (closure != null) { - using (var _ = torch.enable_grad()) - loss = closure(); - } - - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var (name, param) in _parameters) { - - var state = _state[name]; - - var grad = param.grad(); - - if (grad is null) continue; - - if (_weight_decay != 0) { - grad = grad.add(param, alpha: _weight_decay); - } - - if (_momentum != 0) { - var buf = state.momentum_buffer; - - if (buf is null) { - buf = grad.clone().detach().DetatchFromDisposeScope(); - state.momentum_buffer = buf; - } else { - buf.mul_(_momentum).add_(grad, alpha: (1 - _dampening)); - } - - if (_nesterov) { - grad = grad.add(buf, alpha: _momentum); - } else { - grad = buf; - } - - state.momentum_buffer = buf; - } - - var alpha = _maximize ? LearningRate : -LearningRate; - param.add_(grad, alpha: alpha); - - } - - d.DisposeEverything(); - } - } - - return loss; - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - if (state.momentum_buffer is not null) { - state.momentum_buffer.Dispose(); - } - } - } - - private class State - { - public Tensor momentum_buffer; - } - - private Dictionary _state = new Dictionary(); - private double _momentum; - private double _dampening; - private double _weight_decay; - private bool _nesterov; - private bool _maximize; - - public double Momentum { get => _momentum; set => _momentum = value; } - } - public class RpropOptimizer : OptimizerHelper, ILearningRateController { /// @@ -1067,7 +1176,7 @@ public class RpropOptimizer : OptimizerHelper, ILearningRateController /// Minimum allowed step size. /// Maximum allowed step size. /// - public RpropOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) : base(named_parameters, lr) + public RpropOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) : base(named_parameters, lr) { LearningRate = lr; InitialLearningRate = lr; @@ -1173,7 +1282,7 @@ public class AdagradOptimizer : OptimizerHelper /// /// Term added to the denominator to improve numerical stability (default: 1e-10) /// - public AdagradOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) : base(named_parameters, learningRate) + public AdagradOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) : base(named_parameters, learningRate) { if (learningRate < 0) throw new ArgumentException($"Invalid learning rate: {learningRate}"); if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); @@ -1283,7 +1392,7 @@ public class AdamOptimizer : OptimizerHelper, IBetas /// Whether to use the AMSGrad variant of this algorithm /// Maximize the params based on the objective, instead of minimizing /// - public AdamOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) : base(named_parameters, lr) + public AdamOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) : base(named_parameters, lr) { if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); @@ -1423,7 +1532,7 @@ public class AdamWOptimizer : OptimizerHelper, IBetas /// Whether to use the AMSGrad variant of this algorithm /// Maximize the params based on the objective, instead of minimizing /// - public AdamWOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) : base(named_parameters, lr) + public AdamWOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) : base(named_parameters, lr) { if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); @@ -1555,7 +1664,7 @@ public class RMSPropOptimizer : OptimizerHelper, IMomentum /// Weight decay (L2 penalty) (default: 0) /// if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance /// - public RMSPropOptimizer(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double lr = 1e-3, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0.0, bool centered = false) : base(named_parameters, lr) + public RMSPropOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0.0, bool centered = false) : base(named_parameters, lr) { if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index 05744c88a..213c3fd43 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -9,6 +9,8 @@ using Xunit; using static TorchSharp.torch; +using System.Reflection.Metadata.Ecma335; +using TorchSharp.Modules; #nullable enable @@ -845,7 +847,7 @@ public void TestTrainingSGDMomentum() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate, momentum: 0.5); + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, momentum: 0.5); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -878,7 +880,7 @@ public void TestTrainingSGDNesterov() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate, momentum: 0.1, nesterov: true); + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, momentum: 0.1, nesterov: true); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -911,7 +913,44 @@ public void TestTrainingSGDDefaults() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingSGDDefaultsParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var pgs = new Modules.ParameterGroup[] { + new Modules.ParameterGroup { Parameters = lin1.parameters(), Options = new SGDOptimizer.OptimizerOptions { LearningRate = 0.005f } }, + new Modules.ParameterGroup { Parameters = lin2.parameters() } }; + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(pgs, learning_rate); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -944,7 +983,7 @@ public void TestTrainingSGDStepLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.97); var loss = mse_loss(Reduction.Sum); @@ -987,7 +1026,7 @@ public void TestTrainingSGDMultiStepLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, new int[] { 3, 5, 7 }, 0.97); var loss = mse_loss(Reduction.Sum); @@ -1031,7 +1070,7 @@ public void TestTrainingSGDCosineAnnealingLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 10); var loss = mse_loss(Reduction.Sum); @@ -1075,7 +1114,7 @@ public void TestTrainingSGDCyclicLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, 0.0001, 0.0004, step_size_up: 5); var loss = mse_loss(Reduction.Sum); @@ -1113,7 +1152,7 @@ public void TestTrainingSGDOneCycleLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, 0.0004, total_steps: 10); var loss = mse_loss(Reduction.Sum); From e25832cf769844cec770c611c2fc38478fcc10c9 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Wed, 9 Feb 2022 17:28:02 -0800 Subject: [PATCH 09/25] WIP -- adding parameter groups support to optimizers. --- src/Examples/Program.cs | 6 +- src/FSharp.Examples/SequenceToSequence.fs | 13 +- src/TorchSharp/NN/Optimizer.cs | 255 +++++++++++++----- .../TorchSharpTest.WithCudaBinaries.csproj | 1 + test/TorchSharpTest/TestTraining.cs | 53 +++- 5 files changed, 256 insertions(+), 72 deletions(-) diff --git a/src/Examples/Program.cs b/src/Examples/Program.cs index d7c30d373..032e42652 100644 --- a/src/Examples/Program.cs +++ b/src/Examples/Program.cs @@ -11,9 +11,9 @@ public static class Program { public static void Main(string[] args) { - //MNIST.Main(args); - //AdversarialExampleGeneration.Main(args); - //CIFAR10.Main(args); + MNIST.Main(args); + AdversarialExampleGeneration.Main(args); + CIFAR10.Main(args); SequenceToSequence.Main(args); TextClassification.Main(args); //ImageTransforms.Main(args); diff --git a/src/FSharp.Examples/SequenceToSequence.fs b/src/FSharp.Examples/SequenceToSequence.fs index ae6546c7c..c83a9373a 100644 --- a/src/FSharp.Examples/SequenceToSequence.fs +++ b/src/FSharp.Examples/SequenceToSequence.fs @@ -8,6 +8,7 @@ open System.Diagnostics open System.Collections.Generic open TorchSharp +open TorchSharp.Modules open type TorchSharp.torch.nn open type TorchSharp.torch.optim @@ -244,7 +245,17 @@ let run epochs = use model = new TransformerModel(ntokens, device) let lr = 2.50 - let optimizer = SGD(model.parameters(), lr) + + let pgs = [| + ParamsGroup(Parameters = model.parameters(), Options = SGD.Options(momentum = 1.0, dampening = 0.5)); + ParamsGroup(model.parameters(), SGD.Options(momentum = 1.0, dampening = 0.5)) + |] + + let optimizer = SGD([| + ParamsGroup(Parameters = model.parameters(), Options = SGD.Options(LearningRate = 0.005, momentum = 1.0, dampening = 0.5)); + ParamsGroup(model.parameters(), SGD.Options(momentum = 1.0, dampening = 0.5)) + |], lr) + let scheduler = lr_scheduler.StepLR(optimizer, 1, 0.95, last_epoch=15) let totalTime = Stopwatch() diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index c4e42d326..14d9ae12b 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -116,7 +116,7 @@ public virtual Tensor step(Func closure = null) [DllImport("LibTorchSharp")] private static extern void THSNN_Optimizer_getParameters(HType module, AllocatePinnedArray allocator); - public IEnumerable parameters() + public virtual IEnumerable parameters() { IntPtr[] ptrArray; @@ -125,7 +125,7 @@ public IEnumerable parameters() torch.CheckForErrors(); ptrArray = pa.Array; } - return ptrArray.Select(x => new Tensor(x)); + return ptrArray.Select(x => new Parameter(x)); } } @@ -307,16 +307,33 @@ public static AdamaxOptimizer Adamax(IEnumerable<(string name, Parameter paramet /// /// It has been proposed in Acceleration of stochastic approximation by averaging. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Decay term (default: 1e-4) + /// Power for eta update (default: 0.75) + /// Point at which to start averaging (default: 1e6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static ASGD ASGD(IEnumerable parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + { + return new Modules.ASGD(parameters, lr, lambd, alpha, t0, weight_decay); + } + + /// + /// Implements Averaged Stochastic Gradient Descent. + /// + /// It has been proposed in Acceleration of stochastic approximation by averaging. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// Decay term (default: 1e-4) /// Power for eta update (default: 0.75) /// Point at which to start averaging (default: 1e6) /// Weight decay (L2 penalty) (default: 0) /// - public static ASGDOptimizer ASGD(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + public static ASGD ASGD(IEnumerable> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) { - return new ASGDOptimizer(named_parameters, lr, lambd, alpha, t0, weight_decay); + return new Modules.ASGD(parameters, lr, lambd, alpha, t0, weight_decay); } /// @@ -343,15 +360,27 @@ public static RpropOptimizer Rprop(IEnumerable<(string name, Parameter parameter /// Dampening for momentum (default: 0) /// Weight decay (L2 penalty) (default: 0) /// Enables Nesterov momentum (default: False) + /// /// - public static SGDOptimizer SGD(IEnumerable parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false) + public static Modules.SGD SGD(IEnumerable parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) { - return new SGDOptimizer(parameters, learningRate, momentum, dampening, weight_decay, nesterov); + return new Modules.SGD(parameters, learningRate, momentum, dampening, weight_decay, nesterov, maximize); } - public static SGDOptimizer SGD(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public static Modules.SGD SGD(IEnumerable> parameters, double learningRate, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) { - return new SGDOptimizer(parameters, lr, momentum, dampening, weight_decay, nesterov); + return new Modules.SGD(parameters, learningRate, momentum, dampening, weight_decay, nesterov, maximize); } } } @@ -388,6 +417,11 @@ public override void zero_grad() } } + public override IEnumerable parameters() + { + return _parameters.Select(p => p.parameter); + } + public double LearningRate { get; set; } public double InitialLearningRate { get; set; } @@ -416,7 +450,12 @@ public override void zero_grad() } } - public virtual void add_param_group(ParameterGroup param_group) + public override IEnumerable parameters() + { + return _parameter_groups.SelectMany(pg => pg.Parameters); + } + + public virtual void add_param_group(ParamsGroup param_group) { _parameter_groups.Add(param_group); } @@ -426,7 +465,7 @@ public virtual void add_param_group(ParameterGroup param_group) public double InitialLearningRate { get => _defaults.InitialLearningRate; set => _defaults.InitialLearningRate = value; } protected OptimizerOptions _defaults; - protected IList _parameter_groups; + protected IList _parameter_groups; } public class OptimizerOptions @@ -435,7 +474,7 @@ public class OptimizerOptions public double InitialLearningRate { get; set; } } - public class ParameterGroup : ILearningRateController + public class ParamsGroup : ILearningRateController { public IEnumerable Parameters { get; set; } @@ -445,7 +484,22 @@ public class ParameterGroup : ILearningRateController public double InitialLearningRate { get => Options.InitialLearningRate; set => Options.InitialLearningRate = value; } } - public class SGDOptimizer : NewOptimizerHelper, IMomentum + public class ParamsGroup : ParamsGroup where TOptions : OptimizerOptions + { + public ParamsGroup() + { + } + + public ParamsGroup(IEnumerable parameters, TOptions options = null) + { + base.Options = options; + } + + public new OptimizerOptions Options { get => base.Options; set => base.Options = value; } + + } + + public class SGD : NewOptimizerHelper, IMomentum { /// /// Implements stochastic gradient descent (optionally with momentum). @@ -458,18 +512,29 @@ public class SGDOptimizer : NewOptimizerHelper, IMomentum /// Enables Nesterov momentum (default: False) /// /// - public SGDOptimizer(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) - : this(new ParameterGroup[] { new ParameterGroup { Parameters = parameters.ToList() } }, lr, momentum, dampening, weight_decay, nesterov, maximize) + public SGD(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + : this(new ParamsGroup[] { new ParamsGroup { Parameters = parameters.ToList() } }, lr, momentum, dampening, weight_decay, nesterov, maximize) { } - public SGDOptimizer(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public SGD(IEnumerable> parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) { if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); if (nesterov && (momentum <= 0 || dampening != 0)) throw new ArgumentException("Nesterov momentum requires a momentum and zero dampening"); - var options = new OptimizerOptions { + var options = new Options { LearningRate = lr, InitialLearningRate = lr, dampening = dampening, @@ -480,7 +545,7 @@ public SGDOptimizer(IEnumerable parameters, double lr = 1e-3, do }; _defaults = options; - _parameter_groups = new List(); + _parameter_groups = new List(); foreach (var g in parameters) { add_param_group(g); @@ -502,7 +567,7 @@ public override Tensor step(Func closure = null) foreach (var group in _parameter_groups) { - var options = group.Options as OptimizerOptions; + var options = group.Options as Options; var momentum = options.momentum.Value; var dampening = options.dampening.Value; var weight_decay = options.weight_decay.Value; @@ -570,14 +635,14 @@ private class State public Tensor momentum_buffer; } - public override void add_param_group(ParameterGroup param_group) + public override void add_param_group(ParamsGroup param_group) { - var def = _defaults as OptimizerOptions; + var def = _defaults as Options; if (param_group.Options is null) { - param_group.Options = new OptimizerOptions(); + param_group.Options = new Options(); } - var opt = param_group.Options as OptimizerOptions; + var opt = param_group.Options as Options; // Make sure all the options are set. if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; @@ -598,7 +663,7 @@ public override void add_param_group(ParameterGroup param_group) } } - public class OptimizerOptions : Modules.OptimizerOptions + public class Options : Modules.OptimizerOptions { public double? momentum; public double? dampening; @@ -610,7 +675,7 @@ public class OptimizerOptions : Modules.OptimizerOptions private Dictionary _state = new Dictionary(); - public double Momentum { get => (_defaults as OptimizerOptions).momentum.Value; set => (_defaults as OptimizerOptions).momentum = value; } + public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } } public class AdadeltaOptimizer : OptimizerHelper, ILearningRateController @@ -1058,36 +1123,53 @@ private class State } } - public class ASGDOptimizer : OptimizerHelper, ILearningRateController + public class ASGD : NewOptimizerHelper, ILearningRateController { /// /// Implements ASGD algorithm (a variant of Adam based on infinity norm). /// /// It has been proposed in Adam: A Method for Stochastic Optimization. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// Decay term (default: 1e-4) /// Power for eta update (default: 0.75) /// Point at which to start averaging (default: 1e6) /// Weight decay (L2 penalty) (default: 0) /// - public ASGDOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) : base(named_parameters, lr) + public ASGD(IEnumerable parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + : this(new ParamsGroup[] { new ParamsGroup { Parameters = parameters.ToList() } }, lr, lambd, alpha, t0, weight_decay) { - LearningRate = lr; - InitialLearningRate = lr; - _lambd = lambd; - _alpha = alpha; - _t0 = t0; - _weight_decay = weight_decay; + } - foreach (var (name, p) in named_parameters) { - var state = new State(); - _state[name] = state; - state.step = 0; - state.eta = lr; - state.mu = 1; - state.ax = torch.zeros_like(p); + /// + /// Implements ASGD algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Decay term (default: 1e-4) + /// Power for eta update (default: 0.75) + /// Point at which to start averaging (default: 1e6) + /// Weight decay (L2 penalty) (default: 0) + /// + public ASGD(IEnumerable> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + { + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + lambd = lambd, + alpha = alpha, + t0 = t0, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); } } @@ -1103,33 +1185,43 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { - var grad = param.grad(); + var options = group.Options as Options; + var lambd = options.lambd.Value; + var alpha = options.alpha.Value; + var weight_decay = options.weight_decay.Value; + var t0 = options.t0.Value; + var lr = options.LearningRate.Value; - if (grad is null) continue; + foreach (var param in group.Parameters) { - if (grad.is_sparse) throw new ArgumentException("ASGD does not support sparse gradients"); + var grad = param.grad(); - var state = _state[name]; + if (grad is null) continue; - state.step += 1; + if (grad.is_sparse) throw new ArgumentException("ASGD does not support sparse gradients"); - grad = (_weight_decay != 0) - ? grad.add(param, alpha: _weight_decay) - : grad.alias(); + var state = _state[param]; - param.mul_(1 - _lambd * state.eta); - param.add_(grad, alpha: -state.eta); + state.step += 1; - if (state.mu != 1) { - state.ax.add_(param.sub(state.ax).mul(state.mu)); - } else { - state.ax.copy_(param); - } + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - state.eta = LearningRate / Math.Pow((1 + _lambd * LearningRate * state.step), _alpha); - state.mu = 1 / Math.Max(1, state.step - _t0); + param.mul_(1 - lambd * state.eta); + param.add_(grad, alpha: -state.eta); + + if (state.mu != 1) { + state.ax.add_(param.sub(state.ax).mul(state.mu)); + } else { + state.ax.copy_(param); + } + + state.eta = lr / Math.Pow((1 + lambd * lr * state.step), alpha); + state.mu = 1 / Math.Max(1, state.step - t0); + } } d.DisposeEverything(); @@ -1145,6 +1237,7 @@ protected override void Dispose(bool disposing) foreach (var (name, state) in _state) { state.ax.Dispose(); } + _state.Clear(); } private class State @@ -1155,11 +1248,45 @@ private class State public Tensor ax; } - private Dictionary _state = new Dictionary(); - private double _lambd; - private double _alpha; - private double _t0; - private double _weight_decay; + public override void add_param_group(ParamsGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.lambd.HasValue) opt.lambd = def.lambd; + if (!opt.alpha.HasValue) opt.alpha = def.alpha; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.t0.HasValue) opt.t0 = def.t0; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p] = state; + state.step = 0; + state.eta = param_group.LearningRate; + state.mu = 1; + state.ax = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? lambd; + public double? alpha; + public double? weight_decay; + public double? t0; + } + + private Dictionary _state = new Dictionary(); } public class RpropOptimizer : OptimizerHelper, ILearningRateController diff --git a/test/TorchSharpTest.WithCudaBinaries/TorchSharpTest.WithCudaBinaries.csproj b/test/TorchSharpTest.WithCudaBinaries/TorchSharpTest.WithCudaBinaries.csproj index 768c20c7e..08c360e78 100644 --- a/test/TorchSharpTest.WithCudaBinaries/TorchSharpTest.WithCudaBinaries.csproj +++ b/test/TorchSharpTest.WithCudaBinaries/TorchSharpTest.WithCudaBinaries.csproj @@ -14,6 +14,7 @@ + diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index 213c3fd43..a1db424dd 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -632,7 +632,52 @@ public void TestTrainingASGD() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.ASGD(seq.named_parameters()); + var optimizer = torch.optim.ASGD(seq.parameters()); + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + + + [Fact] + public void TestTrainingASGDParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var pgs = new[] + { + new ParamsGroup(lin1.parameters(), new () { LearningRate = 0.005f }), + new ParamsGroup(lin2.parameters()) + }; + + + var optimizer = torch.optim.ASGD(new ParamsGroup[] + { + new () { Parameters = lin1.parameters(), Options = { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -942,9 +987,9 @@ public void TestTrainingSGDDefaultsParamGroups() var lin2 = Linear(100, 10); var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var pgs = new Modules.ParameterGroup[] { - new Modules.ParameterGroup { Parameters = lin1.parameters(), Options = new SGDOptimizer.OptimizerOptions { LearningRate = 0.005f } }, - new Modules.ParameterGroup { Parameters = lin2.parameters() } }; + var pgs = new Modules.ParamsGroup[] { + new () { Parameters = lin1.parameters(), Options = { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } }; var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); From 989520605591fb8b2340ea02b77edafa7eb9fb8e Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Sun, 13 Feb 2022 15:20:02 -0800 Subject: [PATCH 10/25] Further optimizer work. --- src/Examples/CIFAR10.cs | 2 +- src/Examples/MNIST.cs | 2 +- src/FSharp.Examples/AlexNet.fs | 2 +- src/FSharp.Examples/MNIST.fs | 2 +- src/FSharp.Examples/SequenceToSequence.fs | 8 +- src/TorchSharp/NN/Optimizer.cs | 711 ++++++++++++++++------ test/TorchSharpTest/TestTraining.cs | 83 ++- 7 files changed, 584 insertions(+), 226 deletions(-) diff --git a/src/Examples/CIFAR10.cs b/src/Examples/CIFAR10.cs index ce56fb3a0..5aeb3de9b 100644 --- a/src/Examples/CIFAR10.cs +++ b/src/Examples/CIFAR10.cs @@ -120,7 +120,7 @@ internal static void Main(string[] args) using (var train = new CIFARReader(targetDir, false, _trainBatchSize, shuffle: true, device: device, transforms: new torchvision.ITransform[] { })) using (var test = new CIFARReader(targetDir, true, _testBatchSize, device: device)) - using (var optimizer = torch.optim.Adam(model.named_parameters(), 0.001)) { + using (var optimizer = torch.optim.Adam(model.parameters(), 0.001)) { Stopwatch totalSW = new Stopwatch(); totalSW.Start(); diff --git a/src/Examples/MNIST.cs b/src/Examples/MNIST.cs index e82ae9d96..0010d314a 100644 --- a/src/Examples/MNIST.cs +++ b/src/Examples/MNIST.cs @@ -82,7 +82,7 @@ internal static void TrainingLoop(string dataset, Device device, Model model, MN _epochs *= 4; } - var optimizer = torch.optim.Adam(model.named_parameters()); + var optimizer = torch.optim.Adam(model.parameters()); var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.75); diff --git a/src/FSharp.Examples/AlexNet.fs b/src/FSharp.Examples/AlexNet.fs index 7fff03eee..4c410435f 100644 --- a/src/FSharp.Examples/AlexNet.fs +++ b/src/FSharp.Examples/AlexNet.fs @@ -154,7 +154,7 @@ let test (model:Model) (dataLoader:CIFARReader) = let trainingLoop (model:Model) epochs trainData testData = - use optimizer = Adam(model.named_parameters(), 0.001) + use optimizer = Adam(model.parameters(), 0.001) let sw = Stopwatch() sw.Start() diff --git a/src/FSharp.Examples/MNIST.fs b/src/FSharp.Examples/MNIST.fs index c6914640f..297a25d4d 100644 --- a/src/FSharp.Examples/MNIST.fs +++ b/src/FSharp.Examples/MNIST.fs @@ -141,7 +141,7 @@ let trainingLoop (model:Model) epochs dataset trainData testData = let epochs = if device.``type`` = DeviceType.CUDA then epochs * 4 else epochs - let optimizer = Adam(model.named_parameters()) + let optimizer = Adam(model.parameters()) lr_scheduler.StepLR(optimizer, 1, 0.7, last_epoch=5) |> ignore let sw = Stopwatch() diff --git a/src/FSharp.Examples/SequenceToSequence.fs b/src/FSharp.Examples/SequenceToSequence.fs index c83a9373a..e2b119d11 100644 --- a/src/FSharp.Examples/SequenceToSequence.fs +++ b/src/FSharp.Examples/SequenceToSequence.fs @@ -247,13 +247,13 @@ let run epochs = let lr = 2.50 let pgs = [| - ParamsGroup(Parameters = model.parameters(), Options = SGD.Options(momentum = 1.0, dampening = 0.5)); - ParamsGroup(model.parameters(), SGD.Options(momentum = 1.0, dampening = 0.5)) + SGD.ParamGroup(Parameters = model.parameters(), Options = SGD.Options(momentum = 1.0, dampening = 0.5)); + SGD.ParamGroup(model.parameters(), momentum = 1.5, dampening = 0.1) |] let optimizer = SGD([| - ParamsGroup(Parameters = model.parameters(), Options = SGD.Options(LearningRate = 0.005, momentum = 1.0, dampening = 0.5)); - ParamsGroup(model.parameters(), SGD.Options(momentum = 1.0, dampening = 0.5)) + SGD.ParamGroup(model.parameters(), momentum = 1.0, dampening = 0.5); + SGD.ParamGroup(model.parameters(), momentum = 1.5, dampening = 0.1) |], lr) let scheduler = lr_scheduler.StepLR(optimizer, 1, 0.95, last_epoch=15) diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index 14d9ae12b..ed7b0ae36 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -166,7 +166,7 @@ public static LBFGSOptimizer LBFGS(IEnumerable parameters, double lea /// /// Proposed by G.Hinton in his course. /// - /// Parameters to optimize + /// Parameters to optimize /// Learning rate (default: 1e-2) /// Term added to the denominator to improve numerical stability (default: 1e-8) /// Smoothing constant (default: 0.99) @@ -174,9 +174,27 @@ public static LBFGSOptimizer LBFGS(IEnumerable parameters, double lea /// Momentum factor (default: 0) /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance /// - public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 0.01, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0, bool centered = false) + public static RMSProp RMSProp(IEnumerable parameters, double learningRate = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) { - return new RMSPropOptimizer(named_parameters, learningRate, eps, alpha, weight_decay, momentum, centered); + return new RMSProp(parameters, learningRate, alpha, eps, weight_decay, momentum, centered); + } + + /// + /// Implements RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. + /// + /// Parameters to optimize + /// Learning rate (default: 1e-2) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Smoothing constant (default: 0.99) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum factor (default: 0) + /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public static RMSProp RMSProp(IEnumerable parameters, double learningRate = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) + { + return new RMSProp(parameters, learningRate, alpha, eps, weight_decay, momentum, centered); } /// @@ -184,7 +202,7 @@ public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Parameter param /// /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. /// - /// Parameters to optimize + /// Parameters to optimize /// learning rate (default: 1e-3) /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) @@ -193,9 +211,28 @@ public static RMSPropOptimizer RMSProp(IEnumerable<(string name, Parameter param /// Whether to use the AMSGrad variant of this algorithm. (default: False) /// /// - public static AdamOptimizer Adam(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + public static Adam Adam(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { - return new AdamOptimizer(named_parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); + return new Adam(parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + + /// + /// Implements Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static Adam Adam(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new Adam(parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); } /// @@ -203,7 +240,26 @@ public static AdamOptimizer Adam(IEnumerable<(string name, Parameter parameter)> /// /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. /// - /// Parameters to optimize + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static AdamW AdamW(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new AdamW(parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + + /// + /// Implements AdamW algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize /// learning rate (default: 1e-3) /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) @@ -212,9 +268,9 @@ public static AdamOptimizer Adam(IEnumerable<(string name, Parameter parameter)> /// Whether to use the AMSGrad variant of this algorithm. (default: False) /// /// - public static AdamWOptimizer AdamW(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + public static AdamW AdamW(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { - return new AdamWOptimizer(named_parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); + return new AdamW(parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); } /// @@ -331,7 +387,7 @@ public static ASGD ASGD(IEnumerable parameters, double lr = 1e-3, dou /// Point at which to start averaging (default: 1e6) /// Weight decay (L2 penalty) (default: 0) /// - public static ASGD ASGD(IEnumerable> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + public static ASGD ASGD(IEnumerable> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) { return new Modules.ASGD(parameters, lr, lambd, alpha, t0, weight_decay); } @@ -378,7 +434,7 @@ public static Modules.SGD SGD(IEnumerable parameters, double learning /// Enables Nesterov momentum (default: False) /// /// - public static Modules.SGD SGD(IEnumerable> parameters, double learningRate, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + public static Modules.SGD SGD(IEnumerable parameters, double learningRate, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) { return new Modules.SGD(parameters, learningRate, momentum, dampening, weight_decay, nesterov, maximize); } @@ -455,7 +511,7 @@ public override IEnumerable parameters() return _parameter_groups.SelectMany(pg => pg.Parameters); } - public virtual void add_param_group(ParamsGroup param_group) + public virtual void add_param_group(ParamGroup param_group) { _parameter_groups.Add(param_group); } @@ -465,7 +521,7 @@ public virtual void add_param_group(ParamsGroup param_group) public double InitialLearningRate { get => _defaults.InitialLearningRate; set => _defaults.InitialLearningRate = value; } protected OptimizerOptions _defaults; - protected IList _parameter_groups; + protected IList _parameter_groups; } public class OptimizerOptions @@ -474,7 +530,7 @@ public class OptimizerOptions public double InitialLearningRate { get; set; } } - public class ParamsGroup : ILearningRateController + public class ParamGroup : ILearningRateController { public IEnumerable Parameters { get; set; } @@ -484,18 +540,19 @@ public class ParamsGroup : ILearningRateController public double InitialLearningRate { get => Options.InitialLearningRate; set => Options.InitialLearningRate = value; } } - public class ParamsGroup : ParamsGroup where TOptions : OptimizerOptions + public class ParamGroup : ParamGroup where TOptions : OptimizerOptions { - public ParamsGroup() + public ParamGroup() { } - public ParamsGroup(IEnumerable parameters, TOptions options = null) + public ParamGroup(IEnumerable parameters, TOptions options = null) { + base.Parameters = parameters; base.Options = options; } - public new OptimizerOptions Options { get => base.Options; set => base.Options = value; } + public new TOptions Options { get => (TOptions)base.Options; set => base.Options = value; } } @@ -512,8 +569,8 @@ public class SGD : NewOptimizerHelper, IMomentum /// Enables Nesterov momentum (default: False) /// /// - public SGD(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) - : this(new ParamsGroup[] { new ParamsGroup { Parameters = parameters.ToList() } }, lr, momentum, dampening, weight_decay, nesterov, maximize) + public SGD(IEnumerable parameters, double lr, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, momentum, dampening, weight_decay, nesterov, maximize) { } @@ -528,7 +585,7 @@ public SGD(IEnumerable parameters, double lr = 1e-3, double momentum /// Enables Nesterov momentum (default: False) /// /// - public SGD(IEnumerable> parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + public SGD(IEnumerable parameters, double lr, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) { if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); @@ -545,7 +602,7 @@ public SGD(IEnumerable> parameters, double lr = 1e-3, doubl }; _defaults = options; - _parameter_groups = new List(); + _parameter_groups = new List(); foreach (var g in parameters) { add_param_group(g); @@ -577,7 +634,7 @@ public override Tensor step(Func closure = null) foreach (var param in group.Parameters) { - var state = _state[param]; + var state = _state[param.handle]; var grad = param.grad(); @@ -635,7 +692,7 @@ private class State public Tensor momentum_buffer; } - public override void add_param_group(ParamsGroup param_group) + public override void add_param_group(Modules.ParamGroup param_group) { var def = _defaults as Options; if (param_group.Options is null) { @@ -658,7 +715,7 @@ public override void add_param_group(ParamsGroup param_group) foreach (var p in param_group.Parameters) { var state = new State(); - _state[p] = state; + _state[p.Handle] = state; state.momentum_buffer = null; } } @@ -672,8 +729,21 @@ public class Options : Modules.OptimizerOptions public bool? maximize; } + public class ParamGroup : ParamGroup, IMomentum + { + public ParamGroup() { } - private Dictionary _state = new Dictionary(); + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + : base(parameters, new SGD.Options { LearningRate = lr, dampening = dampening, momentum = momentum, weight_decay = weight_decay, nesterov = nesterov, maximize = maximize }) + { + } + + public double Momentum { get => (Options as Options).momentum.Value; set => (Options as Options).momentum = value; } + } + + private Dictionary _state = new Dictionary(); public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } } @@ -1137,8 +1207,8 @@ public class ASGD : NewOptimizerHelper, ILearningRateController /// Point at which to start averaging (default: 1e6) /// Weight decay (L2 penalty) (default: 0) /// - public ASGD(IEnumerable parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) - : this(new ParamsGroup[] { new ParamsGroup { Parameters = parameters.ToList() } }, lr, lambd, alpha, t0, weight_decay) + public ASGD(IEnumerable parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, lambd, alpha, t0, weight_decay) { } @@ -1154,7 +1224,7 @@ public ASGD(IEnumerable parameters, double lr = 1e-3, double lambd = /// Point at which to start averaging (default: 1e6) /// Weight decay (L2 penalty) (default: 0) /// - public ASGD(IEnumerable> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + public ASGD(IEnumerable> parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) { var options = new Options { LearningRate = lr, @@ -1166,7 +1236,7 @@ public ASGD(IEnumerable> parameters, double lr = 1e-3, doub }; _defaults = options; - _parameter_groups = new List(); + _parameter_groups = new List(); foreach (var g in parameters) { add_param_group(g); @@ -1202,7 +1272,7 @@ public override Tensor step(Func closure = null) if (grad.is_sparse) throw new ArgumentException("ASGD does not support sparse gradients"); - var state = _state[param]; + var state = _state[param.handle]; state.step += 1; @@ -1248,7 +1318,7 @@ private class State public Tensor ax; } - public override void add_param_group(ParamsGroup param_group) + public override void add_param_group(Modules.ParamGroup param_group) { var def = _defaults as Options; if (param_group.Options is null) { @@ -1270,7 +1340,7 @@ public override void add_param_group(ParamsGroup param_group) foreach (var p in param_group.Parameters) { var state = new State(); - _state[p] = state; + _state[p.Handle] = state; state.step = 0; state.eta = param_group.LearningRate; state.mu = 1; @@ -1286,7 +1356,19 @@ public class Options : OptimizerOptions public double? t0; } - private Dictionary _state = new Dictionary(); + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + : base(parameters, new ASGD.Options { LearningRate = lr, lambd = lambd, alpha = alpha, t0 = t0, weight_decay = weight_decay }) + { + } + } + + private Dictionary _state = new Dictionary(); } public class RpropOptimizer : OptimizerHelper, ILearningRateController @@ -1503,14 +1585,33 @@ private class State private double _weight_decay; } - public class AdamOptimizer : OptimizerHelper, IBetas + public class Adam : NewOptimizerHelper, IBetas { /// /// Implements Adam algorithm. /// /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// First coefficient used for computing running averages of gradient and its square + /// Second coefficient used for computing running averages of gradient and its square + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm + /// Maximize the params based on the objective, instead of minimizing + /// + public Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize) + { + } + + /// + /// Implements Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// First coefficient used for computing running averages of gradient and its square /// Second coefficient used for computing running averages of gradient and its square @@ -1519,7 +1620,7 @@ public class AdamOptimizer : OptimizerHelper, IBetas /// Whether to use the AMSGrad variant of this algorithm /// Maximize the params based on the objective, instead of minimizing /// - public AdamOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) : base(named_parameters, lr) + public Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); @@ -1527,28 +1628,22 @@ public AdamOptimizer(IEnumerable<(string name, Parameter parameter)> named_param if (beta2 < 0 || beta2 >= 1.0) throw new ArgumentException($"Invalid beta2: {beta2}"); if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - LearningRate = lr; - InitialLearningRate = lr; - _beta1 = beta1; - _beta2 = beta2; - _weight_decay = weight_decay; - _eps = eps; - _amsgrad = amsgrad; - _maximize = maximize; - - foreach (var (name, p) in named_parameters) { + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + maximize = maximize, + eps = eps, + amsgrad = amsgrad, + weight_decay = weight_decay + }; - if (torch.is_complex(p.dtype)) - throw new NotImplementedException("Adam optimizer with complex weights."); + _defaults = options; + _parameter_groups = new List(); - var state = new State(); - _state[name] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_avg_sq = torch.zeros_like(p); - if (_amsgrad) { - state.max_exp_avg_sq = torch.zeros_like(p); - } + foreach (var g in parameters) { + add_param_group(g); } } @@ -1565,42 +1660,53 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { - var state = _state[name]; + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var weight_decay = options.weight_decay.Value; + var amsgrad = options.amsgrad.Value; + var maximize = options.maximize.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; - var grad = (_maximize) ? -param.grad() : param.grad(); + foreach (var param in group.Parameters) { - if (grad is null) continue; + var state = _state[param.handle]; - state.step += 1; + var grad = (maximize) ? -param.grad() : param.grad(); - var bias_correction1 = 1 - Math.Pow(_beta1, state.step); - var bias_correction2 = 1 - Math.Pow(_beta2, state.step); + if (grad is null) continue; - if (_weight_decay != 0) { - grad = grad.add(param, alpha: _weight_decay); - } + state.step += 1; - state.exp_avg.mul_(_beta1).add_(grad, alpha: 1 - _beta1); - // When complex types are supported: - //state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad.conj(), value: 1 - _beta2) - state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad, value: 1 - _beta2); - - Tensor denom = null; - if (_amsgrad) { - var t0 = state.max_exp_avg_sq; - state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); - t0.Dispose(); - denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); - } else { - denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); - } + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); - var step_size = LearningRate / bias_correction1; - param.addcdiv_(state.exp_avg, denom, value: -step_size); - } + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } + + state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + // When complex types are supported: + //state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad.conj(), value: 1 - _beta2) + state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + + Tensor denom = null; + if (amsgrad) { + var t0 = state.max_exp_avg_sq; + state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); + t0.Dispose(); + denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } else { + denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } + var step_size = lr / bias_correction1; + param.addcdiv_(state.exp_avg, denom, value: -step_size); + } + } d.DisposeEverything(); } } @@ -1611,7 +1717,7 @@ public override Tensor step(Func closure = null) protected override void Dispose(bool disposing) { base.Dispose(disposing); - foreach (var (name, state) in _state) { + foreach (var (_, state) in _state) { state.exp_avg.Dispose(); state.exp_avg_sq.Dispose(); if (state.max_exp_avg_sq is not null) { @@ -1628,29 +1734,102 @@ private class State public Tensor max_exp_avg_sq; } - public (double, double) Betas { - get => (_beta1, _beta2); - set { _beta1 = value.Item1; _beta2 = value.Item2; } + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.amsgrad.HasValue) opt.amsgrad = def.amsgrad; + if (!opt.maximize.HasValue) opt.maximize = def.maximize; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + if (opt.amsgrad.Value) { + state.max_exp_avg_sq = torch.zeros_like(p); + } + } } + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? weight_decay; + public double? eps; + public bool? amsgrad; + public bool? maximize; + } - private Dictionary _state = new Dictionary(); - private double _beta1; - private double _beta2; - private double _eps; - private double _weight_decay; - private bool _amsgrad; - private bool _maximize; + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + : base(parameters, new Adam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, amsgrad = amsgrad, maximize = maximize }) + { + } + + public (double, double) Betas { + get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); + set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); } - public class AdamWOptimizer : OptimizerHelper, IBetas + public class AdamW : NewOptimizerHelper, IBetas { /// /// Implements AdamW algorithm. /// /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// First coefficient used for computing running averages of gradient and its square + /// Second coefficient used for computing running averages of gradient and its square + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm + /// Maximize the params based on the objective, instead of minimizing + /// + public AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize) + { + } + + /// + /// Implements AdamW algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// First coefficient used for computing running averages of gradient and its square /// Second coefficient used for computing running averages of gradient and its square @@ -1659,7 +1838,7 @@ public class AdamWOptimizer : OptimizerHelper, IBetas /// Whether to use the AMSGrad variant of this algorithm /// Maximize the params based on the objective, instead of minimizing /// - public AdamWOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) : base(named_parameters, lr) + public AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); @@ -1667,25 +1846,22 @@ public AdamWOptimizer(IEnumerable<(string name, Parameter parameter)> named_para if (beta2 < 0 || beta2 >= 1.0) throw new ArgumentException($"Invalid beta2: {beta2}"); if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - LearningRate = lr; - InitialLearningRate = lr; - _beta1 = beta1; - _beta2 = beta2; - _weight_decay = weight_decay; - _eps = eps; - _amsgrad = amsgrad; - _maximize = maximize; + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + maximize = maximize, + eps = eps, + amsgrad = amsgrad, + weight_decay = weight_decay + }; - foreach (var (name, p) in named_parameters) { + _defaults = options; + _parameter_groups = new List(); - var state = new State(); - _state[name] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_avg_sq = torch.zeros_like(p); - if (_amsgrad) { - state.max_exp_avg_sq = torch.zeros_like(p); - } + foreach (var g in parameters) { + add_param_group(g); } } @@ -1702,38 +1878,49 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { - var state = _state[name]; + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var weight_decay = options.weight_decay.Value; + var amsgrad = options.amsgrad.Value; + var maximize = options.maximize.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; - var grad = (_maximize) ? -param.grad() : param.grad(); + foreach (var param in group.Parameters) { - if (grad is null) continue; + var state = _state[param.handle]; - state.step += 1; + var grad = (maximize) ? -param.grad() : param.grad(); - param.mul_(1 - LearningRate * _weight_decay); + if (grad is null) continue; - var bias_correction1 = 1 - Math.Pow(_beta1, state.step); - var bias_correction2 = 1 - Math.Pow(_beta2, state.step); + state.step += 1; - state.exp_avg.mul_(_beta1).add_(grad, alpha: 1 - _beta1); - state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad, value: 1 - _beta2); + param.mul_(1 - LearningRate * weight_decay); - Tensor denom = null; - if (_amsgrad) { - var t0 = state.max_exp_avg_sq; - state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); - t0.Dispose(); - denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); - } else { - denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(_eps); - } + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); - var step_size = LearningRate / bias_correction1; - param.addcdiv_(state.exp_avg, denom, value: -step_size); - } + state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + + Tensor denom = null; + if (amsgrad) { + var t0 = state.max_exp_avg_sq; + state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); + t0.Dispose(); + denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } else { + denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } + var step_size = lr / bias_correction1; + param.addcdiv_(state.exp_avg, denom, value: -step_size); + } + } d.DisposeEverything(); } } @@ -1744,7 +1931,7 @@ public override Tensor step(Func closure = null) protected override void Dispose(bool disposing) { base.Dispose(disposing); - foreach (var (name, state) in _state) { + foreach (var (_, state) in _state) { state.exp_avg.Dispose(); state.exp_avg_sq.Dispose(); if (state.max_exp_avg_sq is not null) { @@ -1761,29 +1948,84 @@ private class State public Tensor max_exp_avg_sq; } - public (double, double) Betas { - get => (_beta1, _beta2); - set { _beta1 = value.Item1; _beta2 = value.Item2; } + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.amsgrad.HasValue) opt.amsgrad = def.amsgrad; + if (!opt.maximize.HasValue) opt.maximize = def.maximize; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + if (opt.amsgrad.Value) { + state.max_exp_avg_sq = torch.zeros_like(p); + } + } } + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? weight_decay; + public double? eps; + public bool? amsgrad; + public bool? maximize; + } - private Dictionary _state = new Dictionary(); - private double _beta1; - private double _beta2; - private double _eps; - private double _weight_decay; - private bool _amsgrad; - private bool _maximize; + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + : base(parameters, new AdamW.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, amsgrad = amsgrad, maximize = maximize }) + { + } + + public (double, double) Betas { + get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); + set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); } - public class RMSPropOptimizer : OptimizerHelper, IMomentum + public class RMSProp : NewOptimizerHelper, IMomentum { + /// /// Implements RMSprop algorithm. /// /// Proposed by G.Hinton in his course. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the parameters collection. /// Learning rate /// Smoothing constant. /// Momentum factor (default: 0) @@ -1791,7 +2033,25 @@ public class RMSPropOptimizer : OptimizerHelper, IMomentum /// Weight decay (L2 penalty) (default: 0) /// if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance /// - public RMSPropOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-3, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0.0, bool centered = false) : base(named_parameters, lr) + public RMSProp(IEnumerable parameters, double lr = 1e-3, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0.0, bool centered = false) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, alpha, eps, weight_decay, momentum, centered) + { + } + + /// + /// Implements RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. + /// + /// Parameters to optimize. This optimizer requires the parameters collection. + /// Learning rate + /// Smoothing constant. + /// Momentum factor (default: 0) + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public RMSProp(IEnumerable parameters, double lr = 1e-3, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0.0, bool centered = false) { if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); @@ -1799,21 +2059,21 @@ public RMSPropOptimizer(IEnumerable<(string name, Parameter parameter)> named_pa if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - LearningRate = lr; - InitialLearningRate = lr; - _momentum = momentum; - _weight_decay = weight_decay; - _alpha = alpha; - _eps = eps; - _centered = centered; + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + eps = eps, + alpha = alpha, + momentum = momentum, + centered = centered, + weight_decay = weight_decay + }; - foreach (var (name, p) in named_parameters) { - var state = new State(); - _state[name] = state; - state.step = 0; - state.square_avg = torch.zeros_like(p); - state.grad_avg = torch.zeros_like(p); - state.momentum_buffer = torch.zeros_like(p); + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); } } @@ -1830,38 +2090,49 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { - var state = _state[name]; + var options = group.Options as Options; + var momentum = options.momentum.Value; + var alpha = options.alpha.Value; + var weight_decay = options.weight_decay.Value; + var centered = options.centered.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; - var grad = param.grad(); + foreach (var param in group.Parameters) { - if (grad is null) continue; + var state = _state[param.handle]; - state.step += 1; + var grad = param.grad(); - if (_weight_decay != 0) { - grad = grad.add(param, alpha: _weight_decay); - } + if (grad is null) continue; - state.square_avg.mul_(_alpha).addcmul_(grad, grad, value: 1 - _alpha); + state.step += 1; - Tensor avg = null; + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } - if (_centered) { - var grad_avg = state.grad_avg; - grad_avg.mul_(_alpha).add_(grad, alpha: 1 - _alpha); - avg = state.square_avg.addcmul(grad_avg, grad_avg, value: -1).sqrt_().add_(_eps); - } else { - avg = state.square_avg.sqrt().add_(_eps); - } + state.square_avg.mul_(alpha).addcmul_(grad, grad, value: 1 - alpha); - if (_momentum > 0) { - var buf = state.momentum_buffer; - buf.mul_(_momentum).addcdiv_(grad, avg); - param.add_(buf, alpha: -LearningRate); - } else { - param.addcdiv_(grad, avg, -LearningRate); + Tensor avg = null; + + if (centered) { + var grad_avg = state.grad_avg; + grad_avg.mul_(alpha).add_(grad, alpha: 1 - alpha); + avg = state.square_avg.addcmul(grad_avg, grad_avg, value: -1).sqrt_().add_(eps); + } else { + avg = state.square_avg.sqrt().add_(eps); + } + + if (momentum > 0) { + var buf = state.momentum_buffer; + buf.mul_(momentum).addcdiv_(grad, avg); + param.add_(buf, alpha: -lr); + } else { + param.addcdiv_(grad, avg, -lr); + } } } @@ -1875,11 +2146,12 @@ public override Tensor step(Func closure = null) protected override void Dispose(bool disposing) { base.Dispose(disposing); - foreach (var (name, state) in _state) { + foreach (var (_, state) in _state) { state.momentum_buffer.Dispose(); state.square_avg.Dispose(); state.grad_avg.Dispose(); } + _state.Clear(); } private class State @@ -1890,14 +2162,61 @@ private class State public Tensor grad_avg; } - private Dictionary _state = new Dictionary(); - private double _momentum; - private double _alpha; - private double _eps; - private double _weight_decay; - private bool _centered; - public double Momentum { get => _momentum; set => _momentum = value; } + + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.momentum.HasValue) opt.momentum = def.momentum; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.alpha.HasValue) opt.alpha = def.alpha; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.centered.HasValue) opt.centered = def.centered; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.square_avg = torch.zeros_like(p); + state.grad_avg = torch.zeros_like(p); + state.momentum_buffer = torch.zeros_like(p); + } + } + public class Options : Modules.OptimizerOptions + { + public double? momentum; + public double? alpha; + public double? eps; + public double? weight_decay; + public bool? centered; + } + + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-3, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0.0, bool centered = false) + : base(parameters, new RMSProp.Options { LearningRate = lr, eps = eps, alpha = alpha, weight_decay = weight_decay, momentum = momentum, centered = centered }) + { + } + } + + private Dictionary _state = new Dictionary(); + + public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } } // The following optimizers are just wrappers for the native code implementations. diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index a1db424dd..ae3d1eb8f 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -123,7 +123,7 @@ public void TestAdam() double learning_rate = 0.00001; - var optimizer = torch.optim.Adam(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.Adam(seq.parameters(), learning_rate); Assert.NotNull(optimizer); } @@ -137,7 +137,7 @@ public void TestAdamBetas() double learning_rate = 0.00001; - var optimizer = torch.optim.Adam(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.Adam(seq.parameters(), learning_rate); var (beta1, beta2) = optimizer.Betas; Assert.Equal(0.9, beta1); Assert.Equal(0.99, beta2); @@ -163,7 +163,7 @@ public void TestTrainingAdamDefaults() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.named_parameters()); + var optimizer = torch.optim.Adam(seq.parameters()); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -195,7 +195,7 @@ public void TestTrainingAdamAmsGrad() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.named_parameters(), amsgrad: true); + var optimizer = torch.optim.Adam(seq.parameters(), amsgrad: true); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -227,7 +227,7 @@ public void TestTrainingAdamOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.named_parameters(), amsgrad: true); + var optimizer = torch.optim.Adam(seq.parameters(), amsgrad: true); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -268,7 +268,7 @@ public void TestTrainingAdamWDefaults() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.AdamW(seq.named_parameters()); + var optimizer = torch.optim.AdamW(seq.parameters()); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -300,7 +300,7 @@ public void TestTrainingAdamWOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.AdamW(seq.named_parameters(), amsgrad: true); + var optimizer = torch.optim.AdamW(seq.parameters(), amsgrad: true); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -665,16 +665,15 @@ public void TestTrainingASGDParamGroups() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var pgs = new[] + var pgs = new ASGD.ParamGroup[] { - new ParamsGroup(lin1.parameters(), new () { LearningRate = 0.005f }), - new ParamsGroup(lin2.parameters()) + new (lin1.parameters(), new () { LearningRate = 0.005f }), + new (lin2.parameters()) }; - - var optimizer = torch.optim.ASGD(new ParamsGroup[] + var optimizer = torch.optim.ASGD(new ASGD.ParamGroup[] { - new () { Parameters = lin1.parameters(), Options = { LearningRate = 0.005f } }, + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, new () { Parameters = lin2.parameters() } }); @@ -749,7 +748,7 @@ public void TestTrainingRMSLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.RMSProp(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -786,7 +785,7 @@ public void TestTrainingRMSOneCycleLR() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.RMSProp(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -822,7 +821,7 @@ public void TestTrainingRMSAlpha() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.RMSProp(seq.named_parameters(), learning_rate, alpha: 0.75); + var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, alpha: 0.75); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -855,7 +854,46 @@ public void TestTrainingRMSCentered() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.RMSProp(seq.named_parameters(), learning_rate, centered: true); + var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, centered: true); + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingRMSCenteredParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + double learning_rate = 0.00004f; + + var optimizer = torch.optim.RMSProp( + new RMSProp.ParamGroup[] { + new(lin1.parameters()), + new(lin2.parameters(), lr: learning_rate*10) }, + learning_rate, centered: true); + var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -987,9 +1025,10 @@ public void TestTrainingSGDDefaultsParamGroups() var lin2 = Linear(100, 10); var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var pgs = new Modules.ParamsGroup[] { - new () { Parameters = lin1.parameters(), Options = { LearningRate = 0.005f } }, - new () { Parameters = lin2.parameters() } }; + var pgs = new SGD.ParamGroup[] { + new () { Parameters = lin1.parameters(), Options = new() { LearningRate = 0.00005 } }, + new (lin2.parameters(), lr: 0.00003, dampening: 0.05, momentum: 0.25) + }; var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); @@ -1332,7 +1371,7 @@ public void TestTrainingConv2d() var x = torch.randn(new long[] { 64, 3, 28, 28 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.named_parameters()); + var optimizer = torch.optim.Adam(seq.parameters()); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -1376,7 +1415,7 @@ public void TestTrainingConv2dCUDA() seq.to((Device)device); - var optimizer = torch.optim.Adam(seq.named_parameters()); + var optimizer = torch.optim.Adam(seq.parameters()); var loss = mse_loss(Reduction.Sum); using (Tensor x = torch.randn(new long[] { 64, 3, 28, 28 }, device: (Device)device), From e348e8ef20bf87955c8092042a74905576280023 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 15 Feb 2022 09:09:41 -0800 Subject: [PATCH 11/25] Moved more optimizers to support parameter groups. --- RELEASENOTES.md | 14 +- src/TorchSharp/NN/Module.cs | 2 +- src/TorchSharp/NN/Optimizer.cs | 975 +++++++++++++++++++++------- test/TorchSharpTest/TestTraining.cs | 275 +++++++- 4 files changed, 1006 insertions(+), 260 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1c3441044..21d580a7f 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -4,20 +4,18 @@ Releases, starting with 9/2/2021, are listed with the most recent release at the ## NuGet Version 0.96.1 -__Fixed Bugs:__ - -#510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d
- -## NuGet Version 0.96.1 - __API Changes:__ -__NOTE__: This release contains breaking changes. +__NOTE__: This release contains breaking changes.
-The APIs to create optimizers all take 'named_parameters' rather than 'parameters' now. +The APIs to create optimizers all take 'named_parameters' rather than 'parameters' now.
__Fixed Bugs:__ +#510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d
+#515 what's reason for making register_module internal?
+ + ## NuGet Version 0.96.0 __API Changes:__ diff --git a/src/TorchSharp/NN/Module.cs b/src/TorchSharp/NN/Module.cs index 83ecdca41..7bf6a8075 100644 --- a/src/TorchSharp/NN/Module.cs +++ b/src/TorchSharp/NN/Module.cs @@ -615,7 +615,7 @@ public virtual void register_parameter(string name, Modules.Parameter param) /// Name of the submodule. /// The module to register. /// - internal virtual void register_module(string name, Module submodule) + public virtual void register_module(string name, Module submodule) { if (submodule is null || submodule.handle.IsInvalid) { if (_internal_submodules.ContainsKey(name)) { diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index ed7b0ae36..2f5b7f5e4 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -278,16 +278,33 @@ public static AdamW AdamW(IEnumerable parameters, double learn /// /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. /// - /// Parameters to optimize + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public static Adagrad Adagrad(IEnumerable parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + { + return new Adagrad(parameters, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps); + } + + /// + /// Implements Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// + /// Parameters to optimize /// learning rate (default: 1e-2) /// learning rate decay (default: 0) /// weight decay (L2 penalty) (default: 0) /// /// Term added to the denominator to improve numerical stability (default: 1e-10) /// - public static AdagradOptimizer Adagrad(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + public static Adagrad Adagrad(IEnumerable parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) { - return new AdagradOptimizer(named_parameters, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps); + return new Adagrad(parameters, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps); } /// @@ -295,15 +312,31 @@ public static AdagradOptimizer Adagrad(IEnumerable<(string name, Parameter param /// /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing a running average of squared gradients (default: 0.9) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adadelta Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + { + return new Adadelta(parameters, lr, rho, eps, weight_decay); + } + + /// + /// Implements Adadelta algorithm. + /// + /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// Coefficient used for computing a running average of squared gradients (default: 0.9) /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) /// Weight decay (L2 penalty) (default: 0) /// - public static AdadeltaOptimizer Adadelta(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + public static Adadelta Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) { - return new AdadeltaOptimizer(named_parameters, lr, rho, eps, weight_decay); + return new Adadelta(parameters, lr, rho, eps, weight_decay); } /// @@ -319,9 +352,27 @@ public static AdadeltaOptimizer Adadelta(IEnumerable<(string name, Parameter par /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// Momentum decay - public static NAdamOptimizer NAdam(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + public static NAdam NAdam(IEnumerable named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) { - return new NAdamOptimizer(named_parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); + return new NAdam(named_parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); + } + + /// + /// Implements NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + public static NAdam NAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + { + return new NAdam(parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); } /// @@ -330,15 +381,32 @@ public static NAdamOptimizer NAdam(IEnumerable<(string name, Parameter parameter /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' /// https://arxiv.org/abs/1908.03265 /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + public static RAdam RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new RAdam(parameters, lr, beta1, beta2, eps, weight_decay); + } + + /// + /// Implements RAdam algorithm. + /// + /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' + /// https://arxiv.org/abs/1908.03265 + /// + /// Parameters to optimize. /// Learning rate /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) - public static RAdamOptimizer RAdam(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + public static RAdam RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) { - return new RAdamOptimizer(named_parameters, lr, beta1, beta2, eps, weight_decay); + return new RAdam(parameters, lr, beta1, beta2, eps, weight_decay); } /// @@ -346,16 +414,33 @@ public static RAdamOptimizer RAdam(IEnumerable<(string name, Parameter parameter /// /// It has been proposed in Adam: A Method for Stochastic Optimization. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. /// Learning rate /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// - public static AdamaxOptimizer Adamax(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + public static Adamax Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) { - return new AdamaxOptimizer(named_parameters, lr, beta1, beta2, eps, weight_decay); + return new Adamax(parameters, lr, beta1, beta2, eps, weight_decay); + } + + /// + /// Implements Adamax algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adamax Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new Adamax(parameters, lr, beta1, beta2, eps, weight_decay); } /// @@ -402,9 +487,9 @@ public static ASGD ASGD(IEnumerable> parameters, double /// Minimum allowed step size. /// Maximum allowed step size. /// - public static RpropOptimizer Rprop(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + public static Rprop Rprop(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) { - return new RpropOptimizer(named_parameters, lr, etaminus, etaplus, min_step, max_step); + return new Rprop(named_parameters, lr, etaminus, etaplus, min_step, max_step); } /// @@ -587,6 +672,7 @@ public SGD(IEnumerable parameters, double lr, double momentum = 0.0, /// public SGD(IEnumerable parameters, double lr, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); if (nesterov && (momentum <= 0 || dampening != 0)) throw new ArgumentException("Nesterov momentum requires a momentum and zero dampening"); @@ -748,30 +834,49 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double mo public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } } - public class AdadeltaOptimizer : OptimizerHelper, ILearningRateController + public class Adadelta : NewOptimizerHelper, ILearningRateController { /// /// Constructor /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. /// Learning rate /// Coefficient used for computing a running average of squared gradients (default: 0.9) /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) /// Weight decay (L2 penalty) (default: 0) - public AdadeltaOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) : base(named_parameters, lr) + public Adadelta(IEnumerable parameters, double lr, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, rho, eps, weight_decay) { - LearningRate = lr; - InitialLearningRate = lr; - _rho = rho; - _eps = eps; - _weight_decay = weight_decay; + } - foreach (var (name, p) in named_parameters) { - var state = new State(); - _state[name] = state; - state.step = 0; - state.square_avg = torch.zeros_like(p); - state.acc_delta = torch.zeros_like(p); + /// + /// Constructor + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing a running average of squared gradients (default: 0.9) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) + /// Weight decay (L2 penalty) (default: 0) + public Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (rho < 0.0 || rho > 1.0) throw new ArgumentException($"Invalid rho value: {rho}"); + if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + rho = rho, + eps = eps, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); } } @@ -787,30 +892,39 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { + + var options = group.Options as Options; + var rho = options.rho.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { - var grad = param.grad(); + var grad = param.grad(); - if (grad is null) continue; + if (grad is null) continue; - if (grad.is_sparse) throw new ArgumentException("Adadelta does not support sparse gradients"); + if (grad.is_sparse) throw new ArgumentException("Adadelta does not support sparse gradients"); - var state = _state[name]; + var state = _state[param.handle]; - var square_avg = state.square_avg; - var acc_delta = state.acc_delta; + var square_avg = state.square_avg; + var acc_delta = state.acc_delta; - grad = (_weight_decay != 0) - ? grad.add(param, alpha: _weight_decay) - : grad.alias(); + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - square_avg.mul_(_rho).addcmul_(grad, grad, 1 - _rho); + square_avg.mul_(rho).addcmul_(grad, grad, 1 - rho); - var std = square_avg.add(_eps).sqrt_(); - var delta = acc_delta.add(_eps).sqrt_().div_(std).mul_(grad); + var std = square_avg.add(eps).sqrt_(); + var delta = acc_delta.add(eps).sqrt_().div_(std).mul_(grad); - param.add_(delta, alpha: -LearningRate); - acc_delta.mul_(_rho).addcmul_(delta, delta, 1 - _rho); + param.add_(delta, alpha: -lr); + acc_delta.mul_(rho).addcmul_(delta, delta, 1 - rho); + } } d.DisposeEverything(); @@ -836,41 +950,109 @@ private class State public Tensor acc_delta; } - private Dictionary _state = new Dictionary(); - private double _rho; - private double _eps; - private double _weight_decay; + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.rho.HasValue) opt.rho = def.rho; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.square_avg = torch.zeros_like(p); + state.acc_delta = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? rho; + public double? eps; + public double? weight_decay; + } + + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + : base(parameters, new Adadelta.Options { LearningRate = lr, rho = rho, eps = eps, weight_decay = weight_decay }) + { + } + } + + private Dictionary _state = new Dictionary(); } - public class AdamaxOptimizer : OptimizerHelper, ILearningRateController, IBetas + public class Adamax : NewOptimizerHelper, ILearningRateController, IBetas { /// /// Implements Adamax algorithm (a variant of Adam based on infinity norm). /// /// It has been proposed in Adam: A Method for Stochastic Optimization. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// - public AdamaxOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) : base(named_parameters, lr) + public Adamax(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay) { - LearningRate = lr; - InitialLearningRate = lr; - _beta1 = beta1; - _beta2 = beta2; - _eps = eps; - _weight_decay = weight_decay; + } - foreach (var (name, p) in named_parameters) { - var state = new State(); - _state[name] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_inf = torch.zeros_like(p); + /// + /// Implements Adamax algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); + if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); + if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + eps = eps, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); } } @@ -886,35 +1068,46 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { - var grad = param.grad(); + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; - if (grad is null) continue; + foreach (var param in group.Parameters) { - if (grad.is_sparse) throw new ArgumentException("Adamax does not support sparse gradients"); + var grad = param.grad(); - var state = _state[name]; + if (grad is null) continue; - state.step += 1; + if (grad.is_sparse) throw new ArgumentException("Adamax does not support sparse gradients"); - var exp_avg = state.exp_avg; - var exp_inf = state.exp_inf; + var state = _state[param.handle]; - grad = (_weight_decay != 0) - ? grad.add(param, alpha: _weight_decay) - : grad.alias(); + state.step += 1; + + var exp_avg = state.exp_avg; + var exp_inf = state.exp_inf; + + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - exp_avg.mul_(_beta1).add_(grad, alpha: 1 - _beta1); + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - var norm_buf = torch.cat(new Tensor[] { - exp_inf.mul_(_beta2).unsqueeze(0), - grad.abs().add_(_eps).unsqueeze_(0) }, 0); + var norm_buf = torch.cat(new Tensor[] { + exp_inf.mul_(beta2).unsqueeze(0), + grad.abs().add_(eps).unsqueeze_(0) + }, 0); - torch.amax(norm_buf, new long[] { 0 }, false, exp_inf); + torch.amax(norm_buf, new long[] { 0 }, false, exp_inf); - var clr = LearningRate / (1 - Math.Pow(_beta1, state.step)); - param.addcdiv_(exp_avg, exp_inf, value: -clr); + var clr = lr / (1 - Math.Pow(beta1, state.step)); + param.addcdiv_(exp_avg, exp_inf, value: -clr); + } } d.DisposeEverything(); @@ -933,11 +1126,6 @@ protected override void Dispose(bool disposing) } } - public (double, double) Betas { - get => (_beta1, _beta2); - set { _beta1 = value.Item1; _beta2 = value.Item2; } - } - private class State { public int step; @@ -945,46 +1133,126 @@ private class State public Tensor exp_inf; } - private Dictionary _state = new Dictionary(); - private double _beta1; - private double _beta2; - private double _eps; - private double _weight_decay; - } - - public class NAdamOptimizer : OptimizerHelper, ILearningRateController, IBetas - { - /// - /// Implements NAdam algorithm. - /// - /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. - /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum decay - /// - public NAdamOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) : base(named_parameters, lr) + public override void add_param_group(Modules.ParamGroup param_group) { - LearningRate = lr; - InitialLearningRate = lr; - _beta1 = beta1; - _beta2 = beta2; - _eps = eps; - _weight_decay = weight_decay; - _momentum_decay = momentum_decay; + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + + opt.InitialLearningRate = opt.LearningRate.Value; - foreach (var (name, p) in named_parameters) { + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { var state = new State(); - _state[name] = state; + _state[p.Handle] = state; state.step = 0; - state.mu_product = 1; state.exp_avg = torch.zeros_like(p); - state.exp_avg_sq = torch.zeros_like(p); + state.exp_inf = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? eps; + public double? weight_decay; + } + + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + : base(parameters, new Adamax.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay }) + { + } + + public (double, double) Betas { + get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); + set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); + } + + public class NAdam : NewOptimizerHelper, ILearningRateController, IBetas + { + /// + /// Implements NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + public NAdam(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, momentum_decay) + { + } + + /// + /// Implements NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + /// + public NAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); + if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); + if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + if (momentum_decay < 0.0) throw new ArgumentException($"Invalid momentum_decay value: {momentum_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + eps = eps, + weight_decay = weight_decay, + momentum_decay = momentum_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); } } @@ -1000,40 +1268,51 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { - var grad = param.grad(); + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var momentum_decay = options.momentum_decay.Value; + var lr = options.LearningRate.Value; - if (grad is null) continue; + foreach (var param in group.Parameters) { - var state = _state[name]; + var grad = param.grad(); - state.step += 1; + if (grad is null) continue; - var exp_avg = state.exp_avg; - var exp_avg_sq = state.exp_avg_sq; + var state = _state[param.handle]; - var bias_correction2 = 1 - Math.Pow(_beta2, state.step); + state.step += 1; - grad = (_weight_decay != 0) - ? grad.add(param, alpha: _weight_decay) - : grad.alias(); + var exp_avg = state.exp_avg; + var exp_avg_sq = state.exp_avg_sq; - var mu = _beta1 * (1.0 - 0.5 * Math.Pow(0.96, state.step * _momentum_decay)); - var mu_next = _beta1 * (1.0 - 0.5 * Math.Pow(0.96, (state.step + 1) * _momentum_decay)); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); - var mu_product = state.mu_product * mu; - var mu_product_next = mu_product * mu * mu_next; + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); + + var mu = beta1 * (1.0 - 0.5 * Math.Pow(0.96, state.step * momentum_decay)); + var mu_next = beta1 * (1.0 - 0.5 * Math.Pow(0.96, (state.step + 1) * momentum_decay)); + + var mu_product = state.mu_product * mu; + var mu_product_next = mu_product * mu * mu_next; - exp_avg.mul_(_beta1).add_(grad, alpha: 1 - _beta1); - exp_avg_sq.mul_(_beta2).addcmul_(grad, grad, value: 1 - _beta2); + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); - var denom = exp_avg_sq.div(bias_correction2).sqrt_().add_(_eps); + var denom = exp_avg_sq.div(bias_correction2).sqrt_().add_(eps); - param.addcdiv_(grad, denom, value: -LearningRate * (1 - mu) / (1 - mu_product)); - param.addcdiv_(exp_avg, denom, value: -LearningRate * mu_next / (1 - mu_product_next)); + param.addcdiv_(grad, denom, value: -lr * (1 - mu) / (1 - mu_product)); + param.addcdiv_(exp_avg, denom, value: -lr * mu_next / (1 - mu_product_next)); - state.mu_product = mu_product; + state.mu_product = mu_product; + } } d.DisposeEverything(); @@ -1052,11 +1331,6 @@ protected override void Dispose(bool disposing) } } - public (double, double) Betas { - get => (_beta1, _beta2); - set { _beta1 = value.Item1; _beta2 = value.Item2; } - } - private class State { public int step; @@ -1065,15 +1339,71 @@ private class State public Tensor exp_avg_sq; } - private Dictionary _state = new Dictionary(); - private double _beta1; - private double _beta2; - private double _eps; - private double _weight_decay; - private double _momentum_decay; + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.momentum_decay.HasValue) opt.momentum_decay = def.momentum_decay; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? eps; + public double? weight_decay; + public double? momentum_decay; + } + + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + : base(parameters, new NAdam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, momentum_decay = momentum_decay }) + { + } + + public (double, double) Betas { + get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); + set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); } - public class RAdamOptimizer : OptimizerHelper, ILearningRateController, IBetas + public class RAdam : NewOptimizerHelper, ILearningRateController, IBetas { /// /// Implements RAdam algorithm. @@ -1081,28 +1411,53 @@ public class RAdamOptimizer : OptimizerHelper, ILearningRateController, IBetas /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' /// https://arxiv.org/abs/1908.03265 /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) /// Weight decay (L2 penalty) (default: 0) /// - public RAdamOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) : base(named_parameters, lr) + public RAdam(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay) { - LearningRate = lr; - InitialLearningRate = lr; - _beta1 = beta1; - _beta2 = beta2; - _eps = eps; - _weight_decay = weight_decay; + } - foreach (var (name, p) in named_parameters) { - var state = new State(); - _state[name] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_avg_sq = torch.zeros_like(p); + /// + /// Implements RAdam algorithm. + /// + /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' + /// https://arxiv.org/abs/1908.03265 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); + if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); + if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + eps = eps, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); } } @@ -1118,43 +1473,53 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { - var grad = param.grad(); + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; - if (grad is null) continue; + foreach (var param in group.Parameters) { - var state = _state[name]; + var grad = param.grad(); - state.step += 1; + if (grad is null) continue; - var exp_avg = state.exp_avg; - var exp_avg_sq = state.exp_avg_sq; + var state = _state[param.handle]; - var bias_correction1 = 1 - Math.Pow(_beta1, state.step); - var bias_correction2 = 1 - Math.Pow(_beta2, state.step); + state.step += 1; - grad = (_weight_decay != 0) - ? grad.add(param, alpha: _weight_decay) - : grad.alias(); + var exp_avg = state.exp_avg; + var exp_avg_sq = state.exp_avg_sq; + + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); + + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - exp_avg.mul_(_beta1).add_(grad, alpha: 1 - _beta1); - exp_avg_sq.mul_(_beta2).addcmul_(grad, grad, value: 1 - _beta2); + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); - var bias_corrected_exp_avg = exp_avg / bias_correction1; + var bias_corrected_exp_avg = exp_avg / bias_correction1; - var rho_inf = 2 / (1 - _beta2) - 1; - var rho_t = rho_inf - 2 * state.step * Math.Pow(_beta2, state.step) / bias_correction2; + var rho_inf = 2 / (1 - beta2) - 1; + var rho_t = rho_inf - 2 * state.step * Math.Pow(beta2, state.step) / bias_correction2; - var t6 = bias_corrected_exp_avg * LearningRate; + var t6 = bias_corrected_exp_avg * lr; - if (rho_t > 5) { - var rect = Math.Sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t)); - var adaptive_lr = Math.Sqrt(bias_correction2) / exp_avg_sq.sqrt().add_(_eps); + if (rho_t > 5) { + var rect = Math.Sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t)); + var adaptive_lr = Math.Sqrt(bias_correction2) / exp_avg_sq.sqrt().add_(eps); - param.add_(t6 * LearningRate * adaptive_lr * rect, alpha: -1.0); - } else { - param.add_(t6, alpha: -1.0); + param.add_(t6 * lr * adaptive_lr * rect, alpha: -1.0); + } else { + param.add_(t6, alpha: -1.0); + } } } @@ -1181,16 +1546,66 @@ private class State public Tensor exp_avg_sq; } - private Dictionary _state = new Dictionary(); - private double _beta1; - private double _beta2; - private double _eps; - private double _weight_decay; + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? eps; + public double? weight_decay; + } + + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + : base(parameters, new RAdam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay }) + { + } + + public (double, double) Betas { + get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); + set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + } + } public (double, double) Betas { - get => (_beta1, _beta2); - set { _beta1 = value.Item1; _beta2 = value.Item2; } + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } } + + private Dictionary _state = new Dictionary(); } public class ASGD : NewOptimizerHelper, ILearningRateController @@ -1226,6 +1641,8 @@ public ASGD(IEnumerable parameters, double lr = 0.01, double lambd = /// public ASGD(IEnumerable> parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + var options = new Options { LearningRate = lr, InitialLearningRate = lr, @@ -1371,7 +1788,7 @@ public ParamGroup(IEnumerable parameters, double lr = 0.01, double la private Dictionary _state = new Dictionary(); } - public class RpropOptimizer : OptimizerHelper, ILearningRateController + public class Rprop : OptimizerHelper, ILearningRateController { /// /// Implements Rprop algorithm (a variant of Adam based on infinity norm). @@ -1385,7 +1802,7 @@ public class RpropOptimizer : OptimizerHelper, ILearningRateController /// Minimum allowed step size. /// Maximum allowed step size. /// - public RpropOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) : base(named_parameters, lr) + public Rprop(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) : base(named_parameters, lr) { LearningRate = lr; InitialLearningRate = lr; @@ -1477,42 +1894,58 @@ private class State private double _max_step; } - public class AdagradOptimizer : OptimizerHelper + public class Adagrad : NewOptimizerHelper { /// /// Implements Adagrad algorithm. /// /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. /// - /// Parameters to optimize - /// learning rate (default: 1e-2) + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, lr_decay, weight_decay, initial_accumulator_value, eps) + { + } + /// + /// Implements Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// + /// Parameters to optimize + /// learning rate (default: 1e-2) /// learning rate decay (default: 0) /// weight decay (L2 penalty) (default: 0) /// /// Term added to the denominator to improve numerical stability (default: 1e-10) /// - public AdagradOptimizer(IEnumerable<(string name, Parameter parameter)> named_parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) : base(named_parameters, learningRate) + public Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) { - if (learningRate < 0) throw new ArgumentException($"Invalid learning rate: {learningRate}"); + if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (lr_decay < 0.0) throw new ArgumentException($"Invalid lr_decay value: {lr_decay}"); if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + if (initial_accumulator_value < 0) throw new ArgumentException($"Invalid initial_accumulator_value: {initial_accumulator_value}"); - LearningRate = learningRate; - InitialLearningRate = learningRate; - _lr_decay = lr_decay; - _initial_accumulator_value = initial_accumulator_value; - _weight_decay = weight_decay; - _eps = eps; + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + lr_decay = lr_decay, + eps = eps, + initial_accumulator_value = initial_accumulator_value, + weight_decay = weight_decay + }; - foreach (var (name, p) in named_parameters) { + _defaults = options; + _parameter_groups = new List(); - var state = new State(); - _state[name] = state; - state.step = 0; - var init_value = torch.is_complex(p.dtype) - ? (Scalar)new System.Numerics.Complex(initial_accumulator_value, initial_accumulator_value) - : (Scalar)initial_accumulator_value; - state.sum = torch.full_like(p, init_value); + foreach (var g in parameters) { + add_param_group(g); } } @@ -1529,30 +1962,40 @@ public override Tensor step(Func closure = null) using (var d = torch.NewDisposeScope()) { - foreach (var (name, param) in _parameters) { + foreach (var group in _parameter_groups) { - var state = _state[name]; + var options = group.Options as Options; + var lr_decay = options.lr_decay.Value; + var weight_decay = options.weight_decay.Value; + var eps = options.eps.Value; + var initial_accumulator_value = options.initial_accumulator_value.Value; + var lr = options.LearningRate.Value; - var grad = param.grad(); + foreach (var param in group.Parameters) { - if (grad is null) continue; + var state = _state[param.handle]; - state.step += 1; + var grad = param.grad(); - if (_weight_decay != 0) { - grad = grad.add(param, alpha: _weight_decay); - } + if (grad is null) continue; + + state.step += 1; - var clr = LearningRate / (1 + (state.step - 1) * _lr_decay); + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } - if (grad.is_sparse) { - throw new NotImplementedException("Adagrad optimization over sparse parameters"); - } else if (torch.is_complex(param)) { - throw new NotImplementedException("Adagrad optimization over complex parameters"); - } else { - state.sum.addcmul_(grad, grad, value: 1); - var std = state.sum.sqrt().add_(_eps); - param.addcdiv_(grad, std, value: -clr); + var clr = lr / (1 + (state.step - 1) * lr_decay); + + if (grad.is_sparse) { + throw new NotImplementedException("Adagrad optimization over sparse parameters"); + } else if (torch.is_complex(param)) { + throw new NotImplementedException("Adagrad optimization over complex parameters"); + } else { + state.sum.addcmul_(grad, grad, value: 1); + var std = state.sum.sqrt().add_(eps); + param.addcdiv_(grad, std, value: -clr); + } } d.DisposeEverything(); @@ -1577,12 +2020,58 @@ private class State public Tensor sum; } + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; - private Dictionary _state = new Dictionary(); - private double _lr_decay; - private double _initial_accumulator_value; - private double _eps; - private double _weight_decay; + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.lr_decay.HasValue) opt.lr_decay = def.lr_decay; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.initial_accumulator_value.HasValue) opt.initial_accumulator_value = def.initial_accumulator_value; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + var init_value = torch.is_complex(p.dtype) + ? (Scalar)new System.Numerics.Complex((param_group.Options as Options).initial_accumulator_value.Value, (param_group.Options as Options).initial_accumulator_value.Value) + : (Scalar)(param_group.Options as Options).initial_accumulator_value.Value; + state.sum = torch.full_like(p, init_value); + } + } + + public class Options : OptimizerOptions + { + public double? lr_decay; + public double? initial_accumulator_value; + public double? eps; + public double? weight_decay; + } + + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 0.01, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + : base(parameters, new Adagrad.Options { LearningRate = lr, lr_decay = lr_decay, initial_accumulator_value = initial_accumulator_value, weight_decay = weight_decay, eps = eps }) + { + } + } + + private Dictionary _state = new Dictionary(); } public class Adam : NewOptimizerHelper, IBetas diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index ae3d1eb8f..3b693258b 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -185,6 +185,43 @@ public void TestTrainingAdamDefaults() Assert.True(finalLoss < initialLoss); } + [Fact] + public void TestTrainingAdamParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var optimizer = torch.optim.Adam(new Adam.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + [Fact] public void TestTrainingAdamAmsGrad() { @@ -290,6 +327,43 @@ public void TestTrainingAdamWDefaults() Assert.True(finalLoss < initialLoss); } + [Fact] + public void TestTrainingAdamWParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var optimizer = torch.optim.AdamW(new AdamW.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + [Fact] public void TestTrainingAdamWOneCycleLR() { @@ -343,7 +417,7 @@ public void TestTrainingAdagrad() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 0.00004f; - var optimizer = torch.optim.Adagrad(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.Adagrad(seq.parameters(), learning_rate); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -365,6 +439,43 @@ public void TestTrainingAdagrad() Assert.True(finalLoss < initialLoss); } + [Fact] + public void TestTrainingAdagradParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var optimizer = torch.optim.Adagrad(new Adagrad.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + /// /// Fully connected ReLU net with one hidden layer trained using Adadelta optimizer. /// @@ -379,7 +490,44 @@ public void TestTrainingAdadelta() var y = torch.randn(new long[] { 64, 10 }); double learning_rate = 1.0f; - var optimizer = torch.optim.Adadelta(seq.named_parameters(), learning_rate); + var optimizer = torch.optim.Adadelta(seq.parameters(), learning_rate); + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingAdadeltaParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var optimizer = torch.optim.Adadelta(new Adadelta.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -414,7 +562,7 @@ public void TestTrainingAdamax() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adamax(seq.named_parameters()); + var optimizer = torch.optim.Adamax(seq.parameters()); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -436,6 +584,43 @@ public void TestTrainingAdamax() Assert.True(finalLoss < initialLoss); } + [Fact] + public void TestTrainingAdamaxParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var optimizer = torch.optim.Adamax(new Adamax.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + /// /// Fully connected ReLU net with one hidden layer trained using Adamax optimizer. /// @@ -449,7 +634,7 @@ public void TestTrainingAdamaxOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adamax(seq.named_parameters()); + var optimizer = torch.optim.Adamax(seq.parameters()); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 15); var loss = mse_loss(Reduction.Sum); @@ -487,7 +672,44 @@ public void TestTrainingNAdam() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.NAdam(seq.named_parameters()); + var optimizer = torch.optim.NAdam(seq.parameters()); + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingNAdamParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var optimizer = torch.optim.NAdam(new NAdam.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -522,7 +744,7 @@ public void TestTrainingNAdamOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.NAdam(seq.named_parameters()); + var optimizer = torch.optim.NAdam(seq.parameters()); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -559,7 +781,44 @@ public void TestTrainingRAdam() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.RAdam(seq.named_parameters()); + var optimizer = torch.optim.RAdam(seq.parameters()); + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingRAdamParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var optimizer = torch.optim.RAdam(new RAdam.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -594,7 +853,7 @@ public void TestTrainingRAdamOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.RAdam(seq.named_parameters()); + var optimizer = torch.optim.RAdam(seq.parameters()); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 1.5, total_steps: 10); var loss = mse_loss(Reduction.Sum); From 45e196a5d2b07e9f748c324e652146ce088321e7 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 15 Feb 2022 10:40:48 -0800 Subject: [PATCH 12/25] Finished convering optimizers to managed code. --- src/TorchSharp/NN/Optimizer.cs | 1013 ++++++++++++--------------- test/TorchSharpTest/TestTraining.cs | 40 +- 2 files changed, 506 insertions(+), 547 deletions(-) diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index 2f5b7f5e4..2c2806c91 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -134,6 +134,8 @@ public interface ILearningRateController double LearningRate { set; get; } double InitialLearningRate { set; get; } + + IEnumerable ParamGroups { get; } } public interface IMomentum @@ -480,16 +482,31 @@ public static ASGD ASGD(IEnumerable> parameters, double /// /// Implements the resilient backpropagation algorithm. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Multiplicative increase factor. + /// Multiplicative decrease factor. + /// Minimum allowed step size. + /// Maximum allowed step size. + /// + public static Rprop Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + { + return new Rprop(parameters, lr, etaminus, etaplus, min_step, max_step); + } + + /// + /// Implements the resilient backpropagation algorithm. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. /// Learning rate /// Multiplicative increase factor. /// Multiplicative decrease factor. /// Minimum allowed step size. /// Maximum allowed step size. /// - public static Rprop Rprop(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + public static Rprop Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) { - return new Rprop(named_parameters, lr, etaminus, etaplus, min_step, max_step); + return new Rprop(parameters, lr, etaminus, etaplus, min_step, max_step); } /// @@ -539,56 +556,49 @@ namespace Modules public class OptimizerHelper : Optimizer, ILearningRateController { - public OptimizerHelper(IEnumerable<(string name, Modules.Parameter parameter)> named_parameters, double learningRate) : base(IntPtr.Zero) + public OptimizerHelper() : base(IntPtr.Zero) { - LearningRate = learningRate; - InitialLearningRate = learningRate; - _parameters = named_parameters; } public override void zero_grad() { - foreach (var (_, p) in _parameters) { + foreach (var g in _parameter_groups) { - using var grad = p.grad(); + foreach (var p in g.Parameters) { + + using var grad = p.grad(); - if (grad is null) continue; + if (grad is null) continue; - grad.zero_().Dispose(); + grad.zero_().Dispose(); + } } } - public override IEnumerable parameters() + protected Tensor _step(Action body, Func loss_closure = null) where T:ParamGroup { - return _parameters.Select(p => p.parameter); - } - - public double LearningRate { get; set; } - - public double InitialLearningRate { get; set; } + Tensor loss = null; - protected IEnumerable<(string name, Modules.Parameter parameter)> _parameters; - } + if (loss_closure != null) { + using (var _ = torch.enable_grad()) + loss = loss_closure(); + } - public class NewOptimizerHelper : Optimizer, ILearningRateController - { - public NewOptimizerHelper() : base(IntPtr.Zero) - { - } + using (var _ = torch.no_grad()) { - public override void zero_grad() - { - foreach (var g in _parameter_groups) { + using (var d = torch.NewDisposeScope()) { - foreach (var p in g.Parameters) { + foreach (var group in _parameter_groups) { - using var grad = p.grad(); + body(group as T); - if (grad is null) continue; + } - grad.zero_().Dispose(); + d.DisposeEverything(); } } + + return loss; } public override IEnumerable parameters() @@ -605,6 +615,8 @@ public virtual void add_param_group(ParamGroup param_group) public double InitialLearningRate { get => _defaults.InitialLearningRate; set => _defaults.InitialLearningRate = value; } + public IEnumerable ParamGroups { get => _parameter_groups; } + protected OptimizerOptions _defaults; protected IList _parameter_groups; } @@ -623,6 +635,8 @@ public class ParamGroup : ILearningRateController public double LearningRate { get => Options.LearningRate.Value; set => Options.LearningRate = value; } public double InitialLearningRate { get => Options.InitialLearningRate; set => Options.InitialLearningRate = value; } + + public IEnumerable ParamGroups { get => throw new InvalidOperationException("ParamGroups should not be called on a ParamGroup"); } } public class ParamGroup : ParamGroup where TOptions : OptimizerOptions @@ -641,7 +655,7 @@ public ParamGroup(IEnumerable parameters, TOptions options = null) } - public class SGD : NewOptimizerHelper, IMomentum + public class SGD : OptimizerHelper, IMomentum { /// /// Implements stochastic gradient descent (optionally with momentum). @@ -697,69 +711,52 @@ public SGD(IEnumerable parameters, double lr, double momentum = 0.0, public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - using (var _ = torch.enable_grad()) - loss = closure(); - } - - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { - - var options = group.Options as Options; - var momentum = options.momentum.Value; - var dampening = options.dampening.Value; - var weight_decay = options.weight_decay.Value; - var nesterov = options.nesterov.Value; - var maximize = options.maximize.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { + return _step(group => { - var state = _state[param.handle]; + var options = group.Options; + var momentum = options.momentum.Value; + var dampening = options.dampening.Value; + var weight_decay = options.weight_decay.Value; + var nesterov = options.nesterov.Value; + var maximize = options.maximize.Value; + var lr = options.LearningRate.Value; - var grad = param.grad(); + foreach (var param in group.Parameters) { - if (grad is null) continue; + var state = _state[param.handle]; - if (weight_decay != 0) { - grad = grad.add(param, alpha: weight_decay); - } + var grad = param.grad(); - if (momentum != 0) { - var buf = state.momentum_buffer; - - if (buf is null) { - buf = grad.clone().detach().DetatchFromDisposeScope(); - state.momentum_buffer = buf; - } else { - buf.mul_(momentum).add_(grad, alpha: (1 - dampening)); - } + if (grad is null) continue; - if (nesterov) { - grad = grad.add(buf, alpha: momentum); - } else { - grad = buf; - } + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } - state.momentum_buffer = buf; - } + if (momentum != 0) { + var buf = state.momentum_buffer; - var alpha = maximize ? lr : -lr; - param.add_(grad, alpha: alpha); + if (buf is null) { + buf = grad.clone().detach().DetatchFromDisposeScope(); + state.momentum_buffer = buf; + } else { + buf.mul_(momentum).add_(grad, alpha: (1 - dampening)); + } + if (nesterov) { + grad = grad.add(buf, alpha: momentum); + } else { + grad = buf; } + + state.momentum_buffer = buf; } - d.DisposeEverything(); - } - } + var alpha = maximize ? lr : -lr; + param.add_(grad, alpha: alpha); - return loss; + } + }, closure); } protected override void Dispose(bool disposing) @@ -834,7 +831,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double mo public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } } - public class Adadelta : NewOptimizerHelper, ILearningRateController + public class Adadelta : OptimizerHelper, ILearningRateController { /// /// Constructor @@ -882,56 +879,40 @@ public Adadelta(IEnumerable parameters, double lr = 1.0, double rho public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - loss = closure(); - } + return _step(group => { - using (var _ = torch.no_grad()) { + var options = group.Options as Options; + var rho = options.rho.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { + foreach (var param in group.Parameters) { - var options = group.Options as Options; - var rho = options.rho.Value; - var eps = options.eps.Value; - var weight_decay = options.weight_decay.Value; - var lr = options.LearningRate.Value; + var grad = param.grad(); - foreach (var param in group.Parameters) { - - var grad = param.grad(); - - if (grad is null) continue; - - if (grad.is_sparse) throw new ArgumentException("Adadelta does not support sparse gradients"); + if (grad is null) continue; - var state = _state[param.handle]; + if (grad.is_sparse) throw new ArgumentException("Adadelta does not support sparse gradients"); - var square_avg = state.square_avg; - var acc_delta = state.acc_delta; + var state = _state[param.handle]; - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); + var square_avg = state.square_avg; + var acc_delta = state.acc_delta; - square_avg.mul_(rho).addcmul_(grad, grad, 1 - rho); + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - var std = square_avg.add(eps).sqrt_(); - var delta = acc_delta.add(eps).sqrt_().div_(std).mul_(grad); + square_avg.mul_(rho).addcmul_(grad, grad, 1 - rho); - param.add_(delta, alpha: -lr); - acc_delta.mul_(rho).addcmul_(delta, delta, 1 - rho); - } - } + var std = square_avg.add(eps).sqrt_(); + var delta = acc_delta.add(eps).sqrt_().div_(std).mul_(grad); - d.DisposeEverything(); + param.add_(delta, alpha: -lr); + acc_delta.mul_(rho).addcmul_(delta, delta, 1 - rho); } - } - - return loss; + },closure); } protected override void Dispose(bool disposing) @@ -1000,7 +981,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double rho private Dictionary _state = new Dictionary(); } - public class Adamax : NewOptimizerHelper, ILearningRateController, IBetas + public class Adamax : OptimizerHelper, ILearningRateController, IBetas { /// /// Implements Adamax algorithm (a variant of Adam based on infinity norm). @@ -1058,63 +1039,47 @@ public Adamax(IEnumerable parameters, double lr = 0.002, double beta public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - loss = closure(); - } - - using (var _ = torch.no_grad()) { + return _step(group => { - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var eps = options.eps.Value; - var weight_decay = options.weight_decay.Value; - var lr = options.LearningRate.Value; + foreach (var param in group.Parameters) { - foreach (var param in group.Parameters) { + var grad = param.grad(); - var grad = param.grad(); - - if (grad is null) continue; + if (grad is null) continue; - if (grad.is_sparse) throw new ArgumentException("Adamax does not support sparse gradients"); + if (grad.is_sparse) throw new ArgumentException("Adamax does not support sparse gradients"); - var state = _state[param.handle]; + var state = _state[param.handle]; - state.step += 1; + state.step += 1; - var exp_avg = state.exp_avg; - var exp_inf = state.exp_inf; + var exp_avg = state.exp_avg; + var exp_inf = state.exp_inf; - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - var norm_buf = torch.cat(new Tensor[] { + var norm_buf = torch.cat(new Tensor[] { exp_inf.mul_(beta2).unsqueeze(0), grad.abs().add_(eps).unsqueeze_(0) }, 0); - torch.amax(norm_buf, new long[] { 0 }, false, exp_inf); + torch.amax(norm_buf, new long[] { 0 }, false, exp_inf); - var clr = lr / (1 - Math.Pow(beta1, state.step)); - param.addcdiv_(exp_avg, exp_inf, value: -clr); - } - } - - d.DisposeEverything(); + var clr = lr / (1 - Math.Pow(beta1, state.step)); + param.addcdiv_(exp_avg, exp_inf, value: -clr); } - } - - return loss; + }, closure); } protected override void Dispose(bool disposing) @@ -1195,7 +1160,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet private Dictionary _state = new Dictionary(); } - public class NAdam : NewOptimizerHelper, ILearningRateController, IBetas + public class NAdam : OptimizerHelper, ILearningRateController, IBetas { /// /// Implements NAdam algorithm. @@ -1258,68 +1223,52 @@ public NAdam(IEnumerable parameters, double lr = 0.002, double beta1 public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - loss = closure(); - } + return _step(group => { - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var momentum_decay = options.momentum_decay.Value; + var lr = options.LearningRate.Value; - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var eps = options.eps.Value; - var weight_decay = options.weight_decay.Value; - var momentum_decay = options.momentum_decay.Value; - var lr = options.LearningRate.Value; + foreach (var param in group.Parameters) { - foreach (var param in group.Parameters) { + var grad = param.grad(); - var grad = param.grad(); - - if (grad is null) continue; - - var state = _state[param.handle]; + if (grad is null) continue; - state.step += 1; + var state = _state[param.handle]; - var exp_avg = state.exp_avg; - var exp_avg_sq = state.exp_avg_sq; + state.step += 1; - var bias_correction2 = 1 - Math.Pow(beta2, state.step); + var exp_avg = state.exp_avg; + var exp_avg_sq = state.exp_avg_sq; - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); - var mu = beta1 * (1.0 - 0.5 * Math.Pow(0.96, state.step * momentum_decay)); - var mu_next = beta1 * (1.0 - 0.5 * Math.Pow(0.96, (state.step + 1) * momentum_decay)); + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - var mu_product = state.mu_product * mu; - var mu_product_next = mu_product * mu * mu_next; + var mu = beta1 * (1.0 - 0.5 * Math.Pow(0.96, state.step * momentum_decay)); + var mu_next = beta1 * (1.0 - 0.5 * Math.Pow(0.96, (state.step + 1) * momentum_decay)); - exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + var mu_product = state.mu_product * mu; + var mu_product_next = mu_product * mu * mu_next; - var denom = exp_avg_sq.div(bias_correction2).sqrt_().add_(eps); + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); - param.addcdiv_(grad, denom, value: -lr * (1 - mu) / (1 - mu_product)); - param.addcdiv_(exp_avg, denom, value: -lr * mu_next / (1 - mu_product_next)); + var denom = exp_avg_sq.div(bias_correction2).sqrt_().add_(eps); - state.mu_product = mu_product; - } - } + param.addcdiv_(grad, denom, value: -lr * (1 - mu) / (1 - mu_product)); + param.addcdiv_(exp_avg, denom, value: -lr * mu_next / (1 - mu_product_next)); - d.DisposeEverything(); + state.mu_product = mu_product; } - } - - return loss; + }, closure); } protected override void Dispose(bool disposing) @@ -1403,7 +1352,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet private Dictionary _state = new Dictionary(); } - public class RAdam : NewOptimizerHelper, ILearningRateController, IBetas + public class RAdam : OptimizerHelper, ILearningRateController, IBetas { /// /// Implements RAdam algorithm. @@ -1463,71 +1412,55 @@ public RAdam(IEnumerable parameters, double lr = 0.002, double beta1 public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - loss = closure(); - } - - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { + return _step(group => { - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var eps = options.eps.Value; - var weight_decay = options.weight_decay.Value; - var lr = options.LearningRate.Value; + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; - foreach (var param in group.Parameters) { + foreach (var param in group.Parameters) { - var grad = param.grad(); + var grad = param.grad(); - if (grad is null) continue; + if (grad is null) continue; - var state = _state[param.handle]; + var state = _state[param.handle]; - state.step += 1; + state.step += 1; - var exp_avg = state.exp_avg; - var exp_avg_sq = state.exp_avg_sq; + var exp_avg = state.exp_avg; + var exp_avg_sq = state.exp_avg_sq; - var bias_correction1 = 1 - Math.Pow(beta1, state.step); - var bias_correction2 = 1 - Math.Pow(beta2, state.step); + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); - var bias_corrected_exp_avg = exp_avg / bias_correction1; + var bias_corrected_exp_avg = exp_avg / bias_correction1; - var rho_inf = 2 / (1 - beta2) - 1; - var rho_t = rho_inf - 2 * state.step * Math.Pow(beta2, state.step) / bias_correction2; + var rho_inf = 2 / (1 - beta2) - 1; + var rho_t = rho_inf - 2 * state.step * Math.Pow(beta2, state.step) / bias_correction2; - var t6 = bias_corrected_exp_avg * lr; + var t6 = bias_corrected_exp_avg * lr; - if (rho_t > 5) { - var rect = Math.Sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t)); - var adaptive_lr = Math.Sqrt(bias_correction2) / exp_avg_sq.sqrt().add_(eps); + if (rho_t > 5) { + var rect = Math.Sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t)); + var adaptive_lr = Math.Sqrt(bias_correction2) / exp_avg_sq.sqrt().add_(eps); - param.add_(t6 * lr * adaptive_lr * rect, alpha: -1.0); - } else { - param.add_(t6, alpha: -1.0); - } - } + param.add_(t6 * lr * adaptive_lr * rect, alpha: -1.0); + } else { + param.add_(t6, alpha: -1.0); } - - d.DisposeEverything(); } - } - - return loss; + }, closure); } protected override void Dispose(bool disposing) @@ -1608,7 +1541,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet private Dictionary _state = new Dictionary(); } - public class ASGD : NewOptimizerHelper, ILearningRateController + public class ASGD : OptimizerHelper, ILearningRateController { /// /// Implements ASGD algorithm (a variant of Adam based on infinity norm). @@ -1662,60 +1595,44 @@ public ASGD(IEnumerable> parameters, double lr = 0.01, doubl public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - loss = closure(); - } - - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { - - var options = group.Options as Options; - var lambd = options.lambd.Value; - var alpha = options.alpha.Value; - var weight_decay = options.weight_decay.Value; - var t0 = options.t0.Value; - var lr = options.LearningRate.Value; + return _step(group => { - foreach (var param in group.Parameters) { + var options = group.Options as Options; + var lambd = options.lambd.Value; + var alpha = options.alpha.Value; + var weight_decay = options.weight_decay.Value; + var t0 = options.t0.Value; + var lr = options.LearningRate.Value; - var grad = param.grad(); + foreach (var param in group.Parameters) { - if (grad is null) continue; + var grad = param.grad(); - if (grad.is_sparse) throw new ArgumentException("ASGD does not support sparse gradients"); + if (grad is null) continue; - var state = _state[param.handle]; + if (grad.is_sparse) throw new ArgumentException("ASGD does not support sparse gradients"); - state.step += 1; + var state = _state[param.handle]; - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); + state.step += 1; - param.mul_(1 - lambd * state.eta); - param.add_(grad, alpha: -state.eta); + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); - if (state.mu != 1) { - state.ax.add_(param.sub(state.ax).mul(state.mu)); - } else { - state.ax.copy_(param); - } + param.mul_(1 - lambd * state.eta); + param.add_(grad, alpha: -state.eta); - state.eta = lr / Math.Pow((1 + lambd * lr * state.step), alpha); - state.mu = 1 / Math.Max(1, state.step - t0); - } + if (state.mu != 1) { + state.ax.add_(param.sub(state.ax).mul(state.mu)); + } else { + state.ax.copy_(param); } - d.DisposeEverything(); + state.eta = lr / Math.Pow((1 + lambd * lr * state.step), alpha); + state.mu = 1 / Math.Max(1, state.step - t0); } - } - - return loss; + }, closure); } protected override void Dispose(bool disposing) @@ -1795,80 +1712,95 @@ public class Rprop : OptimizerHelper, ILearningRateController /// /// It has been proposed in Adam: A Method for Stochastic Optimization. /// - /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Parameters to optimize. /// Learning rate /// Multiplicative increase factor. /// Multiplicative decrease factor. /// Minimum allowed step size. /// Maximum allowed step size. /// - public Rprop(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) : base(named_parameters, lr) + public Rprop(IEnumerable parameters, double lr = 0.01, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, etaminus, etaplus, min_step, max_step) { - LearningRate = lr; - InitialLearningRate = lr; - _etaminus = etaminus; - _etaplus = etaplus; - _min_step = min_step; - _max_step = max_step; - - foreach (var (name, p) in _parameters) { - var state = new State(); - _state[name] = state; - state.step = 0; - state.prev = torch.zeros_like(p); - state.step_size = p.new_empty(p.shape).fill_(LearningRate); - } } - public override Tensor step(Func closure = null) + /// + /// Implements Rprop algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. + /// Learning rate + /// Multiplicative increase factor. + /// Multiplicative decrease factor. + /// Minimum allowed step size. + /// Maximum allowed step size. + /// + public Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) { - Tensor loss = null; + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (closure != null) { - loss = closure(); - } + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + etaminus = etaminus, + etaplus = etaplus, + min_step = min_step, + max_step = max_step + }; - using (var _ = torch.no_grad()) { + _defaults = options; + _parameter_groups = new List(); - using (var d = torch.NewDisposeScope()) { + foreach (var g in parameters) { + add_param_group(g); + } + } + + public override Tensor step(Func closure = null) + { + return _step(group => { - foreach (var (name, p) in _parameters) { + var options = group.Options as Options; + var etaminus = options.etaminus.Value; + var etaplus = options.etaplus.Value; + var min_step = options.min_step.Value; + var max_step = options.max_step.Value; + var lr = options.LearningRate.Value; - var grad = p.grad(); + foreach (var param in group.Parameters) { - if (grad is null) continue; + var grad = param.grad(); - if (grad.is_sparse) throw new ArgumentException("Rprop does not support sparse gradients"); + if (grad is null) continue; - var state = _state[name]; + if (grad.is_sparse) throw new ArgumentException("Rprop does not support sparse gradients"); - state.step += 1; + var state = _state[param.handle]; - grad = (_max_step != 0) - ? grad.add(p, alpha: _max_step) - : grad.alias(); + state.step += 1; - var sign = grad.mul(state.prev).sign(); - sign[sign.gt(0)] = (Tensor)_etaplus; - sign[sign.lt(0)] = (Tensor)_etaminus; - sign[sign.eq(0)] = (Tensor)1; + grad = (max_step != 0) + ? grad.add(param, alpha: max_step) + : grad.alias(); - state.step_size.mul_(sign).clamp_(_min_step, _max_step); + var sign = grad.mul(state.prev).sign(); + sign[sign.gt(0)] = (Tensor)etaplus; + sign[sign.lt(0)] = (Tensor)etaminus; + sign[sign.eq(0)] = (Tensor)1; - grad = grad.clone(); + state.step_size.mul_(sign).clamp_(min_step, max_step); - grad.index_put_(0, sign.eq(_etaminus)); + grad = grad.clone(); - p.addcmul_(grad.sign(), state.step_size, -1); + grad.index_put_(0, sign.eq(etaminus)); - state.prev.copy_(grad); - } + param.addcmul_(grad.sign(), state.step_size, -1); - d.DisposeEverything(); + state.prev.copy_(grad); } - } - return loss; + }, closure); } protected override void Dispose(bool disposing) @@ -1887,14 +1819,60 @@ private class State public Tensor step_size; } - private Dictionary _state = new Dictionary(); - private double _etaminus; - private double _etaplus; - private double _min_step; - private double _max_step; + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.etaminus.HasValue) opt.etaminus = def.etaminus; + if (!opt.etaplus.HasValue) opt.etaplus = def.etaplus; + if (!opt.min_step.HasValue) opt.min_step = def.min_step; + if (!opt.max_step.HasValue) opt.max_step = def.max_step; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.prev = torch.zeros_like(p); + state.step_size = p.new_empty(p.shape).fill_(LearningRate); + } + } + + public class Options : OptimizerOptions + { + public double? etaminus; + public double? etaplus; + public double? min_step; + public double? max_step; + } + + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + : base(parameters, new Rprop.Options { LearningRate = lr, etaminus = etaminus, etaplus = etaplus, min_step = min_step, max_step = max_step }) + { + } + } + + private Dictionary _state = new Dictionary(); + } - public class Adagrad : NewOptimizerHelper + public class Adagrad : OptimizerHelper { /// /// Implements Adagrad algorithm. @@ -1951,59 +1929,44 @@ public Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_d public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - using (var _ = torch.enable_grad()) - loss = closure(); - } - - using (var _ = torch.no_grad()) { + return _step(group => { - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { - - var options = group.Options as Options; - var lr_decay = options.lr_decay.Value; - var weight_decay = options.weight_decay.Value; - var eps = options.eps.Value; - var initial_accumulator_value = options.initial_accumulator_value.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { + var options = group.Options as Options; + var lr_decay = options.lr_decay.Value; + var weight_decay = options.weight_decay.Value; + var eps = options.eps.Value; + var initial_accumulator_value = options.initial_accumulator_value.Value; + var lr = options.LearningRate.Value; - var state = _state[param.handle]; + foreach (var param in group.Parameters) { - var grad = param.grad(); + var state = _state[param.handle]; - if (grad is null) continue; + var grad = param.grad(); - state.step += 1; + if (grad is null) continue; - if (weight_decay != 0) { - grad = grad.add(param, alpha: weight_decay); - } + state.step += 1; - var clr = lr / (1 + (state.step - 1) * lr_decay); + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } - if (grad.is_sparse) { - throw new NotImplementedException("Adagrad optimization over sparse parameters"); - } else if (torch.is_complex(param)) { - throw new NotImplementedException("Adagrad optimization over complex parameters"); - } else { - state.sum.addcmul_(grad, grad, value: 1); - var std = state.sum.sqrt().add_(eps); - param.addcdiv_(grad, std, value: -clr); - } - } + var clr = lr / (1 + (state.step - 1) * lr_decay); - d.DisposeEverything(); + if (grad.is_sparse) { + throw new NotImplementedException("Adagrad optimization over sparse parameters"); + } else if (torch.is_complex(param)) { + throw new NotImplementedException("Adagrad optimization over complex parameters"); + } else { + state.sum.addcmul_(grad, grad, value: 1); + var std = state.sum.sqrt().add_(eps); + param.addcdiv_(grad, std, value: -clr); } } - } - return loss; + + }, closure); } protected override void Dispose(bool disposing) @@ -2074,7 +2037,7 @@ public ParamGroup(IEnumerable parameters, double lr = 0.01, double lr private Dictionary _state = new Dictionary(); } - public class Adam : NewOptimizerHelper, IBetas + public class Adam : OptimizerHelper, IBetas { /// /// Implements Adam algorithm. @@ -2138,69 +2101,53 @@ public Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - using (var _ = torch.enable_grad()) - loss = closure(); - } - - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { - - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var weight_decay = options.weight_decay.Value; - var amsgrad = options.amsgrad.Value; - var maximize = options.maximize.Value; - var eps = options.eps.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { + return _step(group => { - var state = _state[param.handle]; + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var weight_decay = options.weight_decay.Value; + var amsgrad = options.amsgrad.Value; + var maximize = options.maximize.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; - var grad = (maximize) ? -param.grad() : param.grad(); + foreach (var param in group.Parameters) { - if (grad is null) continue; + var state = _state[param.handle]; - state.step += 1; + var grad = (maximize) ? -param.grad() : param.grad(); - var bias_correction1 = 1 - Math.Pow(beta1, state.step); - var bias_correction2 = 1 - Math.Pow(beta2, state.step); + if (grad is null) continue; - if (weight_decay != 0) { - grad = grad.add(param, alpha: weight_decay); - } + state.step += 1; - state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - // When complex types are supported: - //state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad.conj(), value: 1 - _beta2) - state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); - Tensor denom = null; - if (amsgrad) { - var t0 = state.max_exp_avg_sq; - state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); - t0.Dispose(); - denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); - } else { - denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); - } + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } - var step_size = lr / bias_correction1; - param.addcdiv_(state.exp_avg, denom, value: -step_size); - } + state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + // When complex types are supported: + //state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad.conj(), value: 1 - _beta2) + state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + + Tensor denom = null; + if (amsgrad) { + var t0 = state.max_exp_avg_sq; + state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); + t0.Dispose(); + denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } else { + denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); } - d.DisposeEverything(); - } - } - return loss; + var step_size = lr / bias_correction1; + param.addcdiv_(state.exp_avg, denom, value: -step_size); + } + }, closure); } protected override void Dispose(bool disposing) @@ -2292,7 +2239,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double be private Dictionary _state = new Dictionary(); } - public class AdamW : NewOptimizerHelper, IBetas + public class AdamW : OptimizerHelper, IBetas { /// /// Implements AdamW algorithm. @@ -2356,65 +2303,49 @@ public AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 public override Tensor step(Func closure = null) { - Tensor loss = null; - - if (closure != null) { - using (var _ = torch.enable_grad()) - loss = closure(); - } - - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { - - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var weight_decay = options.weight_decay.Value; - var amsgrad = options.amsgrad.Value; - var maximize = options.maximize.Value; - var eps = options.eps.Value; - var lr = options.LearningRate.Value; + return _step(group => { - foreach (var param in group.Parameters) { + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var weight_decay = options.weight_decay.Value; + var amsgrad = options.amsgrad.Value; + var maximize = options.maximize.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; - var state = _state[param.handle]; + foreach (var param in group.Parameters) { - var grad = (maximize) ? -param.grad() : param.grad(); + var state = _state[param.handle]; - if (grad is null) continue; + var grad = (maximize) ? -param.grad() : param.grad(); - state.step += 1; + if (grad is null) continue; - param.mul_(1 - LearningRate * weight_decay); + state.step += 1; - var bias_correction1 = 1 - Math.Pow(beta1, state.step); - var bias_correction2 = 1 - Math.Pow(beta2, state.step); + param.mul_(1 - LearningRate * weight_decay); - state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); - Tensor denom = null; - if (amsgrad) { - var t0 = state.max_exp_avg_sq; - state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); - t0.Dispose(); - denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); - } else { - denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); - } + state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); - var step_size = lr / bias_correction1; - param.addcdiv_(state.exp_avg, denom, value: -step_size); - } + Tensor denom = null; + if (amsgrad) { + var t0 = state.max_exp_avg_sq; + state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); + t0.Dispose(); + denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } else { + denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); } - d.DisposeEverything(); - } - } - return loss; + var step_size = lr / bias_correction1; + param.addcdiv_(state.exp_avg, denom, value: -step_size); + } + }, closure); } protected override void Dispose(bool disposing) @@ -2506,7 +2437,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double be private Dictionary _state = new Dictionary(); } - public class RMSProp : NewOptimizerHelper, IMomentum + public class RMSProp : OptimizerHelper, IMomentum { /// @@ -2568,68 +2499,51 @@ public RMSProp(IEnumerable parameters, double lr = 1e-3, double alph public override Tensor step(Func closure = null) { - Tensor loss = null; + return _step(group => { - if (closure != null) { - using (var _ = torch.enable_grad()) - loss = closure(); - } - - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { - - var options = group.Options as Options; - var momentum = options.momentum.Value; - var alpha = options.alpha.Value; - var weight_decay = options.weight_decay.Value; - var centered = options.centered.Value; - var eps = options.eps.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { + var options = group.Options as Options; + var momentum = options.momentum.Value; + var alpha = options.alpha.Value; + var weight_decay = options.weight_decay.Value; + var centered = options.centered.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; - var state = _state[param.handle]; + foreach (var param in group.Parameters) { - var grad = param.grad(); + var state = _state[param.handle]; - if (grad is null) continue; + var grad = param.grad(); - state.step += 1; + if (grad is null) continue; - if (weight_decay != 0) { - grad = grad.add(param, alpha: weight_decay); - } + state.step += 1; - state.square_avg.mul_(alpha).addcmul_(grad, grad, value: 1 - alpha); + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } - Tensor avg = null; + state.square_avg.mul_(alpha).addcmul_(grad, grad, value: 1 - alpha); - if (centered) { - var grad_avg = state.grad_avg; - grad_avg.mul_(alpha).add_(grad, alpha: 1 - alpha); - avg = state.square_avg.addcmul(grad_avg, grad_avg, value: -1).sqrt_().add_(eps); - } else { - avg = state.square_avg.sqrt().add_(eps); - } + Tensor avg = null; - if (momentum > 0) { - var buf = state.momentum_buffer; - buf.mul_(momentum).addcdiv_(grad, avg); - param.add_(buf, alpha: -lr); - } else { - param.addcdiv_(grad, avg, -lr); - } - } + if (centered) { + var grad_avg = state.grad_avg; + grad_avg.mul_(alpha).add_(grad, alpha: 1 - alpha); + avg = state.square_avg.addcmul(grad_avg, grad_avg, value: -1).sqrt_().add_(eps); + } else { + avg = state.square_avg.sqrt().add_(eps); } - d.DisposeEverything(); + if (momentum > 0) { + var buf = state.momentum_buffer; + buf.mul_(momentum).addcdiv_(grad, avg); + param.add_(buf, alpha: -lr); + } else { + param.addcdiv_(grad, avg, -lr); + } } - } - - return loss; + }, closure); } protected override void Dispose(bool disposing) @@ -2708,7 +2622,9 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double ep public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } } - // The following optimizers are just wrappers for the native code implementations. + // The following optimizers are wrappers for the native code implementations. + // + // LBGFS does not allow for param groups, so there's no need to use anything but the native implementation. public class LBFGSOptimizer : Optimizer, ILearningRateController { @@ -2716,6 +2632,7 @@ public LBFGSOptimizer(IntPtr handle, double lr) : base(handle) { _rate = lr; InitialLearningRate = lr; + _paramGroups = new ParamGroup[] { new ParamGroup { Parameters = this.parameters(), Options = { LearningRate = lr, InitialLearningRate = lr } } }; } [DllImport("LibTorchSharp")] @@ -2728,6 +2645,9 @@ public double LearningRate { public double InitialLearningRate { get; set; } + public IEnumerable ParamGroups { get => _paramGroups; } + + public override Tensor step(Func closure = null) { if (closure == null) @@ -2735,7 +2655,8 @@ public override Tensor step(Func closure = null) return base.step(closure); } - private double _rate; + public double _rate; + private ParamGroup[] _paramGroups; } } } diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index 3b693258b..b4936ca37 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -970,7 +970,45 @@ public void TestTrainingRprop() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Rprop(seq.named_parameters()); + var optimizer = torch.optim.Rprop(seq.parameters()); + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + } + Assert.True(finalLoss < initialLoss); + } + + + [Fact] + public void TestTrainingRpropParamGroups() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var optimizer = torch.optim.Rprop(new Rprop.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); + var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); From a6c2cdc6fe43fe406cd05f5fc708afa73f39c756 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 15 Feb 2022 16:20:44 -0800 Subject: [PATCH 13/25] Adjusted LR schedulers to handle parameter groups. --- src/Examples/SequenceToSequence.cs | 3 +- src/Examples/TextClassification.cs | 4 +- src/FSharp.Examples/SequenceToSequence.fs | 3 +- src/FSharp.Examples/TextClassification.fs | 3 +- src/TorchSharp/NN/LRScheduler.cs | 593 +++++++++++++++++----- src/TorchSharp/NN/Optimizer.cs | 121 +++-- test/TorchSharpTest/TestTraining.cs | 305 ++++++++++- 7 files changed, 825 insertions(+), 207 deletions(-) diff --git a/src/Examples/SequenceToSequence.cs b/src/Examples/SequenceToSequence.cs index 10f2f8e67..9d2368ecf 100644 --- a/src/Examples/SequenceToSequence.cs +++ b/src/Examples/SequenceToSequence.cs @@ -88,7 +88,8 @@ internal static void Main(string[] args) var val_loss = evaluate(valid_data, model, loss, bptt, ntokens, optimizer); sw.Stop(); - Console.WriteLine($"\nEnd of epoch: {epoch} | lr: {optimizer.LearningRate:0.00} | time: {sw.Elapsed.TotalSeconds:0.0}s | loss: {val_loss:0.00}\n"); + var pgFirst = optimizer.ParamGroups.First(); + Console.WriteLine($"\nEnd of epoch: {epoch} | lr: {pgFirst.LearningRate:0.00} | time: {sw.Elapsed.TotalSeconds:0.0}s | loss: {val_loss:0.00}\n"); scheduler.step(); } diff --git a/src/Examples/TextClassification.cs b/src/Examples/TextClassification.cs index 6e74b6173..eae34de94 100644 --- a/src/Examples/TextClassification.cs +++ b/src/Examples/TextClassification.cs @@ -80,7 +80,9 @@ internal static void Main(string[] args) sw.Stop(); - Console.WriteLine($"\nEnd of epoch: {epoch} | lr: {optimizer.LearningRate:0.0000} | time: {sw.Elapsed.TotalSeconds:0.0}s\n"); + var pgFirst = optimizer.ParamGroups.First(); + + Console.WriteLine($"\nEnd of epoch: {epoch} | lr: {pgFirst.LearningRate:0.00} | time: {sw.Elapsed.TotalSeconds:0.0}s\n"); scheduler.step(); } } diff --git a/src/FSharp.Examples/SequenceToSequence.fs b/src/FSharp.Examples/SequenceToSequence.fs index e2b119d11..0b5732adb 100644 --- a/src/FSharp.Examples/SequenceToSequence.fs +++ b/src/FSharp.Examples/SequenceToSequence.fs @@ -271,7 +271,8 @@ let run epochs = let val_loss = evaluate model valid_data ntokens sw.Stop() - let lrStr = optimizer.LearningRate.ToString("0.00") + let pgFirst = optimizer.ParamGroups.First() + let lrStr = pgFirst.LearningRate.ToString("0.00") let elapsed = sw.Elapsed.TotalSeconds.ToString("0.0") let lossStr = val_loss.ToString("0.00") diff --git a/src/FSharp.Examples/TextClassification.fs b/src/FSharp.Examples/TextClassification.fs index 0213b9708..b12c8e072 100644 --- a/src/FSharp.Examples/TextClassification.fs +++ b/src/FSharp.Examples/TextClassification.fs @@ -151,7 +151,8 @@ let run epochs = sw.Stop() - let lrStr = optimizer.LearningRate.ToString("0.0000") + let pgFirst = optimizer.ParamGroups.First() + let lrStr = pgFirst.LearningRate.ToString("0.0000") let tsStr = sw.Elapsed.TotalSeconds.ToString("0.0") printfn $"\nEnd of epoch: {epoch} | lr: {lrStr} | time: {tsStr}s\n" scheduler.step() |> ignore diff --git a/src/TorchSharp/NN/LRScheduler.cs b/src/TorchSharp/NN/LRScheduler.cs index 3f27f1564..5504ff719 100644 --- a/src/TorchSharp/NN/LRScheduler.cs +++ b/src/TorchSharp/NN/LRScheduler.cs @@ -26,16 +26,18 @@ protected LRScheduler() } - protected LRScheduler(ILearningRateController optimizer, int last_epoch = -1, bool verbose = false) + protected LRScheduler(Optimizer optimizer, int last_epoch = -1, bool verbose = false) { _optimizer = optimizer; _last_epoch = last_epoch; _verbose = verbose; - _base_lr = optimizer.InitialLearningRate; - _last_lr = optimizer.LearningRate; + _base_lrs = optimizer.ParamGroups.Select(pg => pg.InitialLearningRate).ToList(); + _last_lrs = optimizer.ParamGroups.Select(pg => pg.LearningRate).ToList(); if (last_epoch == -1) { - optimizer.InitialLearningRate = optimizer.LearningRate; + foreach (var pg in optimizer.ParamGroups) { + pg.InitialLearningRate = pg.LearningRate; + } } } @@ -48,28 +50,32 @@ public virtual void step() _step_count += 1; _last_epoch += 1; - // NOTE: It is super-important to use the 'get_lr()' method once per step(), since + // NOTE: It is super-important to use the 'get_lr()' method no more than once per step(), since // for many LR schedulers, it will modify the internal state of the scheduler, // as well as that of the controlled optimizer. - var lr = get_lr(); + var lr = get_lr().ToList(); + var pgs = _optimizer.ParamGroups.ToList(); + + for (int i = 0; i < _base_lrs.Count; i++) { - _optimizer.LearningRate = lr; - if (_verbose && _last_lr != lr) - Console.WriteLine($"Adjusting learning rate to {lr}"); - _last_lr = lr; + pgs[i].LearningRate = lr[i]; + if (_verbose && _last_lrs[i] != lr[i]) + Console.WriteLine($"Adjusting learning rate to {lr[i]}"); + _last_lrs[i] = lr[i]; + } } /// /// Compute the current learning rate for the scheduler. /// - protected virtual double get_lr() => _optimizer.LearningRate; + protected virtual IEnumerable get_lr() => _optimizer.ParamGroups.Select(pg => pg.LearningRate); - protected ILearningRateController _optimizer; + protected Optimizer _optimizer; protected int _last_epoch = -1; - protected double _last_lr = 0; protected bool _verbose = false; protected int _step_count = 0; - protected double _base_lr; + protected IList _last_lrs; + protected IList _base_lrs; } public static partial class impl @@ -88,10 +94,26 @@ public class LambdaLR : LRScheduler /// The index of last epoch. Default: -1. /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public LambdaLR(ILearningRateController optimizer, Func lr_lambda, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + public LambdaLR(Optimizer optimizer, Func lr_lambda, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + { + if (optimizer == null) throw new ArgumentNullException("optimizer"); + _lr_lambdas = Enumerable.Repeat(lr_lambda, optimizer.ParamGroups.Count()).ToList(); + + step(); + } + + /// + /// Constructor + /// + /// Wrapped optimizer. + /// A list of functions, one for each paramater group, which computes a multiplicative factor given an integer parameter epoch. + /// The index of last epoch. Default: -1. + /// If true, prints a message to stdout for each update. Default: false. + /// A scheduler + public LambdaLR(Optimizer optimizer, IEnumerable> lr_lambdas, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) { if (optimizer == null) throw new ArgumentNullException("optimizer"); - _lr_lambda = lr_lambda; + _lr_lambdas = lr_lambdas.ToList(); step(); } @@ -99,12 +121,13 @@ public LambdaLR(ILearningRateController optimizer, Func lr_lambda, /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { - return _base_lr * _lr_lambda(_last_epoch); + var pgs = _optimizer.ParamGroups.ToList(); + return Enumerable.Range(0, pgs.Count).Select(i => _base_lrs[i] * _lr_lambdas[i](_last_epoch)); } - private Func _lr_lambda; + private List> _lr_lambdas; } /// @@ -113,6 +136,22 @@ protected override double get_lr() /// public class MultiplicativeLR : LRScheduler { + /// + /// Constructor + /// + /// Wrapped optimizer. + /// A list of functions, one for each paramater group, which computes a multiplicative factor given an integer parameter epoch. + /// The index of last epoch. Default: -1. + /// If true, prints a message to stdout for each update. Default: false. + /// A scheduler + public MultiplicativeLR(Optimizer optimizer, IEnumerable> lr_lambdas, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + { + if (optimizer == null) throw new ArgumentNullException("optimizer"); + _lr_lambdas = lr_lambdas.ToList(); + + step(); + } + /// /// Constructor /// @@ -121,10 +160,10 @@ public class MultiplicativeLR : LRScheduler /// The index of last epoch. Default: -1. /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public MultiplicativeLR(ILearningRateController optimizer, Func lr_lambda, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + public MultiplicativeLR(Optimizer optimizer, Func lr_lambda, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) { if (optimizer == null) throw new ArgumentNullException("optimizer"); - _lr_lambda = lr_lambda; + _lr_lambdas = Enumerable.Repeat(lr_lambda, optimizer.ParamGroups.Count()).ToList(); step(); } @@ -132,14 +171,15 @@ public MultiplicativeLR(ILearningRateController optimizer, Func lr_ /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { + var pgs = _optimizer.ParamGroups.ToList(); return (_last_epoch > 0) - ? _optimizer.LearningRate * _lr_lambda(_last_epoch) - : _optimizer.LearningRate; + ? Enumerable.Range(0, pgs.Count).Select(i => pgs[i].LearningRate * _lr_lambdas[i](_last_epoch)) + : Enumerable.Range(0, pgs.Count).Select(i => pgs[i].LearningRate); } - private Func _lr_lambda; + private List> _lr_lambdas; } /// @@ -158,7 +198,7 @@ public class StepLR : LRScheduler /// The index of last epoch. Default: -1. /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public StepLR(ILearningRateController optimizer, int step_size, double gamma = 0.1, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + public StepLR(Optimizer optimizer, int step_size, double gamma = 0.1, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) { if (optimizer == null) throw new ArgumentNullException("optimizer"); _step_size = step_size; @@ -170,11 +210,11 @@ public StepLR(ILearningRateController optimizer, int step_size, double gamma = 0 /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { return (_last_epoch == 0) || (_last_epoch % _step_size != 0) - ? _optimizer.LearningRate - : _optimizer.LearningRate * _gamma; + ? _optimizer.ParamGroups.Select(pg => pg.LearningRate) + : _optimizer.ParamGroups.Select(pg => pg.LearningRate * _gamma); } private int _step_size; @@ -197,7 +237,7 @@ public class MultiStepLR : LRScheduler /// The index of last epoch. Default: -1. /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public MultiStepLR(ILearningRateController optimizer, IList milestones, double gamma = 0.1, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + public MultiStepLR(Optimizer optimizer, IList milestones, double gamma = 0.1, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) { if (optimizer == null) throw new ArgumentNullException("optimizer"); _milestones = milestones; @@ -209,12 +249,12 @@ public MultiStepLR(ILearningRateController optimizer, IList milestones, dou /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { var idx = _milestones.IndexOf(_last_epoch); return idx == -1 - ? _optimizer.LearningRate - : _optimizer.LearningRate * Math.Pow(_gamma, _milestones[idx]); + ? _optimizer.ParamGroups.Select(pg => pg.LearningRate) + : _optimizer.ParamGroups.Select(pg => pg.LearningRate * Math.Pow(_gamma, _milestones[idx])); } private IList _milestones; @@ -236,7 +276,7 @@ public class ExponentialLR : LRScheduler /// The index of last epoch. Default: -1. /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public ExponentialLR(ILearningRateController optimizer, double gamma = 0.1, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + public ExponentialLR(Optimizer optimizer, double gamma = 0.1, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) { if (optimizer == null) throw new ArgumentNullException("optimizer"); _gamma = gamma; @@ -247,11 +287,11 @@ public ExponentialLR(ILearningRateController optimizer, double gamma = 0.1, int /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { return (_last_epoch == 0) - ? _optimizer.LearningRate - : _optimizer.LearningRate * _gamma; + ? _optimizer.ParamGroups.Select(pg => pg.LearningRate) + : _optimizer.ParamGroups.Select(pg => pg.LearningRate * _gamma); } private double _gamma; @@ -273,7 +313,7 @@ public class ConstantLR : LRScheduler /// The index of last epoch. Default: -1. /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public ConstantLR(ILearningRateController optimizer, double factor = 1.0 / 3, int total_iters = 5, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + public ConstantLR(Optimizer optimizer, double factor = 1.0 / 3, int total_iters = 5, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) { if (optimizer == null) throw new ArgumentNullException("optimizer"); _factor = factor; @@ -285,14 +325,14 @@ public ConstantLR(ILearningRateController optimizer, double factor = 1.0 / 3, in /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { if (_last_epoch == 0) { - return _optimizer.LearningRate * _factor; + return _optimizer.ParamGroups.Select(pg => pg.LearningRate * _factor); } else if (_last_epoch == _total_iters) { - return _optimizer.LearningRate * (1.0 / _factor); + return _optimizer.ParamGroups.Select(pg => pg.LearningRate * (1.0 / _factor)); } else { - return _optimizer.LearningRate; + return _optimizer.ParamGroups.Select(pg => pg.LearningRate); } } @@ -319,7 +359,7 @@ public class LinearLR : LRScheduler /// The index of last epoch. Default: -1. /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public LinearLR(ILearningRateController optimizer, double start_factor = 1.0 / 3, double end_factor = 5, int total_iters = 5, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + public LinearLR(Optimizer optimizer, double start_factor = 1.0 / 3, double end_factor = 5, int total_iters = 5, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) { if (optimizer == null) throw new ArgumentNullException("optimizer"); _start_factor = start_factor; @@ -332,15 +372,16 @@ public LinearLR(ILearningRateController optimizer, double start_factor = 1.0 / 3 /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { if (_last_epoch == 0) { - return _optimizer.LearningRate * _start_factor; - } else if (_last_epoch > _total_iters) { - return _optimizer.LearningRate; - } else { - return (_total_iters * _start_factor + (_last_epoch - 1) * (_end_factor - _start_factor)); - } + return _optimizer.ParamGroups.Select(pg => pg.LearningRate * _start_factor); + } else if (_last_epoch > _total_iters) { + return _optimizer.ParamGroups.Select(pg => pg.LearningRate); + } else { + var factor = (1 + (_end_factor - _start_factor)) / (_total_iters * _start_factor + (_last_epoch - 1) * (_end_factor - _start_factor)); + return _optimizer.ParamGroups.Select(pg => pg.LearningRate * factor); + } } private double _start_factor; @@ -364,7 +405,7 @@ public class CosineAnnealingLR : LRScheduler /// The index of last epoch. Default: -1. /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public CosineAnnealingLR(ILearningRateController optimizer, double T_max, double eta_min = 0, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + public CosineAnnealingLR(Optimizer optimizer, double T_max, double eta_min = 0, int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) { if (optimizer == null) throw new ArgumentNullException("optimizer"); _T_max = T_max; @@ -376,18 +417,17 @@ public CosineAnnealingLR(ILearningRateController optimizer, double T_max, double /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { if (_last_epoch == 0) { - return _optimizer.LearningRate; - } else if ((_last_epoch - 1 - _T_max) % (2 * _T_max) == 0) { - return _optimizer.LearningRate + (_base_lr - _eta_min) * - (1 - Math.Cos(Math.PI / _T_max)) / 2; - } else { - return (1 + Math.Cos(Math.PI * _last_epoch / _T_max)) / - (1 + Math.Cos(Math.PI * (_last_epoch - 1) / _T_max)) * - (_optimizer.LearningRate - _eta_min) + _eta_min; - } + return _optimizer.ParamGroups.Select(pg => pg.LearningRate); + } else if ((_last_epoch - 1 - _T_max) % (2 * _T_max) == 0) { + var pgs = _optimizer.ParamGroups.ToList(); + return Enumerable.Range(0, pgs.Count).Select(i => pgs[i].LearningRate + (_base_lrs[i] - _eta_min) * (1 - Math.Cos(Math.PI / _T_max)) / 2); + } else { + return _optimizer.ParamGroups.Select(pg => (1 + Math.Cos(Math.PI * _last_epoch / _T_max)) / + (1 + Math.Cos(Math.PI * (_last_epoch - 1) / _T_max)) * (pg.LearningRate - _eta_min) + _eta_min); + } } private double _T_max; @@ -419,7 +459,7 @@ public enum ScaleMode /// /// Constructor /// - public CyclicLR(ILearningRateController optimizer, + public CyclicLR(Optimizer optimizer, double base_lr, double max_lr, int step_size_up = 2000, @@ -436,14 +476,73 @@ public CyclicLR(ILearningRateController optimizer, { if (optimizer == null) throw new ArgumentNullException("optimizer"); + var pgCount = optimizer.ParamGroups.Count(); + + Initialize( + optimizer, + Enumerable.Repeat(base_lr, pgCount), + Enumerable.Repeat(max_lr, pgCount), + step_size_up, + step_size_down, + mode, + gamma, + scale_fn, + scale_mode, + cycle_momentum, + Enumerable.Repeat(base_momentum, pgCount), + Enumerable.Repeat(max_momentum, pgCount)); + } + + /// + /// Constructor + /// + public CyclicLR(Optimizer optimizer, + IEnumerable base_lr, + IEnumerable max_lr, + int step_size_up = 2000, + int step_size_down = -1, + Mode mode = Mode.Triangular, + double gamma = 1.0, + Func scale_fn = null, + ScaleMode scale_mode = ScaleMode.Cycle, + bool cycle_momentum = true, + IEnumerable base_momentum = null, + IEnumerable max_momentum = null, + int last_epoch = -1, + bool verbose = false) : base(optimizer, last_epoch, verbose) + { + if (optimizer == null) throw new ArgumentNullException("optimizer"); + + Initialize( + optimizer, + base_lr, + max_lr, + step_size_up, + step_size_down, + mode, + gamma, + scale_fn, + scale_mode, + cycle_momentum, + base_momentum, + max_momentum); + } + + private void Initialize(Optimizer optimizer, IEnumerable base_lr, IEnumerable max_lr, int step_size_up, int step_size_down, Mode mode, double gamma, Func scale_fn, ScaleMode scale_mode, bool cycle_momentum, IEnumerable base_momentum, IEnumerable max_momentum) + { double down = (step_size_down == -1) ? step_size_up : step_size_down; _total_size = step_size_up + down; _step_ratio = step_size_up / _total_size; - _max_lr = max_lr; - _base_lr = base_lr; + var pgs = optimizer.ParamGroups.ToList(); + + _max_lrs = max_lr.ToList(); + _base_lrs = base_lr.ToList(); + if (_last_epoch == -1) { - optimizer.LearningRate = base_lr; + for (int i = 0; i < pgs.Count; i++) { + pgs[i].LearningRate = _base_lrs[i]; + } } _mode = mode; @@ -451,15 +550,17 @@ public CyclicLR(ILearningRateController optimizer, _cycle_momentum = cycle_momentum; if (cycle_momentum) { - _momentum = optimizer as IMomentum; - if (_momentum == null && cycle_momentum) throw new ArgumentException($"optimizer must support momentum with `cycle_momentum` option enabled"); + var momentum = optimizer as IMomentum; + if (momentum == null && cycle_momentum) throw new ArgumentException($"optimizer must support momentum with `cycle_momentum` option enabled"); + + _base_momentum = (base_momentum is null) ? Enumerable.Repeat(0.8, pgs.Count).ToList() : base_momentum.ToList(); + _max_momentum = (max_momentum is null) ? Enumerable.Repeat(0.9, pgs.Count).ToList() : max_momentum.ToList(); if (_last_epoch == -1) { - _momentum.Momentum = base_momentum; + for (int i = 0; i < pgs.Count; i++) { + (pgs[i] as IMomentum).Momentum = _base_momentum[i]; + } } - - _base_momentum = base_momentum; - _max_momentum = max_momentum; } @@ -489,45 +590,50 @@ public CyclicLR(ILearningRateController optimizer, /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { + var pgs = _optimizer.ParamGroups.ToList(); + var cycle = Math.Floor(1.0 + _last_epoch / _total_size); - var x = 1.0 + _last_epoch / _total_size - cycle; + var x = 1.0 + _last_epoch / _total_size - cycle; + + var scale_factor = (x <= _step_ratio) ? x / _step_ratio : (x - 1) / (_step_ratio - 1); - var scale_factor = (x <= _step_ratio) ? x / _step_ratio : (x - 1) / (_step_ratio - 1); - var base_height = (_max_lr - _base_lr) * scale_factor; + return Enumerable.Range(0, pgs.Count).Select(i => { + + var base_height = (_max_lrs[i] - _base_lrs[i]) * scale_factor; var computed_lr = (_scale_mode == ScaleMode.Cycle) - ? _base_lr + base_height * _scale_func(cycle) - : _base_lr + base_height * _scale_func(_last_epoch); + ? _base_lrs[i] + base_height * _scale_func(cycle) + : _base_lrs[i] + base_height * _scale_func(_last_epoch); if (_cycle_momentum) { - base_height = (_max_momentum - _base_momentum) * scale_factor; + base_height = (_max_momentum[i] - _base_momentum[i]) * scale_factor; - _momentum.Momentum = (_scale_mode == ScaleMode.Cycle) - ? _max_momentum + base_height * _scale_func(cycle) - : _max_momentum + base_height * _scale_func(_last_epoch); + (pgs[i] as IMomentum).Momentum = (_scale_mode == ScaleMode.Cycle) + ? _max_momentum[i] + base_height * _scale_func(cycle) + : _max_momentum[i] + base_height * _scale_func(_last_epoch); } return computed_lr; + }); } private double _total_size; private double _step_ratio; - private double _max_lr; - private Func _scale_func; private ScaleMode _scale_mode; private bool _cycle_momentum; - private double _base_momentum; - private double _max_momentum; + private List _max_lrs; + + private List _base_momentum; + private List _max_momentum; - private IMomentum _momentum; private Mode _mode; private double _gamma; } @@ -546,7 +652,7 @@ public enum AnnealStrategy /// /// Constructor /// - public OneCycleLR(ILearningRateController optimizer, + public OneCycleLR(Optimizer optimizer, double max_lr, int total_steps = -1, int epochs = -1, @@ -563,11 +669,68 @@ public OneCycleLR(ILearningRateController optimizer, { if (optimizer == null) throw new ArgumentNullException("optimizer"); + var pgCount = _optimizer.ParamGroups.Count(); + + Initialize(optimizer, + Enumerable.Repeat(max_lr, pgCount), + total_steps, + epochs, + steps_per_epoch, + pct_start, anneal_strategy, + cycle_momentum, + Enumerable.Repeat(base_momentum, pgCount), + Enumerable.Repeat(max_momentum, pgCount), + div_factor, + final_div_factor, + three_phase, + last_epoch); + } + + /// + /// Constructor + /// + public OneCycleLR(Optimizer optimizer, + IEnumerable max_lr, + int total_steps = -1, + int epochs = -1, + int steps_per_epoch = -1, + double pct_start = 0.3, + AnnealStrategy anneal_strategy = AnnealStrategy.Cos, + bool cycle_momentum = true, + IEnumerable base_momentum = null, + IEnumerable max_momentum = null, + double div_factor = 25, + double final_div_factor = 1e4, + bool three_phase = false, + int last_epoch = -1, bool verbose = false) : base(optimizer, last_epoch, verbose) + { + if (optimizer == null) throw new ArgumentNullException("optimizer"); + + Initialize(optimizer, + max_lr, + total_steps, + epochs, + steps_per_epoch, + pct_start, + anneal_strategy, + cycle_momentum, + base_momentum, + max_momentum, + div_factor, + final_div_factor, + three_phase, + last_epoch); + } + + private void Initialize(Optimizer optimizer, IEnumerable max_lr, int total_steps, int epochs, int steps_per_epoch, double pct_start, AnnealStrategy anneal_strategy, bool cycle_momentum, IEnumerable base_momentum, IEnumerable max_momentum, double div_factor, double final_div_factor, bool three_phase, int last_epoch) + { _cycle_momentum = cycle_momentum; + var pgs = optimizer.ParamGroups.ToList(); + if (cycle_momentum) { - _momentum = optimizer as IMomentum; - _betas = optimizer as IBetas; + var _momentum = optimizer as IMomentum; + var _betas = optimizer as IBetas; if (_momentum == null && _betas == null) throw new ArgumentException($"optimizer must support momentum with `cycle_momentum` option enabled"); } @@ -584,11 +747,16 @@ public OneCycleLR(ILearningRateController optimizer, _total_steps = epochs * steps_per_epoch; } - var initial_lr = max_lr / div_factor; + var mlr = max_lr.ToList(); + + _betas = pgs.Select(pg => pg as IBetas).ToList(); + _momentum = pgs.Select(pg => pg as IMomentum).ToList(); - _phase_values["initial_lr"] = initial_lr; - _phase_values["max_lr"] = max_lr; - _phase_values["min_lr"] = initial_lr / final_div_factor; + var initial_lrs = max_lr.Select(mlr => mlr / div_factor).ToList(); + + _phase_values["initial_lr"] = initial_lrs; + _phase_values["max_lr"] = mlr; + _phase_values["min_lr"] = initial_lrs.Select(ilr => ilr / final_div_factor).ToList(); _annealing_func = (anneal_strategy == AnnealStrategy.Cos) ? (start, end, pct) => end + (start - end) / 2.0 * (Math.Cos(Math.PI * pct) + 1) @@ -608,14 +776,21 @@ public OneCycleLR(ILearningRateController optimizer, if (last_epoch == -1) { if (cycle_momentum) { - if (_betas != null) { - var (_, beta2) = _betas.Betas; - _betas.Betas = (max_momentum, beta2); - } else { - _momentum.Momentum = max_momentum; + + var _base_momentum = (base_momentum is null) ? Enumerable.Repeat(0.85, pgs.Count).ToList() : base_momentum.ToList(); + var _max_momentum = (max_momentum is null) ? Enumerable.Repeat(0.95, pgs.Count).ToList() : max_momentum.ToList(); + + for (int i = 0; i < pgs.Count; i++) { + if (_betas[i] != null) { + var (_, beta2) = _betas[i].Betas; + _betas[i].Betas = (_max_momentum[i], beta2); + } else { + if (_momentum[i] != null) + _momentum[i].Momentum = _max_momentum[i]; + } } - _phase_values["max_momentum"] = max_momentum; - _phase_values["base_momentum"] = base_momentum; + _phase_values["max_momentum"] = _max_momentum; + _phase_values["base_momentum"] = _base_momentum; } } step(); @@ -624,27 +799,31 @@ public OneCycleLR(ILearningRateController optimizer, /// /// Compute the current learning rate for the scheduler. /// - protected override double get_lr() + protected override IEnumerable get_lr() { var step_num = _last_epoch; - if (step_num > _total_steps) { - throw new InvalidOperationException($"Tried to step {step_num + 1} times. The specified number of total steps is {_total_steps}"); - } + if (step_num > _total_steps) { + throw new InvalidOperationException($"Tried to step {step_num + 1} times. The specified number of total steps is {_total_steps}"); + } + + var pgs = _optimizer.ParamGroups.ToList(); + + return Enumerable.Range(0, pgs.Count).Select(i => { double start_step = 0; double computed_lr = 0; double computed_momentum = 0; - for (var i = 0; i < _schedule_phases.Length; i++) { + for (var j = 0; j < _schedule_phases.Length; j++) { - var phase = _schedule_phases[i]; + var phase = _schedule_phases[j]; var end_step = phase.end_step; if (step_num <= end_step || i == _schedule_phases.Length - 1) { var pct = (step_num - start_step) / (end_step - start_step); - computed_lr = _annealing_func(_phase_values[phase.start_lr], _phase_values[phase.end_lr], pct); + computed_lr = _annealing_func(_phase_values[phase.start_lr][i], _phase_values[phase.end_lr][i], pct); if (_cycle_momentum) { - computed_momentum = _annealing_func(_phase_values[phase.start_momentum], _phase_values[phase.end_momentum], pct); + computed_momentum = _annealing_func(_phase_values[phase.start_momentum][i], _phase_values[phase.end_momentum][i], pct); } break; } @@ -652,25 +831,26 @@ protected override double get_lr() } if (_cycle_momentum) { - if (_betas != null) { - var (_, beta2) = _betas.Betas; - _betas.Betas = (computed_momentum, beta2); + if (_betas[i] != null) { + var (_, beta2) = _betas[i].Betas; + _betas[i].Betas = (computed_momentum, beta2); } else { - _momentum.Momentum = computed_momentum; + _momentum[i].Momentum = computed_momentum; } } return computed_lr; + }); } private Func _annealing_func; private PhaseDescriptor[] _schedule_phases; - private Dictionary _phase_values = new Dictionary(); + private Dictionary> _phase_values = new Dictionary>(); private bool _cycle_momentum; - private IBetas _betas; - private IMomentum _momentum; + private List _betas; + private List _momentum; private int _total_steps; @@ -725,7 +905,7 @@ public override void step() /// /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler StepLR(ILearningRateController optimizer, int step_size, double gamma = 0.1, int last_epoch = -1, bool verbose = false) + public static LRScheduler StepLR(Optimizer optimizer, int step_size, double gamma = 0.1, int last_epoch = -1, bool verbose = false) { return new impl.StepLR(optimizer, step_size, gamma, last_epoch, verbose); } @@ -743,7 +923,7 @@ public static LRScheduler StepLR(ILearningRateController optimizer, int step_siz /// /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler LambdaLR(ILearningRateController optimizer, Func lr_lambda, int last_epoch = -1, bool verbose = false) + public static LRScheduler LambdaLR(Optimizer optimizer, Func lr_lambda, int last_epoch = -1, bool verbose = false) { return new impl.LambdaLR(optimizer, lr_lambda, last_epoch, verbose); @@ -764,7 +944,7 @@ public static LRScheduler LambdaLR(ILearningRateController optimizer, Func /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler MultiStepLR(ILearningRateController optimizer, IList milestones, double gamma = 0.1, int last_epoch = -1, bool verbose = false) + public static LRScheduler MultiStepLR(Optimizer optimizer, IList milestones, double gamma = 0.1, int last_epoch = -1, bool verbose = false) { return new impl.MultiStepLR(optimizer, milestones, gamma, last_epoch, verbose); @@ -783,7 +963,7 @@ public static LRScheduler MultiStepLR(ILearningRateController optimizer, IList /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler MultiplicativeLR(ILearningRateController optimizer, Func lr_lambda, int last_epoch = -1, bool verbose = false) + public static LRScheduler MultiplicativeLR(Optimizer optimizer, Func lr_lambda, int last_epoch = -1, bool verbose = false) { return new impl.MultiplicativeLR(optimizer, lr_lambda, last_epoch, verbose); @@ -803,7 +983,7 @@ public static LRScheduler MultiplicativeLR(ILearningRateController optimizer, Fu /// /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler ExponentialLR(ILearningRateController optimizer, double gamma = 0.1, int last_epoch = -1, bool verbose = false) + public static LRScheduler ExponentialLR(Optimizer optimizer, double gamma = 0.1, int last_epoch = -1, bool verbose = false) { return new impl.ExponentialLR(optimizer, gamma, last_epoch, verbose); } @@ -823,7 +1003,7 @@ public static LRScheduler ExponentialLR(ILearningRateController optimizer, doubl /// /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler ConstantLR(ILearningRateController optimizer, double factor = 1.0 / 3, int total_iters = 5, int last_epoch = -1, bool verbose = false) + public static LRScheduler ConstantLR(Optimizer optimizer, double factor = 1.0 / 3, int total_iters = 5, int last_epoch = -1, bool verbose = false) { return new impl.ConstantLR(optimizer, factor, total_iters, last_epoch, verbose); } @@ -843,7 +1023,7 @@ public static LRScheduler ConstantLR(ILearningRateController optimizer, double f /// /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler LinearLR(ILearningRateController optimizer, double start_factor = 1.0 / 3, double end_factor = 5, int total_iters = 5, int last_epoch = -1, bool verbose = false) + public static LRScheduler LinearLR(Optimizer optimizer, double start_factor = 1.0 / 3, double end_factor = 5, int total_iters = 5, int last_epoch = -1, bool verbose = false) { return new impl.LinearLR(optimizer, start_factor, end_factor, total_iters, last_epoch, verbose); } @@ -871,7 +1051,7 @@ public static LRScheduler ChainedLR(IEnumerable schedulers) /// /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler CosineAnnealingLR(ILearningRateController optimizer, double T_max, double eta_min = 0, int last_epoch = -1, bool verbose = false) + public static LRScheduler CosineAnnealingLR(Optimizer optimizer, double T_max, double eta_min = 0, int last_epoch = -1, bool verbose = false) { return new impl.CosineAnnealingLR(optimizer, T_max, eta_min, last_epoch, verbose); } @@ -953,7 +1133,7 @@ public static LRScheduler CosineAnnealingLR(ILearningRateController optimizer, d /// claims that "unpublished work has shown even better results by using only two phases". To /// mimic the behaviour of the original paper instead, set ``three_phase= True``. /// - public static LRScheduler OneCycleLR(ILearningRateController optimizer, + public static LRScheduler OneCycleLR(Optimizer optimizer, double max_lr, int total_steps = -1, int epochs = -1, @@ -971,6 +1151,101 @@ public static LRScheduler OneCycleLR(ILearningRateController optimizer, return new impl.OneCycleLR(optimizer, max_lr, total_steps, epochs, steps_per_epoch, pct_start, anneal_strategy, cycle_momentum, base_momentum, max_momentum, div_factor, final_div_factor, three_phase, last_epoch, verbose); } + /// + /// Sets the learning rate of each parameter group according to the + /// 1cycle learning rate policy.The 1cycle policy anneals the learning + /// rate from an initial learning rate to some maximum learning rate and then + /// from that maximum learning rate to some minimum learning rate much lower + /// than the initial learning rate. + /// + /// This policy was initially described in the paper `Super-Convergence: + /// Very Fast Training of Neural Networks Using Large Learning Rates`_. + /// + /// The 1cycle learning rate policy changes the learning rate after every batch. + /// `step` should be called after a batch has been used for training. + /// + /// This scheduler is not chainable. + /// + /// Wrapped optimizer. + /// Upper learning rate boundaries in the cycle + /// + /// The total number of steps in the cycle. + /// Note that if a value is not provided here, then it must be inferred by providing a value for epochs and steps_per_epoch. + /// + /// + /// The number of epochs to train for. This is used along + /// with steps_per_epoch in order to infer the total number of steps in the cycle + /// if a value for total_steps is not provided. + /// + /// + /// The number of steps per epoch to train for. This is + /// used along with epochs in order to infer the total number of steps in the + /// cycle if a value for total_steps is not provided. + /// + /// The percentage of the cycle (in number of steps) spent increasing the learning rate. + /// Specifies the annealing strategy: "cos" for cosine annealing, "linear" for linear annealing. + /// If true, momentum is cycled inversely to learning rate between 'base_momentum' and 'max_momentum'. + /// + /// Lower momentum boundaries in the cycle + /// for each parameter group.Note that momentum is cycled inversely + /// to learning rate; at the peak of a cycle, momentum is + /// 'base_momentum' and learning rate is 'max_lr'. + /// + /// + /// Upper momentum boundaries in the cycle for each parameter group. + /// Functionally, it defines the cycle amplitude(max_momentum - base_momentum). + /// Note that momentum is cycled inversely to learning rate; at the start of a cycle, momentum is 'max_momentum' + /// and learning rate is 'base_lr' + /// + /// Determines the initial learning rate via initial_lr = max_lr/div_factor + /// Determines the minimum learning rate via min_lr = initial_lr/final_div_factor + /// + /// If ``True``, use a third phase of the schedule to annihilate the + /// learning rate according to 'final_div_factor' instead of modifying the second + /// phase (the first two phases will be symmetrical about the step indicated by 'pct_start'). + /// + /// + /// The index of the last batch. This parameter is used when resuming a training job.Since `step()` should be invoked after each + /// batch instead of after each epoch, this number represents the total number of *batches* computed, not the total number of epochs computed. + /// When last_epoch = -1, the schedule is started from the beginning. + /// + /// If true, prints a message to stdout for each update. Default: false. + /// A scheduler + /// + /// Note also that the total number of steps in the cycle can be determined in one + /// of two ways (listed in order of precedence): + /// + /// #. A value for total_steps is explicitly provided. + /// #. A number of epochs (epochs) and a number of steps per epoch + /// (steps_per_epoch) are provided. + /// In this case, the number of total steps is inferred by + /// total_steps = epochs * steps_per_epoch + /// + /// You must either provide a value for total_steps or provide a value for both + /// epochs and steps_per_epoch. + /// + /// The default behaviour of this scheduler follows the fastai implementation of 1cycle, which + /// claims that "unpublished work has shown even better results by using only two phases". To + /// mimic the behaviour of the original paper instead, set ``three_phase= True``. + /// + public static LRScheduler OneCycleLR(Optimizer optimizer, + IEnumerable max_lr, + int total_steps = -1, + int epochs = -1, + int steps_per_epoch = -1, + double pct_start = 0.3, + impl.OneCycleLR.AnnealStrategy anneal_strategy = impl.OneCycleLR.AnnealStrategy.Cos, + bool cycle_momentum = true, + IEnumerable base_momentum = null, + IEnumerable max_momentum = null, + double div_factor = 25, + double final_div_factor = 1e4, + bool three_phase = false, + int last_epoch = -1, bool verbose = false) + { + return new impl.OneCycleLR(optimizer, max_lr, total_steps, epochs, steps_per_epoch, pct_start, anneal_strategy, cycle_momentum, base_momentum, max_momentum, div_factor, final_div_factor, three_phase, last_epoch, verbose); + } + /// /// Sets the learning rate of each parameter group according to /// cyclical learning rate policy(CLR). The policy cycles the learning @@ -1021,7 +1296,7 @@ public static LRScheduler OneCycleLR(ILearningRateController optimizer, /// /// If true, prints a message to stdout for each update. Default: false. /// A scheduler - public static LRScheduler CyclicLR(ILearningRateController optimizer, + public static LRScheduler CyclicLR(Optimizer optimizer, double base_lr, double max_lr, int step_size_up = 2000, @@ -1038,6 +1313,74 @@ public static LRScheduler CyclicLR(ILearningRateController optimizer, { return new impl.CyclicLR(optimizer, base_lr, max_lr, step_size_up, step_size_down, mode, gamma, scale_fn, scale_mode, cycle_momentum, base_momentum, max_momentum, last_epoch, verbose); } + + /// + /// Sets the learning rate of each parameter group according to + /// cyclical learning rate policy(CLR). The policy cycles the learning + /// rate between two boundaries with a constant frequency, as detailed in + /// the paper `Cyclical Learning Rates for Training Neural Networks`_. + /// The distance between the two boundaries can be scaled on a per-iteration + /// or per-cycle basis. + /// + /// Cyclical learning rate policy changes the learning rate after every batch. + /// `step` should be called after a batch has been used for training. + /// + /// This class has three built-in policies, as put forth in the paper: + /// * "triangular": A basic triangular cycle without amplitude scaling. + /// * "triangular2": A basic triangular cycle that scales initial amplitude by half each cycle. + /// * "exp_range": A cycle that scales initial amplitude by gamma^(cycle iterations). + /// }` + /// at each cycle iteration. + /// /// + /// Wrapped optimizer. + /// Initial learning rate which is the lower boundary in the cycle + /// + /// Upper learning rate boundaries in the cycle for each parameter group. + /// Functionally, it defines the cycle amplitude(max_lr - base_lr). + /// The lr at any cycle is the sum of base_lr and some scaling of the amplitude; therefore + /// max_lr may not actually be reached depending on the scaling function. + /// + /// Number of training iterations in the increasing half of a cycle. + /// + /// Number of training iterations in the decreasing half of a cycle. + /// If step_size_down is -1, it is set to step_size_up. + /// + /// Values correspond to policies detailed above. If scale_fn is non-null, this argument is ignored. + /// Constant in 'exp_range' scaling function + /// Custom scaling policy defined by a single argument lambda function. If specified, then 'mode' is ignored. + /// Defines whether scale_fn is evaluated on cycle number or cycle iterations(training iterations since start of cycle) + /// If true, momentum is cycled inversely to learning rate between 'base_momentum' and 'max_momentum'. + /// Lower momentum boundaries in the cycle. Note that momentum is cycled inversely to learning rate + /// + /// Upper momentum boundaries in the cycle. + /// Functionally, it defines the cycle amplitude(max_momentum - base_momentum). + /// The momentum at any cycle is the difference of max_momentum and some scaling of the amplitude; therefore + /// base_momentum may not actually be reached depending on the scaling function. + /// + /// + /// The index of the last batch. This parameter is used when resuming a training job.Since `step()` should be invoked after each + /// batch instead of after each epoch, this number represents the total number of *batches* computed, not the total number of epochs computed. + /// When last_epoch = -1, the schedule is started from the beginning. + /// + /// If true, prints a message to stdout for each update. Default: false. + /// A scheduler + public static LRScheduler CyclicLR(Optimizer optimizer, + IEnumerable base_lr, + IEnumerable max_lr, + int step_size_up = 2000, + int step_size_down = -1, + impl.CyclicLR.Mode mode = impl.CyclicLR.Mode.Triangular, + double gamma = 1.0, + Func scale_fn = null, + impl.CyclicLR.ScaleMode scale_mode = impl.CyclicLR.ScaleMode.Cycle, + bool cycle_momentum = true, + IEnumerable base_momentum = null, + IEnumerable max_momentum = null, + int last_epoch = -1, + bool verbose = false) + { + return new impl.CyclicLR(optimizer, base_lr, max_lr, step_size_up, step_size_down, mode, gamma, scale_fn, scale_mode, cycle_momentum, base_momentum, max_momentum, last_epoch, verbose); + } } } } diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index 2c2806c91..0ec8b15e8 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -127,6 +127,12 @@ public virtual IEnumerable parameters() } return ptrArray.Select(x => new Parameter(x)); } + + public virtual IEnumerable ParamGroups { + get => _parameter_groups; + } + + protected IList _parameter_groups; } public interface ILearningRateController @@ -134,8 +140,6 @@ public interface ILearningRateController double LearningRate { set; get; } double InitialLearningRate { set; get; } - - IEnumerable ParamGroups { get; } } public interface IMomentum @@ -151,16 +155,16 @@ public interface IBetas [DllImport("LibTorchSharp")] private static extern IntPtr THSNN_LBFGS_ctor(IntPtr parameters, int len, double learningRate, long max_iter, long max_eval, double tolerange_grad, double tolerance_change, long history_size); - public static LBFGSOptimizer LBFGS(IEnumerable parameters, double learningRate = 0.01, long max_iter = 20, long? max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100) + public static LBFGS LBFGS(IEnumerable parameters, double lr = 0.01, long max_iter = 20, long? max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100) { if (!max_eval.HasValue) max_eval = 5 * max_iter / 4; var parray = new PinnedArray(); IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray()); - var res = THSNN_LBFGS_ctor(paramsRef, parray.Array.Length, learningRate, max_iter, max_eval.Value, tolerange_grad, tolerance_change, history_size); + var res = THSNN_LBFGS_ctor(paramsRef, parray.Array.Length, lr, max_iter, max_eval.Value, tolerange_grad, tolerance_change, history_size); if (res == IntPtr.Zero) { torch.CheckForErrors(); } - return new LBFGSOptimizer(res, learningRate); + return new LBFGS(res, lr); } /// @@ -169,16 +173,16 @@ public static LBFGSOptimizer LBFGS(IEnumerable parameters, double lea /// Proposed by G.Hinton in his course. /// /// Parameters to optimize - /// Learning rate (default: 1e-2) + /// Learning rate (default: 1e-2) /// Term added to the denominator to improve numerical stability (default: 1e-8) /// Smoothing constant (default: 0.99) /// Weight decay (L2 penalty) (default: 0) /// Momentum factor (default: 0) /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance /// - public static RMSProp RMSProp(IEnumerable parameters, double learningRate = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) + public static RMSProp RMSProp(IEnumerable parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) { - return new RMSProp(parameters, learningRate, alpha, eps, weight_decay, momentum, centered); + return new RMSProp(parameters, lr, alpha, eps, weight_decay, momentum, centered); } /// @@ -187,16 +191,16 @@ public static RMSProp RMSProp(IEnumerable parameters, double learning /// Proposed by G.Hinton in his course. /// /// Parameters to optimize - /// Learning rate (default: 1e-2) + /// Learning rate (default: 1e-2) /// Term added to the denominator to improve numerical stability (default: 1e-8) /// Smoothing constant (default: 0.99) /// Weight decay (L2 penalty) (default: 0) /// Momentum factor (default: 0) /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance /// - public static RMSProp RMSProp(IEnumerable parameters, double learningRate = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) + public static RMSProp RMSProp(IEnumerable parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) { - return new RMSProp(parameters, learningRate, alpha, eps, weight_decay, momentum, centered); + return new RMSProp(parameters, lr, alpha, eps, weight_decay, momentum, centered); } /// @@ -205,7 +209,7 @@ public static RMSProp RMSProp(IEnumerable parameters, double /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. /// /// Parameters to optimize - /// learning rate (default: 1e-3) + /// learning rate (default: 1e-3) /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability (default: 1e-8) @@ -213,9 +217,9 @@ public static RMSProp RMSProp(IEnumerable parameters, double /// Whether to use the AMSGrad variant of this algorithm. (default: False) /// /// - public static Adam Adam(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + public static Adam Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { - return new Adam(parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); + return new Adam(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); } /// @@ -224,7 +228,7 @@ public static Adam Adam(IEnumerable parameters, double learningRate = /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. /// /// Parameters to optimize - /// learning rate (default: 1e-3) + /// learning rate (default: 1e-3) /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability (default: 1e-8) @@ -232,9 +236,9 @@ public static Adam Adam(IEnumerable parameters, double learningRate = /// Whether to use the AMSGrad variant of this algorithm. (default: False) /// /// - public static Adam Adam(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + public static Adam Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { - return new Adam(parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); + return new Adam(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); } /// @@ -243,7 +247,7 @@ public static Adam Adam(IEnumerable parameters, double learning /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. /// /// Parameters to optimize - /// learning rate (default: 1e-3) + /// learning rate (default: 1e-3) /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability (default: 1e-8) @@ -251,9 +255,9 @@ public static Adam Adam(IEnumerable parameters, double learning /// Whether to use the AMSGrad variant of this algorithm. (default: False) /// /// - public static AdamW AdamW(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + public static AdamW AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { - return new AdamW(parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); + return new AdamW(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); } /// @@ -262,7 +266,7 @@ public static AdamW AdamW(IEnumerable parameters, double learningRate /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. /// /// Parameters to optimize - /// learning rate (default: 1e-3) + /// learning rate (default: 1e-3) /// Coefficient used for computing running averages of gradient and its square (default: 0.9) /// Coefficient used for computing running averages of gradient and its square (default: 0.999) /// Term added to the denominator to improve numerical stability (default: 1e-8) @@ -270,9 +274,9 @@ public static AdamW AdamW(IEnumerable parameters, double learningRate /// Whether to use the AMSGrad variant of this algorithm. (default: False) /// /// - public static AdamW AdamW(IEnumerable parameters, double learningRate = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + public static AdamW AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) { - return new AdamW(parameters, learningRate, beta1, beta2, eps, weight_decay, amsgrad, maximize); + return new AdamW(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); } /// @@ -281,15 +285,15 @@ public static AdamW AdamW(IEnumerable parameters, double learn /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. /// /// Parameters to optimize - /// learning rate (default: 1e-2) + /// learning rate (default: 1e-2) /// learning rate decay (default: 0) /// weight decay (L2 penalty) (default: 0) /// /// Term added to the denominator to improve numerical stability (default: 1e-10) /// - public static Adagrad Adagrad(IEnumerable parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + public static Adagrad Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) { - return new Adagrad(parameters, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps); + return new Adagrad(parameters, lr, lr_decay, weight_decay, initial_accumulator_value, eps); } /// @@ -298,15 +302,15 @@ public static Adagrad Adagrad(IEnumerable parameters, double learning /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. /// /// Parameters to optimize - /// learning rate (default: 1e-2) + /// learning rate (default: 1e-2) /// learning rate decay (default: 0) /// weight decay (L2 penalty) (default: 0) /// /// Term added to the denominator to improve numerical stability (default: 1e-10) /// - public static Adagrad Adagrad(IEnumerable parameters, double learningRate = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + public static Adagrad Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) { - return new Adagrad(parameters, learningRate, lr_decay, weight_decay, initial_accumulator_value, eps); + return new Adagrad(parameters, lr, lr_decay, weight_decay, initial_accumulator_value, eps); } /// @@ -554,7 +558,7 @@ namespace Modules /// Base class to help with a couple of the things that managed-code implementations need. /// - public class OptimizerHelper : Optimizer, ILearningRateController + public class OptimizerHelper : Optimizer { public OptimizerHelper() : base(IntPtr.Zero) { @@ -611,14 +615,7 @@ public virtual void add_param_group(ParamGroup param_group) _parameter_groups.Add(param_group); } - public double LearningRate { get => _defaults.LearningRate.Value; set => _defaults.LearningRate = value; } - - public double InitialLearningRate { get => _defaults.InitialLearningRate; set => _defaults.InitialLearningRate = value; } - - public IEnumerable ParamGroups { get => _parameter_groups; } - protected OptimizerOptions _defaults; - protected IList _parameter_groups; } public class OptimizerOptions @@ -823,7 +820,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double mo { } - public double Momentum { get => (Options as Options).momentum.Value; set => (Options as Options).momentum = value; } + public double Momentum { get => Options.momentum.Value; set => Options.momentum = value; } } private Dictionary _state = new Dictionary(); @@ -831,7 +828,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double mo public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } } - public class Adadelta : OptimizerHelper, ILearningRateController + public class Adadelta : OptimizerHelper { /// /// Constructor @@ -981,7 +978,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double rho private Dictionary _state = new Dictionary(); } - public class Adamax : OptimizerHelper, ILearningRateController, IBetas + public class Adamax : OptimizerHelper, IBetas { /// /// Implements Adamax algorithm (a variant of Adam based on infinity norm). @@ -1147,8 +1144,8 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet } public (double, double) Betas { - get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); - set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } } } @@ -1160,7 +1157,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet private Dictionary _state = new Dictionary(); } - public class NAdam : OptimizerHelper, ILearningRateController, IBetas + public class NAdam : OptimizerHelper, IBetas { /// /// Implements NAdam algorithm. @@ -1339,8 +1336,8 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet } public (double, double) Betas { - get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); - set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } } } @@ -1352,7 +1349,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet private Dictionary _state = new Dictionary(); } - public class RAdam : OptimizerHelper, ILearningRateController, IBetas + public class RAdam : OptimizerHelper, IBetas { /// /// Implements RAdam algorithm. @@ -1528,8 +1525,8 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet } public (double, double) Betas { - get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); - set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } } } @@ -1541,7 +1538,7 @@ public ParamGroup(IEnumerable parameters, double lr = 1.0, double bet private Dictionary _state = new Dictionary(); } - public class ASGD : OptimizerHelper, ILearningRateController + public class ASGD : OptimizerHelper { /// /// Implements ASGD algorithm (a variant of Adam based on infinity norm). @@ -1705,7 +1702,7 @@ public ParamGroup(IEnumerable parameters, double lr = 0.01, double la private Dictionary _state = new Dictionary(); } - public class Rprop : OptimizerHelper, ILearningRateController + public class Rprop : OptimizerHelper { /// /// Implements Rprop algorithm (a variant of Adam based on infinity norm). @@ -1844,7 +1841,7 @@ public override void add_param_group(Modules.ParamGroup param_group) _state[p.Handle] = state; state.step = 0; state.prev = torch.zeros_like(p); - state.step_size = p.new_empty(p.shape).fill_(LearningRate); + state.step_size = p.new_empty(p.shape).fill_(opt.LearningRate); } } @@ -2226,8 +2223,8 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double be } public (double, double) Betas { - get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); - set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } } } @@ -2324,7 +2321,7 @@ public override Tensor step(Func closure = null) state.step += 1; - param.mul_(1 - LearningRate * weight_decay); + param.mul_(1 - lr * weight_decay); var bias_correction1 = 1 - Math.Pow(beta1, state.step); var bias_correction2 = 1 - Math.Pow(beta2, state.step); @@ -2424,8 +2421,8 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double be } public (double, double) Betas { - get => ((Options as Options).beta1.Value, (Options as Options).beta2.Value); - set { (Options as Options).beta1 = value.Item1; (Options as Options).beta2 = value.Item2; } + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } } } @@ -2605,7 +2602,7 @@ public class Options : Modules.OptimizerOptions public bool? centered; } - public class ParamGroup : ParamGroup + public class ParamGroup : ParamGroup, IMomentum { public ParamGroup() { } @@ -2615,6 +2612,8 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double ep : base(parameters, new RMSProp.Options { LearningRate = lr, eps = eps, alpha = alpha, weight_decay = weight_decay, momentum = momentum, centered = centered }) { } + + public double Momentum { get => Options.momentum.Value; set => Options.momentum = value; } } private Dictionary _state = new Dictionary(); @@ -2626,13 +2625,13 @@ public ParamGroup(IEnumerable parameters, double lr = 1e-3, double ep // // LBGFS does not allow for param groups, so there's no need to use anything but the native implementation. - public class LBFGSOptimizer : Optimizer, ILearningRateController + public class LBFGS : Optimizer, ILearningRateController { - public LBFGSOptimizer(IntPtr handle, double lr) : base(handle) + public LBFGS(IntPtr handle, double lr) : base(handle) { _rate = lr; InitialLearningRate = lr; - _paramGroups = new ParamGroup[] { new ParamGroup { Parameters = this.parameters(), Options = { LearningRate = lr, InitialLearningRate = lr } } }; + _paramGroups = new ParamGroup[] { new ParamGroup { Parameters = this.parameters(), Options = new() { LearningRate = lr, InitialLearningRate = lr } } }; } [DllImport("LibTorchSharp")] @@ -2645,7 +2644,7 @@ public double LearningRate { public double InitialLearningRate { get; set; } - public IEnumerable ParamGroups { get => _paramGroups; } + public override IEnumerable ParamGroups { get => _paramGroups; } public override Tensor step(Func closure = null) diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index b4936ca37..a01687288 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -264,8 +264,9 @@ public void TestTrainingAdamOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adam(seq.parameters(), amsgrad: true); - var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); + var lr = 0.001; + var optimizer = torch.optim.Adam(seq.parameters(), lr: lr, amsgrad: true); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -374,8 +375,9 @@ public void TestTrainingAdamWOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.AdamW(seq.parameters(), amsgrad: true); - var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); + var lr = 0.001; + var optimizer = torch.optim.AdamW(seq.parameters(), lr: lr, amsgrad: true); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -634,8 +636,9 @@ public void TestTrainingAdamaxOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.Adamax(seq.parameters()); - var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 15); + var lr = 0.002; + var optimizer = torch.optim.Adamax(seq.parameters(), lr: lr); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 15); var loss = mse_loss(Reduction.Sum); @@ -744,8 +747,9 @@ public void TestTrainingNAdamOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.NAdam(seq.parameters()); - var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); + var lr = 0.002; + var optimizer = torch.optim.NAdam(seq.parameters(), lr: lr); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -853,8 +857,95 @@ public void TestTrainingRAdamOneCycleLR() var x = torch.randn(new long[] { 64, 1000 }); var y = torch.randn(new long[] { 64, 10 }); - var optimizer = torch.optim.RAdam(seq.parameters()); - var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 1.5, total_steps: 10); + var lr = 0.002; + var optimizer = torch.optim.RAdam(seq.parameters(), lr: lr); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 1.5, total_steps: 10); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + scheduler.step(); + } + Assert.True(finalLoss < initialLoss); + } + + /// + /// Fully connected ReLU net with one hidden layer trained using RAdam optimizer. + /// + [Fact] + public void TestTrainingRAdamOneCycleLR_PG() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var lr = 0.002; + var optimizer = torch.optim.RAdam(new RAdam.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.003f } }, + new () { Parameters = lin2.parameters(), Options = new () { LearningRate = lr } } + }); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, new double[] { lr * 1.5, lr * 2.25 }, total_steps: 10); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + scheduler.step(); + } + Assert.True(finalLoss < initialLoss); + } + + /// + /// Fully connected ReLU net with one hidden layer trained using RAdam optimizer. + /// + [Fact] + public void TestTrainingRAdamCyclicLR_PG() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + var lr = 0.0002; + var optimizer = torch.optim.SGD(new SGD.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.003f } }, + new () { Parameters = lin2.parameters(), Options = new () { LearningRate = lr } } + }, lr); + var scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, new double[] { lr / 2, lr * 2 }, new double[] { lr * 1.5, lr * 2.25 }); var loss = mse_loss(Reduction.Sum); @@ -1083,7 +1174,7 @@ public void TestTrainingRMSOneCycleLR() double learning_rate = 0.00004f; var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate); - var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, optimizer.LearningRate * 10, total_steps: 10); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, learning_rate * 10, total_steps: 10); var loss = mse_loss(Reduction.Sum); @@ -1388,9 +1479,187 @@ public void TestTrainingSGDStepLR() optimizer.step(); scheduler.step(); - Assert.True(optimizer.LearningRate < lastLR); + var pgFirst = optimizer.ParamGroups.First(); + Assert.True(pgFirst.LearningRate < lastLR); + + lastLR = pgFirst.LearningRate; + } + + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingSGDLambdaLR() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, i => Math.Pow(0.95,(1+i))); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + double lastLR = learning_rate; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + scheduler.step(); + + var pgFirst = optimizer.ParamGroups.First(); + Assert.True(pgFirst.LearningRate < lastLR); + + lastLR = pgFirst.LearningRate; + } + + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingSGDMultiplicativeLR() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer, i => 0.95); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + double lastLR = learning_rate; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + scheduler.step(); + + var pgFirst = optimizer.ParamGroups.First(); + Assert.True(pgFirst.LearningRate < lastLR); + + lastLR = pgFirst.LearningRate; + } + + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingSGDExponentialLR() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + double lastLR = learning_rate; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + scheduler.step(); + + var pgFirst = optimizer.ParamGroups.First(); + Assert.True(pgFirst.LearningRate < lastLR); + + lastLR = pgFirst.LearningRate; + } + + Assert.True(finalLoss < initialLoss); + } + + [Fact] + public void TestTrainingSGDLinearLR() + { + var lin1 = Linear(1000, 100); + var lin2 = Linear(100, 10); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var x = torch.randn(new long[] { 64, 1000 }); + var y = torch.randn(new long[] { 64, 10 }); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var scheduler = torch.optim.lr_scheduler.LinearLR(optimizer, end_factor: 0.75, total_iters: 10); + + var loss = mse_loss(Reduction.Sum); + + float initialLoss = loss(seq.forward(x), y).ToSingle(); + float finalLoss = float.MaxValue; + + var pgFirst = optimizer.ParamGroups.First(); + + double lastLR = pgFirst.LearningRate; + + for (int i = 0; i < 10; i++) { + using var eval = seq.forward(x); + using var output = loss(eval, y); + var lossVal = output.ToSingle(); + + finalLoss = lossVal; + + optimizer.zero_grad(); + + output.backward(); + + optimizer.step(); + scheduler.step(); + + Assert.True(pgFirst.LearningRate < lastLR); - lastLR = optimizer.LearningRate; + lastLR = pgFirst.LearningRate; } Assert.True(finalLoss < initialLoss); @@ -1431,10 +1700,11 @@ public void TestTrainingSGDMultiStepLR() optimizer.step(); scheduler.step(); + var pgFirst = optimizer.ParamGroups.First(); if (i == 2 || i == 4 || i == 6) { - Assert.True(optimizer.LearningRate < lastLR); + Assert.True(pgFirst.LearningRate < lastLR); } - lastLR = optimizer.LearningRate; + lastLR = pgFirst.LearningRate; } Assert.True(finalLoss < initialLoss); @@ -1475,10 +1745,11 @@ public void TestTrainingSGDCosineAnnealingLR() optimizer.step(); scheduler.step(); + var pgFirst = optimizer.ParamGroups.First(); if (i == 2 || i == 4 || i == 6) { - Assert.True(optimizer.LearningRate < lastLR); + Assert.True(pgFirst.LearningRate < lastLR); } - lastLR = optimizer.LearningRate; + lastLR = pgFirst.LearningRate; } Assert.True(finalLoss < initialLoss); From 61ae75731d109d31fae4b056c228325c326e5ffd Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Wed, 16 Feb 2022 09:01:46 -0800 Subject: [PATCH 14/25] Added test for #516 --- test/TorchSharpTest/TestTorchTensorBugs.cs | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/test/TorchSharpTest/TestTorchTensorBugs.cs b/test/TorchSharpTest/TestTorchTensorBugs.cs index cf8b41220..a94b60bde 100644 --- a/test/TorchSharpTest/TestTorchTensorBugs.cs +++ b/test/TorchSharpTest/TestTorchTensorBugs.cs @@ -501,5 +501,79 @@ public override torch.Tensor forward(torch.Tensor t) return this.stack.forward(t); } } + + [Fact] + public void ValidateIssue516() + { + using var module = new LMHead(128, 1000, "tanh", torch.rand(128, 1000)); + using var optim = torch.optim.Adam(module.parameters(), 0.002); + } + + internal abstract class BaseModule : torch.nn.Module + { + public int? InstanceId = null; + + protected BaseModule(string name) : base(name) + { + } + } + + internal sealed class ActivationFunction : BaseModule + { + private torch.nn.Module Function; + + public ActivationFunction(string name) : base(name) + { + Function = name?.ToLower() switch { + "relu" => torch.nn.ReLU(), + "gelu" => torch.nn.GELU(), + "tanh" => torch.nn.Tanh(), + "linear" => torch.nn.Identity(), + _ => throw new NotSupportedException($"Activation function {name} not supported.") + }; + } + + public override torch.Tensor forward(torch.Tensor x) + { + return Function.forward(x); + } + + public override string GetName() + { + return Function.GetName(); + } + } + + internal sealed class LMHead : BaseModule + { + public readonly Modules.Sequential Projection; + public readonly TorchSharp.Modules.Parameter Weight; + public readonly TorchSharp.Modules.Parameter Bias; + + public LMHead(int inSize, int embedDim, string activationFn, torch.Tensor outputWeight) + : base(nameof(LMHead)) + { + if (outputWeight is null) { + throw new ArgumentNullException(nameof(outputWeight)); + } + + Projection = torch.nn.Sequential( + ("dense", torch.nn.Linear(inSize, embedDim)), + ("activation", new ActivationFunction(activationFn)), + ("layernorm", torch.nn.LayerNorm(new long[] { embedDim })) + ); + + var outputDim = outputWeight.shape[^2]; + Weight = new TorchSharp.Modules.Parameter(outputWeight); + Bias = new TorchSharp.Modules.Parameter(torch.zeros(outputDim, dtype: torch.float32, requiresGrad: true)); + + RegisterComponents(); + } + + public override torch.Tensor forward(torch.Tensor features) + { + return torch.randn(10); + } + } } } \ No newline at end of file From dec5dc44393e426484e86027e50558a2a83ac2ac Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Wed, 16 Feb 2022 09:17:12 -0800 Subject: [PATCH 15/25] Update version number. --- azure-pipelines.yml | 28 ++++++++++++++-------------- build/BranchInfo.props | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b971c9f32..5ae2cf53e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -518,7 +518,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp.*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -527,7 +527,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp-*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -537,7 +537,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -547,7 +547,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -557,7 +557,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -602,7 +602,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -613,7 +613,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -624,7 +624,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -635,7 +635,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -646,7 +646,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -689,7 +689,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -700,7 +700,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -711,7 +711,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -722,7 +722,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true \ No newline at end of file diff --git a/build/BranchInfo.props b/build/BranchInfo.props index 3c8e510e0..20628973c 100644 --- a/build/BranchInfo.props +++ b/build/BranchInfo.props @@ -2,7 +2,7 @@ 0 96 - 0 + 1 From 3ccdeb6bf5f9ca8d5fbbe04f2612717e9a6dfe87 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Wed, 16 Feb 2022 09:27:29 -0800 Subject: [PATCH 16/25] Reverted version number. --- RELEASENOTES.md | 2 +- azure-pipelines.yml | 28 ++++++++++++++-------------- build/BranchInfo.props | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 21d580a7f..b5484b34d 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -14,7 +14,7 @@ __Fixed Bugs:__ #510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d
#515 what's reason for making register_module internal?
- +#495 Add support for OptimizerParamGroup ## NuGet Version 0.96.0 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5ae2cf53e..b971c9f32 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -518,7 +518,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp.*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -527,7 +527,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp-*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -537,7 +537,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -547,7 +547,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -557,7 +557,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -602,7 +602,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -613,7 +613,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -624,7 +624,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -635,7 +635,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -646,7 +646,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -689,7 +689,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -700,7 +700,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -711,7 +711,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -722,7 +722,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true \ No newline at end of file diff --git a/build/BranchInfo.props b/build/BranchInfo.props index 20628973c..3c8e510e0 100644 --- a/build/BranchInfo.props +++ b/build/BranchInfo.props @@ -2,7 +2,7 @@ 0 96 - 1 + 0 From 9d861eb67e6f52bc0a2c1997dc9502d647136b3a Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Wed, 16 Feb 2022 14:31:42 -0800 Subject: [PATCH 17/25] Adding a couple of minor APIs that were missing. Support complex types in Adam. --- RELEASENOTES.md | 3 + src/TorchSharp/NN/Optimizer.cs | 17 +-- src/TorchSharp/Tensor/Tensor.cs | 221 ++++++++++++++++---------------- 3 files changed, 121 insertions(+), 120 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index b5484b34d..4b6fa599c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -9,12 +9,15 @@ __API Changes:__ __NOTE__: This release contains breaking changes.
The APIs to create optimizers all take 'named_parameters' rather than 'parameters' now.
+Support for parameter groups in most optimizers. +Support for parameter groups in LR schedulers. __Fixed Bugs:__ #510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d
#515 what's reason for making register_module internal?
#495 Add support for OptimizerParamGroup +#509 Tensor.conj() not implemented ## NuGet Version 0.96.0 diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index 0ec8b15e8..d0ed4352d 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -1951,15 +1951,14 @@ public override Tensor step(Func closure = null) var clr = lr / (1 + (state.step - 1) * lr_decay); - if (grad.is_sparse) { + if (grad.is_sparse) throw new NotImplementedException("Adagrad optimization over sparse parameters"); - } else if (torch.is_complex(param)) { + if (torch.is_complex(grad)) throw new NotImplementedException("Adagrad optimization over complex parameters"); - } else { - state.sum.addcmul_(grad, grad, value: 1); - var std = state.sum.sqrt().add_(eps); - param.addcdiv_(grad, std, value: -clr); - } + + state.sum.addcmul_(grad, grad, value: 1); + var std = state.sum.sqrt().add_(eps); + param.addcdiv_(grad, std, value: -clr); } @@ -2127,9 +2126,7 @@ public override Tensor step(Func closure = null) } state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - // When complex types are supported: - //state.exp_avg_sq.mul_(_beta2).addcmul_(grad, grad.conj(), value: 1 - _beta2) - state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad.conj(), value: 1 - beta2); Tensor denom = null; if (amsgrad) { diff --git a/src/TorchSharp/Tensor/Tensor.cs b/src/TorchSharp/Tensor/Tensor.cs index 3ae1becb9..176a919c7 100644 --- a/src/TorchSharp/Tensor/Tensor.cs +++ b/src/TorchSharp/Tensor/Tensor.cs @@ -43,17 +43,15 @@ internal Tensor(IntPtr handle) /// TBD /// /// - /// + public override bool Equals(object? obj) { return (obj is Tensor) && this.Equals((obj as Tensor)!); } - /// /// TBD /// - /// public override int GetHashCode() { return base.GetHashCode(); @@ -128,7 +126,6 @@ public torch.Tensor DetatchFromDisposeScope() /// /// See the torch.nn.Module.Module(string name) constructor for an example of its use. /// - /// public IntPtr DecoupleFromNativeHandle() { GC.SuppressFinalize(this); @@ -409,56 +406,48 @@ public Tensor imag { /// Read the double-precision value at the given index. /// /// The index. - /// public double ReadCpuDouble(long i) => Utils.TensorAccessor.ReadItemAt(this, i); /// /// Read the single-precision float value at the given index. /// /// The index. - /// public float ReadCpuSingle(long i) => Utils.TensorAccessor.ReadItemAt(this, i); /// /// Read the 32-bit integer float value at the given index. /// /// The index. - /// public int ReadCpuInt32(long i) => Utils.TensorAccessor.ReadItemAt(this, i); /// /// Read the 64-bit integer value at the given index. /// /// The index. - /// public long ReadCpuInt64(long i) => Utils.TensorAccessor.ReadItemAt(this, i); /// /// Read the byte value at the given index. /// /// The index. - /// public byte ReadCpuByte(long i) => Utils.TensorAccessor.ReadItemAt(this, i); /// /// Read the short value at the given index. /// /// The index. - /// public sbyte ReadCpuSByte(long i) => Utils.TensorAccessor.ReadItemAt(this, i); /// /// Read the int16 value at the given index. /// /// The index. - /// public short ReadCpuInt16(long i) => Utils.TensorAccessor.ReadItemAt(this, i); /// /// Read the Boolean value at the given index. /// /// The index. - /// public bool ReadCpuBool(long i) => Utils.TensorAccessor.ReadItemAt(this, i); /// @@ -466,7 +455,6 @@ public Tensor imag { /// /// The type of the element to read. /// The index. - /// public T ReadCpuValue(long i) where T : unmanaged => Utils.TensorAccessor.ReadItemAt(this, i); [DllImport("LibTorchSharp")] @@ -476,7 +464,6 @@ public Tensor imag { /// Read the Float16 value at the given index. /// /// The index. - /// public float ReadCpuFloat16(long i) { if (i >= NumberOfElements) { @@ -492,7 +479,6 @@ public float ReadCpuFloat16(long i) /// Read the BFloat16 value at the given index. /// /// The index. - /// public float ReadCpuBFloat16(long i) { if (i >= NumberOfElements) { @@ -507,7 +493,6 @@ public float ReadCpuBFloat16(long i) /// /// Convert to a scalar. /// - /// public Scalar ToScalar() { var res = THSTensor_item(Handle); @@ -522,7 +507,6 @@ public Scalar ToScalar() /// Fill the tensor with the provided scalar value. /// /// A scalar value - /// public Tensor fill_(Scalar value) { var res = THSTensor_fill_(Handle, value.Handle); @@ -607,7 +591,6 @@ public bool is_sparse { /// Creates a tensor by loading it from a file. /// /// The file path where tensor values are stored. - /// public static Tensor load(string location) { var res = THSTensor_load(location); @@ -652,7 +635,6 @@ public bool requires_grad { /// Change if autograd should record operations on this tensor: sets this tensor’s requires_grad attribute in-place. Returns this tensor. /// /// - /// public Tensor requires_grad_(bool requires_grad = true) { this.requires_grad = true; @@ -674,7 +656,6 @@ public Tensor with_requires_grad(bool requires_grad = true) /// /// Moves the tensor data to the CPU device /// - /// public Tensor cpu() { var res = THSTensor_cpu(Handle); @@ -690,7 +671,6 @@ public Tensor cpu() /// Returns a copy of this object in CUDA memory. /// If this object is already in CUDA memory and on the correct device, then no copy is performed and the original object is returned. /// - /// public Tensor cuda() { torch.InitializeDeviceType(DeviceType.CUDA); @@ -734,7 +714,6 @@ public Tensor to_type(ScalarType type, bool copy = false) /// Overwrite an existing tensor with the contents of another tensor. /// /// The source tensor - /// public Tensor set_(Tensor source) { var res = THSTensor_set_(Handle, source.Handle); @@ -749,7 +728,6 @@ public Tensor set_(Tensor source) /// The device type, e.g. 'CPU' or 'CUDA'. /// The optional device index. /// When copy is set, a new Tensor is created even when the Tensor already matches the desired conversion. - /// public Tensor to(DeviceType deviceType, int deviceIndex = -1, bool copy = false) { torch.InitializeDeviceType(deviceType); @@ -765,7 +743,6 @@ public Tensor to(DeviceType deviceType, int deviceIndex = -1, bool copy = false) /// The target type /// The target device /// When copy is set, a new Tensor is created even when the Tensor already matches the desired conversion. - /// public Tensor to(ScalarType type, torch.Device device, bool copy = false) { torch.InitializeDevice(device); @@ -793,21 +770,19 @@ public Tensor to(ScalarType type, torch.Device device, bool copy = false) /// Moves the tensor data. /// /// A string denoting the target device. - /// public Tensor to(string device) => to(new torch.Device(device)); /// /// Moves the tensor data. /// /// The target device - /// + public Tensor to(torch.Device device) => to(device.type, device.index); - /// /// Moves the tensor data. /// /// The tensor serving as a template. - /// + public Tensor to(Tensor other) => to(other.device_type, other.device_index); [DllImport("LibTorchSharp")] @@ -817,7 +792,6 @@ public Tensor to(ScalarType type, torch.Device device, bool copy = false) /// Retrieves the size of the specified dimension in the tensor. /// /// - /// public long size(int dim) { var res = THSTensor_size(Handle, dim); @@ -975,7 +949,7 @@ public void backward() /// /// Creates a strided copy of self. /// - /// + public Tensor to_dense() { var res = THSTensor_to_dense(Handle); @@ -990,7 +964,7 @@ public Tensor to_dense() /// /// Returns a copy of the tensor input. /// - /// + public Tensor clone() { var res = THSTensor_clone(Handle); @@ -1005,7 +979,7 @@ public Tensor clone() /// /// Copies the elements from source into the tensor and returns it. /// - /// + /// The src tensor must be broadcastable with the target 'this' tensor. It may be of a different data type or reside on a different device. public Tensor copy_(Tensor source, bool nonBlocking = false) { @@ -1021,7 +995,7 @@ public Tensor copy_(Tensor source, bool nonBlocking = false) /// /// Returns true if the tensor is contiguous. /// - /// + public bool is_contiguous() { var res = THSTensor_is_contiguous(Handle); @@ -1036,7 +1010,7 @@ public bool is_contiguous() /// Returns a contiguous in memory tensor containing the same data as the input tensor. /// If tensor is already in the specified memory format, this function returns the original tensor. /// - /// + public Tensor contiguous() { var res = THSTensor_contiguous(Handle); @@ -1052,7 +1026,7 @@ public Tensor contiguous() /// This attribute is null by default and becomes a Tensor the first time a call to backward() computes gradients for the tensor. /// The attribute will then contain the gradients computed and future calls to backward() will accumulate (add) gradients into it. /// - /// + public Tensor? grad() { var res = THSTensor_grad(Handle); @@ -1142,7 +1116,7 @@ public Tensor this[params Tensor[] indices] { /// Tensor indexer. /// /// The first-dimension index. - /// + [IndexerName("TensorItems")] public Tensor this[long i1] { get { @@ -1167,7 +1141,7 @@ public Tensor this[long i1] { /// /// The first-dimension index. /// The second-dimension index. - /// + [IndexerName("TensorItems")] public Tensor this[long i1, long i2] { get { @@ -1193,7 +1167,7 @@ public Tensor this[long i1] { /// The first-dimension index. /// The second-dimension index. /// The third-dimension index - /// + [IndexerName("TensorItems")] public Tensor this[long i1, long i2, long i3] { get { @@ -1221,7 +1195,7 @@ public Tensor this[long i1] { /// The second-dimension index. /// The third-dimension index /// The fourth-dimension index - /// + [IndexerName("TensorItems")] public Tensor this[long i1, long i2, long i3, long i4] { get { @@ -1250,7 +1224,7 @@ public Tensor this[long i1] { /// The third-dimension index /// The fourth-dimension index /// The fifth-dimension index - /// + [IndexerName("TensorItems")] public Tensor this[long i1, long i2, long i3, long i4, long i5] { get { @@ -1281,7 +1255,7 @@ public Tensor this[long i1] { /// The fourth-dimension index /// The fifth-dimension index /// The sixth-dimension index - /// + [IndexerName("TensorItems")] public Tensor this[long i1, long i2, long i3, long i4, long i5, long i6] { get { @@ -1299,7 +1273,7 @@ public Tensor this[long i1] { /// /// Index into the tensor using Python-like indexing expressions. /// - /// + public Tensor index(params TensorIndex[] indices) { EncodeIndices(indices, out var arrKindAndStarts, out var arrStops, out var arrSteps, out var arrTensors); @@ -1320,7 +1294,7 @@ public Tensor index(params TensorIndex[] indices) /// /// Index into the tensor using Python-like indexing expressions. /// - /// + public Tensor index(params Tensor[] indices) { return index(indices.Select(t => TensorIndex.Tensor(t)).ToArray()); @@ -1329,7 +1303,7 @@ public Tensor index(params Tensor[] indices) /// /// Index into the tensor using Python-like indexing expressions and place a tensor at the index. /// - /// + public Tensor index_put_(Tensor value, params TensorIndex[] indices) { EncodeIndices(indices, out var arrKindAndStarts, out var arrStops, out var arrSteps, out var arrTensors); @@ -1350,7 +1324,7 @@ public Tensor index_put_(Tensor value, params TensorIndex[] indices) /// /// Index into the tensor using Python-like indexing expressions and place a tensor at the index. /// - /// + public Tensor index_put_(Tensor value, params Tensor[] indices) { return index_put_(value, indices.Select(t => TensorIndex.Tensor(t)).ToArray()); @@ -1360,7 +1334,7 @@ public Tensor index_put_(Tensor value, params Tensor[] indices) /// /// Index into the tensor using Python-like indexing expressions and place a scalar tensor at the index. /// - /// + public Tensor index_put_(Scalar value, params TensorIndex[] indices) { EncodeIndices(indices, out var arrKindAndStarts, out var arrStops, out var arrSteps, out var arrTensors); @@ -1381,7 +1355,7 @@ public Tensor index_put_(Scalar value, params TensorIndex[] indices) /// /// Index into the tensor using Python-like indexing expressions and place a scalar tensor at the index. /// - /// + public Tensor index_put_(Scalar value, params Tensor[] indices) { return index_put_(value, indices.Select(t => TensorIndex.Tensor(t)).ToArray()); @@ -1395,7 +1369,7 @@ public Tensor index_put_(Scalar value, params Tensor[] indices) /// /// /// - /// + public Tensor index_select(long dimension, Tensor index) { var res = THSTensor_index_select(Handle, dimension, index.Handle); @@ -1413,7 +1387,7 @@ public Tensor index_select(long dimension, Tensor index) /// /// The dimension to slice /// The index to select with - /// + public Tensor select(long dim, long index) { var res = THSTensor_select(Handle, dim, index); @@ -1430,7 +1404,7 @@ public Tensor select(long dim, long index) /// The result takes the same shape as the indices. /// /// The indices into tensor, an Int64 tensor. - /// + public Tensor take(Tensor index) { var res = THSTensor_take(Handle, index.Handle); @@ -1449,7 +1423,7 @@ public Tensor take(Tensor index) /// Selects values from input at the 1-dimensional indices from indices along the given dim. /// /// The indices into input. Must have long dtype. - /// + /// Functions that return indices along a dimension, like torch.argmax() and torch.argsort(), are designed to work with this function. public Tensor take_along_dim(Tensor indices) { @@ -1463,7 +1437,7 @@ public Tensor take_along_dim(Tensor indices) /// Selects values from input at the 1-dimensional indices from indices along the given dim. /// /// The indices into input. Must have long dtype. - /// + /// Functions that return indices along a dimension, like torch.argmax() and torch.argsort(), are designed to work with this function. public Tensor take_along_dim(IEnumerable indices) => take_along_dim(torch.tensor(indices.ToArray())); @@ -1472,7 +1446,7 @@ public Tensor take_along_dim(Tensor indices) /// /// The indices into input. Must have long dtype. /// Dimension to select along. - /// + /// Functions that return indices along a dimension, like torch.argmax() and torch.argsort(), are designed to work with this function. public Tensor take_along_dim(Tensor indices, long dimension) { @@ -1487,7 +1461,7 @@ public Tensor take_along_dim(Tensor indices, long dimension) /// /// The indices into input. Must have long dtype. /// Dimension to select along. - /// + /// Functions that return indices along a dimension, like torch.argmax() and torch.argsort(), are designed to work with this function. public Tensor take_along_dim(IEnumerable indices, long dim) => take_along_dim(torch.tensor(indices.ToArray()), dim); @@ -1498,7 +1472,7 @@ public Tensor take_along_dim(Tensor indices, long dimension) /// Returns a tensor with the same data and number of elements as self but with the specified shape. /// /// The new tensor shape. - /// + public Tensor reshape(params long[] shape) { unsafe { @@ -1521,7 +1495,7 @@ public Tensor reshape(params long[] shape) /// Returns a tensor with all the dimensions of input of size 1 removed. When dim is given, a squeeze operation is done only in the given dimension. /// /// If given, the input will be squeezed only in this dimension - /// + public Tensor squeeze(long? dim = null) { var res = dim.HasValue ? THSTensor_squeeze(Handle, dim.Value) : THSTensor_squeeze_no_dim(Handle); @@ -1536,7 +1510,7 @@ public Tensor squeeze(long? dim = null) /// /// Expects input to be 1- or 2-D tensor and transposes dimensions 0 and 1. /// - /// + public Tensor t() { var res = THSTensor_t(Handle); @@ -1548,7 +1522,7 @@ public Tensor t() /// /// Is this Tensor with its dimensions reversed. /// - /// + public Tensor T { get { return this.permute(Enumerable.Range(0, (int)ndim).Reverse().Select(i => (long)i).ToArray()); @@ -1563,7 +1537,7 @@ public Tensor T { /// /// /// - /// + public Tensor transpose(long dim0, long dim1) { var res = THSTensor_transpose(Handle, dim0, dim1); @@ -1580,7 +1554,7 @@ public Tensor transpose(long dim0, long dim1) /// The lower triangular part of the matrix is defined as the elements on and below the diagonal. /// /// The diagonal to consider - /// + public Tensor tril(long diagonal = 0) { var res = THSTensor_tril(Handle, diagonal); @@ -1597,7 +1571,7 @@ public Tensor tril(long diagonal = 0) /// The upper triangular part of the matrix is defined as the elements on and above the diagonal. /// /// The diagonal to consider - /// + public Tensor triu(long diagonal = 0) { var res = THSTensor_triu(Handle, diagonal); @@ -1626,7 +1600,7 @@ public Tensor triu(long diagonal = 0) /// /// /// - /// + public Tensor transpose_(long dim0, long dim1) { var res = THSTensor_transpose_(Handle, dim0, dim1); @@ -1642,7 +1616,7 @@ public Tensor transpose_(long dim0, long dim1) /// Returns a new tensor with the same data as the input tensor but of a different shape. /// /// The shape of the view - /// + public Tensor view(params long[] shape) { unsafe { @@ -1655,6 +1629,19 @@ public Tensor view(params long[] shape) } } + /// + /// View this tensor as the same size as other. + /// + /// The result tensor has the same size as other. + /// + /// self.view_as(other) is equivalent to self.view(other.size()). + /// Please see view() for more information about view. + /// + public Tensor view_as(Tensor other) + { + return view(other.shape); + } + [DllImport("LibTorchSharp")] static extern IntPtr THSTensor_view_as_complex(IntPtr tensor); @@ -1689,7 +1676,7 @@ public Tensor view_as_real() /// /// /// - /// + public Tensor all() { var res = THSTensor_all(Handle); @@ -1706,7 +1693,7 @@ public Tensor all() /// /// /// - /// + public Tensor all(long dimension, bool keepDim = false) { var res = THSTensor_all_along_dimension(Handle, dimension, keepDim); @@ -1727,7 +1714,7 @@ public Tensor all(long dimension, bool keepDim = false) /// The dimension or dimensions to reduce. /// Whether the output tensor has dim retained or not. /// The output tensor -- optional. - /// + public Tensor amax(long[] dims, bool keepDim = false, Tensor? @out = null) { unsafe { @@ -1752,7 +1739,7 @@ public Tensor amax(long[] dims, bool keepDim = false, Tensor? @out = null) /// The dimension or dimensions to reduce. /// Whether the output tensor has dim retained or not. /// The output tensor -- optional. - /// + public Tensor amin(long[] dims, bool keepDim = false, Tensor? @out = null) { unsafe { @@ -1775,7 +1762,7 @@ public Tensor amin(long[] dims, bool keepDim = false, Tensor? @out = null) /// /// The dimension along which to compute the values. If null, computes the values over the entire input tensor /// If true, the reduced dimensions will be kept in the output tensor as dimensions with size 1 for broadcasting. - /// + public (Tensor min, Tensor max) aminmax(long? dim = null, bool keepDim = false) { var res = THSTensor_aminmax(Handle, (dim is null) ? -1 : dim.Value, keepDim, out IntPtr maxHandle); @@ -1789,7 +1776,7 @@ public Tensor amin(long[] dims, bool keepDim = false, Tensor? @out = null) /// /// /// - /// + public Tensor any() { var res = THSTensor_any(Handle); @@ -1806,7 +1793,7 @@ public Tensor any() /// /// /// - /// + public Tensor any(long dimension, bool keepDim = false) { var res = THSTensor_any_along_dimension(Handle, dimension, keepDim); @@ -1821,7 +1808,7 @@ public Tensor any(long dimension, bool keepDim = false) /// /// /// - /// + public Tensor argmax() { var res = THSTensor_argmax(Handle); @@ -1838,7 +1825,7 @@ public Tensor argmax() /// /// /// - /// + public Tensor argmax(long dimension, bool keepDim = false) { var res = THSTensor_argmax_along_dimension(Handle, dimension, keepDim); @@ -1853,7 +1840,7 @@ public Tensor argmax(long dimension, bool keepDim = false) /// /// /// - /// + public Tensor argmin() { var res = THSTensor_argmin(Handle); @@ -1870,7 +1857,7 @@ public Tensor argmin() /// /// /// - /// + public Tensor argmin(long dimension, bool keepDim = false) { var res = THSTensor_argmin_along_dimension(Handle, dimension, keepDim); @@ -1887,7 +1874,7 @@ public Tensor argmin(long dimension, bool keepDim = false) /// /// The dimension to sort along /// Controls the sorting order (ascending or descending) - /// + public Tensor argsort(long dimension = -1, bool descending = false) { var res = THSTensor_argsort(Handle, dimension, descending); @@ -1902,7 +1889,7 @@ public Tensor argsort(long dimension = -1, bool descending = false) /// /// Convert each element from degrees to radians. /// - /// + public Tensor deg2rad() { var res = THSTensor_deg2rad(Handle); @@ -1917,7 +1904,7 @@ public Tensor deg2rad() /// /// Convert each element from radians to degrees. /// - /// + public Tensor rad2deg() { var res = THSTensor_rad2deg(Handle); @@ -1932,7 +1919,7 @@ public Tensor rad2deg() /// /// /// - /// + public Tensor copysign(Tensor other) { var res = THSTensor_copysign(Handle, other.Handle); @@ -2009,7 +1996,7 @@ public Tensor corrcoef() /// Constructs a tensor by repeating the elements of input. The reps argument specifies the number of repetitions in each dimension. /// /// The number of repetitions per dimension. - /// + public Tensor tile(long[] reps) { unsafe { @@ -2027,7 +2014,7 @@ public Tensor tile(long[] reps) /// /// Computes the logarithmic derivative of the gamma function on input. /// - /// + public Tensor digamma() { var res = THSTensor_digamma(Handle); @@ -2042,7 +2029,7 @@ public Tensor digamma() /// /// Computes the logarithmic derivative of the gamma function on input, in place. /// - /// + public Tensor digamma_() { var res = THSTensor_digamma_(Handle); @@ -2057,7 +2044,7 @@ public Tensor digamma_() /// /// Computes the logarithm of the gamma function on input. /// - /// + public Tensor lgamma() { var res = THSTensor_lgamma(Handle); @@ -2072,7 +2059,7 @@ public Tensor lgamma() /// /// Computes the logarithm of the gamma function on input, in place. /// - /// + public Tensor lgamma_() { var res = THSTensor_lgamma_(Handle); @@ -2088,7 +2075,7 @@ public Tensor lgamma_() /// Computes the multivariate log-gamma function) with dimension pp element-wise /// /// The number of dimensions - /// + public Tensor mvlgamma(long p) { var res = THSTensor_mvlgamma(Handle, p); @@ -2104,7 +2091,7 @@ public Tensor mvlgamma(long p) /// Computes the multivariate log-gamma function) with dimension pp element-wise, in place. /// /// The number of dimensions - /// + public Tensor mvlgamma_(long p) { var res = THSTensor_mvlgamma_(Handle, p); @@ -2141,7 +2128,7 @@ public Tensor polygamma_(long p) /// /// Returns input. Throws a runtime error if input is a bool tensor. /// - /// + public Tensor positive() { if (this.dtype == ScalarType.Bool) throw new ArgumentException("Boolean tensor"); @@ -2356,7 +2343,7 @@ public Tensor heaviside(Tensor other) /// Computes the regularized lower incomplete gamma function /// /// The second non-negative input tensor - /// + public Tensor igamma(Tensor other) { var res = THSTensor_igamma(Handle, other.Handle); @@ -2372,7 +2359,7 @@ public Tensor igamma(Tensor other) /// Computes the regularized upper incomplete gamma function. /// /// The second non-negative input tensor - /// + public Tensor igammac(Tensor other) { var res = THSTensor_igammac(Handle, other.Handle); @@ -2387,7 +2374,7 @@ public Tensor igammac(Tensor other) /// /// Computes the zeroth order modified Bessel function of the first kind for each element of input. /// - /// + public Tensor i0() { var res = THSTensor_i0(Handle); @@ -2623,7 +2610,7 @@ public Tensor bmm(Tensor batch2) /// If no suitable index found, return 0 for non-numerical value (eg. nan, inf) or the size of boundaries (one pass the last index). /// In other words, if false, gets the lower bound index for each value in input from boundaries. /// If true, gets the upper bound index instead. Default value is False. - /// + public Tensor bucketize(Tensor boundaries, bool outInt32 = false, bool right = false) { var res = THSTensor_bucketize(Handle, boundaries.Handle, outInt32, right); @@ -2907,7 +2894,7 @@ public bool Equals(Tensor target) /// Relative tolerance /// Absolute tolerance /// If true, then two NaN s will be considered equal - /// + public bool allclose(Tensor target, double rtol = 1e-05, double atol = 1e-08, bool equal_nan = false) { var res = THSTensor_allclose(Handle, target.Handle, rtol, atol, equal_nan); @@ -3036,7 +3023,7 @@ public Tensor lcm_(Tensor other) /// Multiplies input by pow(2,other). /// /// A tensor of exponents, typically integers - /// + /// Typically this function is used to construct floating point numbers by multiplying mantissas in input with integral powers of two created from the exponents in other. public Tensor ldexp(Tensor other) { @@ -3229,7 +3216,7 @@ public Tensor[] unbind(int dimension = 0) /// /// The size of a single chunk /// The dimension along which to split the tensor. - /// + public Tensor[] split(long size, int dimension = 0) { IntPtr[] ptrArray; @@ -3251,7 +3238,7 @@ public Tensor[] split(long size, int dimension = 0) /// /// A list of sizes for each chunk /// The dimension along which to split the tensor. - /// + public Tensor[] split(long[] sizes, int dimension = 0) { IntPtr[] ptrArray; @@ -3329,7 +3316,7 @@ public Tensor[] tensor_split(Tensor indices, int dimension = 0) /// Splits input, a tensor with one or more dimensions, into multiple tensors vertically according to sizes. /// /// The size of each chunk - /// + public Tensor[] vsplit(long size) { if (this.shape[0] % size != 0) throw new ArgumentException("The first dimension must be evenly divisible by the size"); @@ -3351,7 +3338,7 @@ public Tensor[] vsplit(long size) /// Splits input, a tensor with one or more dimensions, into multiple tensors vertically according to sizes. /// /// A list of split points - /// + public Tensor[] vsplit(params long[] sizes) { IntPtr[] ptrArray; @@ -3373,7 +3360,7 @@ public Tensor[] vsplit(params long[] sizes) /// Splits input, a tensor with one or more dimensions, into multiple tensors vertically according to indices. /// /// A list of split points - /// + public Tensor[] vsplit(Tensor indices) => tensor_split(indices, 0); @@ -3384,7 +3371,7 @@ public Tensor[] vsplit(params long[] sizes) /// Splits input, a tensor with one or more dimensions, into multiple tensors horizontally according to sizes. /// /// The size of each chunk - /// + public Tensor[] hsplit(long size) { if (this.shape[1] % size != 0) throw new ArgumentException("The second dimension must be evenly divisible by the size"); @@ -3406,7 +3393,7 @@ public Tensor[] hsplit(long size) /// Splits input, a tensor with one or more dimensions, into multiple tensors horizontally according to sizes. /// /// A list of split points - /// + public Tensor[] hsplit(params long[] sizes) { IntPtr[] ptrArray; @@ -3428,7 +3415,7 @@ public Tensor[] hsplit(params long[] sizes) /// Splits input, a tensor with one or more dimensions, into multiple tensors horizontally according to indices. /// /// A list of split points - /// + public Tensor[] hsplit(Tensor indices) => tensor_split(indices, 1); [DllImport("LibTorchSharp")] @@ -3438,7 +3425,7 @@ public Tensor[] hsplit(params long[] sizes) /// Splits input, a tensor with three or more dimensions, into multiple tensors depthwise according to indices_or_sections. Each split is a view of input. /// /// The size of each chunk - /// + public Tensor[] dsplit(long size) { if (this.shape[2] % size != 0) throw new ArgumentException("The third dimension must be evenly divisible by the size"); @@ -3492,7 +3479,7 @@ public Tensor[] dsplit(params long[] sizes) /// /// The number of chunks to return /// Dimension along which to split the tensor - /// + /// The last chunk will be smaller if the tensor size along the given dimension dim is not divisible by chunks. public Tensor[] chunk(long chunks, long dim = 0L) { @@ -3519,7 +3506,7 @@ public Tensor[] chunk(long chunks, long dim = 0L) /// k for the k-th smallest element /// The dimension to find the kth value along /// Whether the output tensor has dim retained or not. - /// + public static (Tensor, Tensor) kthvalue(Tensor input, long k, long? dim, bool keepdim = false) { var values = THSTensor_kthvalue(input.Handle, k, dim.HasValue ? dim.Value : -1, keepdim, out var indices); @@ -3586,7 +3573,7 @@ public Tensor mean() /// 1D tensor of quantile values in the range [0, 1] /// The dimension to reduce. /// Whether the output tensor has dim retained or not. - /// + Tensor quantile(Tensor q, long dim = -1, bool keepdim = false) { var res = THSTensor_quantile(Handle, q.Handle, dim, keepdim); @@ -3605,7 +3592,7 @@ Tensor quantile(Tensor q, long dim = -1, bool keepdim = false) /// 1D tensor of quantile values in the range [0, 1] /// The dimension to reduce. /// Whether the output tensor has dim retained or not. - /// + Tensor nanquantile(Tensor q, long dim = -1, bool keepdim = false) { var res = THSTensor_nanquantile(Handle, q.Handle, dim, keepdim); @@ -3622,7 +3609,7 @@ Tensor nanquantile(Tensor q, long dim = -1, bool keepdim = false) /// /// The dimension to reduce, the last dimension by default. /// Whether the output tensor has dim retained or not - /// + public (Tensor values, Tensor indices) mode(long dim = -1L, bool keepdim = false) { @@ -3980,7 +3967,7 @@ public Tensor expand(long[] sizes, bool isImplicit = false) /// /// Expand this tensor to the same size as other. /// - /// + public Tensor expand_as(Tensor other) => expand(other.shape); @@ -4410,7 +4397,7 @@ public Tensor zeros(params long[] sizes) /// /// Fills the tensor with zeros. /// - /// + public Tensor zero_() { return zeros(shape); @@ -4909,7 +4896,7 @@ public Tensor narrow(long dimension, long start, long length) /// /// /// - /// + public Tensor nonzero() { var res = THSTensor_nonzero(Handle); @@ -5212,7 +5199,7 @@ public override string ToString() /// The format string to use for floating point values. /// The width of each line of the output string. /// The CulturInfo to use when formatting the text - /// + public string ToString(bool withData, string fltFormat = "g5", int width = 100, CultureInfo? cultureInfo = null) { var actualCulturInfo = cultureInfo ?? CultureInfo.CurrentCulture; @@ -5643,6 +5630,20 @@ public static bool is_complex(ScalarType type) public static bool is_floating_point(Tensor t) => is_floating_point(t.dtype); public static bool is_complex(Tensor t) => is_complex(t.dtype); + /// + /// Returns a view of input as a real tensor. + /// For an input complex tensor of size m1, m2, …, mi, this function returns a new real tensor of size m1, m2, …, mi, 2, where the last dimension of size 2 represents the real and imaginary components of complex numbers. + /// + /// The input tensor + public static Tensor view_as_real(Tensor input) => input.view_as_real(); + + /// + /// Returns a view of input as a complex tensor. + /// For an input complex tensor of size m1, m2, …, mi, 2, this function returns a new complex tensor of size m1, m2, …, mi where the last dimension of the input tensor is expected to represent the real and imaginary components of complex numbers. + /// + /// The input tensor + public static Tensor view_as_complex(Tensor input) => input.view_as_complex(); + public static ScalarType @bool = ScalarType.Bool; public static ScalarType uint8 = ScalarType.Byte; From 275400feb6f14b0b3132a7abef89817e2b2b82f2 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Wed, 16 Feb 2022 14:35:31 -0800 Subject: [PATCH 18/25] Update version number. --- azure-pipelines.yml | 28 ++++++++++++++-------------- build/BranchInfo.props | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b971c9f32..5ae2cf53e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -518,7 +518,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp.*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -527,7 +527,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp-*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -537,7 +537,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -547,7 +547,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -557,7 +557,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -602,7 +602,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -613,7 +613,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -624,7 +624,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -635,7 +635,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -646,7 +646,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -689,7 +689,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -700,7 +700,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -711,7 +711,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -722,7 +722,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true \ No newline at end of file diff --git a/build/BranchInfo.props b/build/BranchInfo.props index 3c8e510e0..20628973c 100644 --- a/build/BranchInfo.props +++ b/build/BranchInfo.props @@ -2,7 +2,7 @@ 0 96 - 0 + 1 From dc022eeef3064cc74f2685e7630f5a6cb843eec2 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Thu, 17 Feb 2022 10:23:16 -0800 Subject: [PATCH 19/25] Added is_leaf and retain_grad functions. --- azure-pipelines.yml | 28 +++---- src/Native/LibTorchSharp/THSTensor.cpp | 10 +++ src/Native/LibTorchSharp/THSTensor.h | 4 + src/TorchSharp/NN/Module.cs | 10 ++- src/TorchSharp/Tensor/Tensor.cs | 26 +++++- test/TorchSharpTest/TestTorchTensorBugs.cs | 96 +++++++++++----------- 6 files changed, 105 insertions(+), 69 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5ae2cf53e..b971c9f32 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -518,7 +518,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp.*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -527,7 +527,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp-*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -537,7 +537,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -547,7 +547,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -557,7 +557,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -602,7 +602,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -613,7 +613,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -624,7 +624,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -635,7 +635,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -646,7 +646,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -689,7 +689,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -700,7 +700,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -711,7 +711,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -722,7 +722,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true \ No newline at end of file diff --git a/src/Native/LibTorchSharp/THSTensor.cpp b/src/Native/LibTorchSharp/THSTensor.cpp index 6e009792d..9e150080d 100644 --- a/src/Native/LibTorchSharp/THSTensor.cpp +++ b/src/Native/LibTorchSharp/THSTensor.cpp @@ -1030,6 +1030,16 @@ int THSTensor_requires_grad(const Tensor tensor) CATCH_RETURN(int, 0, tensor->requires_grad()); } +void THSTensor_retain_grad(const Tensor tensor) +{ + CATCH(tensor->retain_grad();); +} + +int64_t THSTensor_is_leaf(const Tensor tensor) +{ + CATCH_RETURN(int64_t, 0, tensor->is_leaf();); +} + Tensor THSTensor_reshape(const Tensor tensor, const int64_t* shape, const int length) { CATCH_TENSOR(tensor->reshape(at::ArrayRef(shape, length))); diff --git a/src/Native/LibTorchSharp/THSTensor.h b/src/Native/LibTorchSharp/THSTensor.h index a75661fca..c21bc396f 100644 --- a/src/Native/LibTorchSharp/THSTensor.h +++ b/src/Native/LibTorchSharp/THSTensor.h @@ -586,6 +586,8 @@ EXPORT_API(Tensor) THSTensor_inverse(const Tensor tensor); EXPORT_API(int) THSTensor_is_contiguous(const Tensor input); +EXPORT_API(int64_t) THSTensor_is_leaf(const Tensor tensor); + EXPORT_API(int) THSTensor_is_sparse(const Tensor tensor); EXPORT_API(Tensor) THSTensor_isclose(const Tensor tensor, const Tensor other, const double rtol, const double atol, const bool equal_nan); @@ -988,6 +990,8 @@ EXPORT_API(Tensor) THSTensor_remainder_scalar(const Tensor left, const Scalar ri EXPORT_API(Tensor) THSTensor_remainder_scalar_(const Tensor left, const Scalar right); +EXPORT_API(void) THSTensor_retain_grad(const Tensor tensor); + EXPORT_API(Tensor) THSTensor_rsqrt(const Tensor tensor); EXPORT_API(Tensor) THSTensor_rsqrt_(const Tensor tensor); diff --git a/src/TorchSharp/NN/Module.cs b/src/TorchSharp/NN/Module.cs index 7bf6a8075..87ef9d657 100644 --- a/src/TorchSharp/NN/Module.cs +++ b/src/TorchSharp/NN/Module.cs @@ -133,6 +133,8 @@ public virtual Module to(DeviceType deviceType, int deviceIndex = -1) { if (deviceType != DeviceType.CUDA) deviceIndex = -1; + if (deviceType == DeviceType.CUDA && !torch.cuda.is_available()) throw new InvalidOperationException("CUDA is not available."); + if (deviceType != _deviceType || deviceIndex != _deviceIndex) { torch.InitializeDeviceType(deviceType); @@ -151,7 +153,9 @@ public virtual Module to(DeviceType deviceType, int deviceIndex = -1) if (param is not null) { // This test must come before the Tensor test if (deviceType != param.device_type || deviceIndex != param.device_index) { - var p = new Modules.Parameter(param.to(deviceType, deviceIndex), param.requires_grad); + var t = param.to(deviceType, deviceIndex); + t.retain_grad(); + var p = new Modules.Parameter(t, param.requires_grad); field.SetValue(this, p); ConditionallyRegisterParameter(name, p); } @@ -218,7 +222,9 @@ public virtual Module to(ScalarType dtype) if (param is not null) { // This test must come before the Tensor test if (dtype != param.dtype) { - var p = new Modules.Parameter(param.to(dtype), param.requires_grad); + var t = param.to(dtype); + t.retain_grad(); + var p = new Modules.Parameter(t, param.requires_grad); field.SetValue(this, p); ConditionallyRegisterParameter(name, p); } diff --git a/src/TorchSharp/Tensor/Tensor.cs b/src/TorchSharp/Tensor/Tensor.cs index 176a919c7..62f8fc0a4 100644 --- a/src/TorchSharp/Tensor/Tensor.cs +++ b/src/TorchSharp/Tensor/Tensor.cs @@ -238,6 +238,16 @@ internal IntPtr MoveHandle() public bool is_cuda { get { return device.type == DeviceType.CUDA; } } + [DllImport("LibTorchSharp")] + internal static extern long THSTensor_is_leaf(IntPtr handle); + + /// + /// All Tensors that have requires_grad which is true will be leaf Tensors by convention. + /// For Tensors that have requires_grad which is true, they will be leaf Tensors if they were created by the user.This means that they are not the result of an operation and so grad_fn is None. + /// Only leaf Tensors will have their grad populated during a call to backward(). To get grad populated for non-leaf Tensors, you can use retain_grad(). + /// + public bool is_leaf { get => THSTensor_is_leaf(Handle) != 0; } + [DllImport("LibTorchSharp")] internal static extern IntPtr THSTensor_alias(IntPtr handle); @@ -631,16 +641,24 @@ public bool requires_grad { } } - /// - /// Change if autograd should record operations on this tensor: sets this tensor’s requires_grad attribute in-place. Returns this tensor. - /// - /// public Tensor requires_grad_(bool requires_grad = true) { this.requires_grad = true; return this; } + [DllImport("LibTorchSharp")] + static extern void THSTensor_retain_grad(IntPtr handle); + + /// + /// Enables this Tensor to have their grad populated during backward(). This is a no-op for leaf tensors. + /// + public void retain_grad() + { + THSTensor_retain_grad(Handle); + torch.CheckForErrors(); + } + /// /// Adds gradient tracking. /// diff --git a/test/TorchSharpTest/TestTorchTensorBugs.cs b/test/TorchSharpTest/TestTorchTensorBugs.cs index a94b60bde..a1021c0b2 100644 --- a/test/TorchSharpTest/TestTorchTensorBugs.cs +++ b/test/TorchSharpTest/TestTorchTensorBugs.cs @@ -502,11 +502,44 @@ public override torch.Tensor forward(torch.Tensor t) } } - [Fact] + [Fact]//(Skip ="")] public void ValidateIssue516() { - using var module = new LMHead(128, 1000, "tanh", torch.rand(128, 1000)); - using var optim = torch.optim.Adam(module.parameters(), 0.002); + if (torch.cuda.is_available()) { + // *** Note1 *** + // When cuda == false, everything all right. + // When cuda == true, the following warning raised. + var cuda = true; + + // All message printed in console: + // [W TensorBody.h:417] Warning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. + // Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to + // be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf + // Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 + // for more informations. (function grad) + + var model = new TestGradWarningModel(); + if (cuda) { + model.cuda(); + } + var optimizer = torch.optim.Adam(model.parameters()); + optimizer.zero_grad(); // Raise a warning + + var x = torch.ones(5, 3); + var y = torch.ones(5, 4); + if (cuda) { + x = x.cuda(); + y = y.cuda(); + } + var z = model.forward(x); + var lossFunc = torch.nn.functional.cross_entropy_loss(); + var loss = lossFunc(y, z); + loss.backward(); + optimizer.step(); // Raise a warning + + var grad1 = optimizer.parameters().ToArray()[0].grad(); // Raise a warning + var grad2 = model.Weight.grad(); // Raise a warning + } } internal abstract class BaseModule : torch.nn.Module @@ -518,61 +551,26 @@ protected BaseModule(string name) : base(name) } } - internal sealed class ActivationFunction : BaseModule - { - private torch.nn.Module Function; - - public ActivationFunction(string name) : base(name) - { - Function = name?.ToLower() switch { - "relu" => torch.nn.ReLU(), - "gelu" => torch.nn.GELU(), - "tanh" => torch.nn.Tanh(), - "linear" => torch.nn.Identity(), - _ => throw new NotSupportedException($"Activation function {name} not supported.") - }; - } - - public override torch.Tensor forward(torch.Tensor x) - { - return Function.forward(x); - } - - public override string GetName() - { - return Function.GetName(); - } - } - - internal sealed class LMHead : BaseModule + public class TestGradWarningModel : torch.nn.Module { - public readonly Modules.Sequential Projection; - public readonly TorchSharp.Modules.Parameter Weight; - public readonly TorchSharp.Modules.Parameter Bias; + public readonly Modules.Parameter Weight; - public LMHead(int inSize, int embedDim, string activationFn, torch.Tensor outputWeight) - : base(nameof(LMHead)) + public TestGradWarningModel() : base(nameof(TestGradWarningModel)) { - if (outputWeight is null) { - throw new ArgumentNullException(nameof(outputWeight)); - } - Projection = torch.nn.Sequential( - ("dense", torch.nn.Linear(inSize, embedDim)), - ("activation", new ActivationFunction(activationFn)), - ("layernorm", torch.nn.LayerNorm(new long[] { embedDim })) - ); - - var outputDim = outputWeight.shape[^2]; - Weight = new TorchSharp.Modules.Parameter(outputWeight); - Bias = new TorchSharp.Modules.Parameter(torch.zeros(outputDim, dtype: torch.float32, requiresGrad: true)); + // *** Note2 *** + // If I set the tensor device here, then setting cuda = true at line 20 won't cause the warnings. + // The warnings won't appear even when setting cuda = false at line 20 and execute a "model.cpu();". + // So I guess this is a single-directional problem (cpu -> cuda). + Weight = torch.zeros(new long[] { 3, 4 }).AsParameter(); + //Weight = torch.zeros(new long[] { 3, 4 }, device: torch.CUDA).AsParameter(); RegisterComponents(); } - public override torch.Tensor forward(torch.Tensor features) + public override torch.Tensor forward(torch.Tensor t) { - return torch.randn(10); + return torch.matmul(t, Weight); } } } From 376eeb6913e39d310165ecc84d85ba72cdca52a4 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Thu, 17 Feb 2022 11:58:28 -0800 Subject: [PATCH 20/25] Temporary fix. --- azure-pipelines.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b971c9f32..5ae2cf53e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -518,7 +518,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp.*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -527,7 +527,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp-*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -537,7 +537,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -547,7 +547,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -557,7 +557,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true continueOnError: true @@ -602,7 +602,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -613,7 +613,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -624,7 +624,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -635,7 +635,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -646,7 +646,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -689,7 +689,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -700,7 +700,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -711,7 +711,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -722,7 +722,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/SignedPackages' + publishVstsFeed: 'TorchSharp/Testing' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true \ No newline at end of file From 2dfb4f1c5e51aaf3d50298bac715fef490c8d1ec Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Thu, 17 Feb 2022 18:56:39 -0800 Subject: [PATCH 21/25] Adding "named_parameters" version for all optimizer factories. Updated release notes. --- README.md | 2 + RELEASENOTES.md | 15 +-- src/TorchSharp/NN/Optimizer.cs | 194 +++++++++++++++++++++++++++++++++ 3 files changed, 204 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8afbe8ac7..01a1d2f5a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ [![Build Status](https://dotnet.visualstudio.com/TorchSharp/_apis/build/status/dotnet.TorchSharp?branchName=main)](https://dotnet.visualstudio.com/TorchSharp/_build/latest?definitionId=174&branchName=main) +Please check the [Release Notes](RELEASENOTES.md) file for news on what's been updated in each new release. + __TorchSharp is now in the .NET Foundation!__ If you are using TorchSharp from NuGet, you should be using a version >= 0.95.1 of TorchSharp, and >= 1.10.0.1 of the libtorch-xxx redistributable packages. We recommend using one of the 'bundled' packages: TorchSharp-cpu, TorchSharp-cuda-windows, or TorchSharp-cuda-linux. They will pull in the right libtorch backends. diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 4b6fa599c..2f2f800ee 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -8,16 +8,17 @@ __API Changes:__ __NOTE__: This release contains breaking changes.
-The APIs to create optimizers all take 'named_parameters' rather than 'parameters' now.
-Support for parameter groups in most optimizers. -Support for parameter groups in LR schedulers. +The APIs to create optimizers all take 'parameters()' as well as 'named_parameters()' now.
+Support for parameter groups in most optimizers.
+Support for parameter groups in LR schedulers.
__Fixed Bugs:__ +#495 Add support for OptimizerParamGroup
+#509 Tensor.conj() not implemented
#510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d
#515 what's reason for making register_module internal?
-#495 Add support for OptimizerParamGroup -#509 Tensor.conj() not implemented +#516 AdamW bug on v0.96.0
## NuGet Version 0.96.0 @@ -31,9 +32,9 @@ Lower-cased names: Module.Train --> Module.train and Module.Eval --> Module.eval __Fixed Bugs:__ -#500 BatchNorm1d throws exception during eval with batch size of 1
-#499 Setting Linear.weight is not reflected in 'parameters()'
#496 Wrong output shape of torch.nn.Conv2d with 2d stride overload
+#499 Setting Linear.weight is not reflected in 'parameters()'
+#500 BatchNorm1d throws exception during eval with batch size of 1
## NuGet Version 0.95.4 diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs index d0ed4352d..f872e10fb 100644 --- a/src/TorchSharp/NN/Optimizer.cs +++ b/src/TorchSharp/NN/Optimizer.cs @@ -155,6 +155,11 @@ public interface IBetas [DllImport("LibTorchSharp")] private static extern IntPtr THSNN_LBFGS_ctor(IntPtr parameters, int len, double learningRate, long max_iter, long max_eval, double tolerange_grad, double tolerance_change, long history_size); + public static LBFGS LBFGS(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.01, long max_iter = 20, long? max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100) + { + return LBFGS(parameters.Select(np => np.parameter), lr, max_iter, max_eval.Value, tolerange_grad, tolerance_change, history_size); + } + public static LBFGS LBFGS(IEnumerable parameters, double lr = 0.01, long max_iter = 20, long? max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100) { if (!max_eval.HasValue) max_eval = 5 * max_iter / 4; @@ -185,6 +190,24 @@ public static RMSProp RMSProp(IEnumerable parameters, double lr = 0.0 return new RMSProp(parameters, lr, alpha, eps, weight_decay, momentum, centered); } + /// + /// Implements RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. + /// + /// Parameters to optimize + /// Learning rate (default: 1e-2) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Smoothing constant (default: 0.99) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum factor (default: 0) + /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public static RMSProp RMSProp(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) + { + return new RMSProp(parameters.Select(np => np.parameter), lr, alpha, eps, weight_decay, momentum, centered); + } + /// /// Implements RMSprop algorithm. /// @@ -222,6 +245,25 @@ public static Adam Adam(IEnumerable parameters, double lr = 1e-3, dou return new Adam(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); } + /// + /// Implements Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static Adam Adam(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new Adam(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + /// /// Implements Adam algorithm. /// @@ -260,6 +302,25 @@ public static AdamW AdamW(IEnumerable parameters, double lr = 1e-3, d return new AdamW(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); } + /// + /// Implements AdamW algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static AdamW AdamW(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new AdamW(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + /// /// Implements AdamW algorithm. /// @@ -296,6 +357,23 @@ public static Adagrad Adagrad(IEnumerable parameters, double lr = 1e- return new Adagrad(parameters, lr, lr_decay, weight_decay, initial_accumulator_value, eps); } + /// + /// Implements Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public static Adagrad Adagrad(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + { + return new Adagrad(parameters.Select(np => np.parameter), lr, lr_decay, weight_decay, initial_accumulator_value, eps); + } + /// /// Implements Adagrad algorithm. /// @@ -329,6 +407,22 @@ public static Adadelta Adadelta(IEnumerable parameters, double lr = 1 return new Adadelta(parameters, lr, rho, eps, weight_decay); } + /// + /// Implements Adadelta algorithm. + /// + /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing a running average of squared gradients (default: 0.9) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adadelta Adadelta(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + { + return new Adadelta(parameters.Select(np => np.parameter), lr, rho, eps, weight_decay); + } + /// /// Implements Adadelta algorithm. /// @@ -363,6 +457,24 @@ public static NAdam NAdam(IEnumerable named_parameters, double lr = 0 return new NAdam(named_parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); } + /// + /// Implements NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + public static NAdam NAdam(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + { + return new NAdam(named_parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, momentum_decay); + } + /// /// Implements NAdam algorithm. /// @@ -398,6 +510,23 @@ public static RAdam RAdam(IEnumerable parameters, double lr = 0.002, return new RAdam(parameters, lr, beta1, beta2, eps, weight_decay); } + /// + /// Implements RAdam algorithm. + /// + /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' + /// https://arxiv.org/abs/1908.03265 + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + public static RAdam RAdam(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new RAdam(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay); + } + /// /// Implements RAdam algorithm. /// @@ -432,6 +561,23 @@ public static Adamax Adamax(IEnumerable parameters, double lr = 0.002 return new Adamax(parameters, lr, beta1, beta2, eps, weight_decay); } + /// + /// Implements Adamax algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adamax Adamax(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new Adamax(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay); + } + /// /// Implements Adamax algorithm (a variant of Adam based on infinity norm). /// @@ -466,6 +612,23 @@ public static ASGD ASGD(IEnumerable parameters, double lr = 1e-3, dou return new Modules.ASGD(parameters, lr, lambd, alpha, t0, weight_decay); } + /// + /// Implements Averaged Stochastic Gradient Descent. + /// + /// It has been proposed in Acceleration of stochastic approximation by averaging. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Decay term (default: 1e-4) + /// Power for eta update (default: 0.75) + /// Point at which to start averaging (default: 1e6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static ASGD ASGD(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + { + return new Modules.ASGD(parameters.Select(np => np.parameter), lr, lambd, alpha, t0, weight_decay); + } + /// /// Implements Averaged Stochastic Gradient Descent. /// @@ -498,6 +661,21 @@ public static Rprop Rprop(IEnumerable parameters, double lr = 1e-2, d return new Rprop(parameters, lr, etaminus, etaplus, min_step, max_step); } + /// + /// Implements the resilient backpropagation algorithm. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Multiplicative increase factor. + /// Multiplicative decrease factor. + /// Minimum allowed step size. + /// Maximum allowed step size. + /// + public static Rprop Rprop(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + { + return new Rprop(parameters.Select(np => np.parameter), lr, etaminus, etaplus, min_step, max_step); + } + /// /// Implements the resilient backpropagation algorithm. /// @@ -529,6 +707,22 @@ public static Modules.SGD SGD(IEnumerable parameters, double learning return new Modules.SGD(parameters, learningRate, momentum, dampening, weight_decay, nesterov, maximize); } + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public static Modules.SGD SGD(IEnumerable<(string name, Parameter parameter)> parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + { + return new Modules.SGD(parameters.Select(np => np.parameter), learningRate, momentum, dampening, weight_decay, nesterov, maximize); + } + /// /// Implements stochastic gradient descent (optionally with momentum). /// From 07f4657bc980f286228df0df75f117377da41e65 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Thu, 17 Feb 2022 19:14:53 -0800 Subject: [PATCH 22/25] Updates to release notes and developer guide. --- DEVGUIDE.md | 4 ++++ RELEASENOTES.md | 1 + azure-pipelines.yml | 28 ++++++++++++++-------------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/DEVGUIDE.md b/DEVGUIDE.md index 1d35a0d99..12a5bc8da 100644 --- a/DEVGUIDE.md +++ b/DEVGUIDE.md @@ -231,3 +231,7 @@ version of PyTorch then quite a lot of careful work needs to be done. 10. Remember to delete all massive artifacts from Azure DevOps and reset this `BuildLibTorchPackages` in in [azure-pipelines.yml](azure-pipelines.yml) + +## Building with Visual Studio + +In order for builds to work properly using Visual Studio 2019 or 2022, you must start VS from the 'x64 Native Tools Command Prompt for VS 2022' (or 2019) in order for the full environment to be set up correctly. Starting VS from the desktop or taskbar will not work properly. \ No newline at end of file diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 2f2f800ee..d2b291bcc 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -19,6 +19,7 @@ __Fixed Bugs:__ #510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d
#515 what's reason for making register_module internal?
#516 AdamW bug on v0.96.0
+#521 Can't set Tensor slice using indexing
## NuGet Version 0.96.0 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5ae2cf53e..b971c9f32 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -518,7 +518,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp.*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -527,7 +527,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/TorchSharp-*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -537,7 +537,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -547,7 +547,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -557,7 +557,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/libtorch-cpu*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true continueOnError: true @@ -602,7 +602,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -613,7 +613,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -624,7 +624,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -635,7 +635,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -646,7 +646,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*win*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -689,7 +689,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -700,7 +700,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -711,7 +711,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true @@ -722,7 +722,7 @@ jobs: inputs: command: push packagesToPush: '$(Pipeline.Workspace)/**/*cuda*linux*.nupkg' - publishVstsFeed: 'TorchSharp/Testing' + publishVstsFeed: 'TorchSharp/SignedPackages' allowPackageConflicts: true # often fails - try but ignore the error until we sort it out continueOnError: true \ No newline at end of file From e0718d5dd102b98b1012573075e759382dc5e51c Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Fri, 18 Feb 2022 17:22:43 -0800 Subject: [PATCH 23/25] Initial round of PR responses. --- src/TorchSharp/NN/Init.cs | 2 +- src/TorchSharp/NN/Optimizer.cs | 2853 ----------------- src/TorchSharp/Optimizers/ASGD.cs | 256 ++ src/TorchSharp/Optimizers/Adadelta.cs | 239 ++ src/TorchSharp/Optimizers/Adagrad.cs | 256 ++ src/TorchSharp/Optimizers/Adam.cs | 298 ++ src/TorchSharp/Optimizers/AdamW.cs | 296 ++ src/TorchSharp/Optimizers/Adamax.cs | 271 ++ src/TorchSharp/Optimizers/LBFGS.cs | 112 + .../{NN => Optimizers}/LRScheduler.cs | 0 src/TorchSharp/Optimizers/NAdam.cs | 284 ++ src/TorchSharp/Optimizers/Optimizer.cs | 357 +++ src/TorchSharp/Optimizers/RAdam.cs | 278 ++ src/TorchSharp/Optimizers/RMSprop.cs | 282 ++ src/TorchSharp/Optimizers/Rprop.cs | 259 ++ src/TorchSharp/Optimizers/SGD.cs | 264 ++ test/TorchSharpTest/TestTorchTensorBugs.cs | 45 +- test/TorchSharpTest/TestTraining.cs | 1680 ++++------ 18 files changed, 4066 insertions(+), 3966 deletions(-) delete mode 100644 src/TorchSharp/NN/Optimizer.cs create mode 100644 src/TorchSharp/Optimizers/ASGD.cs create mode 100644 src/TorchSharp/Optimizers/Adadelta.cs create mode 100644 src/TorchSharp/Optimizers/Adagrad.cs create mode 100644 src/TorchSharp/Optimizers/Adam.cs create mode 100644 src/TorchSharp/Optimizers/AdamW.cs create mode 100644 src/TorchSharp/Optimizers/Adamax.cs create mode 100644 src/TorchSharp/Optimizers/LBFGS.cs rename src/TorchSharp/{NN => Optimizers}/LRScheduler.cs (100%) create mode 100644 src/TorchSharp/Optimizers/NAdam.cs create mode 100644 src/TorchSharp/Optimizers/Optimizer.cs create mode 100644 src/TorchSharp/Optimizers/RAdam.cs create mode 100644 src/TorchSharp/Optimizers/RMSprop.cs create mode 100644 src/TorchSharp/Optimizers/Rprop.cs create mode 100644 src/TorchSharp/Optimizers/SGD.cs diff --git a/src/TorchSharp/NN/Init.cs b/src/TorchSharp/NN/Init.cs index 4d15852e5..d16e93f15 100644 --- a/src/TorchSharp/NN/Init.cs +++ b/src/TorchSharp/NN/Init.cs @@ -181,7 +181,7 @@ public static Tensor xavier_normal_(Tensor tensor, double gain = 1.0) /// public static Tensor glorot_normal_(Tensor tensor, double gain = 1.0) => xavier_normal_(tensor, gain); - public static (long fanIn, long fanOut) CalculateFanInAndFanOut(Tensor tensor) + public static (long fanIn, long fanOut) CalculateFanInAndFanOut(Tensor tensor) { var dimensions = tensor.Dimensions; diff --git a/src/TorchSharp/NN/Optimizer.cs b/src/TorchSharp/NN/Optimizer.cs deleted file mode 100644 index f872e10fb..000000000 --- a/src/TorchSharp/NN/Optimizer.cs +++ /dev/null @@ -1,2853 +0,0 @@ -// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using static TorchSharp.torch; - -namespace TorchSharp -{ - using Modules; - - public static partial class torch - { - public static partial class optim - { - public partial class Optimizer : IDisposable - { - /// - /// Class wrapping PyTorch's optimzer object reference. - /// - internal sealed class HType : SafeHandle - { - public HType(IntPtr preexistingHandle, bool ownsHandle) : base(IntPtr.Zero, ownsHandle) - { - SetHandle(preexistingHandle); - } - - public override bool IsInvalid => handle == IntPtr.Zero; - - // This is just for marshalling - internal HType() : base(IntPtr.Zero, true) - { - } - - [DllImport("LibTorchSharp")] - private static extern void THSNN_Optimizer_dispose(HType handle); - - protected override bool ReleaseHandle() - { - THSNN_Optimizer_dispose(this); - return true; - } - - protected override void Dispose(bool disposing) - { - if (disposing) { - ReleaseHandle(); - } - } - } - - internal HType handle; - - protected Optimizer(IntPtr handle) - { - if (handle != IntPtr.Zero) { - this.handle = new HType(handle, true); - } - } - - ~Optimizer() - { - Dispose(false); - } - - /// - /// Releases the storage. - /// - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - /// - /// Implements the .NET Dispose pattern. - /// - protected virtual void Dispose(bool disposing) - { - if (disposing && handle != null && !handle.IsInvalid) { - handle.Dispose(); - handle.SetHandleAsInvalid(); - } - } - - [DllImport("LibTorchSharp")] - private static extern void THSNN_Optimizer_zero_grad(HType module); - - public virtual void zero_grad() - { - THSNN_Optimizer_zero_grad(handle); - torch.CheckForErrors(); - } - - [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - public delegate IntPtr LossClosure(); - - - [DllImport("LibTorchSharp")] - private static extern IntPtr THSNN_Optimizer_step(HType module, LossClosure closure); - - public virtual Tensor step(Func closure = null) - { - IntPtr res = (closure == null) ? - THSNN_Optimizer_step(handle, null) : - THSNN_Optimizer_step(handle, () => { - return closure().DecoupleFromNativeHandle(); - }); - - if (res == IntPtr.Zero) - torch.CheckForErrors(); - - return (res == IntPtr.Zero) ? null : new Tensor(res); - } - - [DllImport("LibTorchSharp")] - private static extern void THSNN_Optimizer_getParameters(HType module, AllocatePinnedArray allocator); - - public virtual IEnumerable parameters() - { - IntPtr[] ptrArray; - - using (var pa = new PinnedArray()) { - THSNN_Optimizer_getParameters(handle, pa.CreateArray); - torch.CheckForErrors(); - ptrArray = pa.Array; - } - return ptrArray.Select(x => new Parameter(x)); - } - - public virtual IEnumerable ParamGroups { - get => _parameter_groups; - } - - protected IList _parameter_groups; - } - - public interface ILearningRateController - { - double LearningRate { set; get; } - - double InitialLearningRate { set; get; } - } - - public interface IMomentum - { - double Momentum { get; set; } - } - - public interface IBetas - { - (double, double) Betas { get; set; } - } - - [DllImport("LibTorchSharp")] - private static extern IntPtr THSNN_LBFGS_ctor(IntPtr parameters, int len, double learningRate, long max_iter, long max_eval, double tolerange_grad, double tolerance_change, long history_size); - - public static LBFGS LBFGS(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.01, long max_iter = 20, long? max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100) - { - return LBFGS(parameters.Select(np => np.parameter), lr, max_iter, max_eval.Value, tolerange_grad, tolerance_change, history_size); - } - - public static LBFGS LBFGS(IEnumerable parameters, double lr = 0.01, long max_iter = 20, long? max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100) - { - if (!max_eval.HasValue) max_eval = 5 * max_iter / 4; - - var parray = new PinnedArray(); - IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray()); - - var res = THSNN_LBFGS_ctor(paramsRef, parray.Array.Length, lr, max_iter, max_eval.Value, tolerange_grad, tolerance_change, history_size); - if (res == IntPtr.Zero) { torch.CheckForErrors(); } - return new LBFGS(res, lr); - } - - /// - /// Implements RMSprop algorithm. - /// - /// Proposed by G.Hinton in his course. - /// - /// Parameters to optimize - /// Learning rate (default: 1e-2) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Smoothing constant (default: 0.99) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum factor (default: 0) - /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance - /// - public static RMSProp RMSProp(IEnumerable parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) - { - return new RMSProp(parameters, lr, alpha, eps, weight_decay, momentum, centered); - } - - /// - /// Implements RMSprop algorithm. - /// - /// Proposed by G.Hinton in his course. - /// - /// Parameters to optimize - /// Learning rate (default: 1e-2) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Smoothing constant (default: 0.99) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum factor (default: 0) - /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance - /// - public static RMSProp RMSProp(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) - { - return new RMSProp(parameters.Select(np => np.parameter), lr, alpha, eps, weight_decay, momentum, centered); - } - - /// - /// Implements RMSprop algorithm. - /// - /// Proposed by G.Hinton in his course. - /// - /// Parameters to optimize - /// Learning rate (default: 1e-2) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Smoothing constant (default: 0.99) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum factor (default: 0) - /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance - /// - public static RMSProp RMSProp(IEnumerable parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) - { - return new RMSProp(parameters, lr, alpha, eps, weight_decay, momentum, centered); - } - - /// - /// Implements Adam algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-3) - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm. (default: False) - /// - /// - public static Adam Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - { - return new Adam(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); - } - - /// - /// Implements Adam algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-3) - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm. (default: False) - /// - /// - public static Adam Adam(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - { - return new Adam(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); - } - - /// - /// Implements Adam algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-3) - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm. (default: False) - /// - /// - public static Adam Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - { - return new Adam(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); - } - - /// - /// Implements AdamW algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-3) - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm. (default: False) - /// - /// - public static AdamW AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - { - return new AdamW(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); - } - - /// - /// Implements AdamW algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-3) - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm. (default: False) - /// - /// - public static AdamW AdamW(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - { - return new AdamW(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); - } - - /// - /// Implements AdamW algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-3) - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm. (default: False) - /// - /// - public static AdamW AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - { - return new AdamW(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); - } - - /// - /// Implements Adagrad algorithm. - /// - /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-2) - /// learning rate decay (default: 0) - /// weight decay (L2 penalty) (default: 0) - /// - /// Term added to the denominator to improve numerical stability (default: 1e-10) - /// - public static Adagrad Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) - { - return new Adagrad(parameters, lr, lr_decay, weight_decay, initial_accumulator_value, eps); - } - - /// - /// Implements Adagrad algorithm. - /// - /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-2) - /// learning rate decay (default: 0) - /// weight decay (L2 penalty) (default: 0) - /// - /// Term added to the denominator to improve numerical stability (default: 1e-10) - /// - public static Adagrad Adagrad(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) - { - return new Adagrad(parameters.Select(np => np.parameter), lr, lr_decay, weight_decay, initial_accumulator_value, eps); - } - - /// - /// Implements Adagrad algorithm. - /// - /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-2) - /// learning rate decay (default: 0) - /// weight decay (L2 penalty) (default: 0) - /// - /// Term added to the denominator to improve numerical stability (default: 1e-10) - /// - public static Adagrad Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) - { - return new Adagrad(parameters, lr, lr_decay, weight_decay, initial_accumulator_value, eps); - } - - /// - /// Implements Adadelta algorithm. - /// - /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing a running average of squared gradients (default: 0.9) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) - /// Weight decay (L2 penalty) (default: 0) - /// - public static Adadelta Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) - { - return new Adadelta(parameters, lr, rho, eps, weight_decay); - } - - /// - /// Implements Adadelta algorithm. - /// - /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing a running average of squared gradients (default: 0.9) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) - /// Weight decay (L2 penalty) (default: 0) - /// - public static Adadelta Adadelta(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) - { - return new Adadelta(parameters.Select(np => np.parameter), lr, rho, eps, weight_decay); - } - - /// - /// Implements Adadelta algorithm. - /// - /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing a running average of squared gradients (default: 0.9) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) - /// Weight decay (L2 penalty) (default: 0) - /// - public static Adadelta Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) - { - return new Adadelta(parameters, lr, rho, eps, weight_decay); - } - - /// - /// Implements NAdam algorithm. - /// - /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. - /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum decay - public static NAdam NAdam(IEnumerable named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) - { - return new NAdam(named_parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); - } - - /// - /// Implements NAdam algorithm. - /// - /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. - /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum decay - public static NAdam NAdam(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) - { - return new NAdam(named_parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, momentum_decay); - } - - /// - /// Implements NAdam algorithm. - /// - /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. - /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum decay - public static NAdam NAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) - { - return new NAdam(parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); - } - - /// - /// Implements RAdam algorithm. - /// - /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' - /// https://arxiv.org/abs/1908.03265 - /// - /// Parameters to optimize. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - public static RAdam RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - { - return new RAdam(parameters, lr, beta1, beta2, eps, weight_decay); - } - - /// - /// Implements RAdam algorithm. - /// - /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' - /// https://arxiv.org/abs/1908.03265 - /// - /// Parameters to optimize. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - public static RAdam RAdam(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - { - return new RAdam(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay); - } - - /// - /// Implements RAdam algorithm. - /// - /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' - /// https://arxiv.org/abs/1908.03265 - /// - /// Parameters to optimize. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - public static RAdam RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - { - return new RAdam(parameters, lr, beta1, beta2, eps, weight_decay); - } - - /// - /// Implements Adamax algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// - public static Adamax Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - { - return new Adamax(parameters, lr, beta1, beta2, eps, weight_decay); - } - - /// - /// Implements Adamax algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// - public static Adamax Adamax(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - { - return new Adamax(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay); - } - - /// - /// Implements Adamax algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// - public static Adamax Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - { - return new Adamax(parameters, lr, beta1, beta2, eps, weight_decay); - } - - /// - /// Implements Averaged Stochastic Gradient Descent. - /// - /// It has been proposed in Acceleration of stochastic approximation by averaging. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Decay term (default: 1e-4) - /// Power for eta update (default: 0.75) - /// Point at which to start averaging (default: 1e6) - /// Weight decay (L2 penalty) (default: 0) - /// - public static ASGD ASGD(IEnumerable parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) - { - return new Modules.ASGD(parameters, lr, lambd, alpha, t0, weight_decay); - } - - /// - /// Implements Averaged Stochastic Gradient Descent. - /// - /// It has been proposed in Acceleration of stochastic approximation by averaging. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Decay term (default: 1e-4) - /// Power for eta update (default: 0.75) - /// Point at which to start averaging (default: 1e6) - /// Weight decay (L2 penalty) (default: 0) - /// - public static ASGD ASGD(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) - { - return new Modules.ASGD(parameters.Select(np => np.parameter), lr, lambd, alpha, t0, weight_decay); - } - - /// - /// Implements Averaged Stochastic Gradient Descent. - /// - /// It has been proposed in Acceleration of stochastic approximation by averaging. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Decay term (default: 1e-4) - /// Power for eta update (default: 0.75) - /// Point at which to start averaging (default: 1e6) - /// Weight decay (L2 penalty) (default: 0) - /// - public static ASGD ASGD(IEnumerable> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) - { - return new Modules.ASGD(parameters, lr, lambd, alpha, t0, weight_decay); - } - - /// - /// Implements the resilient backpropagation algorithm. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Multiplicative increase factor. - /// Multiplicative decrease factor. - /// Minimum allowed step size. - /// Maximum allowed step size. - /// - public static Rprop Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) - { - return new Rprop(parameters, lr, etaminus, etaplus, min_step, max_step); - } - - /// - /// Implements the resilient backpropagation algorithm. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Multiplicative increase factor. - /// Multiplicative decrease factor. - /// Minimum allowed step size. - /// Maximum allowed step size. - /// - public static Rprop Rprop(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) - { - return new Rprop(parameters.Select(np => np.parameter), lr, etaminus, etaplus, min_step, max_step); - } - - /// - /// Implements the resilient backpropagation algorithm. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Multiplicative increase factor. - /// Multiplicative decrease factor. - /// Minimum allowed step size. - /// Maximum allowed step size. - /// - public static Rprop Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) - { - return new Rprop(parameters, lr, etaminus, etaplus, min_step, max_step); - } - - /// - /// Implements stochastic gradient descent (optionally with momentum). - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Momentum factor (default: 0) - /// Dampening for momentum (default: 0) - /// Weight decay (L2 penalty) (default: 0) - /// Enables Nesterov momentum (default: False) - /// - /// - public static Modules.SGD SGD(IEnumerable parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) - { - return new Modules.SGD(parameters, learningRate, momentum, dampening, weight_decay, nesterov, maximize); - } - - /// - /// Implements stochastic gradient descent (optionally with momentum). - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Momentum factor (default: 0) - /// Dampening for momentum (default: 0) - /// Weight decay (L2 penalty) (default: 0) - /// Enables Nesterov momentum (default: False) - /// - /// - public static Modules.SGD SGD(IEnumerable<(string name, Parameter parameter)> parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) - { - return new Modules.SGD(parameters.Select(np => np.parameter), learningRate, momentum, dampening, weight_decay, nesterov, maximize); - } - - /// - /// Implements stochastic gradient descent (optionally with momentum). - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Momentum factor (default: 0) - /// Dampening for momentum (default: 0) - /// Weight decay (L2 penalty) (default: 0) - /// Enables Nesterov momentum (default: False) - /// - /// - public static Modules.SGD SGD(IEnumerable parameters, double learningRate, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) - { - return new Modules.SGD(parameters, learningRate, momentum, dampening, weight_decay, nesterov, maximize); - } - } - } - - namespace Modules - { - using static torch.optim; - - // Most optimizers are implemented in native code, but a few of them are directly implemented in - // managed code. - - /// - /// Base class to help with a couple of the things that managed-code implementations need. - /// - - public class OptimizerHelper : Optimizer - { - public OptimizerHelper() : base(IntPtr.Zero) - { - } - - public override void zero_grad() - { - foreach (var g in _parameter_groups) { - - foreach (var p in g.Parameters) { - - using var grad = p.grad(); - - if (grad is null) continue; - - grad.zero_().Dispose(); - } - } - } - - protected Tensor _step(Action body, Func loss_closure = null) where T:ParamGroup - { - Tensor loss = null; - - if (loss_closure != null) { - using (var _ = torch.enable_grad()) - loss = loss_closure(); - } - - using (var _ = torch.no_grad()) { - - using (var d = torch.NewDisposeScope()) { - - foreach (var group in _parameter_groups) { - - body(group as T); - - } - - d.DisposeEverything(); - } - } - - return loss; - } - - public override IEnumerable parameters() - { - return _parameter_groups.SelectMany(pg => pg.Parameters); - } - - public virtual void add_param_group(ParamGroup param_group) - { - _parameter_groups.Add(param_group); - } - - protected OptimizerOptions _defaults; - } - - public class OptimizerOptions - { - public double? LearningRate { get; set; } - public double InitialLearningRate { get; set; } - } - - public class ParamGroup : ILearningRateController - { - public IEnumerable Parameters { get; set; } - - public OptimizerOptions Options { get; set; } - - public double LearningRate { get => Options.LearningRate.Value; set => Options.LearningRate = value; } - public double InitialLearningRate { get => Options.InitialLearningRate; set => Options.InitialLearningRate = value; } - - public IEnumerable ParamGroups { get => throw new InvalidOperationException("ParamGroups should not be called on a ParamGroup"); } - } - - public class ParamGroup : ParamGroup where TOptions : OptimizerOptions - { - public ParamGroup() - { - } - - public ParamGroup(IEnumerable parameters, TOptions options = null) - { - base.Parameters = parameters; - base.Options = options; - } - - public new TOptions Options { get => (TOptions)base.Options; set => base.Options = value; } - - } - - public class SGD : OptimizerHelper, IMomentum - { - /// - /// Implements stochastic gradient descent (optionally with momentum). - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Momentum factor (default: 0) - /// Dampening for momentum (default: 0) - /// Weight decay (L2 penalty) (default: 0) - /// Enables Nesterov momentum (default: False) - /// - /// - public SGD(IEnumerable parameters, double lr, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) - : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, momentum, dampening, weight_decay, nesterov, maximize) - { - } - - /// - /// Implements stochastic gradient descent (optionally with momentum). - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Momentum factor (default: 0) - /// Dampening for momentum (default: 0) - /// Weight decay (L2 penalty) (default: 0) - /// Enables Nesterov momentum (default: False) - /// - /// - public SGD(IEnumerable parameters, double lr, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) - { - if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - if (nesterov && (momentum <= 0 || dampening != 0)) throw new ArgumentException("Nesterov momentum requires a momentum and zero dampening"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - dampening = dampening, - maximize = maximize, - momentum = momentum, - nesterov = nesterov, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options; - var momentum = options.momentum.Value; - var dampening = options.dampening.Value; - var weight_decay = options.weight_decay.Value; - var nesterov = options.nesterov.Value; - var maximize = options.maximize.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var state = _state[param.handle]; - - var grad = param.grad(); - - if (grad is null) continue; - - if (weight_decay != 0) { - grad = grad.add(param, alpha: weight_decay); - } - - if (momentum != 0) { - var buf = state.momentum_buffer; - - if (buf is null) { - buf = grad.clone().detach().DetatchFromDisposeScope(); - state.momentum_buffer = buf; - } else { - buf.mul_(momentum).add_(grad, alpha: (1 - dampening)); - } - - if (nesterov) { - grad = grad.add(buf, alpha: momentum); - } else { - grad = buf; - } - - state.momentum_buffer = buf; - } - - var alpha = maximize ? lr : -lr; - param.add_(grad, alpha: alpha); - - } - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - if (state.momentum_buffer is not null) { - state.momentum_buffer.Dispose(); - } - } - _state.Clear(); - } - - private class State - { - public Tensor momentum_buffer; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.momentum.HasValue) opt.momentum = def.momentum; - if (!opt.dampening.HasValue) opt.dampening = def.dampening; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - if (!opt.nesterov.HasValue) opt.nesterov = def.nesterov; - if (!opt.maximize.HasValue) opt.maximize = def.maximize; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.momentum_buffer = null; - } - } - - public class Options : Modules.OptimizerOptions - { - public double? momentum; - public double? dampening; - public double? weight_decay; - public bool? nesterov; - public bool? maximize; - } - - public class ParamGroup : ParamGroup, IMomentum - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) - : base(parameters, new SGD.Options { LearningRate = lr, dampening = dampening, momentum = momentum, weight_decay = weight_decay, nesterov = nesterov, maximize = maximize }) - { - } - - public double Momentum { get => Options.momentum.Value; set => Options.momentum = value; } - } - - private Dictionary _state = new Dictionary(); - - public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } - } - - public class Adadelta : OptimizerHelper - { - /// - /// Constructor - /// - /// Parameters to optimize. - /// Learning rate - /// Coefficient used for computing a running average of squared gradients (default: 0.9) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) - /// Weight decay (L2 penalty) (default: 0) - public Adadelta(IEnumerable parameters, double lr, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) - : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, rho, eps, weight_decay) - { - } - - /// - /// Constructor - /// - /// Parameters to optimize. - /// Learning rate - /// Coefficient used for computing a running average of squared gradients (default: 0.9) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) - /// Weight decay (L2 penalty) (default: 0) - public Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) - { - if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (rho < 0.0 || rho > 1.0) throw new ArgumentException($"Invalid rho value: {rho}"); - if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - rho = rho, - eps = eps, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var rho = options.rho.Value; - var eps = options.eps.Value; - var weight_decay = options.weight_decay.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var grad = param.grad(); - - if (grad is null) continue; - - if (grad.is_sparse) throw new ArgumentException("Adadelta does not support sparse gradients"); - - var state = _state[param.handle]; - - var square_avg = state.square_avg; - var acc_delta = state.acc_delta; - - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); - - square_avg.mul_(rho).addcmul_(grad, grad, 1 - rho); - - var std = square_avg.add(eps).sqrt_(); - var delta = acc_delta.add(eps).sqrt_().div_(std).mul_(grad); - - param.add_(delta, alpha: -lr); - acc_delta.mul_(rho).addcmul_(delta, delta, 1 - rho); - } - },closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - state.square_avg.Dispose(); - state.acc_delta.Dispose(); - } - } - - private class State - { - public int step; - public Tensor square_avg; - public Tensor acc_delta; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.rho.HasValue) opt.rho = def.rho; - if (!opt.eps.HasValue) opt.eps = def.eps; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - state.square_avg = torch.zeros_like(p); - state.acc_delta = torch.zeros_like(p); - } - } - - public class Options : OptimizerOptions - { - public double? rho; - public double? eps; - public double? weight_decay; - } - - public class ParamGroup : ParamGroup - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) - : base(parameters, new Adadelta.Options { LearningRate = lr, rho = rho, eps = eps, weight_decay = weight_decay }) - { - } - } - - private Dictionary _state = new Dictionary(); - } - - public class Adamax : OptimizerHelper, IBetas - { - /// - /// Implements Adamax algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// - public Adamax(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay) - { - } - - /// - /// Implements Adamax algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// - public Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - { - if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); - if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); - if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - beta1 = beta1, - beta2 = beta2, - eps = eps, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var eps = options.eps.Value; - var weight_decay = options.weight_decay.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var grad = param.grad(); - - if (grad is null) continue; - - if (grad.is_sparse) throw new ArgumentException("Adamax does not support sparse gradients"); - - var state = _state[param.handle]; - - state.step += 1; - - var exp_avg = state.exp_avg; - var exp_inf = state.exp_inf; - - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); - - exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - - var norm_buf = torch.cat(new Tensor[] { - exp_inf.mul_(beta2).unsqueeze(0), - grad.abs().add_(eps).unsqueeze_(0) - }, 0); - - torch.amax(norm_buf, new long[] { 0 }, false, exp_inf); - - var clr = lr / (1 - Math.Pow(beta1, state.step)); - param.addcdiv_(exp_avg, exp_inf, value: -clr); - } - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - state.exp_avg.Dispose(); - state.exp_inf.Dispose(); - } - } - - private class State - { - public int step; - public Tensor exp_avg; - public Tensor exp_inf; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.beta1.HasValue) opt.beta1 = def.beta1; - if (!opt.beta2.HasValue) opt.beta2 = def.beta2; - if (!opt.eps.HasValue) opt.eps = def.eps; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_inf = torch.zeros_like(p); - } - } - - public class Options : OptimizerOptions - { - public double? beta1; - public double? beta2; - public double? eps; - public double? weight_decay; - } - - public class ParamGroup : ParamGroup, IBetas - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - : base(parameters, new Adamax.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay }) - { - } - - public (double, double) Betas { - get => (Options.beta1.Value, Options.beta2.Value); - set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } - } - } - - public (double, double) Betas { - get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); - set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } - } - - private Dictionary _state = new Dictionary(); - } - - public class NAdam : OptimizerHelper, IBetas - { - /// - /// Implements NAdam algorithm. - /// - /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. - /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum decay - public NAdam(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) - : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, momentum_decay) - { - } - - /// - /// Implements NAdam algorithm. - /// - /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. - /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// Momentum decay - /// - public NAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) - { - if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); - if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); - if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - if (momentum_decay < 0.0) throw new ArgumentException($"Invalid momentum_decay value: {momentum_decay}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - beta1 = beta1, - beta2 = beta2, - eps = eps, - weight_decay = weight_decay, - momentum_decay = momentum_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var eps = options.eps.Value; - var weight_decay = options.weight_decay.Value; - var momentum_decay = options.momentum_decay.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var grad = param.grad(); - - if (grad is null) continue; - - var state = _state[param.handle]; - - state.step += 1; - - var exp_avg = state.exp_avg; - var exp_avg_sq = state.exp_avg_sq; - - var bias_correction2 = 1 - Math.Pow(beta2, state.step); - - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); - - var mu = beta1 * (1.0 - 0.5 * Math.Pow(0.96, state.step * momentum_decay)); - var mu_next = beta1 * (1.0 - 0.5 * Math.Pow(0.96, (state.step + 1) * momentum_decay)); - - var mu_product = state.mu_product * mu; - var mu_product_next = mu_product * mu * mu_next; - - exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); - - var denom = exp_avg_sq.div(bias_correction2).sqrt_().add_(eps); - - param.addcdiv_(grad, denom, value: -lr * (1 - mu) / (1 - mu_product)); - param.addcdiv_(exp_avg, denom, value: -lr * mu_next / (1 - mu_product_next)); - - state.mu_product = mu_product; - } - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - state.exp_avg.Dispose(); - state.exp_avg_sq.Dispose(); - } - } - - private class State - { - public int step; - public double mu_product; - public Tensor exp_avg; - public Tensor exp_avg_sq; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.beta1.HasValue) opt.beta1 = def.beta1; - if (!opt.beta2.HasValue) opt.beta2 = def.beta2; - if (!opt.eps.HasValue) opt.eps = def.eps; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - if (!opt.momentum_decay.HasValue) opt.momentum_decay = def.momentum_decay; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_avg_sq = torch.zeros_like(p); - } - } - - public class Options : OptimizerOptions - { - public double? beta1; - public double? beta2; - public double? eps; - public double? weight_decay; - public double? momentum_decay; - } - - public class ParamGroup : ParamGroup, IBetas - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) - : base(parameters, new NAdam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, momentum_decay = momentum_decay }) - { - } - - public (double, double) Betas { - get => (Options.beta1.Value, Options.beta2.Value); - set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } - } - } - - public (double, double) Betas { - get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); - set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } - } - - private Dictionary _state = new Dictionary(); - } - - public class RAdam : OptimizerHelper, IBetas - { - /// - /// Implements RAdam algorithm. - /// - /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' - /// https://arxiv.org/abs/1908.03265 - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// - public RAdam(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay) - { - } - - /// - /// Implements RAdam algorithm. - /// - /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' - /// https://arxiv.org/abs/1908.03265 - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Coefficient used for computing running averages of gradient and its square (default: 0.9) - /// Coefficient used for computing running averages of gradient and its square (default: 0.999) - /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) - /// Weight decay (L2 penalty) (default: 0) - /// - public RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - { - if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); - if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); - if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - beta1 = beta1, - beta2 = beta2, - eps = eps, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var eps = options.eps.Value; - var weight_decay = options.weight_decay.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var grad = param.grad(); - - if (grad is null) continue; - - var state = _state[param.handle]; - - state.step += 1; - - var exp_avg = state.exp_avg; - var exp_avg_sq = state.exp_avg_sq; - - var bias_correction1 = 1 - Math.Pow(beta1, state.step); - var bias_correction2 = 1 - Math.Pow(beta2, state.step); - - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); - - exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); - - var bias_corrected_exp_avg = exp_avg / bias_correction1; - - var rho_inf = 2 / (1 - beta2) - 1; - var rho_t = rho_inf - 2 * state.step * Math.Pow(beta2, state.step) / bias_correction2; - - var t6 = bias_corrected_exp_avg * lr; - - if (rho_t > 5) { - var rect = Math.Sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t)); - var adaptive_lr = Math.Sqrt(bias_correction2) / exp_avg_sq.sqrt().add_(eps); - - param.add_(t6 * lr * adaptive_lr * rect, alpha: -1.0); - } else { - param.add_(t6, alpha: -1.0); - } - } - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - state.exp_avg.Dispose(); - state.exp_avg_sq.Dispose(); - } - } - - private class State - { - public int step; - public Tensor exp_avg; - public Tensor exp_avg_sq; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.beta1.HasValue) opt.beta1 = def.beta1; - if (!opt.beta2.HasValue) opt.beta2 = def.beta2; - if (!opt.eps.HasValue) opt.eps = def.eps; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_avg_sq = torch.zeros_like(p); - } - } - - public class Options : OptimizerOptions - { - public double? beta1; - public double? beta2; - public double? eps; - public double? weight_decay; - } - - public class ParamGroup : ParamGroup, IBetas - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) - : base(parameters, new RAdam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay }) - { - } - - public (double, double) Betas { - get => (Options.beta1.Value, Options.beta2.Value); - set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } - } - } - - public (double, double) Betas { - get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); - set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } - } - - private Dictionary _state = new Dictionary(); - } - - public class ASGD : OptimizerHelper - { - /// - /// Implements ASGD algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Decay term (default: 1e-4) - /// Power for eta update (default: 0.75) - /// Point at which to start averaging (default: 1e6) - /// Weight decay (L2 penalty) (default: 0) - /// - public ASGD(IEnumerable parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) - : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, lambd, alpha, t0, weight_decay) - { - } - - /// - /// Implements ASGD algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// Decay term (default: 1e-4) - /// Power for eta update (default: 0.75) - /// Point at which to start averaging (default: 1e6) - /// Weight decay (L2 penalty) (default: 0) - /// - public ASGD(IEnumerable> parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) - { - if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - lambd = lambd, - alpha = alpha, - t0 = t0, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var lambd = options.lambd.Value; - var alpha = options.alpha.Value; - var weight_decay = options.weight_decay.Value; - var t0 = options.t0.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var grad = param.grad(); - - if (grad is null) continue; - - if (grad.is_sparse) throw new ArgumentException("ASGD does not support sparse gradients"); - - var state = _state[param.handle]; - - state.step += 1; - - grad = (weight_decay != 0) - ? grad.add(param, alpha: weight_decay) - : grad.alias(); - - param.mul_(1 - lambd * state.eta); - param.add_(grad, alpha: -state.eta); - - if (state.mu != 1) { - state.ax.add_(param.sub(state.ax).mul(state.mu)); - } else { - state.ax.copy_(param); - } - - state.eta = lr / Math.Pow((1 + lambd * lr * state.step), alpha); - state.mu = 1 / Math.Max(1, state.step - t0); - } - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - state.ax.Dispose(); - } - _state.Clear(); - } - - private class State - { - public int step; - public double eta; - public double mu; - public Tensor ax; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.lambd.HasValue) opt.lambd = def.lambd; - if (!opt.alpha.HasValue) opt.alpha = def.alpha; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - if (!opt.t0.HasValue) opt.t0 = def.t0; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - state.eta = param_group.LearningRate; - state.mu = 1; - state.ax = torch.zeros_like(p); - } - } - - public class Options : OptimizerOptions - { - public double? lambd; - public double? alpha; - public double? weight_decay; - public double? t0; - } - - public class ParamGroup : ParamGroup - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) - : base(parameters, new ASGD.Options { LearningRate = lr, lambd = lambd, alpha = alpha, t0 = t0, weight_decay = weight_decay }) - { - } - } - - private Dictionary _state = new Dictionary(); - } - - public class Rprop : OptimizerHelper - { - /// - /// Implements Rprop algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. - /// Learning rate - /// Multiplicative increase factor. - /// Multiplicative decrease factor. - /// Minimum allowed step size. - /// Maximum allowed step size. - /// - public Rprop(IEnumerable parameters, double lr = 0.01, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) - : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, etaminus, etaplus, min_step, max_step) - { - } - - /// - /// Implements Rprop algorithm (a variant of Adam based on infinity norm). - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. - /// - /// Parameters to optimize. - /// Learning rate - /// Multiplicative increase factor. - /// Multiplicative decrease factor. - /// Minimum allowed step size. - /// Maximum allowed step size. - /// - public Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) - { - if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - etaminus = etaminus, - etaplus = etaplus, - min_step = min_step, - max_step = max_step - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var etaminus = options.etaminus.Value; - var etaplus = options.etaplus.Value; - var min_step = options.min_step.Value; - var max_step = options.max_step.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var grad = param.grad(); - - if (grad is null) continue; - - if (grad.is_sparse) throw new ArgumentException("Rprop does not support sparse gradients"); - - var state = _state[param.handle]; - - state.step += 1; - - grad = (max_step != 0) - ? grad.add(param, alpha: max_step) - : grad.alias(); - - var sign = grad.mul(state.prev).sign(); - sign[sign.gt(0)] = (Tensor)etaplus; - sign[sign.lt(0)] = (Tensor)etaminus; - sign[sign.eq(0)] = (Tensor)1; - - state.step_size.mul_(sign).clamp_(min_step, max_step); - - grad = grad.clone(); - - grad.index_put_(0, sign.eq(etaminus)); - - param.addcmul_(grad.sign(), state.step_size, -1); - - state.prev.copy_(grad); - } - - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - state.prev.Dispose(); - state.step_size.Dispose(); - } - } - - private class State - { - public int step; - public Tensor prev; - public Tensor step_size; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.etaminus.HasValue) opt.etaminus = def.etaminus; - if (!opt.etaplus.HasValue) opt.etaplus = def.etaplus; - if (!opt.min_step.HasValue) opt.min_step = def.min_step; - if (!opt.max_step.HasValue) opt.max_step = def.max_step; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - state.prev = torch.zeros_like(p); - state.step_size = p.new_empty(p.shape).fill_(opt.LearningRate); - } - } - - public class Options : OptimizerOptions - { - public double? etaminus; - public double? etaplus; - public double? min_step; - public double? max_step; - } - - public class ParamGroup : ParamGroup - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) - : base(parameters, new Rprop.Options { LearningRate = lr, etaminus = etaminus, etaplus = etaplus, min_step = min_step, max_step = max_step }) - { - } - } - - private Dictionary _state = new Dictionary(); - - } - - public class Adagrad : OptimizerHelper - { - /// - /// Implements Adagrad algorithm. - /// - /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-2) - /// learning rate decay (default: 0) - /// weight decay (L2 penalty) (default: 0) - /// - /// Term added to the denominator to improve numerical stability (default: 1e-10) - /// - public Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) - : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, lr_decay, weight_decay, initial_accumulator_value, eps) - { - } - /// - /// Implements Adagrad algorithm. - /// - /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. - /// - /// Parameters to optimize - /// learning rate (default: 1e-2) - /// learning rate decay (default: 0) - /// weight decay (L2 penalty) (default: 0) - /// - /// Term added to the denominator to improve numerical stability (default: 1e-10) - /// - public Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) - { - if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); - if (lr_decay < 0.0) throw new ArgumentException($"Invalid lr_decay value: {lr_decay}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - if (initial_accumulator_value < 0) throw new ArgumentException($"Invalid initial_accumulator_value: {initial_accumulator_value}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - lr_decay = lr_decay, - eps = eps, - initial_accumulator_value = initial_accumulator_value, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var lr_decay = options.lr_decay.Value; - var weight_decay = options.weight_decay.Value; - var eps = options.eps.Value; - var initial_accumulator_value = options.initial_accumulator_value.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var state = _state[param.handle]; - - var grad = param.grad(); - - if (grad is null) continue; - - state.step += 1; - - if (weight_decay != 0) { - grad = grad.add(param, alpha: weight_decay); - } - - var clr = lr / (1 + (state.step - 1) * lr_decay); - - if (grad.is_sparse) - throw new NotImplementedException("Adagrad optimization over sparse parameters"); - if (torch.is_complex(grad)) - throw new NotImplementedException("Adagrad optimization over complex parameters"); - - state.sum.addcmul_(grad, grad, value: 1); - var std = state.sum.sqrt().add_(eps); - param.addcdiv_(grad, std, value: -clr); - } - - - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (name, state) in _state) { - state.sum.Dispose(); - } - } - - private class State - { - public int step; - public Tensor sum; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.lr_decay.HasValue) opt.lr_decay = def.lr_decay; - if (!opt.eps.HasValue) opt.eps = def.eps; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - if (!opt.initial_accumulator_value.HasValue) opt.initial_accumulator_value = def.initial_accumulator_value; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - var init_value = torch.is_complex(p.dtype) - ? (Scalar)new System.Numerics.Complex((param_group.Options as Options).initial_accumulator_value.Value, (param_group.Options as Options).initial_accumulator_value.Value) - : (Scalar)(param_group.Options as Options).initial_accumulator_value.Value; - state.sum = torch.full_like(p, init_value); - } - } - - public class Options : OptimizerOptions - { - public double? lr_decay; - public double? initial_accumulator_value; - public double? eps; - public double? weight_decay; - } - - public class ParamGroup : ParamGroup - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 0.01, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) - : base(parameters, new Adagrad.Options { LearningRate = lr, lr_decay = lr_decay, initial_accumulator_value = initial_accumulator_value, weight_decay = weight_decay, eps = eps }) - { - } - } - - private Dictionary _state = new Dictionary(); - } - - public class Adam : OptimizerHelper, IBetas - { - /// - /// Implements Adam algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// First coefficient used for computing running averages of gradient and its square - /// Second coefficient used for computing running averages of gradient and its square - /// Term added to the denominator to improve numerical stability - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm - /// Maximize the params based on the objective, instead of minimizing - /// - public Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize) - { - } - - /// - /// Implements Adam algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// First coefficient used for computing running averages of gradient and its square - /// Second coefficient used for computing running averages of gradient and its square - /// Term added to the denominator to improve numerical stability - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm - /// Maximize the params based on the objective, instead of minimizing - /// - public Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - { - if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); - if (beta1 < 0 || beta1 >= 1.0) throw new ArgumentException($"Invalid beta1: {beta1}"); - if (beta2 < 0 || beta2 >= 1.0) throw new ArgumentException($"Invalid beta2: {beta2}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - beta1 = beta1, - beta2 = beta2, - maximize = maximize, - eps = eps, - amsgrad = amsgrad, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var weight_decay = options.weight_decay.Value; - var amsgrad = options.amsgrad.Value; - var maximize = options.maximize.Value; - var eps = options.eps.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var state = _state[param.handle]; - - var grad = (maximize) ? -param.grad() : param.grad(); - - if (grad is null) continue; - - state.step += 1; - - var bias_correction1 = 1 - Math.Pow(beta1, state.step); - var bias_correction2 = 1 - Math.Pow(beta2, state.step); - - if (weight_decay != 0) { - grad = grad.add(param, alpha: weight_decay); - } - - state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad.conj(), value: 1 - beta2); - - Tensor denom = null; - if (amsgrad) { - var t0 = state.max_exp_avg_sq; - state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); - t0.Dispose(); - denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); - } else { - denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); - } - - var step_size = lr / bias_correction1; - param.addcdiv_(state.exp_avg, denom, value: -step_size); - } - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (_, state) in _state) { - state.exp_avg.Dispose(); - state.exp_avg_sq.Dispose(); - if (state.max_exp_avg_sq is not null) { - state.max_exp_avg_sq.Dispose(); - } - } - } - - private class State - { - public int step; - public Tensor exp_avg; - public Tensor exp_avg_sq; - public Tensor max_exp_avg_sq; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.beta1.HasValue) opt.beta1 = def.beta1; - if (!opt.beta2.HasValue) opt.beta2 = def.beta2; - if (!opt.eps.HasValue) opt.eps = def.eps; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - if (!opt.amsgrad.HasValue) opt.amsgrad = def.amsgrad; - if (!opt.maximize.HasValue) opt.maximize = def.maximize; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_avg_sq = torch.zeros_like(p); - if (opt.amsgrad.Value) { - state.max_exp_avg_sq = torch.zeros_like(p); - } - } - } - - public class Options : OptimizerOptions - { - public double? beta1; - public double? beta2; - public double? weight_decay; - public double? eps; - public bool? amsgrad; - public bool? maximize; - } - - public class ParamGroup : ParamGroup, IBetas - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - : base(parameters, new Adam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, amsgrad = amsgrad, maximize = maximize }) - { - } - - public (double, double) Betas { - get => (Options.beta1.Value, Options.beta2.Value); - set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } - } - } - - public (double, double) Betas { - get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); - set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } - } - - private Dictionary _state = new Dictionary(); - } - - public class AdamW : OptimizerHelper, IBetas - { - /// - /// Implements AdamW algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// First coefficient used for computing running averages of gradient and its square - /// Second coefficient used for computing running averages of gradient and its square - /// Term added to the denominator to improve numerical stability - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm - /// Maximize the params based on the objective, instead of minimizing - /// - public AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize) - { - } - - /// - /// Implements AdamW algorithm. - /// - /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. - /// - /// Parameters to optimize. This optimizer requires the named parameters collection. - /// Learning rate - /// First coefficient used for computing running averages of gradient and its square - /// Second coefficient used for computing running averages of gradient and its square - /// Term added to the denominator to improve numerical stability - /// Weight decay (L2 penalty) (default: 0) - /// Whether to use the AMSGrad variant of this algorithm - /// Maximize the params based on the objective, instead of minimizing - /// - public AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - { - if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); - if (beta1 < 0 || beta1 >= 1.0) throw new ArgumentException($"Invalid beta1: {beta1}"); - if (beta2 < 0 || beta2 >= 1.0) throw new ArgumentException($"Invalid beta2: {beta2}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - beta1 = beta1, - beta2 = beta2, - maximize = maximize, - eps = eps, - amsgrad = amsgrad, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var beta1 = options.beta1.Value; - var beta2 = options.beta2.Value; - var weight_decay = options.weight_decay.Value; - var amsgrad = options.amsgrad.Value; - var maximize = options.maximize.Value; - var eps = options.eps.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var state = _state[param.handle]; - - var grad = (maximize) ? -param.grad() : param.grad(); - - if (grad is null) continue; - - state.step += 1; - - param.mul_(1 - lr * weight_decay); - - var bias_correction1 = 1 - Math.Pow(beta1, state.step); - var bias_correction2 = 1 - Math.Pow(beta2, state.step); - - state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); - state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); - - Tensor denom = null; - if (amsgrad) { - var t0 = state.max_exp_avg_sq; - state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); - t0.Dispose(); - denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); - } else { - denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); - } - - var step_size = lr / bias_correction1; - param.addcdiv_(state.exp_avg, denom, value: -step_size); - } - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (_, state) in _state) { - state.exp_avg.Dispose(); - state.exp_avg_sq.Dispose(); - if (state.max_exp_avg_sq is not null) { - state.max_exp_avg_sq.Dispose(); - } - } - } - - private class State - { - public int step; - public Tensor exp_avg; - public Tensor exp_avg_sq; - public Tensor max_exp_avg_sq; - } - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.beta1.HasValue) opt.beta1 = def.beta1; - if (!opt.beta2.HasValue) opt.beta2 = def.beta2; - if (!opt.eps.HasValue) opt.eps = def.eps; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - if (!opt.amsgrad.HasValue) opt.amsgrad = def.amsgrad; - if (!opt.maximize.HasValue) opt.maximize = def.maximize; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.step = 0; - state.exp_avg = torch.zeros_like(p); - state.exp_avg_sq = torch.zeros_like(p); - if (opt.amsgrad.Value) { - state.max_exp_avg_sq = torch.zeros_like(p); - } - } - } - - public class Options : OptimizerOptions - { - public double? beta1; - public double? beta2; - public double? weight_decay; - public double? eps; - public bool? amsgrad; - public bool? maximize; - } - - public class ParamGroup : ParamGroup, IBetas - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) - : base(parameters, new AdamW.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, amsgrad = amsgrad, maximize = maximize }) - { - } - - public (double, double) Betas { - get => (Options.beta1.Value, Options.beta2.Value); - set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } - } - } - - public (double, double) Betas { - get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); - set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } - } - - private Dictionary _state = new Dictionary(); - } - - public class RMSProp : OptimizerHelper, IMomentum - { - - /// - /// Implements RMSprop algorithm. - /// - /// Proposed by G.Hinton in his course. - /// - /// Parameters to optimize. This optimizer requires the parameters collection. - /// Learning rate - /// Smoothing constant. - /// Momentum factor (default: 0) - /// Term added to the denominator to improve numerical stability - /// Weight decay (L2 penalty) (default: 0) - /// if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance - /// - public RMSProp(IEnumerable parameters, double lr = 1e-3, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0.0, bool centered = false) - : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, alpha, eps, weight_decay, momentum, centered) - { - } - - /// - /// Implements RMSprop algorithm. - /// - /// Proposed by G.Hinton in his course. - /// - /// Parameters to optimize. This optimizer requires the parameters collection. - /// Learning rate - /// Smoothing constant. - /// Momentum factor (default: 0) - /// Term added to the denominator to improve numerical stability - /// Weight decay (L2 penalty) (default: 0) - /// if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance - /// - public RMSProp(IEnumerable parameters, double lr = 1e-3, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0.0, bool centered = false) - { - if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); - if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); - if (alpha < 0) throw new ArgumentException($"Invalid alpha: {alpha}"); - if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); - if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); - - var options = new Options { - LearningRate = lr, - InitialLearningRate = lr, - eps = eps, - alpha = alpha, - momentum = momentum, - centered = centered, - weight_decay = weight_decay - }; - - _defaults = options; - _parameter_groups = new List(); - - foreach (var g in parameters) { - add_param_group(g); - } - } - - public override Tensor step(Func closure = null) - { - return _step(group => { - - var options = group.Options as Options; - var momentum = options.momentum.Value; - var alpha = options.alpha.Value; - var weight_decay = options.weight_decay.Value; - var centered = options.centered.Value; - var eps = options.eps.Value; - var lr = options.LearningRate.Value; - - foreach (var param in group.Parameters) { - - var state = _state[param.handle]; - - var grad = param.grad(); - - if (grad is null) continue; - - state.step += 1; - - if (weight_decay != 0) { - grad = grad.add(param, alpha: weight_decay); - } - - state.square_avg.mul_(alpha).addcmul_(grad, grad, value: 1 - alpha); - - Tensor avg = null; - - if (centered) { - var grad_avg = state.grad_avg; - grad_avg.mul_(alpha).add_(grad, alpha: 1 - alpha); - avg = state.square_avg.addcmul(grad_avg, grad_avg, value: -1).sqrt_().add_(eps); - } else { - avg = state.square_avg.sqrt().add_(eps); - } - - if (momentum > 0) { - var buf = state.momentum_buffer; - buf.mul_(momentum).addcdiv_(grad, avg); - param.add_(buf, alpha: -lr); - } else { - param.addcdiv_(grad, avg, -lr); - } - } - }, closure); - } - - protected override void Dispose(bool disposing) - { - base.Dispose(disposing); - foreach (var (_, state) in _state) { - state.momentum_buffer.Dispose(); - state.square_avg.Dispose(); - state.grad_avg.Dispose(); - } - _state.Clear(); - } - - private class State - { - public int step; - public Tensor square_avg; - public Tensor momentum_buffer; - public Tensor grad_avg; - } - - - - public override void add_param_group(Modules.ParamGroup param_group) - { - var def = _defaults as Options; - if (param_group.Options is null) { - param_group.Options = new Options(); - } - - var opt = param_group.Options as Options; - - // Make sure all the options are set. - if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; - if (!opt.momentum.HasValue) opt.momentum = def.momentum; - if (!opt.eps.HasValue) opt.eps = def.eps; - if (!opt.alpha.HasValue) opt.alpha = def.alpha; - if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; - if (!opt.centered.HasValue) opt.centered = def.centered; - - opt.InitialLearningRate = opt.LearningRate.Value; - - _parameter_groups.Add(param_group); - - foreach (var p in param_group.Parameters) { - var state = new State(); - _state[p.Handle] = state; - state.square_avg = torch.zeros_like(p); - state.grad_avg = torch.zeros_like(p); - state.momentum_buffer = torch.zeros_like(p); - } - } - public class Options : Modules.OptimizerOptions - { - public double? momentum; - public double? alpha; - public double? eps; - public double? weight_decay; - public bool? centered; - } - - public class ParamGroup : ParamGroup, IMomentum - { - public ParamGroup() { } - - public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } - - public ParamGroup(IEnumerable parameters, double lr = 1e-3, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0.0, bool centered = false) - : base(parameters, new RMSProp.Options { LearningRate = lr, eps = eps, alpha = alpha, weight_decay = weight_decay, momentum = momentum, centered = centered }) - { - } - - public double Momentum { get => Options.momentum.Value; set => Options.momentum = value; } - } - - private Dictionary _state = new Dictionary(); - - public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } - } - - // The following optimizers are wrappers for the native code implementations. - // - // LBGFS does not allow for param groups, so there's no need to use anything but the native implementation. - - public class LBFGS : Optimizer, ILearningRateController - { - public LBFGS(IntPtr handle, double lr) : base(handle) - { - _rate = lr; - InitialLearningRate = lr; - _paramGroups = new ParamGroup[] { new ParamGroup { Parameters = this.parameters(), Options = new() { LearningRate = lr, InitialLearningRate = lr } } }; - } - - [DllImport("LibTorchSharp")] - private static extern void THSNN_LBFGS_set_lr(HType optimizer, double lr); - - public double LearningRate { - get { return _rate; } - set { THSNN_LBFGS_set_lr(handle, value); torch.CheckForErrors(); _rate = value; } - } - - public double InitialLearningRate { get; set; } - - public override IEnumerable ParamGroups { get => _paramGroups; } - - - public override Tensor step(Func closure = null) - { - if (closure == null) - throw new ArgumentNullException("'closure' must be non-null when using the LBFGS optimizer. See: https://pytorch.org/docs/1.9.0/optim.html"); - return base.step(closure); - } - - public double _rate; - private ParamGroup[] _paramGroups; - } - } -} - diff --git a/src/TorchSharp/Optimizers/ASGD.cs b/src/TorchSharp/Optimizers/ASGD.cs new file mode 100644 index 000000000..446adab78 --- /dev/null +++ b/src/TorchSharp/Optimizers/ASGD.cs @@ -0,0 +1,256 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the Averaged Stochastic Gradient Descent. + /// + /// It has been proposed in Acceleration of stochastic approximation by averaging. + /// https://dl.acm.org/citation.cfm?id=131098 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Decay term (default: 1e-4) + /// Power for eta update (default: 0.75) + /// Point at which to start averaging (default: 1e6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static ASGD ASGD(IEnumerable parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + { + return new Modules.ASGD(parameters, lr, lambd, alpha, t0, weight_decay); + } + + /// + /// Implements the Averaged Stochastic Gradient Descent. + /// + /// It has been proposed in Acceleration of stochastic approximation by averaging. + /// https://dl.acm.org/citation.cfm?id=131098 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Decay term (default: 1e-4) + /// Power for eta update (default: 0.75) + /// Point at which to start averaging (default: 1e6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static ASGD ASGD(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + { + return new Modules.ASGD(parameters.Select(np => np.parameter), lr, lambd, alpha, t0, weight_decay); + } + + /// + /// Implements the Averaged Stochastic Gradient Descent. + /// + /// It has been proposed in Acceleration of stochastic approximation by averaging. + /// https://dl.acm.org/citation.cfm?id=131098 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Decay term (default: 1e-4) + /// Power for eta update (default: 0.75) + /// Point at which to start averaging (default: 1e6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static ASGD ASGD(IEnumerable> parameters, double lr = 1e-3, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + { + return new Modules.ASGD(parameters, lr, lambd, alpha, t0, weight_decay); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class ASGD : OptimizerHelper + { + /// + /// Implements ASGD algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Decay term (default: 1e-4) + /// Power for eta update (default: 0.75) + /// Point at which to start averaging (default: 1e6) + /// Weight decay (L2 penalty) (default: 0) + /// + public ASGD(IEnumerable parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, lambd, alpha, t0, weight_decay) + { + } + + /// + /// Implements ASGD algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Decay term (default: 1e-4) + /// Power for eta update (default: 0.75) + /// Point at which to start averaging (default: 1e6) + /// Weight decay (L2 penalty) (default: 0) + /// + public ASGD(IEnumerable> parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + lambd = lambd, + alpha = alpha, + t0 = t0, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var lambd = options.lambd.Value; + var alpha = options.alpha.Value; + var weight_decay = options.weight_decay.Value; + var t0 = options.t0.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var grad = param.grad(); + + if (grad is null) continue; + + if (grad.is_sparse) throw new ArgumentException("ASGD does not support sparse gradients"); + + var state = _state[param.handle]; + + state.step += 1; + + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); + + param.mul_(1 - lambd * state.eta); + param.add_(grad, alpha: -state.eta); + + if (state.mu != 1) { + state.ax.add_(param.sub(state.ax).mul(state.mu)); + } else { + state.ax.copy_(param); + } + + state.eta = lr / Math.Pow((1 + lambd * lr * state.step), alpha); + state.mu = 1 / Math.Max(1, state.step - t0); + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.Dispose(); + } + _state.Clear(); + } + + private class State : IDisposable + { + public long step; + public double eta; + public double mu; + public Tensor ax; + + public void Dispose() + { + ax.Dispose(); + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.lambd.HasValue) opt.lambd = def.lambd; + if (!opt.alpha.HasValue) opt.alpha = def.alpha; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.t0.HasValue) opt.t0 = def.t0; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.eta = param_group.LearningRate; + state.mu = 1; + state.ax = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? lambd; + public double? alpha; + public double? weight_decay; + public double? t0; + } + + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 0.01, double lambd = 1e-4, double alpha = 0.75, double t0 = 1e6, double weight_decay = 0) + : base(parameters, new ASGD.Options { LearningRate = lr, lambd = lambd, alpha = alpha, t0 = t0, weight_decay = weight_decay }) + { + } + } + + private Dictionary _state = new Dictionary(); + } + } +} diff --git a/src/TorchSharp/Optimizers/Adadelta.cs b/src/TorchSharp/Optimizers/Adadelta.cs new file mode 100644 index 000000000..85b548f72 --- /dev/null +++ b/src/TorchSharp/Optimizers/Adadelta.cs @@ -0,0 +1,239 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the Adadelta algorithm. + /// + /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. + /// https://arxiv.org/abs/1212.5701 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing a running average of squared gradients (default: 0.9) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adadelta Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + { + return new Adadelta(parameters, lr, rho, eps, weight_decay); + } + + /// + /// Implements the Adadelta algorithm. + /// + /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. + /// https://arxiv.org/abs/1212.5701 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing a running average of squared gradients (default: 0.9) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adadelta Adadelta(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + { + return new Adadelta(parameters.Select(np => np.parameter), lr, rho, eps, weight_decay); + } + + /// + /// Implements the Adadelta algorithm. + /// + /// It has been proposed in ADADELTA: An Adaptive Learning Rate Method. + /// https://arxiv.org/abs/1212.5701 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing a running average of squared gradients (default: 0.9) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adadelta Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + { + return new Adadelta(parameters, lr, rho, eps, weight_decay); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class Adadelta : OptimizerHelper + { + /// + /// Constructor + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing a running average of squared gradients (default: 0.9) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) + /// Weight decay (L2 penalty) (default: 0) + public Adadelta(IEnumerable parameters, double lr, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, rho, eps, weight_decay) + { + } + + /// + /// Constructor + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing a running average of squared gradients (default: 0.9) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-6) + /// Weight decay (L2 penalty) (default: 0) + public Adadelta(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (rho < 0.0 || rho > 1.0) throw new ArgumentException($"Invalid rho value: {rho}"); + if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + rho = rho, + eps = eps, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var rho = options.rho.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var grad = param.grad(); + + if (grad is null) continue; + + if (grad.is_sparse) throw new ArgumentException("Adadelta does not support sparse gradients"); + + var state = _state[param.handle]; + + var square_avg = state.square_avg; + var acc_delta = state.acc_delta; + + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); + + square_avg.mul_(rho).addcmul_(grad, grad, 1 - rho); + + var std = square_avg.add(eps).sqrt_(); + var delta = acc_delta.add(eps).sqrt_().div_(std).mul_(grad); + + param.add_(delta, alpha: -lr); + acc_delta.mul_(rho).addcmul_(delta, delta, 1 - rho); + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.Dispose(); + } + } + + private class State : IDisposable + { + public long step; + public Tensor square_avg; + public Tensor acc_delta; + + public void Dispose() + { + square_avg.Dispose(); + acc_delta.Dispose(); + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.rho.HasValue) opt.rho = def.rho; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.square_avg = torch.zeros_like(p); + state.acc_delta = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? rho; + public double? eps; + public double? weight_decay; + } + + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1.0, double rho = 0.9, double eps = 1e-6, double weight_decay = 0) + : base(parameters, new Adadelta.Options { LearningRate = lr, rho = rho, eps = eps, weight_decay = weight_decay }) + { + } + } + + private Dictionary _state = new Dictionary(); + } + } +} diff --git a/src/TorchSharp/Optimizers/Adagrad.cs b/src/TorchSharp/Optimizers/Adagrad.cs new file mode 100644 index 000000000..9026e94ec --- /dev/null +++ b/src/TorchSharp/Optimizers/Adagrad.cs @@ -0,0 +1,256 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// http://jmlr.org/papers/v12/duchi11a.html + /// + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public static Adagrad Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + { + return new Adagrad(parameters, lr, lr_decay, weight_decay, initial_accumulator_value, eps); + } + + /// + /// Implements the Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// http://jmlr.org/papers/v12/duchi11a.html + /// + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public static Adagrad Adagrad(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + { + return new Adagrad(parameters.Select(np => np.parameter), lr, lr_decay, weight_decay, initial_accumulator_value, eps); + } + + /// + /// Implements the Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// http://jmlr.org/papers/v12/duchi11a.html + /// + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public static Adagrad Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + { + return new Adagrad(parameters, lr, lr_decay, weight_decay, initial_accumulator_value, eps); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class Adagrad : OptimizerHelper + { + /// + /// Implements Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, lr_decay, weight_decay, initial_accumulator_value, eps) + { + } + /// + /// Implements Adagrad algorithm. + /// + /// It has been proposed in Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. + /// + /// Parameters to optimize + /// learning rate (default: 1e-2) + /// learning rate decay (default: 0) + /// weight decay (L2 penalty) (default: 0) + /// + /// Term added to the denominator to improve numerical stability (default: 1e-10) + /// + public Adagrad(IEnumerable parameters, double lr = 1e-2, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + { + if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (lr_decay < 0.0) throw new ArgumentException($"Invalid lr_decay value: {lr_decay}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + if (initial_accumulator_value < 0) throw new ArgumentException($"Invalid initial_accumulator_value: {initial_accumulator_value}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + lr_decay = lr_decay, + eps = eps, + initial_accumulator_value = initial_accumulator_value, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var lr_decay = options.lr_decay.Value; + var weight_decay = options.weight_decay.Value; + var eps = options.eps.Value; + var initial_accumulator_value = options.initial_accumulator_value.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var state = _state[param.handle]; + + var grad = param.grad(); + + if (grad is null) continue; + + state.step += 1; + + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } + + var clr = lr / (1 + (state.step - 1) * lr_decay); + + if (grad.is_sparse) + throw new NotImplementedException("Adagrad optimization over sparse parameters"); + if (torch.is_complex(grad)) + throw new NotImplementedException("Adagrad optimization over complex parameters"); + + state.sum.addcmul_(grad, grad, value: 1); + var std = state.sum.sqrt().add_(eps); + param.addcdiv_(grad, std, value: -clr); + } + + + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.Dispose(); + } + } + + private class State : IDisposable + { + public long step; + public Tensor sum; + + public void Dispose() + { + sum.Dispose(); + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.lr_decay.HasValue) opt.lr_decay = def.lr_decay; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.initial_accumulator_value.HasValue) opt.initial_accumulator_value = def.initial_accumulator_value; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + var init_value = torch.is_complex(p.dtype) + ? (Scalar)new System.Numerics.Complex((param_group.Options as Options).initial_accumulator_value.Value, (param_group.Options as Options).initial_accumulator_value.Value) + : (Scalar)(param_group.Options as Options).initial_accumulator_value.Value; + state.sum = torch.full_like(p, init_value); + } + } + + public class Options : OptimizerOptions + { + public double? lr_decay; + public double? initial_accumulator_value; + public double? eps; + public double? weight_decay; + } + + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 0.01, double lr_decay = 0, double weight_decay = 0, double initial_accumulator_value = 0, double eps = 1e-10) + : base(parameters, new Adagrad.Options { LearningRate = lr, lr_decay = lr_decay, initial_accumulator_value = initial_accumulator_value, weight_decay = weight_decay, eps = eps }) + { + } + } + + private Dictionary _state = new Dictionary(); + } + } +} diff --git a/src/TorchSharp/Optimizers/Adam.cs b/src/TorchSharp/Optimizers/Adam.cs new file mode 100644 index 000000000..e5f6e3fd6 --- /dev/null +++ b/src/TorchSharp/Optimizers/Adam.cs @@ -0,0 +1,298 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// https://arxiv.org/abs/1412.6980 + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static Adam Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new Adam(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + + /// + /// Implements the Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// https://arxiv.org/abs/1412.6980 + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static Adam Adam(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new Adam(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + + /// + /// Implements the Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// https://arxiv.org/abs/1412.6980 + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static Adam Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new Adam(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class Adam : OptimizerHelper, IBetas + { + /// + /// Implements Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// First coefficient used for computing running averages of gradient and its square + /// Second coefficient used for computing running averages of gradient and its square + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm + /// Maximize the params based on the objective, instead of minimizing + /// + public Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize) + { + } + + /// + /// Implements Adam algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization.The implementation of the L2 penalty follows changes proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// First coefficient used for computing running averages of gradient and its square + /// Second coefficient used for computing running averages of gradient and its square + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm + /// Maximize the params based on the objective, instead of minimizing + /// + public Adam(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (beta1 < 0 || beta1 >= 1.0) throw new ArgumentException($"Invalid beta1: {beta1}"); + if (beta2 < 0 || beta2 >= 1.0) throw new ArgumentException($"Invalid beta2: {beta2}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + maximize = maximize, + eps = eps, + amsgrad = amsgrad, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var weight_decay = options.weight_decay.Value; + var amsgrad = options.amsgrad.Value; + var maximize = options.maximize.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var state = _state[param.handle]; + + var grad = (maximize) ? -param.grad() : param.grad(); + + if (grad is null) continue; + + state.step += 1; + + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); + + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } + + state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad.conj(), value: 1 - beta2); + + Tensor denom = null; + if (amsgrad) { + var t0 = state.max_exp_avg_sq; + state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); + t0.Dispose(); + denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } else { + denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } + + var step_size = lr / bias_correction1; + param.addcdiv_(state.exp_avg, denom, value: -step_size); + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (_, state) in _state) { + state.Dispose(); + } + } + + private class State : IDisposable + { + public long step; + public Tensor exp_avg; + public Tensor exp_avg_sq; + public Tensor max_exp_avg_sq; + + public void Dispose() + { + exp_avg.Dispose(); + exp_avg_sq.Dispose(); + if (max_exp_avg_sq is not null) { + max_exp_avg_sq.Dispose(); + } + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.amsgrad.HasValue) opt.amsgrad = def.amsgrad; + if (!opt.maximize.HasValue) opt.maximize = def.maximize; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + if (opt.amsgrad.Value) { + state.max_exp_avg_sq = torch.zeros_like(p); + } + } + } + + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? weight_decay; + public double? eps; + public bool? amsgrad; + public bool? maximize; + } + + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + : base(parameters, new Adam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, amsgrad = amsgrad, maximize = maximize }) + { + } + + public (double, double) Betas { + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); + } + } +} diff --git a/src/TorchSharp/Optimizers/AdamW.cs b/src/TorchSharp/Optimizers/AdamW.cs new file mode 100644 index 000000000..69bfe2095 --- /dev/null +++ b/src/TorchSharp/Optimizers/AdamW.cs @@ -0,0 +1,296 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the AdamW algorithm. + /// + /// The AdamW variant was proposed in Decoupled Weight Decay Regularization. + /// https://arxiv.org/abs/1711.05101 + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static AdamW AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new AdamW(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + + /// + /// Implements the AdamW algorithm. + /// + /// The AdamW variant was proposed in Decoupled Weight Decay Regularization. + /// https://arxiv.org/abs/1711.05101 + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static AdamW AdamW(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new AdamW(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + + /// + /// Implements the AdamW algorithm. + /// + /// The AdamW variant was proposed in Decoupled Weight Decay Regularization. + /// https://arxiv.org/abs/1711.05101 + /// + /// Parameters to optimize + /// learning rate (default: 1e-3) + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm. (default: False) + /// + /// + public static AdamW AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.99, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + return new AdamW(parameters, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class AdamW : OptimizerHelper, IBetas + { + /// + /// Implements AdamW algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// First coefficient used for computing running averages of gradient and its square + /// Second coefficient used for computing running averages of gradient and its square + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm + /// Maximize the params based on the objective, instead of minimizing + /// + public AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, amsgrad, maximize) + { + } + + /// + /// Implements AdamW algorithm. + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. The AdamW variant was proposed in Decoupled Weight Decay Regularization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// First coefficient used for computing running averages of gradient and its square + /// Second coefficient used for computing running averages of gradient and its square + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// Whether to use the AMSGrad variant of this algorithm + /// Maximize the params based on the objective, instead of minimizing + /// + public AdamW(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + { + if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (beta1 < 0 || beta1 >= 1.0) throw new ArgumentException($"Invalid beta1: {beta1}"); + if (beta2 < 0 || beta2 >= 1.0) throw new ArgumentException($"Invalid beta2: {beta2}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + maximize = maximize, + eps = eps, + amsgrad = amsgrad, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var weight_decay = options.weight_decay.Value; + var amsgrad = options.amsgrad.Value; + var maximize = options.maximize.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var state = _state[param.handle]; + + var grad = (maximize) ? -param.grad() : param.grad(); + + if (grad is null) continue; + + state.step += 1; + + param.mul_(1 - lr * weight_decay); + + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); + + state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + + Tensor denom = null; + if (amsgrad) { + var t0 = state.max_exp_avg_sq; + state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetatchFromDisposeScope(); + t0.Dispose(); + denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } else { + denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps); + } + + var step_size = lr / bias_correction1; + param.addcdiv_(state.exp_avg, denom, value: -step_size); + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (_, state) in _state) { + state.Dispose(); + } + } + + private class State : IDisposable + { + public long step; + public Tensor exp_avg; + public Tensor exp_avg_sq; + public Tensor max_exp_avg_sq; + + public void Dispose() + { + exp_avg.Dispose(); + exp_avg_sq.Dispose(); + if (max_exp_avg_sq is not null) { + max_exp_avg_sq.Dispose(); + } + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.amsgrad.HasValue) opt.amsgrad = def.amsgrad; + if (!opt.maximize.HasValue) opt.maximize = def.maximize; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + if (opt.amsgrad.Value) { + state.max_exp_avg_sq = torch.zeros_like(p); + } + } + } + + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? weight_decay; + public double? eps; + public bool? amsgrad; + public bool? maximize; + } + + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-3, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, bool amsgrad = false, bool maximize = false) + : base(parameters, new AdamW.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, amsgrad = amsgrad, maximize = maximize }) + { + } + + public (double, double) Betas { + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); + } + } +} diff --git a/src/TorchSharp/Optimizers/Adamax.cs b/src/TorchSharp/Optimizers/Adamax.cs new file mode 100644 index 000000000..befd66e04 --- /dev/null +++ b/src/TorchSharp/Optimizers/Adamax.cs @@ -0,0 +1,271 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the Adamax algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// https://arxiv.org/abs/1412.6980 + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adamax Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new Adamax(parameters, lr, beta1, beta2, eps, weight_decay); + } + + /// + /// Implements the Adamax algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// https://arxiv.org/abs/1412.6980 + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adamax Adamax(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new Adamax(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay); + } + + /// + /// Implements the Adamax algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// https://arxiv.org/abs/1412.6980 + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public static Adamax Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new Adamax(parameters, lr, beta1, beta2, eps, weight_decay); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class Adamax : OptimizerHelper, IBetas + { + /// + /// Implements Adamax algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public Adamax(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay) + { + } + + /// + /// Implements Adamax algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public Adamax(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); + if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); + if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + eps = eps, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var grad = param.grad(); + + if (grad is null) continue; + + if (grad.is_sparse) throw new ArgumentException("Adamax does not support sparse gradients"); + + var state = _state[param.handle]; + + state.step += 1; + + var exp_avg = state.exp_avg; + var exp_inf = state.exp_inf; + + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); + + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + + var norm_buf = torch.cat(new Tensor[] { + exp_inf.mul_(beta2).unsqueeze(0), + grad.abs().add_(eps).unsqueeze_(0) + }, 0); + + torch.amax(norm_buf, new long[] { 0 }, false, exp_inf); + + var clr = lr / (1 - Math.Pow(beta1, state.step)); + param.addcdiv_(exp_avg, exp_inf, value: -clr); + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.Dispose(); + } + } + + private class State : IDisposable + { + public long step; + public Tensor exp_avg; + public Tensor exp_inf; + + public void Dispose() + { + exp_avg.Dispose(); + exp_inf.Dispose(); + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_inf = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? eps; + public double? weight_decay; + } + + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + : base(parameters, new Adamax.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay }) + { + } + + public (double, double) Betas { + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); + } + } +} diff --git a/src/TorchSharp/Optimizers/LBFGS.cs b/src/TorchSharp/Optimizers/LBFGS.cs new file mode 100644 index 000000000..908f88c4f --- /dev/null +++ b/src/TorchSharp/Optimizers/LBFGS.cs @@ -0,0 +1,112 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + [DllImport("LibTorchSharp")] + private static extern IntPtr THSNN_LBFGS_ctor(IntPtr parameters, int len, double learningRate, long max_iter, long max_eval, double tolerange_grad, double tolerance_change, long history_size); + + /// + /// Implements the L-BFGS algorithm, heavily inspired by `minFunc` + /// + /// https://www.cs.ubc.ca/~schmidtm/Software/minFunc.html + /// + /// Parameters to optimize + /// Learning rate (default: 1e-2) + /// Maximal number of iterations per optimization step + /// Maximal number of function evaluations per optimization step + /// Termination tolerance on first order optimality + /// Termination tolerance on function value/parameter changes + /// Update history size + public static LBFGS LBFGS(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.01, long max_iter = 20, long? max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100) + { + return LBFGS(parameters.Select(np => np.parameter), lr, max_iter, max_eval.Value, tolerange_grad, tolerance_change, history_size); + } + + /// + /// Implements the L-BFGS algorithm, heavily inspired by `minFunc` + /// + /// https://www.cs.ubc.ca/~schmidtm/Software/minFunc.html + /// + /// Parameters to optimize + /// Learning rate (default: 1e-2) + /// Maximal number of iterations per optimization step + /// Maximal number of function evaluations per optimization step + /// Termination tolerance on first order optimality + /// Termination tolerance on function value/parameter changes + /// Update history size + public static LBFGS LBFGS(IEnumerable parameters, double lr = 0.01, long max_iter = 20, long? max_eval = null, double tolerange_grad = 1e-5, double tolerance_change = 1e-9, long history_size = 100) + { + if (!max_eval.HasValue) max_eval = 5 * max_iter / 4; + + var parray = new PinnedArray(); + IntPtr paramsRef = parray.CreateArray(parameters.Select(p => p.Handle).ToArray()); + + var res = THSNN_LBFGS_ctor(paramsRef, parray.Array.Length, lr, max_iter, max_eval.Value, tolerange_grad, tolerance_change, history_size); + if (res == IntPtr.Zero) { torch.CheckForErrors(); } + return new LBFGS(res, lr); + } + } + } + + namespace Modules + { + using static torch.optim; + + // LBGFS does not allow for param groups, so there's no need to use anything but the native implementation. + public class LBFGS : Optimizer, ILearningRateController + { + /// + /// Implements L-BFGS algorithm, heavily inspired by `minFunc` + /// + /// + /// + /// + public LBFGS(IntPtr handle, double lr) : base(handle) + { + _rate = lr; + InitialLearningRate = lr; + _paramGroups = new ParamGroup[] { new ParamGroup { Parameters = this.parameters(), Options = new() { LearningRate = lr, InitialLearningRate = lr } } }; + } + + [DllImport("LibTorchSharp")] + private static extern void THSNN_LBFGS_set_lr(HType optimizer, double lr); + + public double LearningRate { + get { return _rate; } + set { THSNN_LBFGS_set_lr(handle, value); torch.CheckForErrors(); _rate = value; } + } + + public double InitialLearningRate { get; set; } + + public override IEnumerable ParamGroups { get => _paramGroups; } + + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + if (closure == null) + throw new ArgumentNullException("'closure' must be non-null when using the LBFGS optimizer. See: https://pytorch.org/docs/1.9.0/optim.html"); + return base.step(closure); + } + + public double _rate; + private ParamGroup[] _paramGroups; + } + } +} diff --git a/src/TorchSharp/NN/LRScheduler.cs b/src/TorchSharp/Optimizers/LRScheduler.cs similarity index 100% rename from src/TorchSharp/NN/LRScheduler.cs rename to src/TorchSharp/Optimizers/LRScheduler.cs diff --git a/src/TorchSharp/Optimizers/NAdam.cs b/src/TorchSharp/Optimizers/NAdam.cs new file mode 100644 index 000000000..94d7a5ace --- /dev/null +++ b/src/TorchSharp/Optimizers/NAdam.cs @@ -0,0 +1,284 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + public static NAdam NAdam(IEnumerable named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + { + return new NAdam(named_parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); + } + + /// + /// Implements the NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + public static NAdam NAdam(IEnumerable<(string name, Parameter parameter)> named_parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + { + return new NAdam(named_parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay, momentum_decay); + } + + /// + /// Implements the NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + public static NAdam NAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + { + return new NAdam(parameters, lr, beta1, beta2, eps, weight_decay, momentum_decay); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class NAdam : OptimizerHelper, IBetas + { + /// + /// Implements NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + public NAdam(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay, momentum_decay) + { + } + + /// + /// Implements NAdam algorithm. + /// + /// For further details regarding the algorithm we refer to Incorporating Nesterov Momentum into Adam. + /// https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum decay + /// + public NAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); + if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); + if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + if (momentum_decay < 0.0) throw new ArgumentException($"Invalid momentum_decay value: {momentum_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + eps = eps, + weight_decay = weight_decay, + momentum_decay = momentum_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var momentum_decay = options.momentum_decay.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var grad = param.grad(); + + if (grad is null) continue; + + var state = _state[param.handle]; + + state.step += 1; + + var exp_avg = state.exp_avg; + var exp_avg_sq = state.exp_avg_sq; + + var bias_correction2 = 1 - Math.Pow(beta2, state.step); + + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); + + var mu = beta1 * (1.0 - 0.5 * Math.Pow(0.96, state.step * momentum_decay)); + var mu_next = beta1 * (1.0 - 0.5 * Math.Pow(0.96, (state.step + 1) * momentum_decay)); + + var mu_product = state.mu_product * mu; + var mu_product_next = mu_product * mu * mu_next; + + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + + var denom = exp_avg_sq.div(bias_correction2).sqrt_().add_(eps); + + param.addcdiv_(grad, denom, value: -lr * (1 - mu) / (1 - mu_product)); + param.addcdiv_(exp_avg, denom, value: -lr * mu_next / (1 - mu_product_next)); + + state.mu_product = mu_product; + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.Dispose(); + } + } + + private class State : IDisposable + { + public long step; + public double mu_product; + public Tensor exp_avg; + public Tensor exp_avg_sq; + + public void Dispose() + { + exp_avg.Dispose(); + exp_avg_sq.Dispose(); + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.momentum_decay.HasValue) opt.momentum_decay = def.momentum_decay; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? eps; + public double? weight_decay; + public double? momentum_decay; + } + + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0, double momentum_decay = 4e-3) + : base(parameters, new NAdam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay, momentum_decay = momentum_decay }) + { + } + + public (double, double) Betas { + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); + } + } +} diff --git a/src/TorchSharp/Optimizers/Optimizer.cs b/src/TorchSharp/Optimizers/Optimizer.cs new file mode 100644 index 000000000..393d83d35 --- /dev/null +++ b/src/TorchSharp/Optimizers/Optimizer.cs @@ -0,0 +1,357 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + /// + /// Base class for all optimizers. + /// + public abstract partial class Optimizer : IDisposable + { + /// + /// Class wrapping PyTorch's optimzer object reference. + /// + internal sealed class HType : SafeHandle + { + public HType(IntPtr preexistingHandle, bool ownsHandle) : base(IntPtr.Zero, ownsHandle) + { + SetHandle(preexistingHandle); + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + // This is just for marshalling + internal HType() : base(IntPtr.Zero, true) + { + } + + [DllImport("LibTorchSharp")] + private static extern void THSNN_Optimizer_dispose(HType handle); + + protected override bool ReleaseHandle() + { + THSNN_Optimizer_dispose(this); + return true; + } + + protected override void Dispose(bool disposing) + { + if (disposing) { + ReleaseHandle(); + } + } + } + + internal HType handle; + + /// + /// Constructor used for optimizers implemented in native code. + /// + /// + protected Optimizer(IntPtr handle) + { + if (handle != IntPtr.Zero) { + this.handle = new HType(handle, true); + } + } + + ~Optimizer() + { + Dispose(false); + } + + /// + /// Releases the storage. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Implements the .NET Dispose pattern. + /// + protected virtual void Dispose(bool disposing) + { + if (disposing && handle != null && !handle.IsInvalid) { + handle.Dispose(); + handle.SetHandleAsInvalid(); + } + } + + [DllImport("LibTorchSharp")] + private static extern void THSNN_Optimizer_zero_grad(HType module); + + /// + /// Sets the gradients of all parameters to zero. + /// + public virtual void zero_grad() + { + THSNN_Optimizer_zero_grad(handle); + torch.CheckForErrors(); + } + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + public delegate IntPtr LossClosure(); + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public virtual void add_param_group(ParamGroup param_group) + { + throw new NotImplementedException($"add_param_group"); + } + + + [DllImport("LibTorchSharp")] + private static extern IntPtr THSNN_Optimizer_step(HType module, LossClosure closure); + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public virtual Tensor step(Func closure = null) + { + IntPtr res = (closure == null) ? + THSNN_Optimizer_step(handle, null) : + THSNN_Optimizer_step(handle, () => { + return closure().DecoupleFromNativeHandle(); + }); + + if (res == IntPtr.Zero) + torch.CheckForErrors(); + + return (res == IntPtr.Zero) ? null : new Tensor(res); + } + + [DllImport("LibTorchSharp")] + private static extern void THSNN_Optimizer_getParameters(HType module, AllocatePinnedArray allocator); + + /// + /// Get the parameters that the optimizer is handling. + /// + public virtual IEnumerable parameters() + { + IntPtr[] ptrArray; + + using (var pa = new PinnedArray()) { + THSNN_Optimizer_getParameters(handle, pa.CreateArray); + torch.CheckForErrors(); + ptrArray = pa.Array; + } + return ptrArray.Select(x => new Parameter(x)); + } + + public virtual IEnumerable ParamGroups { + get => _parameter_groups; + } + + protected IList _parameter_groups; + } + + /// + /// This interfce is used by learning rate schedulers to access and control + /// the rates used by optimizers. + /// + public interface ILearningRateController + { + /// + /// The current LR + /// + double LearningRate { set; get; } + + /// + /// The initial LR + /// + double InitialLearningRate { set; get; } + } + + /// + /// Indicates optimizers with support for momentum, which some LR schedulers require. + /// + public interface IMomentum + { + double Momentum { get; set; } + } + + /// + /// Indicates optimizers with support for betas instead of momentum. + /// + public interface IBetas + { + (double, double) Betas { get; set; } + } + } + } + + namespace Modules + { + using static torch.optim; + + /// + /// Base class to help with a couple of the things that managed-code implementations need. + /// + public class OptimizerHelper : Optimizer + { + public OptimizerHelper() : base(IntPtr.Zero) + { + } + + /// + /// Sets the gradients of all parameters to zero. + /// + public override void zero_grad() + { + foreach (var g in _parameter_groups) { + + foreach (var p in g.Parameters) { + + using var grad = p.grad(); + + if (grad is null) continue; + + grad.zero_().Dispose(); + } + } + } + + /// + /// Support routine for implementation of step() in all optimizers that support parameter groups. + /// + /// The ParamGroup type in use + /// The body of the step update. + /// The closure, if any, for computing the loss. + /// + protected Tensor _step(Action body, Func loss_closure = null) where T:ParamGroup + { + Tensor loss = null; + + if (loss_closure != null) { + using (var _ = torch.enable_grad()) + loss = loss_closure(); + } + + using (var _ = torch.no_grad()) { + + using (var d = torch.NewDisposeScope()) { + + foreach (var group in _parameter_groups) { + + body(group as T); + + } + } + } + + return loss; + } + + /// + /// Get the parameters that the optimizer is handling. + /// + public override IEnumerable parameters() + { + return _parameter_groups.SelectMany(pg => pg.Parameters); + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(ParamGroup param_group) + { + _parameter_groups.Add(param_group); + } + + protected OptimizerOptions _defaults; + } + + /// + /// Base class for optimizer options. + /// + public class OptimizerOptions + { + public double? LearningRate { get; set; } + public double InitialLearningRate { get; set; } + } + + /// + /// Base class for parameter groups + /// + public class ParamGroup : ILearningRateController + { + public IEnumerable Parameters { get; set; } + + public OptimizerOptions Options { get; set; } + + public double LearningRate { get => Options.LearningRate.Value; set => Options.LearningRate = value; } + public double InitialLearningRate { get => Options.InitialLearningRate; set => Options.InitialLearningRate = value; } + } + + /// + /// Generic-typed version of ParamGroup + /// + /// The type of options used for the parameter group. + public class ParamGroup : ParamGroup where TOptions : OptimizerOptions + { + /// + /// Constructor + /// + public ParamGroup() + { + } + + /// + /// Constructor + /// + /// The parameters of the parameter group + /// The options of the parameter group + public ParamGroup(IEnumerable parameters, TOptions options = null) + { + base.Parameters = parameters; + base.Options = options; + } + + /// + /// Parameter group options / hyperparameters, used to control the optimizer algorithm. + /// + public new TOptions Options { get => (TOptions)base.Options; set => base.Options = value; } + } + + + + + + + + + + + + + + + + + + + + + } +} + diff --git a/src/TorchSharp/Optimizers/RAdam.cs b/src/TorchSharp/Optimizers/RAdam.cs new file mode 100644 index 000000000..78b632908 --- /dev/null +++ b/src/TorchSharp/Optimizers/RAdam.cs @@ -0,0 +1,278 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the RAdam algorithm. + /// + /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' + /// https://arxiv.org/abs/1908.03265 + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + public static RAdam RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new RAdam(parameters, lr, beta1, beta2, eps, weight_decay); + } + + /// + /// Implements the RAdam algorithm. + /// + /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' + /// https://arxiv.org/abs/1908.03265 + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + public static RAdam RAdam(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new RAdam(parameters.Select(np => np.parameter), lr, beta1, beta2, eps, weight_decay); + } + + /// + /// Implements the RAdam algorithm. + /// + /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' + /// https://arxiv.org/abs/1908.03265 + /// + /// Parameters to optimize. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + public static RAdam RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + return new RAdam(parameters, lr, beta1, beta2, eps, weight_decay); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class RAdam : OptimizerHelper, IBetas + { + /// + /// Implements RAdam algorithm. + /// + /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' + /// https://arxiv.org/abs/1908.03265 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public RAdam(IEnumerable parameters, double lr, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, beta1, beta2, eps, weight_decay) + { + } + + /// + /// Implements RAdam algorithm. + /// + /// For further details regarding the algorithm we refer to 'On the variance of the adaptive learning rate and beyond.' + /// https://arxiv.org/abs/1908.03265 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Coefficient used for computing running averages of gradient and its square (default: 0.9) + /// Coefficient used for computing running averages of gradient and its square (default: 0.999) + /// Term added to the denominator to improve numerical stability, i.e. avoid division-by-zero (default: 1e-8) + /// Weight decay (L2 penalty) (default: 0) + /// + public RAdam(IEnumerable parameters, double lr = 0.002, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (beta1 < 0.0 || beta1 > 1.0) throw new ArgumentException($"Invalid beta1 value: {beta1}"); + if (beta2 < 0.0 || beta2 > 1.0) throw new ArgumentException($"Invalid beta2 value: {beta2}"); + if (eps < 0.0) throw new ArgumentException($"Invalid eps value: {eps}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + beta1 = beta1, + beta2 = beta2, + eps = eps, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var beta1 = options.beta1.Value; + var beta2 = options.beta2.Value; + var eps = options.eps.Value; + var weight_decay = options.weight_decay.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var grad = param.grad(); + + if (grad is null) continue; + + var state = _state[param.handle]; + + state.step += 1; + + var exp_avg = state.exp_avg; + var exp_avg_sq = state.exp_avg_sq; + + var bias_correction1 = 1 - Math.Pow(beta1, state.step); + var bias_correction2 = 1 - Math.Pow(beta2, state.step); + + grad = (weight_decay != 0) + ? grad.add(param, alpha: weight_decay) + : grad.alias(); + + exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1); + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value: 1 - beta2); + + var bias_corrected_exp_avg = exp_avg / bias_correction1; + + var rho_inf = 2 / (1 - beta2) - 1; + var rho_t = rho_inf - 2 * state.step * Math.Pow(beta2, state.step) / bias_correction2; + + var t6 = bias_corrected_exp_avg * lr; + + if (rho_t > 5) { + var rect = Math.Sqrt((rho_t - 4) * (rho_t - 2) * rho_inf / ((rho_inf - 4) * (rho_inf - 2) * rho_t)); + var adaptive_lr = Math.Sqrt(bias_correction2) / exp_avg_sq.sqrt().add_(eps); + + param.add_(t6 * lr * adaptive_lr * rect, alpha: -1.0); + } else { + param.add_(t6, alpha: -1.0); + } + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.Dispose(); + } + } + + private class State : IDisposable + { + public long step; + public Tensor exp_avg; + public Tensor exp_avg_sq; + + public void Dispose() + { + exp_avg.Dispose(); + exp_avg_sq.Dispose(); + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.beta1.HasValue) opt.beta1 = def.beta1; + if (!opt.beta2.HasValue) opt.beta2 = def.beta2; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.exp_avg = torch.zeros_like(p); + state.exp_avg_sq = torch.zeros_like(p); + } + } + + public class Options : OptimizerOptions + { + public double? beta1; + public double? beta2; + public double? eps; + public double? weight_decay; + } + + public class ParamGroup : ParamGroup, IBetas + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1.0, double beta1 = 0.9, double beta2 = 0.999, double eps = 1e-8, double weight_decay = 0) + : base(parameters, new RAdam.Options { LearningRate = lr, beta1 = beta1, beta2 = beta2, eps = eps, weight_decay = weight_decay }) + { + } + + public (double, double) Betas { + get => (Options.beta1.Value, Options.beta2.Value); + set { Options.beta1 = value.Item1; Options.beta2 = value.Item2; } + } + } + + public (double, double) Betas { + get => ((_defaults as Options).beta1.Value, (_defaults as Options).beta2.Value); + set { (_defaults as Options).beta1 = value.Item1; (_defaults as Options).beta2 = value.Item2; } + } + + private Dictionary _state = new Dictionary(); + } + } +} diff --git a/src/TorchSharp/Optimizers/RMSprop.cs b/src/TorchSharp/Optimizers/RMSprop.cs new file mode 100644 index 000000000..5479f560f --- /dev/null +++ b/src/TorchSharp/Optimizers/RMSprop.cs @@ -0,0 +1,282 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. + /// https://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf + /// + /// Parameters to optimize + /// Learning rate (default: 1e-2) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Smoothing constant (default: 0.99) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum factor (default: 0) + /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public static RMSProp RMSProp(IEnumerable parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) + { + return new RMSProp(parameters, lr, alpha, eps, weight_decay, momentum, centered); + } + + /// + /// Implements the RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. + /// https://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf + /// + /// Parameters to optimize + /// Learning rate (default: 1e-2) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Smoothing constant (default: 0.99) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum factor (default: 0) + /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public static RMSProp RMSProp(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) + { + return new RMSProp(parameters.Select(np => np.parameter), lr, alpha, eps, weight_decay, momentum, centered); + } + + /// + /// Implements the RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. + /// https://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf + /// + /// Parameters to optimize + /// Learning rate (default: 1e-2) + /// Term added to the denominator to improve numerical stability (default: 1e-8) + /// Smoothing constant (default: 0.99) + /// Weight decay (L2 penalty) (default: 0) + /// Momentum factor (default: 0) + /// if true, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public static RMSProp RMSProp(IEnumerable parameters, double lr = 0.01, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0, bool centered = false) + { + return new RMSProp(parameters, lr, alpha, eps, weight_decay, momentum, centered); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class RMSProp : OptimizerHelper, IMomentum + { + + /// + /// Implements RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. + /// + /// Parameters to optimize. This optimizer requires the parameters collection. + /// Learning rate + /// Smoothing constant. + /// Momentum factor (default: 0) + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public RMSProp(IEnumerable parameters, double lr = 1e-3, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0.0, bool centered = false) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, alpha, eps, weight_decay, momentum, centered) + { + } + + /// + /// Implements RMSprop algorithm. + /// + /// Proposed by G.Hinton in his course. + /// + /// Parameters to optimize. This optimizer requires the parameters collection. + /// Learning rate + /// Smoothing constant. + /// Momentum factor (default: 0) + /// Term added to the denominator to improve numerical stability + /// Weight decay (L2 penalty) (default: 0) + /// if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance + /// + public RMSProp(IEnumerable parameters, double lr = 1e-3, double alpha = 0.99, double eps = 1e-8, double weight_decay = 0, double momentum = 0.0, bool centered = false) + { + if (lr < 0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (eps < 0) throw new ArgumentException($"Invalid ε: {eps}"); + if (alpha < 0) throw new ArgumentException($"Invalid alpha: {alpha}"); + if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + eps = eps, + alpha = alpha, + momentum = momentum, + centered = centered, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var momentum = options.momentum.Value; + var alpha = options.alpha.Value; + var weight_decay = options.weight_decay.Value; + var centered = options.centered.Value; + var eps = options.eps.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var state = _state[param.handle]; + + var grad = param.grad(); + + if (grad is null) continue; + + state.step += 1; + + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } + + state.square_avg.mul_(alpha).addcmul_(grad, grad, value: 1 - alpha); + + Tensor avg = null; + + if (centered) { + var grad_avg = state.grad_avg; + grad_avg.mul_(alpha).add_(grad, alpha: 1 - alpha); + avg = state.square_avg.addcmul(grad_avg, grad_avg, value: -1).sqrt_().add_(eps); + } else { + avg = state.square_avg.sqrt().add_(eps); + } + + if (momentum > 0) { + var buf = state.momentum_buffer; + buf.mul_(momentum).addcdiv_(grad, avg); + param.add_(buf, alpha: -lr); + } else { + param.addcdiv_(grad, avg, -lr); + } + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (_, state) in _state) { + state.Dispose(); + } + _state.Clear(); + } + + private class State : IDisposable + { + public long step; + public Tensor square_avg; + public Tensor momentum_buffer; + public Tensor grad_avg; + + public void Dispose() + { + momentum_buffer.Dispose(); + square_avg.Dispose(); + grad_avg.Dispose(); + } + } + + + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.momentum.HasValue) opt.momentum = def.momentum; + if (!opt.eps.HasValue) opt.eps = def.eps; + if (!opt.alpha.HasValue) opt.alpha = def.alpha; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.centered.HasValue) opt.centered = def.centered; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.square_avg = torch.zeros_like(p); + state.grad_avg = torch.zeros_like(p); + state.momentum_buffer = torch.zeros_like(p); + } + } + public class Options : Modules.OptimizerOptions + { + public double? momentum; + public double? alpha; + public double? eps; + public double? weight_decay; + public bool? centered; + } + + public class ParamGroup : ParamGroup, IMomentum + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-3, double eps = 1e-8, double alpha = 0.99, double weight_decay = 0, double momentum = 0.0, bool centered = false) + : base(parameters, new RMSProp.Options { LearningRate = lr, eps = eps, alpha = alpha, weight_decay = weight_decay, momentum = momentum, centered = centered }) + { + } + + public double Momentum { get => Options.momentum.Value; set => Options.momentum = value; } + } + + private Dictionary _state = new Dictionary(); + + public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } + } + } +} diff --git a/src/TorchSharp/Optimizers/Rprop.cs b/src/TorchSharp/Optimizers/Rprop.cs new file mode 100644 index 000000000..9cf1bd338 --- /dev/null +++ b/src/TorchSharp/Optimizers/Rprop.cs @@ -0,0 +1,259 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + + /// + /// Implements the the resilient backpropagation algorithm. + /// + /// A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm + /// http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.21.1417 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Multiplicative increase factor. + /// Multiplicative decrease factor. + /// Minimum allowed step size. + /// Maximum allowed step size. + /// + public static Rprop Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + { + return new Rprop(parameters, lr, etaminus, etaplus, min_step, max_step); + } + + /// + /// Implements the the resilient backpropagation algorithm. + /// + /// A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm + /// http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.21.1417 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Multiplicative increase factor. + /// Multiplicative decrease factor. + /// Minimum allowed step size. + /// Maximum allowed step size. + /// + public static Rprop Rprop(IEnumerable<(string name, Parameter parameter)> parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + { + return new Rprop(parameters.Select(np => np.parameter), lr, etaminus, etaplus, min_step, max_step); + } + + /// + /// Implements the the resilient backpropagation algorithm. + /// + /// A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm + /// http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.21.1417 + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Multiplicative increase factor. + /// Multiplicative decrease factor. + /// Minimum allowed step size. + /// Maximum allowed step size. + /// + public static Rprop Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + { + return new Rprop(parameters, lr, etaminus, etaplus, min_step, max_step); + } + } + } + + namespace Modules + { + using static torch.optim; + + public class Rprop : OptimizerHelper + { + /// + /// Implements Rprop algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. + /// Learning rate + /// Multiplicative increase factor. + /// Multiplicative decrease factor. + /// Minimum allowed step size. + /// Maximum allowed step size. + /// + public Rprop(IEnumerable parameters, double lr = 0.01, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + : this(new ParamGroup[] { new() { Parameters = parameters } }, lr, etaminus, etaplus, min_step, max_step) + { + } + + /// + /// Implements Rprop algorithm (a variant of Adam based on infinity norm). + /// + /// It has been proposed in Adam: A Method for Stochastic Optimization. + /// + /// Parameters to optimize. + /// Learning rate + /// Multiplicative increase factor. + /// Multiplicative decrease factor. + /// Minimum allowed step size. + /// Maximum allowed step size. + /// + public Rprop(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + etaminus = etaminus, + etaplus = etaplus, + min_step = min_step, + max_step = max_step + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options as Options; + var etaminus = options.etaminus.Value; + var etaplus = options.etaplus.Value; + var min_step = options.min_step.Value; + var max_step = options.max_step.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var grad = param.grad(); + + if (grad is null) continue; + + if (grad.is_sparse) throw new ArgumentException("Rprop does not support sparse gradients"); + + var state = _state[param.handle]; + + state.step += 1; + + grad = (max_step != 0) + ? grad.add(param, alpha: max_step) + : grad.alias(); + + var sign = grad.mul(state.prev).sign(); + sign[sign.gt(0)] = (Tensor)etaplus; + sign[sign.lt(0)] = (Tensor)etaminus; + sign[sign.eq(0)] = (Tensor)1; + + state.step_size.mul_(sign).clamp_(min_step, max_step); + + grad = grad.clone(); + + grad.index_put_(0, sign.eq(etaminus)); + + param.addcmul_(grad.sign(), state.step_size, -1); + + state.prev.copy_(grad); + } + + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + state.Dispose(); + } + } + + private class State : IDisposable + { + public long step; + public Tensor prev; + public Tensor step_size; + + public void Dispose() + { + prev.Dispose(); + step_size.Dispose(); + } + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.etaminus.HasValue) opt.etaminus = def.etaminus; + if (!opt.etaplus.HasValue) opt.etaplus = def.etaplus; + if (!opt.min_step.HasValue) opt.min_step = def.min_step; + if (!opt.max_step.HasValue) opt.max_step = def.max_step; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.step = 0; + state.prev = torch.zeros_like(p); + state.step_size = p.new_empty(p.shape).fill_(opt.LearningRate); + } + } + + public class Options : OptimizerOptions + { + public double? etaminus; + public double? etaplus; + public double? min_step; + public double? max_step; + } + + public class ParamGroup : ParamGroup + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-2, double etaminus = 0.5, double etaplus = 1.2, double min_step = 1e-6, double max_step = 50) + : base(parameters, new Rprop.Options { LearningRate = lr, etaminus = etaminus, etaplus = etaplus, min_step = min_step, max_step = max_step }) + { + } + } + + private Dictionary _state = new Dictionary(); + + } + } +} diff --git a/src/TorchSharp/Optimizers/SGD.cs b/src/TorchSharp/Optimizers/SGD.cs new file mode 100644 index 000000000..4d3f697a3 --- /dev/null +++ b/src/TorchSharp/Optimizers/SGD.cs @@ -0,0 +1,264 @@ +// Copyright (c) .NET Foundation and Contributors. All Rights Reserved. See LICENSE in the project root for license information. +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using static TorchSharp.torch; + +namespace TorchSharp +{ + using Modules; + + public static partial class torch + { + public static partial class optim + { + /// + /// Implements the stochastic gradient descent (optionally with momentum). + /// + /// The use of momentum is covered in: + /// http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public static Modules.SGD SGD(IEnumerable parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + { + return new Modules.SGD(parameters, learningRate, momentum, dampening, weight_decay, nesterov, maximize); + } + + /// + /// Implements the stochastic gradient descent (optionally with momentum). + /// + /// The use of momentum is covered in: + /// http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public static Modules.SGD SGD(IEnumerable<(string name, Parameter parameter)> parameters, double learningRate, double momentum = 0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + { + return new Modules.SGD(parameters.Select(np => np.parameter), learningRate, momentum, dampening, weight_decay, nesterov, maximize); + } + + /// + /// Implements the stochastic gradient descent (optionally with momentum). + /// + /// The use of momentum is covered in: + /// http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public static Modules.SGD SGD(IEnumerable parameters, double learningRate, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + { + return new Modules.SGD(parameters, learningRate, momentum, dampening, weight_decay, nesterov, maximize); + } + } + } + namespace Modules + { + using static torch.optim; + + public class SGD : OptimizerHelper, IMomentum + { + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public SGD(IEnumerable parameters, double lr, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + : this(new ParamGroup[] { new ParamGroup { Parameters = parameters } }, lr, momentum, dampening, weight_decay, nesterov, maximize) + { + } + + /// + /// Implements stochastic gradient descent (optionally with momentum). + /// + /// Parameters to optimize. This optimizer requires the named parameters collection. + /// Learning rate + /// Momentum factor (default: 0) + /// Dampening for momentum (default: 0) + /// Weight decay (L2 penalty) (default: 0) + /// Enables Nesterov momentum (default: False) + /// + /// + public SGD(IEnumerable parameters, double lr, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + { + if (lr < 0.0) throw new ArgumentException($"Invalid learning rate: {lr}"); + if (momentum < 0.0) throw new ArgumentException($"Invalid momentum value: {momentum}"); + if (weight_decay < 0.0) throw new ArgumentException($"Invalid weight_decay value: {weight_decay}"); + if (nesterov && (momentum <= 0 || dampening != 0)) throw new ArgumentException("Nesterov momentum requires a momentum and zero dampening"); + + var options = new Options { + LearningRate = lr, + InitialLearningRate = lr, + dampening = dampening, + maximize = maximize, + momentum = momentum, + nesterov = nesterov, + weight_decay = weight_decay + }; + + _defaults = options; + _parameter_groups = new List(); + + foreach (var g in parameters) { + add_param_group(g); + } + } + + /// + /// Performs a single optimization step (parameter update). + /// + /// A closure that reevaluates the model and returns the loss. Optional for most optimizers. + /// + public override Tensor step(Func closure = null) + { + return _step(group => { + + var options = group.Options; + var momentum = options.momentum.Value; + var dampening = options.dampening.Value; + var weight_decay = options.weight_decay.Value; + var nesterov = options.nesterov.Value; + var maximize = options.maximize.Value; + var lr = options.LearningRate.Value; + + foreach (var param in group.Parameters) { + + var state = _state[param.handle]; + + var grad = param.grad(); + + if (grad is null) continue; + + if (weight_decay != 0) { + grad = grad.add(param, alpha: weight_decay); + } + + if (momentum != 0) { + var buf = state.momentum_buffer; + + if (buf is null) { + buf = grad.clone().detach().DetatchFromDisposeScope(); + state.momentum_buffer = buf; + } else { + buf.mul_(momentum).add_(grad, alpha: (1 - dampening)); + } + + if (nesterov) { + grad = grad.add(buf, alpha: momentum); + } else { + grad = buf; + } + + state.momentum_buffer = buf; + } + + var alpha = maximize ? lr : -lr; + param.add_(grad, alpha: alpha); + + } + }, closure); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + foreach (var (name, state) in _state) { + if (state.momentum_buffer is not null) { + state.momentum_buffer.Dispose(); + } + } + _state.Clear(); + } + + private class State + { + public Tensor momentum_buffer; + } + + /// + /// Add a param group to the Optimizer s param_groups. + /// + /// + /// This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the Optimizer as training progresses. + public override void add_param_group(Modules.ParamGroup param_group) + { + var def = _defaults as Options; + if (param_group.Options is null) { + param_group.Options = new Options(); + } + + var opt = param_group.Options as Options; + + // Make sure all the options are set. + if (!opt.LearningRate.HasValue) opt.LearningRate = def.LearningRate; + if (!opt.momentum.HasValue) opt.momentum = def.momentum; + if (!opt.dampening.HasValue) opt.dampening = def.dampening; + if (!opt.weight_decay.HasValue) opt.weight_decay = def.weight_decay; + if (!opt.nesterov.HasValue) opt.nesterov = def.nesterov; + if (!opt.maximize.HasValue) opt.maximize = def.maximize; + + opt.InitialLearningRate = opt.LearningRate.Value; + + _parameter_groups.Add(param_group); + + foreach (var p in param_group.Parameters) { + var state = new State(); + _state[p.Handle] = state; + state.momentum_buffer = null; + } + } + + public class Options : Modules.OptimizerOptions + { + public double? momentum; + public double? dampening; + public double? weight_decay; + public bool? nesterov; + public bool? maximize; + } + + public class ParamGroup : ParamGroup, IMomentum + { + public ParamGroup() { } + + public ParamGroup(IEnumerable parameters, Options options) : base(parameters, options) { } + + public ParamGroup(IEnumerable parameters, double lr = 1e-3, double momentum = 0.0, double dampening = 0, double weight_decay = 0, bool nesterov = false, bool maximize = false) + : base(parameters, new SGD.Options { LearningRate = lr, dampening = dampening, momentum = momentum, weight_decay = weight_decay, nesterov = nesterov, maximize = maximize }) + { + } + + public double Momentum { get => Options.momentum.Value; set => Options.momentum = value; } + } + + private Dictionary _state = new Dictionary(); + + public double Momentum { get => (_defaults as Options).momentum.Value; set => (_defaults as Options).momentum = value; } + } + } +} diff --git a/test/TorchSharpTest/TestTorchTensorBugs.cs b/test/TorchSharpTest/TestTorchTensorBugs.cs index a1021c0b2..88c7a9077 100644 --- a/test/TorchSharpTest/TestTorchTensorBugs.cs +++ b/test/TorchSharpTest/TestTorchTensorBugs.cs @@ -502,43 +502,31 @@ public override torch.Tensor forward(torch.Tensor t) } } - [Fact]//(Skip ="")] + [Fact] public void ValidateIssue516() { if (torch.cuda.is_available()) { - // *** Note1 *** - // When cuda == false, everything all right. - // When cuda == true, the following warning raised. - var cuda = true; - - // All message printed in console: - // [W TensorBody.h:417] Warning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. - // Its .grad attribute won't be populated during autograd.backward(). If you indeed want the .grad field to - // be populated for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf - // Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 - // for more informations. (function grad) var model = new TestGradWarningModel(); - if (cuda) { - model.cuda(); - } + model.cuda(); + var optimizer = torch.optim.Adam(model.parameters()); - optimizer.zero_grad(); // Raise a warning + optimizer.zero_grad(); + + var x = torch.ones(5, 3).cuda(); + var y = torch.ones(5, 4).cuda(); - var x = torch.ones(5, 3); - var y = torch.ones(5, 4); - if (cuda) { - x = x.cuda(); - y = y.cuda(); - } var z = model.forward(x); var lossFunc = torch.nn.functional.cross_entropy_loss(); var loss = lossFunc(y, z); loss.backward(); - optimizer.step(); // Raise a warning + optimizer.step(); + + var grad1 = optimizer.parameters().ToArray()[0].grad(); + Assert.NotNull(grad1); - var grad1 = optimizer.parameters().ToArray()[0].grad(); // Raise a warning - var grad2 = model.Weight.grad(); // Raise a warning + var grad2 = model.Weight.grad(); + Assert.NotNull(grad2); } } @@ -557,14 +545,7 @@ public class TestGradWarningModel : torch.nn.Module public TestGradWarningModel() : base(nameof(TestGradWarningModel)) { - - // *** Note2 *** - // If I set the tensor device here, then setting cuda = true at line 20 won't cause the warnings. - // The warnings won't appear even when setting cuda = false at line 20 and execute a "model.cpu();". - // So I guess this is a single-directional problem (cpu -> cuda). Weight = torch.zeros(new long[] { 3, 4 }).AsParameter(); - //Weight = torch.zeros(new long[] { 3, 4 }, device: torch.CUDA).AsParameter(); - RegisterComponents(); } diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index a01687288..d77ef6847 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -149,21 +149,45 @@ public void TestAdamBetas() Assert.Equal(0.975, beta2); } - /// - /// Fully connected ReLU net with one hidden layer trained using Adam optimizer. - /// Taken from . - /// - [Fact] - public void TestTrainingAdamDefaults() + + private static void CreateDataAndLabels(Generator gen, out Tensor data, out Tensor labels, int batchSize = 64, int inputSize = 1000, int categories = 10) { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("drop1", Dropout(0.1)), ("lin2", lin2)); + data = torch.rand(new long[] { 64, inputSize }, generator: gen); + labels = torch.rand(new long[] { 64, categories }, generator: gen); + } - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + private static void CreateLinearLayers(Generator gen, out Linear linear1, out Linear linear2, int inputSize = 1000, int categories = 10, int hiddenSize = 100) + { + linear1 = Linear(inputSize, hiddenSize); + linear2 = Linear(hiddenSize, categories); - var optimizer = torch.optim.Adam(seq.parameters()); + ReInitializeLinear(gen, linear1); + ReInitializeLinear(gen, linear2); + } + + private static void ReInitializeLinear(Generator gen, Linear linear) + { + // The Linear module will have been created with some RNG that we don't have control over, specifically + // the default RNG, which does not work well in a parallel environment (which the unit test framework is). + // + // Therefore, we need to set the initial parameters based on the generator we do have control over + // and which is unique to the unit test. + + var w = rand(linear.weight.shape, generator: gen); + var wBound = 1 / Math.Sqrt(w.shape[1]); + + linear.weight = Parameter(w * (2 * wBound) - wBound); + + if (linear.bias is not null) { + var (fanIn, _) = init.CalculateFanInAndFanOut(linear.weight); + var bBound = (fanIn > 0) ? 1 / Math.Sqrt(fanIn) : 0; + var b = rand(linear.bias.shape, generator: gen); + linear.bias = Parameter(b * (2 * bBound) - bBound); + } + } + + private static float TrainLoop(Sequential seq, Tensor x, Tensor y, optim.Optimizer optimizer) + { var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); @@ -180,32 +204,25 @@ public void TestTrainingAdamDefaults() output.backward(); - using var _ = optimizer.step(); + optimizer.step(); } + + // After 10 iterations, the final loss should always be less than the initial loss. Assert.True(finalLoss < initialLoss); + + return finalLoss; } - [Fact] - public void TestTrainingAdamParamGroups() + private static float TrainLoop(Sequential seq, Tensor x, Tensor y, optim.Optimizer optimizer, optim.lr_scheduler.LRScheduler scheduler, bool check_lr = true) { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); - - var optimizer = torch.optim.Adam(new Adam.ParamGroup[] - { - new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, - new () { Parameters = lin2.parameters() } - }); - var loss = mse_loss(Reduction.Sum); float initialLoss = loss(seq.forward(x), y).ToSingle(); float finalLoss = float.MaxValue; + var pgFirst = optimizer.ParamGroups.First(); + var lastLR = pgFirst.LearningRate; + for (int i = 0; i < 10; i++) { using var eval = seq.forward(x); using var output = loss(eval, y); @@ -218,78 +235,118 @@ public void TestTrainingAdamParamGroups() output.backward(); optimizer.step(); + scheduler.step(); + + if (check_lr) { + // For most LR schedulers, the LR decreases monotonically with each step. However, + // that is not always the case, so the test must be disabled in some circumstances. + Assert.True(pgFirst.LearningRate < lastLR); + lastLR = pgFirst.LearningRate; + } } + + // After 10 iterations, the final loss should always be less than the initial loss. Assert.True(finalLoss < initialLoss); + + return finalLoss; } + private void LossIsClose(float expected, float actual, float tolerance = 0.001f) + { + // The error tolerance should be relative, not absolute. + tolerance *= actual; + Assert.True(MathF.Abs(actual - expected) <= tolerance, $"Expected {expected}, but got {actual}"); + } + + + /// + /// Fully connected ReLU net with one hidden layer trained using Adam optimizer. + /// Taken from . + /// [Fact] - public void TestTrainingAdamAmsGrad() + public void TestTrainingAdamDefaults() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var optimizer = torch.optim.Adam(seq.parameters()); - var optimizer = torch.optim.Adam(seq.parameters(), amsgrad: true); - var loss = mse_loss(Reduction.Sum); + var loss = TrainLoop(seq, x, y, optimizer); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + LossIsClose(53.3606f, loss); + } - for (int i = 0; i < 10; i++) { - var eval = seq.forward(x); - var output = loss(eval, y); - var lossVal = output.ToSingle(); + [Fact] + public void TestTrainingAdamParamGroups() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - finalLoss = lossVal; + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - optimizer.zero_grad(); + var optimizer = torch.optim.Adam(new Adam.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin2.parameters() } + }); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(105.82f, loss); } [Fact] - public void TestTrainingAdamOneCycleLR() + public void TestTrainingAdamAmsGrad() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var optimizer = torch.optim.Adam(seq.parameters(), amsgrad: true); - var lr = 0.001; - var optimizer = torch.optim.Adam(seq.parameters(), lr: lr, amsgrad: true); - var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 10); + var loss = TrainLoop(seq, x, y, optimizer); - var loss = mse_loss(Reduction.Sum); + LossIsClose(53.34f, loss); + } - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + [Fact] + public void TestTrainingAdamWeightDecay() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - for (int i = 0; i < 10; i++) { + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + var optimizer = torch.optim.Adam(seq.parameters(), weight_decay: 0.5); - finalLoss = lossVal; + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.zero_grad(); + LossIsClose(53.28f, loss); + } - output.backward(); + [Fact] + public void TestTrainingAdamOneCycleLR() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.step(); - scheduler.step(); - } + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - Assert.True(finalLoss < initialLoss); + var lr = 0.0005; + var optimizer = torch.optim.Adam(seq.parameters(), lr: lr, amsgrad: true); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 10); + + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); + + LossIsClose(197.63f, loss); } /// @@ -299,44 +356,27 @@ public void TestTrainingAdamOneCycleLR() [Fact] public void TestTrainingAdamWDefaults() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("drop1", Dropout(0.1)), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.AdamW(seq.parameters()); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - var eval = seq.forward(x); - var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(53.3606f, loss); } [Fact] public void TestTrainingAdamWParamGroups() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.AdamW(new AdamW.ParamGroup[] { @@ -344,63 +384,59 @@ public void TestTrainingAdamWParamGroups() new () { Parameters = lin2.parameters() } }); - var loss = mse_loss(Reduction.Sum); + var loss = TrainLoop(seq, x, y, optimizer); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + LossIsClose(105.82f, loss); + } - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + [Fact] + public void TestTrainingAdamWAmsGrad() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - finalLoss = lossVal; + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - optimizer.zero_grad(); + var optimizer = torch.optim.AdamW(seq.parameters(), amsgrad: true); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(53.34f, loss); } [Fact] - public void TestTrainingAdamWOneCycleLR() + public void TestTrainingAdamWWeightDecay() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); - - var lr = 0.001; - var optimizer = torch.optim.AdamW(seq.parameters(), lr: lr, amsgrad: true); - var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 10); - - var loss = mse_loss(Reduction.Sum); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + var optimizer = torch.optim.AdamW(seq.parameters(), weight_decay: 0.5); - for (int i = 0; i < 10; i++) { + var loss = TrainLoop(seq, x, y, optimizer); - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + LossIsClose(53.28f, loss); + } - finalLoss = lossVal; + [Fact] + public void TestTrainingAdamWOneCycleLR() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.zero_grad(); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - output.backward(); + var lr = 0.001; + var optimizer = torch.optim.AdamW(seq.parameters(), lr: lr, amsgrad: true); + var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 10); - optimizer.step(); - scheduler.step(); - } + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - Assert.True(finalLoss < initialLoss); + LossIsClose(197.63f, loss); } @@ -411,45 +447,62 @@ public void TestTrainingAdamWOneCycleLR() [Fact] public void TestTrainingAdagrad() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.Adagrad(seq.parameters(), learning_rate); - var loss = mse_loss(Reduction.Sum); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + var loss = TrainLoop(seq, x, y, optimizer); - for (int i = 0; i < 10; i++) { - var eval = seq.forward(x); - var output = loss(eval, y); - var lossVal = output.ToSingle(); + LossIsClose(203.20f, loss); + } - finalLoss = lossVal; + [Fact] + public void TestTrainingAdagradWeightDecay() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.zero_grad(); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - output.backward(); + double learning_rate = 0.00004f; + var optimizer = torch.optim.Adagrad(seq.parameters(), learning_rate, weight_decay: 0.5); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(203.21f, loss); } [Fact] - public void TestTrainingAdagradParamGroups() + public void TestTrainingAdagradLRDecay() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + double learning_rate = 0.00004f; + var optimizer = torch.optim.Adagrad(seq.parameters(), learning_rate, lr_decay: 0.1); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(211.065f, loss); + } + + [Fact] + public void TestTrainingAdagradParamGroups() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.Adagrad(new Adagrad.ParamGroup[] { @@ -457,72 +510,90 @@ public void TestTrainingAdagradParamGroups() new () { Parameters = lin2.parameters() } }); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + var loss = TrainLoop(seq, x, y, optimizer); - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + LossIsClose(51.51f, loss); + } - finalLoss = lossVal; + [Fact] + public void TestTrainingAdagradParamGroupsWD() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.zero_grad(); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - output.backward(); + var optimizer = torch.optim.Adagrad(new Adagrad.ParamGroup[] + { + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f, weight_decay = 0.25 } }, + new () { Parameters = lin2.parameters() } + }); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(51.45f, loss); } - /// - /// Fully connected ReLU net with one hidden layer trained using Adadelta optimizer. - /// [Fact] public void TestTrainingAdadelta() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 1.0f; var optimizer = torch.optim.Adadelta(seq.parameters(), learning_rate); - var loss = mse_loss(Reduction.Sum); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + var loss = TrainLoop(seq, x, y, optimizer); - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + LossIsClose(74.754f, loss); + } - finalLoss = lossVal; + [Fact] + public void TestTrainingAdadeltaWD() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.zero_grad(); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - output.backward(); + double learning_rate = 1.0f; + var optimizer = torch.optim.Adadelta(seq.parameters(), learning_rate, weight_decay: 0.35); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(73.226f, loss); } [Fact] - public void TestTrainingAdadeltaParamGroups() + public void TestTrainingAdadeltaRHO() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + double learning_rate = 1.0f; + var optimizer = torch.optim.Adadelta(seq.parameters(), learning_rate, rho: 0.75); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(73.027f, loss); + } + + [Fact] + public void TestTrainingAdadeltaParamGroups() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.Adadelta(new Adadelta.ParamGroup[] { @@ -530,71 +601,67 @@ public void TestTrainingAdadeltaParamGroups() new () { Parameters = lin2.parameters() } }); - var loss = mse_loss(Reduction.Sum); + var loss = TrainLoop(seq, x, y, optimizer); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + LossIsClose(77.891f, loss); + } - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + [Fact] + public void TestTrainingAdamax() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - finalLoss = lossVal; + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - optimizer.zero_grad(); + var optimizer = torch.optim.Adamax(seq.parameters()); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(55.559f, loss); } - /// - /// Fully connected ReLU net with one hidden layer trained using Adamax optimizer. - /// [Fact] - public void TestTrainingAdamax() + public void TestTrainingAdamaxWD() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var optimizer = torch.optim.Adamax(seq.parameters(), weight_decay: 0.35); - var optimizer = torch.optim.Adamax(seq.parameters()); - var loss = mse_loss(Reduction.Sum); + var loss = TrainLoop(seq, x, y, optimizer); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + LossIsClose(55.47f, loss); + } - for (int i = 0; i < 15; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + [Fact] + public void TestTrainingAdamaxBetas() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - finalLoss = lossVal; + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - optimizer.zero_grad(); + var optimizer = torch.optim.Adamax(seq.parameters(), beta1: 0.75, beta2: 0.95); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(51.00f, loss); } [Fact] public void TestTrainingAdamaxParamGroups() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.Adamax(new Adamax.ParamGroup[] { @@ -602,25 +669,9 @@ public void TestTrainingAdamaxParamGroups() new () { Parameters = lin2.parameters() } }); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(141.519f, loss); } /// @@ -629,83 +680,93 @@ public void TestTrainingAdamaxParamGroups() [Fact] public void TestTrainingAdamaxOneCycleLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var lr = 0.002; var optimizer = torch.optim.Adamax(seq.parameters(), lr: lr); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 15); - var loss = mse_loss(Reduction.Sum); + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + LossIsClose(61.45f, loss); + } - for (int i = 0; i < 15; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + [Fact] + public void TestTrainingNAdam() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - finalLoss = lossVal; + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - optimizer.zero_grad(); + var optimizer = torch.optim.NAdam(seq.parameters()); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - scheduler.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(63.9739f, loss); } - /// - /// Fully connected ReLU net with one hidden layer trained using NAdam optimizer. - /// [Fact] - public void TestTrainingNAdam() + public void TestTrainingNAdamWD() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var optimizer = torch.optim.NAdam(seq.parameters(), weight_decay: 0.45); - var optimizer = torch.optim.NAdam(seq.parameters()); - var loss = mse_loss(Reduction.Sum); + var loss = TrainLoop(seq, x, y, optimizer); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + LossIsClose(63.281f, loss); + } - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + [Fact] + public void TestTrainingNAdamBetas() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - finalLoss = lossVal; + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - optimizer.zero_grad(); + var optimizer = torch.optim.NAdam(seq.parameters(), beta1: 0.75, beta2: 0.95); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(56.387f, loss); } [Fact] - public void TestTrainingNAdamParamGroups() + public void TestTrainingNAdamMD() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var optimizer = torch.optim.NAdam(seq.parameters(), momentum_decay: 0.04); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(63.46877f, loss); + } + + [Fact] + public void TestTrainingNAdamParamGroups() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.NAdam(new NAdam.ParamGroup[] { @@ -713,25 +774,9 @@ public void TestTrainingNAdamParamGroups() new () { Parameters = lin2.parameters() } }); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(59.230f, loss); } /// @@ -740,36 +785,19 @@ public void TestTrainingNAdamParamGroups() [Fact] public void TestTrainingNAdamOneCycleLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var lr = 0.002; var optimizer = torch.optim.NAdam(seq.parameters(), lr: lr); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 10, total_steps: 10); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - optimizer.step(); - scheduler.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(214.188f, loss); } /// @@ -778,70 +806,37 @@ public void TestTrainingNAdamOneCycleLR() [Fact] public void TestTrainingRAdam() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var optimizer = torch.optim.RAdam(seq.parameters()); - var loss = mse_loss(Reduction.Sum); + var optimizer = torch.optim.RAdam(seq.parameters(), 0.0005); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + var loss = TrainLoop(seq, x, y, optimizer); - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + LossIsClose(66.2651f, loss); + } - finalLoss = lossVal; + [Fact] + public void TestTrainingRAdamParamGroups() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); - } - - [Fact] - public void TestTrainingRAdamParamGroups() - { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.RAdam(new RAdam.ParamGroup[] { - new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.001f } }, new () { Parameters = lin2.parameters() } - }); - - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); + }, 0.0005); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(170.338f, loss); } /// @@ -850,37 +845,19 @@ public void TestTrainingRAdamParamGroups() [Fact] public void TestTrainingRAdamOneCycleLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var lr = 0.002; + var lr = 0.0005; var optimizer = torch.optim.RAdam(seq.parameters(), lr: lr); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, lr * 1.5, total_steps: 10); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - output.backward(); - - optimizer.step(); - scheduler.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(124.478f, loss); } /// @@ -889,14 +866,13 @@ public void TestTrainingRAdamOneCycleLR() [Fact] public void TestTrainingRAdamOneCycleLR_PG() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var lr = 0.002; + var lr = 0.0005; var optimizer = torch.optim.RAdam(new RAdam.ParamGroup[] { new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.003f } }, @@ -904,26 +880,9 @@ public void TestTrainingRAdamOneCycleLR_PG() }); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, new double[] { lr * 1.5, lr * 2.25 }, total_steps: 10); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - optimizer.step(); - scheduler.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(92.047f, loss); } /// @@ -932,12 +891,11 @@ public void TestTrainingRAdamOneCycleLR_PG() [Fact] public void TestTrainingRAdamCyclicLR_PG() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var lr = 0.0002; var optimizer = torch.optim.SGD(new SGD.ParamGroup[] @@ -947,26 +905,9 @@ public void TestTrainingRAdamCyclicLR_PG() }, lr); var scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, new double[] { lr / 2, lr * 2 }, new double[] { lr * 1.5, lr * 2.25 }); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - optimizer.step(); - scheduler.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(58.589f, loss); } /// @@ -975,45 +916,28 @@ public void TestTrainingRAdamCyclicLR_PG() [Fact] public void TestTrainingASGD() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.ASGD(seq.parameters()); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(57.748f, loss); } [Fact] public void TestTrainingASGDParamGroups() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var pgs = new ASGD.ParamGroup[] { @@ -1023,29 +947,13 @@ public void TestTrainingASGDParamGroups() var optimizer = torch.optim.ASGD(new ASGD.ParamGroup[] { - new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.005f } }, + new () { Parameters = lin1.parameters(), Options = new () { LearningRate = 0.0002f } }, new () { Parameters = lin2.parameters() } }); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(56.653f, loss); } /// @@ -1054,45 +962,28 @@ public void TestTrainingASGDParamGroups() [Fact] public void TestTrainingRprop() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.Rprop(seq.parameters()); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(229.68f, loss); } [Fact] public void TestTrainingRpropParamGroups() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); var optimizer = torch.optim.Rprop(new Rprop.ParamGroup[] { @@ -1100,25 +991,9 @@ public void TestTrainingRpropParamGroups() new () { Parameters = lin2.parameters() } }); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(78.619f, loss); } /// @@ -1128,34 +1003,18 @@ public void TestTrainingRpropParamGroups() [Fact] public void TestTrainingRMSLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(56.589f, loss); } /// @@ -1165,114 +1024,63 @@ public void TestTrainingRMSLR() [Fact] public void TestTrainingRMSOneCycleLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, learning_rate * 10, total_steps: 10); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - optimizer.step(); - scheduler.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(207.87f, loss); } [Fact] public void TestTrainingRMSAlpha() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, alpha: 0.75); - var loss = mse_loss(Reduction.Sum); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + var loss = TrainLoop(seq, x, y, optimizer); - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(156.339f, loss); } [Fact] public void TestTrainingRMSCentered() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, centered: true); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + var loss = TrainLoop(seq, x, y, optimizer); - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(56.189f, loss); } [Fact] public void TestTrainingRMSCenteredParamGroups() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; @@ -1282,25 +1090,9 @@ public void TestTrainingRMSCenteredParamGroups() new(lin2.parameters(), lr: learning_rate*10) }, learning_rate, centered: true); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + LossIsClose(53.6049f, loss); } /// @@ -1310,370 +1102,174 @@ public void TestTrainingRMSCenteredParamGroups() [Fact] public void TestTrainingSGDMomentum() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); - - double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, momentum: 0.5); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); - } - - [Fact()]//(Skip = "Fails with an exception in native code.")] - public void TestTrainingSGDNesterov() - { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); - - double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, momentum: 0.1, nesterov: true); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); - } - - [Fact] - public void TestTrainingSGDDefaults() - { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); - - double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); - } - - [Fact] - public void TestTrainingSGDDefaultsParamGroups() - { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - - var pgs = new SGD.ParamGroup[] { - new () { Parameters = lin1.parameters(), Options = new() { LearningRate = 0.00005 } }, - new (lin2.parameters(), lr: 0.00003, dampening: 0.05, momentum: 0.25) - }; - - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); - - double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(pgs, learning_rate); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); - } - - [Fact] - public void TestTrainingSGDStepLR() - { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); - - double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); - var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.97); - - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - double lastLR = learning_rate; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - scheduler.step(); - - var pgFirst = optimizer.ParamGroups.First(); - Assert.True(pgFirst.LearningRate < lastLR); - - lastLR = pgFirst.LearningRate; - } - - Assert.True(finalLoss < initialLoss); - } - - [Fact] - public void TestTrainingSGDLambdaLR() - { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); - - double learning_rate = 0.00004f; - var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); - var scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, i => Math.Pow(0.95,(1+i))); - - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - double lastLR = learning_rate; + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, momentum: 0.5); - finalLoss = lossVal; + var loss = TrainLoop(seq, x, y, optimizer); - optimizer.zero_grad(); + LossIsClose(53.711f, loss); + } - output.backward(); + [Fact()]//(Skip = "Fails with an exception in native code.")] + public void TestTrainingSGDNesterov() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.step(); - scheduler.step(); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var pgFirst = optimizer.ParamGroups.First(); - Assert.True(pgFirst.LearningRate < lastLR); + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, momentum: 0.1, nesterov: true); - lastLR = pgFirst.LearningRate; - } + var loss = TrainLoop(seq, x, y, optimizer); - Assert.True(finalLoss < initialLoss); + LossIsClose(59.270f, loss); } [Fact] - public void TestTrainingSGDMultiplicativeLR() + public void TestTrainingSGDDefaults() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); - var scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer, i => 0.95); - - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - double lastLR = learning_rate; - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + var loss = TrainLoop(seq, x, y, optimizer); - finalLoss = lossVal; + LossIsClose(62.494f, loss); + } - optimizer.zero_grad(); + [Fact] + public void TestTrainingSGDDefaultsParamGroups() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - output.backward(); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - optimizer.step(); - scheduler.step(); + var pgs = new SGD.ParamGroup[] { + new () { Parameters = lin1.parameters(), Options = new() { LearningRate = 0.00005 } }, + new (lin2.parameters(), lr: 0.00003, dampening: 0.05, momentum: 0.25) + }; - var pgFirst = optimizer.ParamGroups.First(); - Assert.True(pgFirst.LearningRate < lastLR); + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(pgs, learning_rate); - lastLR = pgFirst.LearningRate; - } + var loss = TrainLoop(seq, x, y, optimizer); - Assert.True(finalLoss < initialLoss); + LossIsClose(58.698f, loss); } [Fact] - public void TestTrainingSGDExponentialLR() + public void TestTrainingSGDStepLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); - var scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer); - - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - double lastLR = learning_rate; - - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + var scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, 0.97); - finalLoss = lossVal; + var loss = TrainLoop(seq, x, y, optimizer, scheduler); - optimizer.zero_grad(); + LossIsClose(67.36136f, loss); + } - output.backward(); + [Fact] + public void TestTrainingSGDLambdaLR() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.step(); - scheduler.step(); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - var pgFirst = optimizer.ParamGroups.First(); - Assert.True(pgFirst.LearningRate < lastLR); + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, i => Math.Pow(0.95, (1 + i))); - lastLR = pgFirst.LearningRate; - } + var loss = TrainLoop(seq, x, y, optimizer, scheduler); - Assert.True(finalLoss < initialLoss); + LossIsClose(73.87f, loss); } [Fact] - public void TestTrainingSGDLinearLR() + public void TestTrainingSGDMultiplicativeLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); - var scheduler = torch.optim.lr_scheduler.LinearLR(optimizer, end_factor: 0.75, total_iters: 10); + var scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer, i => 0.95); - var loss = mse_loss(Reduction.Sum); + var loss = TrainLoop(seq, x, y, optimizer, scheduler); - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; + LossIsClose(71.1419f, loss); + } - var pgFirst = optimizer.ParamGroups.First(); + [Fact] + public void TestTrainingSGDExponentialLR() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - double lastLR = pgFirst.LearningRate; + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - for (int i = 0; i < 10; i++) { - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer); - finalLoss = lossVal; + var loss = TrainLoop(seq, x, y, optimizer, scheduler); - optimizer.zero_grad(); + LossIsClose(180.249f, loss); + } - output.backward(); + [Fact] + public void TestTrainingSGDLinearLR() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - optimizer.step(); - scheduler.step(); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); - Assert.True(pgFirst.LearningRate < lastLR); + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); + var scheduler = torch.optim.lr_scheduler.LinearLR(optimizer, end_factor: 0.75, total_iters: 10); - lastLR = pgFirst.LearningRate; - } + var loss = TrainLoop(seq, x, y, optimizer, scheduler); - Assert.True(finalLoss < initialLoss); + LossIsClose(209.71f, loss); } [Fact] public void TestTrainingSGDMultiStepLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); @@ -1708,17 +1304,18 @@ public void TestTrainingSGDMultiStepLR() } Assert.True(finalLoss < initialLoss); + + LossIsClose(69.423f, finalLoss); } [Fact] public void TestTrainingSGDCosineAnnealingLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); @@ -1753,93 +1350,54 @@ public void TestTrainingSGDCosineAnnealingLR() } Assert.True(finalLoss < initialLoss); + + LossIsClose(88.98f, finalLoss); } [Fact] public void TestTrainingSGDCyclicLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, 0.0001, 0.0004, step_size_up: 5); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - scheduler.step(); - } + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - Assert.True(finalLoss < initialLoss); + LossIsClose(65.459f, loss); } [Fact] public void TestTrainingSGDOneCycleLR() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.SGD(seq.parameters(), learning_rate); var scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, 0.0004, total_steps: 10); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - - using var eval = seq.forward(x); - using var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - - optimizer.step(); - scheduler.step(); - } + var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - Assert.True(finalLoss < initialLoss); + LossIsClose(112.4992f, loss); } [Fact] public void TestTrainingLBFGSDefaults() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.LBFGS(seq.parameters(), learning_rate); @@ -1873,9 +1431,12 @@ public void TestTrainingLBFGSDefaults() [Fact] public void TestTrainingLBFGSNoClosure() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + double learning_rate = 0.00004f; var optimizer = torch.optim.LBFGS(seq.parameters(), learning_rate); Assert.Throws(() => optimizer.step()); @@ -1884,12 +1445,11 @@ public void TestTrainingLBFGSNoClosure() [Fact] public void TestTrainingLBFGS_ME() { - var lin1 = Linear(1000, 100); - var lin2 = Linear(100, 10); - var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); - var x = torch.randn(new long[] { 64, 1000 }); - var y = torch.randn(new long[] { 64, 10 }); + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); double learning_rate = 0.00004f; var optimizer = torch.optim.LBFGS(seq.parameters(), learning_rate, max_iter: 15, max_eval: 15); @@ -1940,25 +1500,8 @@ public void TestTrainingConv2d() var y = torch.randn(new long[] { 64, 10 }); var optimizer = torch.optim.Adam(seq.parameters()); - var loss = mse_loss(Reduction.Sum); - - float initialLoss = loss(seq.forward(x), y).ToSingle(); - float finalLoss = float.MaxValue; - - for (int i = 0; i < 10; i++) { - var eval = seq.forward(x); - var output = loss(eval, y); - var lossVal = output.ToSingle(); - - finalLoss = lossVal; - - optimizer.zero_grad(); - - output.backward(); - optimizer.step(); - } - Assert.True(finalLoss < initialLoss); + TrainLoop(seq, x, y, optimizer); } @@ -2012,28 +1555,5 @@ public void TestTrainingConv2dCUDA() Assert.Throws(() => torch.randn(new long[] { 64, 3, 28, 28 }).cuda()); } } - - [Fact(Skip = "MNIST data too big to keep in repo")] - public void TestMNISTLoader() - { - using (var train = Data.Loader.MNIST("../../../../test/data/MNIST", 32)) { - Assert.NotNull(train); - - var size = train.Size(); - int i = 0; - - foreach (var (data, target) in train) { - i++; - - Assert.Equal(data.shape, new long[] { 32, 1, 28, 28 }); - Assert.Equal(target.shape, new long[] { 32 }); - - data.Dispose(); - target.Dispose(); - } - - Assert.Equal(size, i * 32); - } - } } } From ada3ef3c6b8349c096c7fe95388bc6b81a6acd9b Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Fri, 18 Feb 2022 18:05:53 -0800 Subject: [PATCH 24/25] More unit tests. --- test/TorchSharpTest/TestTraining.cs | 207 ++++++++++++++++++++++++++-- 1 file changed, 196 insertions(+), 11 deletions(-) diff --git a/test/TorchSharpTest/TestTraining.cs b/test/TorchSharpTest/TestTraining.cs index d77ef6847..6491f54c0 100644 --- a/test/TorchSharpTest/TestTraining.cs +++ b/test/TorchSharpTest/TestTraining.cs @@ -346,7 +346,7 @@ public void TestTrainingAdamOneCycleLR() var loss = TrainLoop(seq, x, y, optimizer, scheduler, false); - LossIsClose(197.63f, loss); + LossIsClose(94.941f, loss); } /// @@ -819,6 +819,38 @@ public void TestTrainingRAdam() LossIsClose(66.2651f, loss); } + [Fact] + public void TestTrainingRAdamWD() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var optimizer = torch.optim.RAdam(seq.parameters(), 0.0005, weight_decay: 0.05); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(66.263f, loss); + } + + [Fact] + public void TestTrainingRAdamBetas() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var optimizer = torch.optim.RAdam(seq.parameters(), 0.0005, beta1: 0.9, beta2: 0.999); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(66.265f, loss); + } + [Fact] public void TestTrainingRAdamParamGroups() { @@ -910,9 +942,6 @@ public void TestTrainingRAdamCyclicLR_PG() LossIsClose(58.589f, loss); } - /// - /// Fully connected ReLU net with one hidden layer trained using ASGD optimizer. - /// [Fact] public void TestTrainingASGD() { @@ -929,6 +958,69 @@ public void TestTrainingASGD() LossIsClose(57.748f, loss); } + [Fact] + public void TestTrainingASGDLambda() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var optimizer = torch.optim.ASGD(seq.parameters(), lambd: 0.00025); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(57.748f, loss); + } + + [Fact] + public void TestTrainingASGDAlpha() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var optimizer = torch.optim.ASGD(seq.parameters(), alpha: 0.65); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(57.748f, loss); + } + + [Fact] + public void TestTrainingASGDWD() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var optimizer = torch.optim.ASGD(seq.parameters(), weight_decay: 0.25); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(57.748f, loss); + } + + [Fact] + public void TestTrainingASGDT0() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var optimizer = torch.optim.ASGD(seq.parameters(), t0: 100000); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(57.748f, loss); + } [Fact] public void TestTrainingASGDParamGroups() @@ -956,9 +1048,6 @@ public void TestTrainingASGDParamGroups() LossIsClose(56.653f, loss); } - /// - /// Fully connected ReLU net with one hidden layer trained using ASGD optimizer. - /// [Fact] public void TestTrainingRprop() { @@ -975,6 +1064,38 @@ public void TestTrainingRprop() LossIsClose(229.68f, loss); } + [Fact] + public void TestTrainingRpropEtam() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var optimizer = torch.optim.Rprop(seq.parameters(), etaminus: 0.55); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(201.417f, loss); + } + + [Fact] + public void TestTrainingRpropEtap() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + var optimizer = torch.optim.Rprop(seq.parameters(), etaplus: 1.25); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(221.365f, loss); + } + [Fact] public void TestTrainingRpropParamGroups() @@ -1056,6 +1177,23 @@ public void TestTrainingRMSAlpha() LossIsClose(156.339f, loss); } + [Fact] + public void TestTrainingRMSWD() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, weight_decay: 0.15); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(56.608f, loss); + } + [Fact] public void TestTrainingRMSCentered() { @@ -1073,6 +1211,40 @@ public void TestTrainingRMSCentered() LossIsClose(56.189f, loss); } + [Fact] + public void TestTrainingRMSCenteredWD() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, weight_decay: 0.15, centered: true); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(56.189f, loss); + } + + [Fact] + public void TestTrainingRMSMomentum() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.RMSProp(seq.parameters(), learning_rate, momentum: 0.15); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(53.50f, loss); + } + [Fact] public void TestTrainingRMSCenteredParamGroups() { @@ -1095,10 +1267,6 @@ public void TestTrainingRMSCenteredParamGroups() LossIsClose(53.6049f, loss); } - /// - /// Fully connected ReLU net with one hidden layer trained using SGD optimizer. - /// Taken from . - /// [Fact] public void TestTrainingSGDMomentum() { @@ -1116,6 +1284,23 @@ public void TestTrainingSGDMomentum() LossIsClose(53.711f, loss); } + [Fact] + public void TestTrainingSGDDampening() + { + var gen = new Generator(4711); + CreateLinearLayers(gen, out var lin1, out var lin2); + CreateDataAndLabels(gen, out var x, out var y); + + var seq = Sequential(("lin1", lin1), ("relu1", ReLU()), ("lin2", lin2)); + + double learning_rate = 0.00004f; + var optimizer = torch.optim.SGD(seq.parameters(), learning_rate, dampening: 0.5); + + var loss = TrainLoop(seq, x, y, optimizer); + + LossIsClose(62.494f, loss); + } + [Fact()]//(Skip = "Fails with an exception in native code.")] public void TestTrainingSGDNesterov() { From 9aabb84c7a9604ca86fc86b4a6c6a367fc212250 Mon Sep 17 00:00:00 2001 From: Niklas Gustafsson Date: Tue, 22 Feb 2022 12:10:08 -0800 Subject: [PATCH 25/25] Updated version number and release notes. --- RELEASENOTES.md | 15 +++++++++++---- build/BranchInfo.props | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 06143e745..3dfc122f2 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -2,7 +2,7 @@ Releases, starting with 9/2/2021, are listed with the most recent release at the top. -## NuGet Version 0.96.1 +## NuGet Version 0.96.2 __API Changes:__ @@ -16,13 +16,20 @@ __Fixed Bugs:__ #495 Add support for OptimizerParamGroup
#509 Tensor.conj() not implemented
-Using libtorch CPU packages from F# Interactive required explicit native loads - -#510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d
#515 what's reason for making register_module internal?
#516 AdamW bug on v0.96.0
#521 Can't set Tensor slice using indexing
+## NuGet Version 0.96.1 + +__API Changes:__ + +__Fixed Bugs:__ + +Using libtorch CPU packages from F# Interactive required explicit native loads + +#510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d
+ ## NuGet Version 0.96.0 __API Changes:__ diff --git a/build/BranchInfo.props b/build/BranchInfo.props index 20628973c..a18a9a875 100644 --- a/build/BranchInfo.props +++ b/build/BranchInfo.props @@ -2,7 +2,7 @@ 0 96 - 1 + 2