Optimizers support for parameter groups

DEVGUIDE.md

@@ -231,3 +231,7 @@ version of PyTorch then quite a lot of careful work needs to be done.
 
 10. Remember to delete all massive artifacts from Azure DevOps and reset this `BuildLibTorchPackages` in in [azure-pipelines.yml](azure-pipelines.yml)
 
+
+## Building with Visual Studio
+
+In order for builds to work properly using Visual Studio 2019 or 2022, you must start VS from the 'x64 Native Tools Command Prompt for VS 2022' (or 2019) in order for the full environment to be set up correctly. Starting VS from the desktop or taskbar will not work properly.

README.md

@@ -1,5 +1,7 @@
 [![Build Status](https://dotnet.visualstudio.com/TorchSharp/_apis/build/status/dotnet.TorchSharp?branchName=main)](https://dotnet.visualstudio.com/TorchSharp/_build/latest?definitionId=174&branchName=main)
 
+Please check the [Release Notes](RELEASENOTES.md) file for news on what's been updated in each new release.
+
 __TorchSharp is now in the .NET Foundation!__
 
 If you are using TorchSharp from NuGet, you should be using a version >= 0.95.1 of TorchSharp, and >= 1.10.0.1 of the libtorch-xxx redistributable packages. We recommend using one of the 'bundled' packages: TorchSharp-cpu, TorchSharp-cuda-windows, or TorchSharp-cuda-linux. They will pull in the right libtorch backends.

RELEASENOTES.md

@@ -4,9 +4,22 @@ Releases, starting with 9/2/2021, are listed with the most recent release at the
 
 ## NuGet Version 0.96.1
 
+__API Changes:__
+
+__NOTE__: This release contains breaking changes.<br/>
+
+The APIs to create optimizers all take 'parameters()' as well as 'named_parameters()' now.<br/>
+Support for parameter groups in most optimizers.<br/>
+Support for parameter groups in LR schedulers.<br/>
+
 __Fixed Bugs:__
 
+#495 Add support for OptimizerParamGroup<br/>
+#509 Tensor.conj() not implemented<br/>
 #510 Module.Load throws Mismatched state_dict sizes exception on BatchNorm1d<br/>
+#515 what's reason for making register_module internal?<br/>
+#516 AdamW bug on v0.96.0<br/>
+#521 Can't set Tensor slice using indexing<br/>
 
 ## NuGet Version 0.96.0
 
@@ -20,9 +33,9 @@ Lower-cased names: Module.Train --> Module.train and Module.Eval --> Module.eval
 
 __Fixed Bugs:__
 
-#500 BatchNorm1d throws exception during eval with batch size of 1<br/>
-#499 Setting Linear.weight is not reflected in 'parameters()'<br/>
 #496 Wrong output shape of torch.nn.Conv2d with 2d stride overload<br/>
+#499 Setting Linear.weight is not reflected in 'parameters()'<br/>
+#500 BatchNorm1d throws exception during eval with batch size of 1<br/>
 
 ## NuGet Version 0.95.4
 

build/BranchInfo.props

@@ -2,7 +2,7 @@
   <PropertyGroup>
     <MajorVersion>0</MajorVersion>
     <MinorVersion>96</MinorVersion>
-    <PatchVersion>0</PatchVersion>
+    <PatchVersion>1</PatchVersion>
   </PropertyGroup>
 
 </Project>

src/Examples/SequenceToSequence.cs

@@ -88,7 +88,8 @@ internal static void Main(string[] args)
                 var val_loss = evaluate(valid_data, model, loss, bptt, ntokens, optimizer);
                 sw.Stop();
 
-                Console.WriteLine($"\nEnd of epoch: {epoch} | lr: {optimizer.LearningRate:0.00} | time: {sw.Elapsed.TotalSeconds:0.0}s | loss: {val_loss:0.00}\n");
+                var pgFirst = optimizer.ParamGroups.First();
+                Console.WriteLine($"\nEnd of epoch: {epoch} | lr: {pgFirst.LearningRate:0.00} | time: {sw.Elapsed.TotalSeconds:0.0}s | loss: {val_loss:0.00}\n");
                 scheduler.step();
             }
 

src/Examples/TextClassification.cs

@@ -80,7 +80,9 @@ internal static void Main(string[] args)
 
                         sw.Stop();
 
-                        Console.WriteLine($"\nEnd of epoch: {epoch} | lr: {optimizer.LearningRate:0.0000} | time: {sw.Elapsed.TotalSeconds:0.0}s\n");
+                        var pgFirst = optimizer.ParamGroups.First();
+
+                        Console.WriteLine($"\nEnd of epoch: {epoch} | lr: {pgFirst.LearningRate:0.00} | time: {sw.Elapsed.TotalSeconds:0.0}s\n");
                         scheduler.step();
                     }
                 }

src/FSharp.Examples/SequenceToSequence.fs

@@ -8,6 +8,7 @@ open System.Diagnostics
 open System.Collections.Generic
 
 open TorchSharp
+open TorchSharp.Modules
 open type TorchSharp.torch.nn
 open type TorchSharp.torch.optim
 
@@ -244,7 +245,17 @@ let run epochs =
 
     use model = new TransformerModel(ntokens, device)
     let lr = 2.50
-    let optimizer = SGD(model.parameters(), lr)
+
+    let pgs = [|
+        SGD.ParamGroup(Parameters = model.parameters(), Options = SGD.Options(momentum = 1.0, dampening = 0.5));
+        SGD.ParamGroup(model.parameters(), momentum = 1.5, dampening = 0.1)
+    |]
+
+    let optimizer = SGD([|
+        SGD.ParamGroup(model.parameters(), momentum = 1.0, dampening = 0.5);
+        SGD.ParamGroup(model.parameters(), momentum = 1.5, dampening = 0.1)
+    |], lr)
+
     let scheduler = lr_scheduler.StepLR(optimizer, 1, 0.95, last_epoch=15)
 
     let totalTime = Stopwatch()
@@ -260,7 +271,8 @@ let run epochs =
         let val_loss = evaluate model valid_data ntokens
         sw.Stop()
 
-        let lrStr = optimizer.LearningRate.ToString("0.00")
+        let pgFirst = optimizer.ParamGroups.First()
+        let lrStr = pgFirst.LearningRate.ToString("0.00")
         let elapsed = sw.Elapsed.TotalSeconds.ToString("0.0")
         let lossStr = val_loss.ToString("0.00")
 

src/FSharp.Examples/TextClassification.fs

@@ -151,7 +151,8 @@ let run epochs =
 
         sw.Stop()
 
-        let lrStr = optimizer.LearningRate.ToString("0.0000")
+        let pgFirst = optimizer.ParamGroups.First()
+        let lrStr = pgFirst.LearningRate.ToString("0.0000")
         let tsStr = sw.Elapsed.TotalSeconds.ToString("0.0")
         printfn $"\nEnd of epoch: {epoch} | lr: {lrStr} | time: {tsStr}s\n"
         scheduler.step() |> ignore

src/Native/LibTorchSharp/THSTensor.cpp

@@ -1030,6 +1030,16 @@ int THSTensor_requires_grad(const Tensor tensor)
     CATCH_RETURN(int, 0, tensor->requires_grad());
 }
 
+void THSTensor_retain_grad(const Tensor tensor)
+{
+    CATCH(tensor->retain_grad(););
+}
+
+int64_t THSTensor_is_leaf(const Tensor tensor)
+{
+    CATCH_RETURN(int64_t, 0, tensor->is_leaf(););
+}
+
 Tensor THSTensor_reshape(const Tensor tensor, const int64_t* shape, const int length)
 {
     CATCH_TENSOR(tensor->reshape(at::ArrayRef<int64_t>(shape, length)));

src/Native/LibTorchSharp/THSTensor.h

@@ -586,6 +586,8 @@ EXPORT_API(Tensor) THSTensor_inverse(const Tensor tensor);
 
 EXPORT_API(int) THSTensor_is_contiguous(const Tensor input);
 
+EXPORT_API(int64_t) THSTensor_is_leaf(const Tensor tensor);
+
 EXPORT_API(int) THSTensor_is_sparse(const Tensor tensor);
 
 EXPORT_API(Tensor) THSTensor_isclose(const Tensor tensor, const Tensor other, const double rtol, const double atol, const bool equal_nan);
@@ -988,6 +990,8 @@ EXPORT_API(Tensor) THSTensor_remainder_scalar(const Tensor left, const Scalar ri
 
 EXPORT_API(Tensor) THSTensor_remainder_scalar_(const Tensor left, const Scalar right);
 
+EXPORT_API(void) THSTensor_retain_grad(const Tensor tensor);
+
 EXPORT_API(Tensor) THSTensor_rsqrt(const Tensor tensor);
 
 EXPORT_API(Tensor) THSTensor_rsqrt_(const Tensor tensor);