From 6306a651a50b1ef3f78eeda956e0f9a75649e847 Mon Sep 17 00:00:00 2001 From: Nikolai Avteniev Date: Fri, 2 Jul 2021 12:25:41 -0400 Subject: [PATCH] Support .Net C# projects using dotnet cli and nuget.org central repository. (#62) * Start to develop dotnet * Lookup specfile from project directory * Added support for list command * Adding info capabilities * "additional package details" * removed debug test, added support for list all * implemented addPackages * implemented search * Implemented Remove, Install, Lock, and set Quirks * Refactored code for testing, added tests * added documentation * Split functions into seperate files * Split tests into seperate files, added more tests. * added api doc link * Handle project references in lock file * fixed typo in error message * Using a struct to parse the lock file data. * Changed language to be dotnet * Use query escape The query string comes directly from the command line and might contain characters that are not valid in query parameters. Co-authored-by: lhchavez * Include actual error in output Co-authored-by: lhchavez * Use url.PathEscape with packageName PackageNames might contain characters that are not valid in URL path. Co-authored-by: lhchavez * Avoid time of check time of use race condition There is no guarantee that after you check for files existence that it will still be there when the code attempts to open it. This way the check and open operation are done in a single call. Co-authored-by: lhchavez * Use appropriate documentation format Co-authored-by: lhchavez * Include actual error in output Co-authored-by: lhchavez * Included err in message and used pathescape * Fix docstrings and bypass TOCTOU race condition. * print unescaped latest version * Include error in message Co-authored-by: lhchavez --- internal/backends/backends.go | 2 + internal/backends/dotnet/dotnet.go | 31 ++++ internal/backends/dotnet/dotnet_cli.go | 35 ++++ internal/backends/dotnet/dotnet_cli_test.go | 97 +++++++++++ internal/backends/dotnet/nuget.go | 152 ++++++++++++++++++ internal/backends/dotnet/nuget_test.go | 33 ++++ internal/backends/dotnet/project_files.go | 148 +++++++++++++++++ .../backends/dotnet/project_files_test.go | 137 ++++++++++++++++ 8 files changed, 635 insertions(+) create mode 100644 internal/backends/dotnet/dotnet.go create mode 100644 internal/backends/dotnet/dotnet_cli.go create mode 100644 internal/backends/dotnet/dotnet_cli_test.go create mode 100644 internal/backends/dotnet/nuget.go create mode 100644 internal/backends/dotnet/nuget_test.go create mode 100644 internal/backends/dotnet/project_files.go create mode 100644 internal/backends/dotnet/project_files_test.go diff --git a/internal/backends/backends.go b/internal/backends/backends.go index 33304e36..71dae7c7 100644 --- a/internal/backends/backends.go +++ b/internal/backends/backends.go @@ -7,6 +7,7 @@ import ( "github.com/replit/upm/internal/api" "github.com/replit/upm/internal/backends/dart" + "github.com/replit/upm/internal/backends/dotnet" "github.com/replit/upm/internal/backends/elisp" "github.com/replit/upm/internal/backends/java" "github.com/replit/upm/internal/backends/nodejs" @@ -31,6 +32,7 @@ var languageBackends = []api.LanguageBackend{ dart.DartPubBackend, java.JavaBackend, rlang.RlangBackend, + dotnet.DotNetBackend, } // matchesLanguage checks if a language backend matches a value for diff --git a/internal/backends/dotnet/dotnet.go b/internal/backends/dotnet/dotnet.go new file mode 100644 index 00000000..b0fddc13 --- /dev/null +++ b/internal/backends/dotnet/dotnet.go @@ -0,0 +1,31 @@ +// Package dotnet provides a backend for c# using dotnet and nuget.org +package dotnet + +import ( + "github.com/replit/upm/internal/api" + "github.com/replit/upm/internal/util" +) + +// DotNetBackend is the UPM language backend .NET languages with support for C# +var DotNetBackend = api.LanguageBackend{ + Name: "dotnet", + Specfile: findSpecFile(), + Lockfile: lockFileName, + FilenamePatterns: []string{"*.cs", "*.csproj"}, // C# support so far + Remove: func(pkgs map[api.PkgName]bool) { removePackages(pkgs, findSpecFile(), util.RunCmd) }, + Add: func(pkgs map[api.PkgName]api.PkgSpec, projectName string) { + addPackages(pkgs, projectName, util.RunCmd) + }, + Search: search, + Info: info, + Install: func() { install(util.RunCmd) }, + Lock: func() { lock(util.RunCmd) }, + ListSpecfile: listSpecfile, + ListLockfile: listLockfile, + GetPackageDir: func() string { + return "bin/" + }, + Quirks: api.QuirksAddRemoveAlsoLocks | + api.QuirksAddRemoveAlsoInstalls | + api.QuirksLockAlsoInstalls, +} diff --git a/internal/backends/dotnet/dotnet_cli.go b/internal/backends/dotnet/dotnet_cli.go new file mode 100644 index 00000000..5f2b63cf --- /dev/null +++ b/internal/backends/dotnet/dotnet_cli.go @@ -0,0 +1,35 @@ +package dotnet + +import ( + "github.com/replit/upm/internal/api" +) + +// removes packages using dotnet command and updates lock file +func removePackages(pkgs map[api.PkgName]bool, specFileName string, cmdRunner func([]string)) { + for packageName := range pkgs { + command := []string{"dotnet", "remove", specFileName, "package", string(packageName)} + cmdRunner(command) + } + lock(cmdRunner) +} + +// adds packages using dotnet command which automatically updates lock files +func addPackages(pkgs map[api.PkgName]api.PkgSpec, projectName string, cmdRunner func([]string)) { + for packageName, spec := range pkgs { + command := []string{"dotnet", "add", "package", string(packageName)} + if string(spec) != "" { + command = append(command, "--version", string(spec)) + } + cmdRunner(command) + } +} + +// installs all packages using dotnet command +func install(cmdRunner func([]string)) { + cmdRunner([]string{"dotnet", "restore"}) +} + +// generates or updates the lock file using dotnet command +func lock(cmdRunner func([]string)) { + cmdRunner([]string{"dotnet", "restore", "--use-lock-file"}) +} diff --git a/internal/backends/dotnet/dotnet_cli_test.go b/internal/backends/dotnet/dotnet_cli_test.go new file mode 100644 index 00000000..4a041855 --- /dev/null +++ b/internal/backends/dotnet/dotnet_cli_test.go @@ -0,0 +1,97 @@ +package dotnet + +import ( + "strings" + "testing" + + "github.com/replit/upm/internal/api" +) + +func TestAddPackages(t *testing.T) { + cmds := []string{} + cmdRunner := func(cmd []string) { + cmds = append(cmds, strings.Join(cmd, " ")) + } + + addPackages(map[api.PkgName]api.PkgSpec{"package": "1.0"}, "", cmdRunner) + + if len(cmds) != 1 { + t.Errorf("Expected one command but got %q", len(cmds)) + } + + if cmds[0] != "dotnet add package package --version 1.0" { + t.Errorf("Wrong command executed %s", cmds[0]) + } +} + +func TestAddPackagesWithoutVersion(t *testing.T) { + cmds := []string{} + cmdRunner := func(cmd []string) { + cmds = append(cmds, strings.Join(cmd, " ")) + } + + addPackages(map[api.PkgName]api.PkgSpec{"package": ""}, "", cmdRunner) + + if len(cmds) != 1 { + t.Errorf("Expected one command but got %q", len(cmds)) + } + + if cmds[0] != "dotnet add package package" { + t.Errorf("Wrong command executed %s", cmds[0]) + } +} + +func TestRemovePackages(t *testing.T) { + cmds := []string{} + cmdRunner := func(cmd []string) { + cmds = append(cmds, strings.Join(cmd, " ")) + } + + removePackages(map[api.PkgName]bool{"package": true}, "specFile.csproj", cmdRunner) + + if len(cmds) != 2 { + t.Errorf("Expected two command but got %q", len(cmds)) + } + + if cmds[0] != "dotnet remove specFile.csproj package package" { + t.Errorf("Wrong remove command executed %s", cmds[0]) + } + + if cmds[1] != "dotnet restore --use-lock-file" { + t.Errorf("Wrong lock command executed %s", cmds[1]) + } +} + +func TestLock(t *testing.T) { + cmds := []string{} + cmdRunner := func(cmd []string) { + cmds = append(cmds, strings.Join(cmd, " ")) + } + + lock(cmdRunner) + + if len(cmds) != 1 { + t.Errorf("Expected one command but got %q", len(cmds)) + } + + if cmds[0] != "dotnet restore --use-lock-file" { + t.Errorf("Wrong command executed %s", cmds[0]) + } +} + +func TestInstall(t *testing.T) { + cmds := []string{} + cmdRunner := func(cmd []string) { + cmds = append(cmds, strings.Join(cmd, " ")) + } + + install(cmdRunner) + + if len(cmds) != 1 { + t.Errorf("Expected one command but got %q", len(cmds)) + } + + if cmds[0] != "dotnet restore" { + t.Errorf("Wrong command executed %s", cmds[0]) + } +} diff --git a/internal/backends/dotnet/nuget.go b/internal/backends/dotnet/nuget.go new file mode 100644 index 00000000..aab0cfc2 --- /dev/null +++ b/internal/backends/dotnet/nuget.go @@ -0,0 +1,152 @@ +package dotnet + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + + "github.com/replit/upm/internal/api" + "github.com/replit/upm/internal/util" +) + +// Used the documentation provided here https://docs.microsoft.com/en-us/nuget/api/overview + +// nuget.org info lookup result +type infoResult struct { + Versions []string `json:"versions"` +} + +// nuget.org .nuspec file package repository data +type repository struct { + XMLName xml.Name `xml:"repository"` + Type string `xml:"type,attr"` + URL string `xml:"url,attr"` + Commit string `xml:"commit,attr"` +} + +// nuget.org .nuspec file package metadata +type packageMetadata struct { + XMLName xml.Name `xml:"metadata"` + ID string `xml:"id"` + Version string `xml:"version"` + Title string `xml:"title"` + Author string `xml:"author"` + Description string `xml:"description"` + License string `xml:"license"` + Repository repository `xml:"repository"` + ProjectURL string `xml:"projectUrl"` +} + +// nuget.org .nuspec file data +type nugetPackage struct { + XMLName xml.Name `xml:"package"` + Metadata packageMetadata `xml:"metadata"` +} + +// nuget.org search service result entry +type searchResultData struct { + ID string + Version string + Description string + ProjectURL string +} + +// nuget.org search service result record +type searchResult struct { + TotalHits int + Data []searchResultData +} + +const searchQueryURL = "https://azuresearch-usnc.nuget.org/query" + +// find the first ten projects that match the query string on nuget.org +func search(query string) []api.PkgInfo { + pkgs := []api.PkgInfo{} + queryURL := fmt.Sprintf("%s?q=%s&take=10", searchQueryURL, url.QueryEscape(query)) + + res, err := http.Get(queryURL) + if err != nil { + util.Die("failed to query for packages: %s", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return pkgs + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + util.Die("Could not read response: %s", err) + } + + var searchResult searchResult + err = json.Unmarshal(body, &searchResult) + if err != nil { + util.Die("Could not unmarshal response data: %s", err) + } + + for _, data := range searchResult.Data { + pkgs = append(pkgs, api.PkgInfo{ + Name: data.ID, + Version: data.Version, + Description: data.Description, + SourceCodeURL: data.ProjectURL, + }) + } + + return pkgs +} + +// looks up all the versions of the package and gets retails for the latest version from nuget.org +func info(pkgName api.PkgName) api.PkgInfo { + lowID := url.PathEscape(strings.ToLower(string(pkgName))) + infoURL := fmt.Sprintf("https://api.nuget.org/v3-flatcontainer/%s/index.json", lowID) + + res, err := http.Get(infoURL) + if err != nil { + util.Die("failed to get the versions: %s", err) + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + util.Die("could not read response: %s", err) + } + var infoResult infoResult + err = json.Unmarshal(body, &infoResult) + if err != nil { + util.Die("could not read json body: %s", err) + } + latestVersion := infoResult.Versions[len(infoResult.Versions)-1] + util.ProgressMsg(fmt.Sprintf("latest version of %s is %s", pkgName, latestVersion)) + specURL := fmt.Sprintf("https://api.nuget.org/v3-flatcontainer/%s/%s/%s.nuspec", lowID, url.PathEscape(latestVersion), lowID) + util.ProgressMsg(fmt.Sprintf("Getting spec from %s", specURL)) + res, err = http.Get(specURL) + if err != nil { + util.Die("failed to get the spec: %s", err) + } + defer res.Body.Close() + body, err = ioutil.ReadAll(res.Body) + if err != nil { + util.Die("could not read response: %s", err) + } + var nugetPackage nugetPackage + err = xml.Unmarshal(body, &nugetPackage) + if err != nil { + util.Die(fmt.Sprintf("failed to read spec %s", err)) + } + + pkgInfo := api.PkgInfo{ + Name: nugetPackage.Metadata.ID, + Version: nugetPackage.Metadata.Version, + Description: nugetPackage.Metadata.Description, + Author: nugetPackage.Metadata.Author, + License: nugetPackage.Metadata.License, + SourceCodeURL: nugetPackage.Metadata.Repository.URL, + HomepageURL: nugetPackage.Metadata.ProjectURL, + } + return pkgInfo +} diff --git a/internal/backends/dotnet/nuget_test.go b/internal/backends/dotnet/nuget_test.go new file mode 100644 index 00000000..e15059bd --- /dev/null +++ b/internal/backends/dotnet/nuget_test.go @@ -0,0 +1,33 @@ +package dotnet + +import ( + "testing" +) + +func TestSearchNuget(t *testing.T) { + pkgs := search("Microsoft.Extensions.Logging") + + if len(pkgs) < 1 { + t.Error("No results found for Micorosft.Extensions.Logging") + } + + for _, pkg := range pkgs { + if pkg.Name == "" { + t.Errorf("pkg %q has no name", pkg) + } + if pkg.Version == "" { + t.Errorf("pkg %q has no version", pkg) + } + } +} + +func TestInfoFromNuget(t *testing.T) { + pkg := info("Microsoft.Extensions.Logging") + + if pkg.Name == "" { + t.Errorf("pkg %q has no name", pkg) + } + if pkg.Version == "" { + t.Errorf("pkg %q has no version", pkg) + } +} diff --git a/internal/backends/dotnet/project_files.go b/internal/backends/dotnet/project_files.go new file mode 100644 index 00000000..1a0b5979 --- /dev/null +++ b/internal/backends/dotnet/project_files.go @@ -0,0 +1,148 @@ +package dotnet + +import ( + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + + "github.com/replit/upm/internal/api" + "github.com/replit/upm/internal/util" +) + +const lockFileName = "packages.lock.json" + +// an individual package record from .csproj file +type packageReference struct { + XMLName xml.Name `xml:"PackageReference"` + Include string `xml:"Include,attr"` + Version string `xml:"Version,attr"` +} + +// .csproj file structure +type project struct { + XMLName xml.Name `xml:"Project"` + Packages []packageReference `xml:"ItemGroup>PackageReference"` +} + +// looks for the .csproj file in the current directory +func findSpecFile() string { + files, err := ioutil.ReadDir("./") + if err != nil { + util.Die("can't read current directory: %s", err) + } + + for _, f := range files { + if strings.HasSuffix(f.Name(), ".csproj") { + return f.Name() + } + } + + return ".csproj" +} + +// loads the details of the project spec file +func listSpecfile() map[api.PkgName]api.PkgSpec { + var pkgs map[api.PkgName]api.PkgSpec + projectFile := findSpecFile() + specReader, err := os.Open(projectFile) + if errors.Is(err, os.ErrNotExist) { + return pkgs + } + if err != nil { + util.Die("Could not open %s, with error: %q", projectFile, err) + } + defer specReader.Close() + + pkgs, err = ReadSpec(specReader) + if err != nil { + util.Die("Failed to read spec file %s, with error: %q", projectFile, err) + } + + return pkgs +} + +// ReadSpec reads the spec and builds up packages. +func ReadSpec(specReader io.Reader) (map[api.PkgName]api.PkgSpec, error) { + xmlbytes, err := ioutil.ReadAll(specReader) + if err != nil { + return nil, err + } + + var project project + err = xml.Unmarshal(xmlbytes, &project) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal file content %q", err) + } + + pkgs := map[api.PkgName]api.PkgSpec{} + for _, packageReference := range project.Packages { + pkgName := packageReference.Include + pkgVersion := api.PkgVersion(packageReference.Version) + pkgs[api.PkgName(pkgName)] = api.PkgSpec(pkgVersion) + } + return pkgs, nil +} + +// loads the details of the lock file +func listLockfile() map[api.PkgName]api.PkgVersion { + pkgs := map[api.PkgName]api.PkgVersion{} + + specReader, err := os.Open(lockFileName) + if errors.Is(err, os.ErrNotExist) { + return pkgs + } + if err != nil { + util.Die("Could not open %s, with error: %q", lockFileName, err) + } + defer specReader.Close() + + pkgs, err = ReadLock(specReader) + if err != nil { + util.Die("error reading lockFile %s: %q", lockFileName, err) + } + + return pkgs +} + +type lockFilePackage struct { + Type string + Resolved string + ContentHash string + Dependencies map[string]string +} + +type lockFile struct { + Version int + Dependencies map[string]map[string]lockFilePackage +} + +// ReadLock reads the lock file and buils up packages. +func ReadLock(lockFileReader io.Reader) (map[api.PkgName]api.PkgVersion, error) { + jsonBytes, err := ioutil.ReadAll(lockFileReader) + if err != nil { + return nil, err + } + var lockFile lockFile + err = json.Unmarshal(jsonBytes, &lockFile) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal lock file data: %q", err) + } + + dependencies := lockFile.Dependencies + pkgs := map[api.PkgName]api.PkgVersion{} + for _, v := range dependencies { + for packageName, packageDetails := range v { + version := packageDetails.Resolved + if version != "" { + pkgs[api.PkgName(packageName)] = api.PkgVersion(packageDetails.Resolved) + } + } + } + + return pkgs, nil +} diff --git a/internal/backends/dotnet/project_files_test.go b/internal/backends/dotnet/project_files_test.go new file mode 100644 index 00000000..53b3618d --- /dev/null +++ b/internal/backends/dotnet/project_files_test.go @@ -0,0 +1,137 @@ +package dotnet + +import ( + "strings" + "testing" + + "github.com/replit/upm/internal/api" +) + +func TestReadSpec(t *testing.T) { + spec := strings.NewReader( + ` + + Exe + net5.0 + + + + + + + `) + + pkgs, err := ReadSpec(spec) + if err != nil { + t.Errorf("Failed to read spec with error %q", err) + } + + for pkg, version := range map[string]string{"Microsoft.Extensions.Logging": "5.0.0", "Newtonsoft.Json": "13.0.1", "SharpYaml": "1.6.6"} { + if pkgs[api.PkgName(pkg)] != api.PkgSpec(version) { + t.Errorf("Wrong version %s for %s", pkgs[api.PkgName(pkg)], pkg) + } + } +} + +func TestReadSpecWithoutDependencies(t *testing.T) { + spec := strings.NewReader( + ` + + Exe + net5.0 + + `) + + pkgs, err := ReadSpec(spec) + if err != nil { + t.Errorf("Failed to read spec with error %q", err) + } + + if len(pkgs) != 0 { + t.Errorf("Unexpected data in spec file %q", pkgs) + } +} + +func TestReadLock(t *testing.T) { + lock := strings.NewReader( + `{ + "version": 1, + "dependencies": { + ".NETCoreApp,Version=v5.0": { + "Microsoft.Extensions.Logging": { + "type": "Direct", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "MgOwK6tPzB6YNH21wssJcw/2MKwee8b2gI7SllYfn6rvTpIrVvVS5HAjSU2vqSku1fwqRvWP0MdIi14qjd93Aw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "5.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "5.0.0", + "Microsoft.Extensions.Logging.Abstractions": "5.0.0", + "Microsoft.Extensions.Options": "5.0.0" + } + }, + "Newtonsoft.Json": { + "type": "Direct", + "requested": "[13.0.1, )", + "resolved": "13.0.1", + "contentHash": "ppPFpBcvxdsfUonNcvITKqLl3bqxWbDCZIzDWHzjpdAHRFfZe0Dw9HmA0+za13IdyrgJwpkDTDA9fHaxOrt20A==" + } + } + } + }`) + + pkgs, err := ReadLock(lock) + + if err != nil { + t.Errorf("Failed to read lock with error %q", err) + } + + for pkg, version := range map[string]string{"Microsoft.Extensions.Logging": "5.0.0", "Newtonsoft.Json": "13.0.1"} { + if pkgs[api.PkgName(pkg)] != api.PkgVersion(version) { + t.Errorf("Wring version %s for %s", pkgs[api.PkgName(pkg)], pkg) + } + } +} + +func TestReadLockWithProjectDependency(t *testing.T) { + lock := strings.NewReader( + `{ + "version": 1, + "dependencies": { + ".NETStandard,Version=v2.0": { + "System.Text.Json": { + "type": "Transitive", + "resolved": "5.0.1", + "contentHash": "/UM3UK1dXKl8Ybysg/21gM4S8DJgkR+yLU8JwqCVbuNqQNImelntgYFAN5QxR8sJJ1kMx//hOUdf0lltosi8cQ==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "5.0.0", + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4", + "System.Numerics.Vectors": "4.5.0", + "System.Runtime.CompilerServices.Unsafe": "5.0.0", + "System.Text.Encodings.Web": "5.0.0", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, + "xunit.v3.common": { + "type": "Project", + "dependencies": { + "System.Text.Json": "5.0.1" + } + } + } + } + }`) + + pkgs, err := ReadLock(lock) + + if err != nil { + t.Errorf("Failed to read lock with error %q", err) + } + + for pkg, version := range map[string]string{"System.Text.Json": "5.0.1"} { + if pkgs[api.PkgName(pkg)] != api.PkgVersion(version) { + t.Errorf("Wrong version %s for %s", pkgs[api.PkgName(pkg)], pkg) + } + } +}