Support .Net C# projects using dotnet cli and nuget.org central repos…

…itory. (#62)

* Start to develop dotnet

* Lookup specfile from project directory

* Added support for list command

* Adding info capabilities

* "additional package details"

* removed debug test, added support for list all

* implemented addPackages

* implemented search

* Implemented Remove, Install, Lock, and set Quirks

* Refactored code for testing, added tests

* added documentation

* Split functions into seperate files

* Split tests into seperate files, added more tests.

* added api doc link

* Handle project references in lock file

* fixed typo in error message

* Using a struct to parse the lock file data.

* Changed language to be dotnet

* Use query escape

The query string comes directly from the command line and might contain characters that are not valid in query parameters.

Co-authored-by: lhchavez <[email protected]>

* Include actual error in output

Co-authored-by: lhchavez <[email protected]>

* Use url.PathEscape with packageName

PackageNames might contain characters that are not valid in URL path.

Co-authored-by: lhchavez <[email protected]>

* Avoid time of check time of use race condition

There is no guarantee that after you check for files existence that it will still be there when the code attempts to open it.  This way the check and open operation are done in a single call.

Co-authored-by: lhchavez <[email protected]>

* Use appropriate documentation format

Co-authored-by: lhchavez <[email protected]>

* Include actual error in output

Co-authored-by: lhchavez <[email protected]>

* Included err in message and used pathescape

* Fix docstrings and bypass TOCTOU race condition.

* print unescaped latest version

* Include error in message

Co-authored-by: lhchavez <[email protected]>

internal/backends/backends.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/replit/upm/internal/api"
 	"github.com/replit/upm/internal/backends/dart"
+	"github.com/replit/upm/internal/backends/dotnet"
 	"github.com/replit/upm/internal/backends/elisp"
 	"github.com/replit/upm/internal/backends/java"
 	"github.com/replit/upm/internal/backends/nodejs"
@@ -31,6 +32,7 @@ var languageBackends = []api.LanguageBackend{
 	dart.DartPubBackend,
 	java.JavaBackend,
 	rlang.RlangBackend,
+	dotnet.DotNetBackend,
 }
 
 // matchesLanguage checks if a language backend matches a value for

internal/backends/dotnet/dotnet.go

@@ -0,0 +1,31 @@
+// Package dotnet provides a backend for c# using dotnet and nuget.org
+package dotnet
+
+import (
+	"github.com/replit/upm/internal/api"
+	"github.com/replit/upm/internal/util"
+)
+
+// DotNetBackend is the UPM language backend .NET languages with support for C#
+var DotNetBackend = api.LanguageBackend{
+	Name:             "dotnet",
+	Specfile:         findSpecFile(),
+	Lockfile:         lockFileName,
+	FilenamePatterns: []string{"*.cs", "*.csproj"}, // C# support so far
+	Remove:           func(pkgs map[api.PkgName]bool) { removePackages(pkgs, findSpecFile(), util.RunCmd) },
+	Add: func(pkgs map[api.PkgName]api.PkgSpec, projectName string) {
+		addPackages(pkgs, projectName, util.RunCmd)
+	},
+	Search:       search,
+	Info:         info,
+	Install:      func() { install(util.RunCmd) },
+	Lock:         func() { lock(util.RunCmd) },
+	ListSpecfile: listSpecfile,
+	ListLockfile: listLockfile,
+	GetPackageDir: func() string {
+		return "bin/"
+	},
+	Quirks: api.QuirksAddRemoveAlsoLocks |
+		api.QuirksAddRemoveAlsoInstalls |
+		api.QuirksLockAlsoInstalls,
+}

internal/backends/dotnet/dotnet_cli.go

@@ -0,0 +1,35 @@
+package dotnet
+
+import (
+	"github.com/replit/upm/internal/api"
+)
+
+// removes packages using dotnet command and updates lock file
+func removePackages(pkgs map[api.PkgName]bool, specFileName string, cmdRunner func([]string)) {
+	for packageName := range pkgs {
+		command := []string{"dotnet", "remove", specFileName, "package", string(packageName)}
+		cmdRunner(command)
+	}
+	lock(cmdRunner)
+}
+
+// adds packages using dotnet command which automatically updates lock files
+func addPackages(pkgs map[api.PkgName]api.PkgSpec, projectName string, cmdRunner func([]string)) {
+	for packageName, spec := range pkgs {
+		command := []string{"dotnet", "add", "package", string(packageName)}
+		if string(spec) != "" {
+			command = append(command, "--version", string(spec))
+		}
+		cmdRunner(command)
+	}
+}
+
+// installs all packages using dotnet command
+func install(cmdRunner func([]string)) {
+	cmdRunner([]string{"dotnet", "restore"})
+}
+
+// generates or updates the lock file using dotnet command
+func lock(cmdRunner func([]string)) {
+	cmdRunner([]string{"dotnet", "restore", "--use-lock-file"})
+}

internal/backends/dotnet/dotnet_cli_test.go

@@ -0,0 +1,97 @@
+package dotnet
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/replit/upm/internal/api"
+)
+
+func TestAddPackages(t *testing.T) {
+	cmds := []string{}
+	cmdRunner := func(cmd []string) {
+		cmds = append(cmds, strings.Join(cmd, " "))
+	}
+
+	addPackages(map[api.PkgName]api.PkgSpec{"package": "1.0"}, "", cmdRunner)
+
+	if len(cmds) != 1 {
+		t.Errorf("Expected one command but got %q", len(cmds))
+	}
+
+	if cmds[0] != "dotnet add package package --version 1.0" {
+		t.Errorf("Wrong command executed %s", cmds[0])
+	}
+}
+
+func TestAddPackagesWithoutVersion(t *testing.T) {
+	cmds := []string{}
+	cmdRunner := func(cmd []string) {
+		cmds = append(cmds, strings.Join(cmd, " "))
+	}
+
+	addPackages(map[api.PkgName]api.PkgSpec{"package": ""}, "", cmdRunner)
+
+	if len(cmds) != 1 {
+		t.Errorf("Expected one command but got %q", len(cmds))
+	}
+
+	if cmds[0] != "dotnet add package package" {
+		t.Errorf("Wrong command executed %s", cmds[0])
+	}
+}
+
+func TestRemovePackages(t *testing.T) {
+	cmds := []string{}
+	cmdRunner := func(cmd []string) {
+		cmds = append(cmds, strings.Join(cmd, " "))
+	}
+
+	removePackages(map[api.PkgName]bool{"package": true}, "specFile.csproj", cmdRunner)
+
+	if len(cmds) != 2 {
+		t.Errorf("Expected two command but got %q", len(cmds))
+	}
+
+	if cmds[0] != "dotnet remove specFile.csproj package package" {
+		t.Errorf("Wrong remove command executed %s", cmds[0])
+	}
+
+	if cmds[1] != "dotnet restore --use-lock-file" {
+		t.Errorf("Wrong lock command executed %s", cmds[1])
+	}
+}
+
+func TestLock(t *testing.T) {
+	cmds := []string{}
+	cmdRunner := func(cmd []string) {
+		cmds = append(cmds, strings.Join(cmd, " "))
+	}
+
+	lock(cmdRunner)
+
+	if len(cmds) != 1 {
+		t.Errorf("Expected one command but got %q", len(cmds))
+	}
+
+	if cmds[0] != "dotnet restore --use-lock-file" {
+		t.Errorf("Wrong command executed %s", cmds[0])
+	}
+}
+
+func TestInstall(t *testing.T) {
+	cmds := []string{}
+	cmdRunner := func(cmd []string) {
+		cmds = append(cmds, strings.Join(cmd, " "))
+	}
+
+	install(cmdRunner)
+
+	if len(cmds) != 1 {
+		t.Errorf("Expected one command but got %q", len(cmds))
+	}
+
+	if cmds[0] != "dotnet restore" {
+		t.Errorf("Wrong command executed %s", cmds[0])
+	}
+}

internal/backends/dotnet/nuget.go

@@ -0,0 +1,152 @@
+package dotnet
+
+import (
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/replit/upm/internal/api"
+	"github.com/replit/upm/internal/util"
+)
+
+// Used the documentation provided here https://docs.microsoft.com/en-us/nuget/api/overview
+
+// nuget.org info lookup result
+type infoResult struct {
+	Versions []string `json:"versions"`
+}
+
+// nuget.org .nuspec file package repository data
+type repository struct {
+	XMLName xml.Name `xml:"repository"`
+	Type    string   `xml:"type,attr"`
+	URL     string   `xml:"url,attr"`
+	Commit  string   `xml:"commit,attr"`
+}
+
+// nuget.org .nuspec file package metadata
+type packageMetadata struct {
+	XMLName     xml.Name   `xml:"metadata"`
+	ID          string     `xml:"id"`
+	Version     string     `xml:"version"`
+	Title       string     `xml:"title"`
+	Author      string     `xml:"author"`
+	Description string     `xml:"description"`
+	License     string     `xml:"license"`
+	Repository  repository `xml:"repository"`
+	ProjectURL  string     `xml:"projectUrl"`
+}
+
+// nuget.org .nuspec file data
+type nugetPackage struct {
+	XMLName  xml.Name        `xml:"package"`
+	Metadata packageMetadata `xml:"metadata"`
+}
+
+// nuget.org search service result entry
+type searchResultData struct {
+	ID          string
+	Version     string
+	Description string
+	ProjectURL  string
+}
+
+// nuget.org search service result record
+type searchResult struct {
+	TotalHits int
+	Data      []searchResultData
+}
+
+const searchQueryURL = "https://azuresearch-usnc.nuget.org/query"
+
+// find the first ten projects that match the query string on nuget.org
+func search(query string) []api.PkgInfo {
+	pkgs := []api.PkgInfo{}
+	queryURL := fmt.Sprintf("%s?q=%s&take=10", searchQueryURL, url.QueryEscape(query))
+
+	res, err := http.Get(queryURL)
+	if err != nil {
+		util.Die("failed to query for packages: %s", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return pkgs
+	}
+
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		util.Die("Could not read response: %s", err)
+	}
+
+	var searchResult searchResult
+	err = json.Unmarshal(body, &searchResult)
+	if err != nil {
+		util.Die("Could not unmarshal response data: %s", err)
+	}
+
+	for _, data := range searchResult.Data {
+		pkgs = append(pkgs, api.PkgInfo{
+			Name:          data.ID,
+			Version:       data.Version,
+			Description:   data.Description,
+			SourceCodeURL: data.ProjectURL,
+		})
+	}
+
+	return pkgs
+}
+
+// looks up all the versions of the package and gets retails for the latest version from nuget.org
+func info(pkgName api.PkgName) api.PkgInfo {
+	lowID := url.PathEscape(strings.ToLower(string(pkgName)))
+	infoURL := fmt.Sprintf("https://api.nuget.org/v3-flatcontainer/%s/index.json", lowID)
+
+	res, err := http.Get(infoURL)
+	if err != nil {
+		util.Die("failed to get the versions: %s", err)
+	}
+	defer res.Body.Close()
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		util.Die("could not read response: %s", err)
+	}
+	var infoResult infoResult
+	err = json.Unmarshal(body, &infoResult)
+	if err != nil {
+		util.Die("could not read json body: %s", err)
+	}
+	latestVersion := infoResult.Versions[len(infoResult.Versions)-1]
+	util.ProgressMsg(fmt.Sprintf("latest version of %s is %s", pkgName, latestVersion))
+	specURL := fmt.Sprintf("https://api.nuget.org/v3-flatcontainer/%s/%s/%s.nuspec", lowID, url.PathEscape(latestVersion), lowID)
+	util.ProgressMsg(fmt.Sprintf("Getting spec from %s", specURL))
+	res, err = http.Get(specURL)
+	if err != nil {
+		util.Die("failed to get the spec: %s", err)
+	}
+	defer res.Body.Close()
+	body, err = ioutil.ReadAll(res.Body)
+	if err != nil {
+		util.Die("could not read response: %s", err)
+	}
+	var nugetPackage nugetPackage
+	err = xml.Unmarshal(body, &nugetPackage)
+	if err != nil {
+		util.Die(fmt.Sprintf("failed to read spec %s", err))
+	}
+
+	pkgInfo := api.PkgInfo{
+		Name:          nugetPackage.Metadata.ID,
+		Version:       nugetPackage.Metadata.Version,
+		Description:   nugetPackage.Metadata.Description,
+		Author:        nugetPackage.Metadata.Author,
+		License:       nugetPackage.Metadata.License,
+		SourceCodeURL: nugetPackage.Metadata.Repository.URL,
+		HomepageURL:   nugetPackage.Metadata.ProjectURL,
+	}
+	return pkgInfo
+}

internal/backends/dotnet/nuget_test.go

@@ -0,0 +1,33 @@
+package dotnet
+
+import (
+	"testing"
+)
+
+func TestSearchNuget(t *testing.T) {
+	pkgs := search("Microsoft.Extensions.Logging")
+
+	if len(pkgs) < 1 {
+		t.Error("No results found for Micorosft.Extensions.Logging")
+	}
+
+	for _, pkg := range pkgs {
+		if pkg.Name == "" {
+			t.Errorf("pkg %q has no name", pkg)
+		}
+		if pkg.Version == "" {
+			t.Errorf("pkg %q has no version", pkg)
+		}
+	}
+}
+
+func TestInfoFromNuget(t *testing.T) {
+	pkg := info("Microsoft.Extensions.Logging")
+
+	if pkg.Name == "" {
+		t.Errorf("pkg %q has no name", pkg)
+	}
+	if pkg.Version == "" {
+		t.Errorf("pkg %q has no version", pkg)
+	}
+}