diff --git a/misc/stdlib_diff/README.md b/misc/stdlib_diff/README.md new file mode 100644 index 00000000000..3ed9c70bfcb --- /dev/null +++ b/misc/stdlib_diff/README.md @@ -0,0 +1,31 @@ +# Stdlibs_diff + +Stdlibs_diff is a tool that generates an html report indicating differences between gno standard libraries and go standrad libraries + +## Usage + +Compare the `go` standard libraries the `gno` standard libraries + +```shell +./stdlibs_diff --src --dst --out +``` + +Compare the `gno` standard libraries the `go` standard libraries + +```shell +./stdlibs_diff --src --dst --out --src_is_gno +``` + + +## Parameters + +| Flag | Description | Default value | +| ---------- | ------------------------------------------------------------------ | ------------- | +| src | Directory containing packages that will be compared to destination | None | +| dst | Directory containing packages; used to compare src packages | None | +| out | Directory where the report will be created | None | +| src_is_gno | Indicates if the src parameters is the gno standard library | false | + +## Tips + +An index.html is generated at the root of the report location. Utilize it to navigate easily through the report. \ No newline at end of file diff --git a/misc/stdlib_diff/algorithm.go b/misc/stdlib_diff/algorithm.go new file mode 100644 index 00000000000..7f832d9fe2c --- /dev/null +++ b/misc/stdlib_diff/algorithm.go @@ -0,0 +1,5 @@ +package main + +type Algorithm interface { + Diff() (srcDiff []LineDifferrence, dstDiff []LineDifferrence) +} diff --git a/misc/stdlib_diff/diffstatus.go b/misc/stdlib_diff/diffstatus.go new file mode 100644 index 00000000000..23829619f64 --- /dev/null +++ b/misc/stdlib_diff/diffstatus.go @@ -0,0 +1,25 @@ +package main + +type diffStatus uint + +const ( + missingInSrc diffStatus = iota + missingInDst + hasDiff + noDiff +) + +func (status diffStatus) String() string { + switch status { + case missingInSrc: + return "missing in src" + case missingInDst: + return "missing in dst" + case hasDiff: + return "files differ" + case noDiff: + return "files are equal" + default: + return "Unknown" + } +} diff --git a/misc/stdlib_diff/filediff.go b/misc/stdlib_diff/filediff.go new file mode 100644 index 00000000000..746c2a689b2 --- /dev/null +++ b/misc/stdlib_diff/filediff.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + "os" + "strings" +) + +// FileDiff is a struct for comparing differences between two files. +type FileDiff struct { + Src []string // Lines of the source file. + Dst []string // Lines of the destination file. + Algorithm // Algorithm used for comparison. +} + +// LineDifferrence represents a difference in a line during file comparison. +type LineDifferrence struct { + Line string // The line content. + Operation operation // The operation performed on the line (e.g., "add", "delete", "equal"). +} + +// NewFileDiff creates a new FileDiff instance for comparing differences between +// the specified source and destination files. It initializes the source and +// destination file lines . +func NewFileDiff(srcPath, dstPath string) (*FileDiff, error) { + src, err := getFileLines(srcPath) + if err != nil { + return nil, fmt.Errorf("can't read src file: %w", err) + } + + dst, err := getFileLines(dstPath) + if err != nil { + return nil, fmt.Errorf("can't read dst file: %w", err) + } + + return &FileDiff{ + Src: src, + Dst: dst, + Algorithm: NewMyers(src, dst), + }, nil +} + +// Differences returns the differences in lines between the source and +// destination files using the configured diff algorithm. +func (f *FileDiff) Differences() (src, dst []LineDifferrence) { + return f.Diff() +} + +// getFileLines reads and returns the lines of a file given its path. +func getFileLines(p string) ([]string, error) { + data, err := os.ReadFile(p) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + lines := strings.Split(strings.ReplaceAll(string(data), "\r\n", "\n"), "\n") + return lines, nil +} diff --git a/misc/stdlib_diff/main.go b/misc/stdlib_diff/main.go new file mode 100644 index 00000000000..fd95ecd5a17 --- /dev/null +++ b/misc/stdlib_diff/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "flag" + "log" +) + +func main() { + var srcPath string + var dstPath string + var outDirectory string + var srcIsGno bool + + flag.StringVar(&srcPath, "src", "", "Directory containing packages that will be compared to destination") + flag.StringVar(&dstPath, "dst", "", "Directory containing packages; used to compare src packages") + flag.StringVar(&outDirectory, "out", "", "Directory where the report will be created") + flag.BoolVar(&srcIsGno, "src_is_gno", false, "If true, indicates that the src parameter corresponds to the gno standard libraries") + flag.Parse() + + reportBuilder, err := NewReportBuilder(srcPath, dstPath, outDirectory, srcIsGno) + if err != nil { + log.Fatal("can't build report builder: ", err.Error()) + } + + log.Println("Building report...") + if err := reportBuilder.Build(); err != nil { + log.Fatalln("can't build report: ", err.Error()) + } + log.Println("Report generation done!") +} diff --git a/misc/stdlib_diff/myers.go b/misc/stdlib_diff/myers.go new file mode 100644 index 00000000000..c967dd639da --- /dev/null +++ b/misc/stdlib_diff/myers.go @@ -0,0 +1,166 @@ +package main + +import ( + "slices" +) + +var _ Algorithm = (*Myers)(nil) + +// Myers is a struct representing the Myers algorithm for line-based difference. +type Myers struct { + src []string // Lines of the source file. + dst []string // Lines of the destination file. +} + +// NewMyers creates a new Myers instance with the specified source and destination lines. +func NewMyers(src, dst []string) *Myers { + return &Myers{ + src: src, + dst: dst, + } +} + +// Do performs the Myers algorithm to find the differences between source and destination files. +// It returns the differences as two slices of LineDifferrence representing source and destination changes. +func (m *Myers) Diff() ([]LineDifferrence, []LineDifferrence) { + var ( + srcIndex, dstIndex int + insertCount, deleteCount int + dstDiff, srcDiff []LineDifferrence + ) + + operations := m.doMyers() + + for _, op := range operations { + switch op { + case insert: + dstDiff = append(dstDiff, LineDifferrence{Line: m.dst[dstIndex], Operation: op}) + srcDiff = append(srcDiff, LineDifferrence{Line: "", Operation: equal}) + dstIndex++ + insertCount++ + continue + + case equal: + dstDiff = append(dstDiff, LineDifferrence{Line: m.src[srcIndex], Operation: op}) + srcDiff = append(srcDiff, LineDifferrence{Line: m.src[srcIndex], Operation: op}) + srcIndex++ + dstIndex++ + continue + + case delete: + dstDiff = append(dstDiff, LineDifferrence{Line: "", Operation: equal}) + srcDiff = append(srcDiff, LineDifferrence{Line: m.src[srcIndex], Operation: op}) + srcIndex++ + deleteCount++ + continue + } + } + + // Means that src file is empty. + if insertCount == len(srcDiff) { + srcDiff = make([]LineDifferrence, 0) + } + // Means that dst file is empty. + if deleteCount == len(dstDiff) { + dstDiff = make([]LineDifferrence, 0) + } + return srcDiff, dstDiff +} + +// doMyers performs the Myers algorithm and returns the list of operations. +func (m *Myers) doMyers() []operation { + var tree []map[int]int + var x, y int + + srcLen := len(m.src) + dstLen := len(m.dst) + max := srcLen + dstLen + + for pathLen := 0; pathLen <= max; pathLen++ { + optimalCoordinates := make(map[int]int, pathLen+2) + tree = append(tree, optimalCoordinates) + + if pathLen == 0 { + commonPrefixLen := 0 + for srcLen > commonPrefixLen && dstLen > commonPrefixLen && m.src[commonPrefixLen] == m.dst[commonPrefixLen] { + commonPrefixLen++ + } + optimalCoordinates[0] = commonPrefixLen + + if commonPrefixLen == srcLen && commonPrefixLen == dstLen { + return m.getAllOperations(tree) + } + continue + } + + lastV := tree[pathLen-1] + + for k := -pathLen; k <= pathLen; k += 2 { + if k == -pathLen || (k != pathLen && lastV[k-1] < lastV[k+1]) { + x = lastV[k+1] + } else { + x = lastV[k-1] + 1 + } + + y = x - k + + for x < srcLen && y < dstLen && m.src[x] == m.dst[y] { + x, y = x+1, y+1 + } + + optimalCoordinates[k] = x + + if x == srcLen && y == dstLen { + return m.getAllOperations(tree) + } + } + } + + return m.getAllOperations(tree) +} + +// getAllOperations retrieves the list of operations from the calculated tree. +func (m *Myers) getAllOperations(tree []map[int]int) []operation { + var operations []operation + var k, prevK, prevX, prevY int + + x := len(m.src) + y := len(m.dst) + + for pathLen := len(tree) - 1; pathLen > 0; pathLen-- { + k = x - y + lastV := tree[pathLen-1] + + if k == -pathLen || (k != pathLen && lastV[k-1] < lastV[k+1]) { + prevK = k + 1 + } else { + prevK = k - 1 + } + + prevX = lastV[prevK] + prevY = prevX - prevK + + for x > prevX && y > prevY { + operations = append(operations, equal) + x -= 1 + y -= 1 + } + + if x == prevX { + operations = append(operations, insert) + } else { + operations = append(operations, delete) + } + + x, y = prevX, prevY + } + + if tree[0][0] != 0 { + for i := 0; i < tree[0][0]; i++ { + operations = append(operations, equal) + } + } + + slices.Reverse(operations) + return operations +} diff --git a/misc/stdlib_diff/operation.go b/misc/stdlib_diff/operation.go new file mode 100644 index 00000000000..5ebc632a90e --- /dev/null +++ b/misc/stdlib_diff/operation.go @@ -0,0 +1,28 @@ +package main + +// operation is an enumeration type representing different types of operations. Used in diff algorithm +// to indicates differences between files. +type operation uint + +const ( + // insert represents an insertion operation. + insert operation = iota + 1 + // delete represents a deletion operation. + delete + // equal represents an equal operation. + equal +) + +// String returns a string representation of the operation. +func (op operation) String() string { + switch op { + case insert: + return "INS" + case delete: + return "DEL" + case equal: + return "EQ" + default: + return "UNKNOWN" + } +} diff --git a/misc/stdlib_diff/packagediff.go b/misc/stdlib_diff/packagediff.go new file mode 100644 index 00000000000..247e49a9a27 --- /dev/null +++ b/misc/stdlib_diff/packagediff.go @@ -0,0 +1,180 @@ +package main + +import ( + "fmt" + "os" + "slices" + "strings" +) + +// PackageDiffChecker is a struct for comparing and identifying differences +// between files in two directories. +type PackageDiffChecker struct { + SrcFiles []string // List of source files. + SrcPath string // Source directory path. + DstFiles []string // List of destination files. + DstPath string // Destination directory path. + SrcIsGno bool // Indicates if the SrcFiles are gno files. +} + +// Differences represents the differences between source and destination packages. +type Differences struct { + SameNumberOfFiles bool // Indicates whether the source and destination have the same number of files. + FilesDifferences []FileDifference // Differences in individual files. +} + +// FileDifference represents the differences between source and destination files. +type FileDifference struct { + Status string // Diff status of the processed files. + SourceName string // Name of the source file. + DestinationName string // Name of the destination file. + SrcLineDiff []LineDifferrence // Differences in source file lines. + DstLineDiff []LineDifferrence // Differences in destination file lines. +} + +// NewPackageDiffChecker creates a new PackageDiffChecker instance with the specified +// source and destination paths. It initializes the SrcFiles and DstFiles fields by +// listing files in the corresponding directories. +func NewPackageDiffChecker(srcPath, dstPath string, srcIsGno bool) (*PackageDiffChecker, error) { + srcFiles, err := listDirFiles(srcPath) + if err != nil { + return nil, err + } + + dstFiles, err := listDirFiles(dstPath) + if err != nil { + return nil, err + } + + return &PackageDiffChecker{ + SrcFiles: srcFiles, + SrcPath: srcPath, + DstFiles: dstFiles, + DstPath: dstPath, + SrcIsGno: srcIsGno, + }, nil +} + +// Differences calculates and returns the differences between source and destination +// packages. It compares files line by line using the Myers algorithm. +func (p *PackageDiffChecker) Differences() (*Differences, error) { + d := &Differences{ + SameNumberOfFiles: p.hasSameNumberOfFiles(), + FilesDifferences: make([]FileDifference, 0), + } + + srcFilesExt, dstFileExt := p.inferFileExtensions() + allFiles := p.listAllPossibleFiles() + + for _, trimmedFileName := range allFiles { + srcFileName := trimmedFileName + srcFilesExt + srcFilePath := p.SrcPath + "/" + srcFileName + dstFileName := trimmedFileName + dstFileExt + dstFilePath := p.DstPath + "/" + dstFileName + + fileDiff, err := NewFileDiff(srcFilePath, dstFilePath) + if err != nil { + return nil, err + } + + srcDiff, dstDiff := fileDiff.Differences() + + d.FilesDifferences = append(d.FilesDifferences, FileDifference{ + Status: p.getStatus(srcDiff, dstDiff).String(), + SourceName: srcFileName, + DestinationName: dstFileName, + SrcLineDiff: srcDiff, + DstLineDiff: dstDiff, + }) + } + + return d, nil +} + +// listAllPossibleFiles returns a list of unique file names without extensions +// from both source and destination directories. +func (p *PackageDiffChecker) listAllPossibleFiles() []string { + files := p.SrcFiles + files = append(files, p.DstFiles...) + + for i := 0; i < len(files); i++ { + files[i] = strings.TrimSuffix(files[i], ".go") + files[i] = strings.TrimSuffix(files[i], ".gno") + } + + unique := make(map[string]bool, len(files)) + uniqueFiles := make([]string, len(unique)) + for _, file := range files { + if len(file) != 0 { + if !unique[file] { + uniqueFiles = append(uniqueFiles, file) + unique[file] = true + } + } + } + + return uniqueFiles +} + +// inferFileExtensions by returning the src and dst files extensions. +func (p *PackageDiffChecker) inferFileExtensions() (string, string) { + if p.SrcIsGno { + return ".gno", ".go" + } + + return ".go", ".gno" +} + +// getStatus determines the diff status based on the differences in source and destination. +// It returns a diffStatus indicating whether there is no difference, missing in source, missing in destination, or differences exist. +func (p *PackageDiffChecker) getStatus(srcDiff, dstDiff []LineDifferrence) diffStatus { + slicesAreEquals := slices.Equal(srcDiff, dstDiff) + if slicesAreEquals { + return noDiff + } + + if len(srcDiff) == 0 { + return missingInSrc + } + + if len(dstDiff) == 0 { + return missingInDst + } + + if !slicesAreEquals { + return hasDiff + } + + return 0 +} + +// hasSameNumberOfFiles checks if the source and destination have the same number of files. +func (p *PackageDiffChecker) hasSameNumberOfFiles() bool { + return len(p.SrcFiles) == len(p.DstFiles) +} + +// listDirFiles returns a list of file names in the specified directory. +func listDirFiles(dirPath string) ([]string, error) { + f, err := os.Open(dirPath) + if err != nil { + return []string{}, nil + } + + defer func() { + if err := f.Close(); err != nil { + fmt.Fprintln(os.Stderr, "can't close "+dirPath) + } + }() + + filesInfo, err := f.Readdir(0) + if err != nil { + return nil, fmt.Errorf("can't list file in directory :%w", err) + } + + fileNames := make([]string, 0) + for _, info := range filesInfo { + fileNames = append(fileNames, info.Name()) + } + + return fileNames, nil +} diff --git a/misc/stdlib_diff/report.go b/misc/stdlib_diff/report.go new file mode 100644 index 00000000000..44ba8feb49e --- /dev/null +++ b/misc/stdlib_diff/report.go @@ -0,0 +1,175 @@ +package main + +import ( + "bytes" + _ "embed" + "fmt" + "html/template" + "os" +) + +var ( + //go:embed templates/package_diff_template.html + packageDiffTemplate string + //go:embed templates/index_template.html + indexTemplate string +) + +// ReportBuilder is a struct for building reports based on the differences +// between source and destination directories. +type ReportBuilder struct { + SrcPath string // Source directory path. + DstPath string // Destination directory path. + OutDir string // Output directory path for the reports. + SrcIsGno bool // Indicates if the Src files are gno files. + packageTemplate *template.Template // Template for generating reports. + indexTemplate *template.Template // Template for generating index file of the reports. +} + +// PackageDiffTemplateData represents the template data structure for a package's +// differences between source and destination directories. +type PackageDiffTemplateData struct { + PackageName string // Package name. + SrcFilesCount int // Number of files in the source package. + SrcPackageLocation string // Location of source files in the source directory. + DstFileCount int // Number of destination files in the package. + DstPackageLocation string // Location of destination files in the destination directory. + FilesDifferences []FileDifference // Differences in individual files. +} + +type IndexTemplate struct { + Reports []LinkToReport +} + +type LinkToReport struct { + PathToReport string + PackageName string +} + +// NewReportBuilder creates a new ReportBuilder instance with the specified +// source path, destination path, and output directory. It also initializes +// the packageTemplate using the provided HTML template file. +func NewReportBuilder(srcPath, dstPath, outDir string, srcIsGno bool) (*ReportBuilder, error) { + packageTemplate, err := template.New("").Parse(packageDiffTemplate) + if err != nil { + return nil, err + } + + indexTemplate, err := template.New("").Parse(indexTemplate) + if err != nil { + return nil, err + } + + return &ReportBuilder{ + SrcPath: srcPath, + DstPath: dstPath, + OutDir: outDir, + SrcIsGno: srcIsGno, + packageTemplate: packageTemplate, + indexTemplate: indexTemplate, + }, nil +} + +// Build generates reports for differences between packages in the source and +// destination directories. It iterates through each directory, calculates +// differences using PackageDiffChecker, and generates reports using the +// packageTemplate. +func (builder *ReportBuilder) Build() error { + directories, err := builder.listSrcDirectories() + if err != nil { + return err + } + + indexTemplateData := &IndexTemplate{ + Reports: make([]LinkToReport, 0), + } + + for _, directory := range directories { + srcPackagePath := builder.SrcPath + "/" + directory + dstPackagePath := builder.DstPath + "/" + directory + + packageChecker, err := NewPackageDiffChecker(srcPackagePath, dstPackagePath, builder.SrcIsGno) + if err != nil { + return fmt.Errorf("can't create new PackageDiffChecker: %w", err) + } + + differences, err := packageChecker.Differences() + if err != nil { + return fmt.Errorf("can't compute differences: %w", err) + } + + data := &PackageDiffTemplateData{ + PackageName: directory, + SrcFilesCount: len(packageChecker.SrcFiles), + SrcPackageLocation: srcPackagePath, + DstFileCount: len(packageChecker.DstFiles), + DstPackageLocation: dstPackagePath, + FilesDifferences: differences.FilesDifferences, + } + + if err := builder.writePackageTemplate(data, directory); err != nil { + return err + } + + indexTemplateData.Reports = append(indexTemplateData.Reports, LinkToReport{ + PathToReport: "./" + directory + "/report.html", + PackageName: directory, + }) + } + + if err := builder.writeIndexTemplate(indexTemplateData); err != nil { + return err + } + + return nil +} + +// listSrcDirectories retrieves a list of directories in the source path. +func (builder *ReportBuilder) listSrcDirectories() ([]string, error) { + dirEntries, err := os.ReadDir(builder.SrcPath) + if err != nil { + return nil, err + } + + directories := make([]string, 0) + for _, dirEntry := range dirEntries { + if dirEntry.IsDir() { + directories = append(directories, dirEntry.Name()) + } + } + + return directories, nil +} + +// writeIndexTemplate generates and writes the index template with the given output paths. +func (builder *ReportBuilder) writeIndexTemplate(data *IndexTemplate) error { + resolvedTemplate := new(bytes.Buffer) + if err := builder.indexTemplate.Execute(resolvedTemplate, data); err != nil { + return err + } + + if err := os.WriteFile(builder.OutDir+"/index.html", resolvedTemplate.Bytes(), 0644); err != nil { + return err + } + + return nil +} + +// writePackageTemplate executes the template with the provided data and +// writes the generated report to the output directory. +func (builder *ReportBuilder) writePackageTemplate(templateData any, packageName string) error { + resolvedTemplate := new(bytes.Buffer) + if err := builder.packageTemplate.Execute(resolvedTemplate, templateData); err != nil { + return err + } + + if err := os.MkdirAll(builder.OutDir+"/"+packageName, 0777); err != nil { + return err + } + + if err := os.WriteFile(builder.OutDir+"/"+packageName+"/report.html", resolvedTemplate.Bytes(), 0644); err != nil { + return err + } + + return nil +} diff --git a/misc/stdlib_diff/templates/index_template.html b/misc/stdlib_diff/templates/index_template.html new file mode 100644 index 00000000000..5ad98f29d3b --- /dev/null +++ b/misc/stdlib_diff/templates/index_template.html @@ -0,0 +1,53 @@ + + + + + + Index + + + +

List of packages processed

+ + + \ No newline at end of file diff --git a/misc/stdlib_diff/templates/package_diff_template.html b/misc/stdlib_diff/templates/package_diff_template.html new file mode 100644 index 00000000000..684e6d7bd1f --- /dev/null +++ b/misc/stdlib_diff/templates/package_diff_template.html @@ -0,0 +1,102 @@ +{{define "file-viewer" }} +
+ {{- range .}} + {{- if eq .Operation 1}} +
+ -

{{.Line}}

+
+ {{- else if eq .Operation 2}} +
+ +

{{.Line}}

+
+ {{- else if eq .Line ""}} +
+ {{- else}} +

{{.Line}}

+ {{- end}} + {{- end}} +
+{{end}} + + + + + + + {{ .PackageName }} + + + + +

{{ .PackageName }} package differences

+ +

Package information

+ +

Sources location

+
    +
  • SRC: {{.SrcPackageLocation}}
    • +
  • DST: {{.DstPackageLocation}}
    • +
+ +

Number of files

+
    +
  • SRC: {{.SrcFilesCount}}
    • +
  • DST: {{.DstFileCount}}
    • +
+ + {{- range .FilesDifferences}} +
+ {{.SourceName}} ({{.Status}}) +
+
+

{{.SourceName}}

+ {{template "file-viewer" .SrcLineDiff}} +
+ +
+

{{.DestinationName}}

+ {{template "file-viewer" .DstLineDiff}} +
+
+
+ {{- end}} + + \ No newline at end of file