Pypi search cleanup (#120)

* implemented pkg search with pypi; porting asadmoosvi/pypi-search * Update internal/backends/python/search_pypi.go Co-authored-by: Ryan Mulligan <[email protected]> * updatde sha * updated sha; use our requests.Do * added http response check * code cleanup for pypi search * removed unused function --------- Co-authored-by: Ryan Mulligan <[email protected]>
replit · Aug 21, 2023 · bf48909 · bf48909
1 parent 90c23bf
commit bf48909
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 50 deletions.
diff --git a/internal/backends/python/gen_pypi_map/db_gen.go b/internal/backends/python/gen_pypi_map/db_gen.go
@@ -64,8 +64,7 @@ func GenerateDB(pkg string, outputFilePath string, cache map[string]PackageInfo,
 	}
 	_, err = db.Exec(`
 	create table module_to_pypi_package (module_name text primary key, guess text, reason text);
-	create table pypi_packages (package_name text primary key, module_list text, downloads int);
-	create index downloads_index on pypi_packages (downloads);
+	create table pypi_packages (package_name text primary key, module_list text);
 	`)
 	if err != nil {
 		return err
@@ -93,20 +92,15 @@ func GenerateDB(pkg string, outputFilePath string, cache map[string]PackageInfo,
 			stmt.Close()
 
 			stmt, err = db.Prepare(`
-			insert into pypi_packages values (?, ?, ?)
+			insert into pypi_packages values (?, ?)
 			on conflict (package_name)
 			do update set
-				module_list = excluded.module_list,
-				downloads = excluded.downloads;
+				module_list = excluded.module_list;
 			`)
 			if err != nil {
 				return err
 			}
-			download, ok := downloadStats[normalizePackageName(guess.Name)]
-			if !ok {
-				download = 0
-			}
-			_, err = stmt.Exec(guess.Name, strings.Join(guess.Modules, ","), download)
+			_, err = stmt.Exec(guess.Name, strings.Join(guess.Modules, ","))
 			if err != nil {
 				return fmt.Errorf("%s on %s", err.Error(), guess.Name)
 			}
@@ -156,10 +150,3 @@ func loadLegacyPypyPackages(filePath string) map[string]LegacyPackageInfo {
 
 	return infoMap
 }
-
-func normalizePackageName(name string) string {
-	nameStr := string(name)
-	nameStr = strings.ToLower(nameStr)
-	nameStr = strings.Replace(nameStr, "_", "-", -1)
-	return nameStr
-}
diff --git a/internal/backends/python/pypi_map.go b/internal/backends/python/pypi_map.go
@@ -83,34 +83,3 @@ func (p *PypiMap) PackageToModules(packageName string) ([]string, bool) {
 	}
 	return strings.Split(moduleList, ","), true
 }
-
-func (p *PypiMap) SearchModules(query string) []string {
-	stmt, err := p.db.Prepare(`
-	select package_name
-	from pypi_packages
-	where package_name like ?
-	order by downloads desc limit 20
-	`)
-	if err != nil {
-		return nil
-	}
-	defer stmt.Close()
-	rows, err := stmt.Query(fmt.Sprintf("%%%s%%", query))
-	if err != nil {
-		return nil
-	}
-	defer rows.Close()
-	var packages []string = nil
-	for {
-		if !rows.Next() {
-			break
-		}
-		var pkg string
-		_ = rows.Scan(&pkg)
-		if err != nil {
-			break
-		}
-		packages = append(packages, pkg)
-	}
-	return packages
-}
diff --git a/internal/backends/python/search_pypi.go b/internal/backends/python/search_pypi.go
@@ -88,10 +88,10 @@ func collectText(node *html.Node) string {
 }
 
 func findNodes(node *html.Node, want func(*html.Node) bool) []*html.Node {
-	var results []*html.Node
 	if want(node) {
-		results = append(results, node)
+		return []*html.Node{node}
 	}
+	var results []*html.Node
 	for child := node.FirstChild; child != nil; child = child.NextSibling {
 		subResults := findNodes(child, want)
 		results = append(results, subResults...)