Merge pull request #51 from replit/cb-lazy-py-package-map

Lazily load python package mappings
replit · Sep 1, 2020 · 88b820a · 88b820a
2 parents f088f70 + 4f2299d
commit 88b820a
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 7 deletions.
diff --git a/internal/backends/python/gen_pypi_map/main.go b/internal/backends/python/gen_pypi_map/main.go
@@ -307,7 +307,11 @@ func main() {
 // moduleToPypiPackage holds a map of all known modules to their corresponding
 // best matching package. This helps us guess which packages should be installed
 // for the given imports.
-var moduleToPypiPackage = map[string]string{
+var moduleToPypiPackageCached = map[string]string{}
+
+func moduleToPypiPackage() map[string]string {
+    if len(moduleToPypiPackageCached) == 0 {
+        moduleToPypiPackageCached = map[string]string{
 `)
 
 	addMap := func(mod, pkg, comment string) {
@@ -368,7 +372,7 @@ nextpkg:
 		}
 	}
 
-	fmt.Fprintf(outgo, "}\n")
+	fmt.Fprintf(outgo, "}\n}\nreturn moduleToPypiPackageCached }\n")
 
 	fmt.Fprintf(outgo, `
 // pypiPackageToModules holds a map of every known python package to the modules
@@ -378,8 +382,11 @@ nextpkg:
 //
 // The module names are comma separated because go's compiler seems to vomit
 // when you create too many slices.
+var pypiPackageToModulesCached = map[string]string{}
 
-var pypiPackageToModules = map[string]string{
+func pypiPackageToModules() map[string]string {
+    if len(pypiPackageToModulesCached) == 0 {
+        pypiPackageToModulesCached = map[string]string{
 		`)
 
 	for _, pkg := range pkgs {
@@ -403,7 +410,7 @@ var pypiPackageToModules = map[string]string{
 		fmt.Fprintf(outgo, "\n")
 	}
 
-	fmt.Fprintf(outgo, "}\n")
+	fmt.Fprintf(outgo, "}\n}\nreturn pypiPackageToModulesCached\n}")
 
 	err = outgo.Close()
 	if err != nil {

diff --git a/internal/backends/python/python.go b/internal/backends/python/python.go
@@ -391,7 +391,7 @@ func guess(python string) (map[api.PkgName]bool, bool) {
 
 	if knownPkgs, err := listSpecfile(); err == nil {
 		for pkgName := range knownPkgs {
-			mods, ok := pypiPackageToModules[string(pkgName)]
+			mods, ok := pypiPackageToModules()[string(pkgName)]
 			if ok {
 				for _, mod := range strings.Split(mods, ",") {
 					availMods[mod] = true
@@ -409,13 +409,13 @@ func guess(python string) (map[api.PkgName]bool, bool) {
 		}
 
 		// If this module has a package pragma, use that
-		if pragmas.Package != ""{
+		if pragmas.Package != "" {
 			name := api.PkgName(pragmas.Package)
 			pkgs[normalizePackageName(name)] = true
 
 		} else {
 			// Otherwise, try and look it up in Pypi
-			pkg, ok := moduleToPypiPackage[modname]
+			pkg, ok := moduleToPypiPackage()[modname]
 			if ok {
 				name := api.PkgName(pkg)
 				pkgs[normalizePackageName(name)] = true