Accept yaml and yml configuration files. (#1181)

algorand · Aug 19, 2022 · 31e7ddf · 31e7ddf
1 parent dc8f994
commit 31e7ddf
Show file tree

Hide file tree

Showing 4 changed files with 141 additions and 57 deletions.
diff --git a/cmd/algorand-indexer/daemon.go b/cmd/algorand-indexer/daemon.go
@@ -32,6 +32,32 @@ import (
 	"github.com/algorand/indexer/util/metrics"
 )
 
+// GetConfigFromDataDir Given the data directory, configuration filename and a list of types, see if
+// a configuration file that matches was located there.  If no configuration file was there then an
+// empty string is returned.  If more than one filetype was matched, an error is returned.
+func GetConfigFromDataDir(dataDirectory string, configFilename string, configFileTypes []string) (string, error) {
+	count := 0
+	fullPath := ""
+	var err error
+
+	for _, configFileType := range configFileTypes {
+		autoloadParamConfigPath := filepath.Join(dataDirectory, configFilename+"."+configFileType)
+		if util.FileExists(autoloadParamConfigPath) {
+			count++
+			fullPath = autoloadParamConfigPath
+		}
+	}
+
+	if count > 1 {
+		return "", fmt.Errorf("config filename (%s) in data directory (%s) matched more than one filetype: %v",
+			configFilename, dataDirectory, configFileTypes)
+	}
+
+	// if count == 0 then the fullpath will be set to "" and error will be nil
+	// if count == 1 then it fullpath will be correct
+	return fullPath, err
+}
+
 type daemonConfig struct {
 	flags                     *pflag.FlagSet
 	algodDataDir              string
@@ -146,17 +172,22 @@ func configureIndexerDataDir(indexerDataDir string) error {
 func loadIndexerConfig(indexerDataDir string, configFile string) error {
 	var err error
 	var resolvedConfigPath string
-	indexerConfigAutoLoadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigName)
-	indexerConfigFound := util.FileExists(indexerConfigAutoLoadPath)
+	potentialIndexerConfigPath, err := GetConfigFromDataDir(indexerDataDir, autoLoadIndexerConfigFileName, config.FileTypes[:])
+	if err != nil {
+		logger.Error(err)
+		return err
+	}
+	indexerConfigFound := potentialIndexerConfigPath != ""
+
 	if indexerConfigFound {
 		//autoload
 		if configFile != "" {
 			err = fmt.Errorf("indexer configuration was found in data directory (%s) as well as supplied via command line.  Only provide one",
-				indexerConfigAutoLoadPath)
+				potentialIndexerConfigPath)
 			logger.Error(err)
 			return err
 		}
-		resolvedConfigPath = indexerConfigAutoLoadPath
+		resolvedConfigPath = potentialIndexerConfigPath
 	} else if configFile != "" {
 		// user specified
 		resolvedConfigPath = configFile
@@ -188,17 +219,21 @@ func loadIndexerParamConfig(cfg *daemonConfig) error {
 		logger.WithError(err).Errorf("API Parameter Error: %v", err)
 		return err
 	}
-	autoloadParamConfigPath := filepath.Join(cfg.indexerDataDir, autoLoadParameterConfigName)
-	paramConfigFound := util.FileExists(autoloadParamConfigPath)
+	potentialParamConfigPath, err := GetConfigFromDataDir(cfg.indexerDataDir, autoLoadParameterConfigFileName, config.FileTypes[:])
+	if err != nil {
+		logger.Error(err)
+		return err
+	}
+	paramConfigFound := potentialParamConfigPath != ""
 	// If we auto-loaded configs but a user supplied them as well, we have an error
 	if paramConfigFound {
 		if cfg.suppliedAPIConfigFile != "" {
 			err = fmt.Errorf("api parameter configuration was found in data directory (%s) as well as supplied via command line.  Only provide one",
-				filepath.Join(cfg.indexerDataDir, autoLoadParameterConfigName))
+				potentialParamConfigPath)
 			logger.WithError(err).Errorf("indexer parameter config error: %v", err)
 			return err
 		}
-		cfg.suppliedAPIConfigFile = autoloadParamConfigPath
+		cfg.suppliedAPIConfigFile = potentialParamConfigPath
 		logger.Infof("Auto-loading parameter configuration file: %s", suppliedAPIConfigFile)
 	}
 	return err

diff --git a/cmd/algorand-indexer/daemon_test.go b/cmd/algorand-indexer/daemon_test.go
@@ -21,6 +21,7 @@ import (
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/rpcs"
 
+	"github.com/algorand/indexer/config"
 	"github.com/algorand/indexer/processor/blockprocessor"
 	itest "github.com/algorand/indexer/util/test"
 )
@@ -90,21 +91,61 @@ func createTempDir(t *testing.T) string {
 	return dir
 }
 
-// Make sure we output and return an error when both an API Config and
-// enable all parameters are provided together.
-func TestConfigWithEnableAllParamsExpectError(t *testing.T) {
+// TestParameterConfigErrorWhenBothFileTypesArePresent test that if both file types are there then it is an error
+func TestParameterConfigErrorWhenBothFileTypesArePresent(t *testing.T) {
+
 	indexerDataDir := createTempDir(t)
 	defer os.RemoveAll(indexerDataDir)
-	autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigName)
-	os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+	for _, configFiletype := range config.FileTypes {
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+	}
+
 	daemonConfig := &daemonConfig{}
 	daemonConfig.flags = pflag.NewFlagSet("indexer", 0)
 	daemonConfig.indexerDataDir = indexerDataDir
-	daemonConfig.enableAllParameters = true
-	daemonConfig.suppliedAPIConfigFile = "foobar"
 	err := runDaemon(daemonConfig)
-	errorStr := "not allowed to supply an api config file and enable all parameters"
-	assert.EqualError(t, err, errorStr)
+	errorStr := fmt.Errorf("config filename (%s) in data directory (%s) matched more than one filetype: %v",
+		autoLoadParameterConfigFileName, indexerDataDir, config.FileTypes)
+	assert.EqualError(t, err, errorStr.Error())
+}
+
+// TestIndexerConfigErrorWhenBothFileTypesArePresent test that if both file types are there then it is an error
+func TestIndexerConfigErrorWhenBothFileTypesArePresent(t *testing.T) {
+
+	indexerDataDir := createTempDir(t)
+	defer os.RemoveAll(indexerDataDir)
+	for _, configFiletype := range config.FileTypes {
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+	}
+
+	daemonConfig := &daemonConfig{}
+	daemonConfig.flags = pflag.NewFlagSet("indexer", 0)
+	daemonConfig.indexerDataDir = indexerDataDir
+	err := runDaemon(daemonConfig)
+	errorStr := fmt.Errorf("config filename (%s) in data directory (%s) matched more than one filetype: %v",
+		autoLoadIndexerConfigFileName, indexerDataDir, config.FileTypes)
+	assert.EqualError(t, err, errorStr.Error())
+}
+
+// Make sure we output and return an error when both an API Config and
+// enable all parameters are provided together.
+func TestConfigWithEnableAllParamsExpectError(t *testing.T) {
+	for _, configFiletype := range config.FileTypes {
+		indexerDataDir := createTempDir(t)
+		defer os.RemoveAll(indexerDataDir)
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+		daemonConfig := &daemonConfig{}
+		daemonConfig.flags = pflag.NewFlagSet("indexer", 0)
+		daemonConfig.indexerDataDir = indexerDataDir
+		daemonConfig.enableAllParameters = true
+		daemonConfig.suppliedAPIConfigFile = "foobar"
+		err := runDaemon(daemonConfig)
+		errorStr := "not allowed to supply an api config file and enable all parameters"
+		assert.EqualError(t, err, errorStr)
+	}
 }
 
 func TestConfigDoesNotExistExpectError(t *testing.T) {
@@ -153,20 +194,23 @@ func TestConfigSpecifiedTwiceExpectError(t *testing.T) {
 }
 
 func TestLoadAPIConfigGivenAutoLoadAndUserSuppliedExpectError(t *testing.T) {
-	indexerDataDir := createTempDir(t)
-	defer os.RemoveAll(indexerDataDir)
-
-	autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigName)
-	userSuppliedPath := filepath.Join(indexerDataDir, "foobar.yml")
-	os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
-	cfg := &daemonConfig{}
-	cfg.indexerDataDir = indexerDataDir
-	cfg.suppliedAPIConfigFile = userSuppliedPath
 
-	err := loadIndexerParamConfig(cfg)
-	errorStr := fmt.Sprintf("api parameter configuration was found in data directory (%s) as well as supplied via command line.  Only provide one",
-		autoloadPath)
-	assert.EqualError(t, err, errorStr)
+	for _, configFiletype := range config.FileTypes {
+		indexerDataDir := createTempDir(t)
+		defer os.RemoveAll(indexerDataDir)
+
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigFileName+"."+configFiletype)
+		userSuppliedPath := filepath.Join(indexerDataDir, "foobar.yml")
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+		cfg := &daemonConfig{}
+		cfg.indexerDataDir = indexerDataDir
+		cfg.suppliedAPIConfigFile = userSuppliedPath
+
+		err := loadIndexerParamConfig(cfg)
+		errorStr := fmt.Sprintf("api parameter configuration was found in data directory (%s) as well as supplied via command line.  Only provide one",
+			autoloadPath)
+		assert.EqualError(t, err, errorStr)
+	}
 }
 
 func TestLoadAPIConfigGivenUserSuppliedExpectSuccess(t *testing.T) {
@@ -183,17 +227,19 @@ func TestLoadAPIConfigGivenUserSuppliedExpectSuccess(t *testing.T) {
 }
 
 func TestLoadAPIConfigGivenAutoLoadExpectSuccess(t *testing.T) {
-	indexerDataDir := createTempDir(t)
-	defer os.RemoveAll(indexerDataDir)
-
-	autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigName)
-	os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
-	cfg := &daemonConfig{}
-	cfg.indexerDataDir = indexerDataDir
-
-	err := loadIndexerParamConfig(cfg)
-	assert.NoError(t, err)
-	assert.Equal(t, autoloadPath, cfg.suppliedAPIConfigFile)
+	for _, configFiletype := range config.FileTypes {
+		indexerDataDir := createTempDir(t)
+		defer os.RemoveAll(indexerDataDir)
+
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadParameterConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+		cfg := &daemonConfig{}
+		cfg.indexerDataDir = indexerDataDir
+
+		err := loadIndexerParamConfig(cfg)
+		assert.NoError(t, err)
+		assert.Equal(t, autoloadPath, cfg.suppliedAPIConfigFile)
+	}
 }
 
 func TestIndexerDataDirNotProvidedExpectError(t *testing.T) {
@@ -217,18 +263,20 @@ func TestIndexerPidFileExpectSuccess(t *testing.T) {
 }
 
 func TestIndexerPidFileCreateFailExpectError(t *testing.T) {
-	indexerDataDir := createTempDir(t)
-	defer os.RemoveAll(indexerDataDir)
-	autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigName)
-	os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
+	for _, configFiletype := range config.FileTypes {
+		indexerDataDir := createTempDir(t)
+		defer os.RemoveAll(indexerDataDir)
+		autoloadPath := filepath.Join(indexerDataDir, autoLoadIndexerConfigFileName+"."+configFiletype)
+		os.WriteFile(autoloadPath, []byte{}, fs.ModePerm)
 
-	invalidDir := filepath.Join(indexerDataDir, "foo", "bar")
-	cfg := &daemonConfig{}
-	cfg.pidFilePath = invalidDir
+		invalidDir := filepath.Join(indexerDataDir, "foo", "bar")
+		cfg := &daemonConfig{}
+		cfg.pidFilePath = invalidDir
 
-	cfg.flags = pflag.NewFlagSet("indexer", 0)
-	cfg.indexerDataDir = indexerDataDir
+		cfg.flags = pflag.NewFlagSet("indexer", 0)
+		cfg.indexerDataDir = indexerDataDir
 
-	assert.ErrorContains(t, runDaemon(cfg), "pid file")
-	assert.Error(t, createIndexerPidFile(cfg.pidFilePath))
+		assert.ErrorContains(t, runDaemon(cfg), "pid file")
+		assert.Error(t, createIndexerPidFile(cfg.pidFilePath))
+	}
 }
diff --git a/cmd/algorand-indexer/main.go b/cmd/algorand-indexer/main.go
@@ -21,8 +21,8 @@ import (
 	"github.com/algorand/indexer/version"
 )
 
-const autoLoadIndexerConfigName = config.FileName + "." + config.FileType
-const autoLoadParameterConfigName = "api_config.yml"
+const autoLoadIndexerConfigFileName = config.FileName
+const autoLoadParameterConfigFileName = "api_config"
 
 // Calling os.Exit() directly will not honor any defer'd statements.
 // Instead, we will create an exit type and handler so that we may panic
@@ -134,7 +134,8 @@ func init() {
 
 	// Setup configuration file
 	viper.SetConfigName(config.FileName)
-	viper.SetConfigType(config.FileType)
+	// just hard-code yaml since we support multiple yaml filetypes
+	viper.SetConfigType("yaml")
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
 
 	if err := viper.ReadInConfig(); err != nil {

diff --git a/config/config.go b/config/config.go
@@ -11,8 +11,8 @@ import (
 // EnvPrefix is the prefix for environment variable configurations.
 const EnvPrefix = "INDEXER"
 
-// FileType is the type of the config file.
-const FileType = "yml"
+// FileTypes is an array of types of the config file.
+var FileTypes = [...]string{"yml", "yaml"}
 
 // FileName is the name of the config file. Don't use 'algorand-indexer', viper
 // gets confused and thinks the binary is a config file with no extension.