From 6be99be5449e27786b5ad144893f059cd3993953 Mon Sep 17 00:00:00 2001 From: Sam Fredrickson Date: Wed, 21 Apr 2021 13:28:59 -0700 Subject: [PATCH 1/2] Acquire DB connection lazily. Instead of connecting to the target database in the provider configuration hook function, wait until a resource actually needs it. To avoid the issue with too many open files, still only open at most one connection. This is achieved by adding a `GetDbConn` helper method to `MySQLConfiguration`, and changing all direct usages of the `Db` field to use this method. This should allow the provider to be used in a Terraform workspace that also creates the target database, and avoid the "Could not connect to server" errors that currently happen. --- mysql/data_source_tables.go | 5 ++++- mysql/provider.go | 20 ++++++++++++-------- mysql/resource_database.go | 28 ++++++++++++++++++++-------- mysql/resource_grant.go | 25 ++++++++++++++++++++----- mysql/resource_role.go | 21 +++++++++++++++------ mysql/resource_sql.go | 14 ++++++++++---- mysql/resource_user.go | 29 ++++++++++++++++++++++------- mysql/resource_user_password.go | 5 ++++- 8 files changed, 107 insertions(+), 40 deletions(-) diff --git a/mysql/data_source_tables.go b/mysql/data_source_tables.go index 7babcbc7e..4f043c112 100644 --- a/mysql/data_source_tables.go +++ b/mysql/data_source_tables.go @@ -30,7 +30,10 @@ func dataSourceTables() *schema.Resource { } func ShowTables(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } database := d.Get("database").(string) pattern := d.Get("pattern").(string) diff --git a/mysql/provider.go b/mysql/provider.go index d0540009d..475f784f3 100644 --- a/mysql/provider.go +++ b/mysql/provider.go @@ -33,6 +33,17 @@ type MySQLConfiguration struct { ConnectRetryTimeoutSec time.Duration } +func (c *MySQLConfiguration) GetDbConn() (*sql.DB, error) { + if c.Db == nil { + db, err := connectToMySQL(c) + if err != nil { + return nil, err + } + c.Db = db + } + return c.Db, nil +} + func Provider() terraform.ResourceProvider { return &schema.Provider{ Schema: map[string]*schema.Schema{ @@ -157,16 +168,9 @@ func providerConfigure(d *schema.ResourceData) (interface{}, error) { MaxConnLifetime: time.Duration(d.Get("max_conn_lifetime_sec").(int)) * time.Second, MaxOpenConns: d.Get("max_open_conns").(int), ConnectRetryTimeoutSec: time.Duration(d.Get("connect_retry_timeout_sec").(int)) * time.Second, + Db: nil, } - db, err := connectToMySQL(mysqlConf) - - if err != nil { - return nil, err - } - - mysqlConf.Db = db - return mysqlConf, nil } diff --git a/mysql/resource_database.go b/mysql/resource_database.go index d7334bb4e..de0fed515 100644 --- a/mysql/resource_database.go +++ b/mysql/resource_database.go @@ -47,12 +47,15 @@ func resourceDatabase() *schema.Resource { } func CreateDatabase(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } stmtSQL := databaseConfigSQL("CREATE", d) log.Println("Executing statement:", stmtSQL) - _, err := db.Exec(stmtSQL) + _, err = db.Exec(stmtSQL) if err != nil { return err } @@ -63,12 +66,15 @@ func CreateDatabase(d *schema.ResourceData, meta interface{}) error { } func UpdateDatabase(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } stmtSQL := databaseConfigSQL("ALTER", d) log.Println("Executing statement:", stmtSQL) - _, err := db.Exec(stmtSQL) + _, err = db.Exec(stmtSQL) if err != nil { return err } @@ -77,7 +83,10 @@ func UpdateDatabase(d *schema.ResourceData, meta interface{}) error { } func ReadDatabase(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } // This is kinda flimsy-feeling, since it depends on the formatting // of the SHOW CREATE DATABASE output... but this data doesn't seem @@ -89,7 +98,7 @@ func ReadDatabase(d *schema.ResourceData, meta interface{}) error { log.Println("Executing statement:", stmtSQL) var createSQL, _database string - err := db.QueryRow(stmtSQL).Scan(&_database, &createSQL) + err = db.QueryRow(stmtSQL).Scan(&_database, &createSQL) if err != nil { if mysqlErr, ok := err.(*mysql.MySQLError); ok { if mysqlErr.Number == unknownDatabaseErrCode { @@ -146,13 +155,16 @@ func ReadDatabase(d *schema.ResourceData, meta interface{}) error { } func DeleteDatabase(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } name := d.Id() stmtSQL := "DROP DATABASE " + quoteIdentifier(name) log.Println("Executing statement:", stmtSQL) - _, err := db.Exec(stmtSQL) + _, err = db.Exec(stmtSQL) if err == nil { d.SetId("") } diff --git a/mysql/resource_grant.go b/mysql/resource_grant.go index 9015c7def..39ecb3efd 100644 --- a/mysql/resource_grant.go +++ b/mysql/resource_grant.go @@ -153,7 +153,10 @@ func supportsRoles(db *sql.DB) (bool, error) { } func CreateGrant(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } hasRoles, err := supportsRoles(db) if err != nil { @@ -229,7 +232,10 @@ func CreateGrant(d *schema.ResourceData, meta interface{}) error { } func ReadGrant(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } hasRoles, err := supportsRoles(db) if err != nil { @@ -276,7 +282,10 @@ func ReadGrant(d *schema.ResourceData, meta interface{}) error { } func UpdateGrant(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } hasRoles, err := supportsRoles(db) @@ -351,7 +360,10 @@ func updatePrivileges(d *schema.ResourceData, db *sql.DB, user string, database } func DeleteGrant(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } database := formatDatabaseName(d.Get("database").(string)) @@ -422,7 +434,10 @@ func ImportGrant(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceDa user := userHost[0] host := userHost[1] - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return nil, err + } grants, err := showGrants(db, fmt.Sprintf("'%s'@'%s'", user, host)) diff --git a/mysql/resource_role.go b/mysql/resource_role.go index ad22e0be5..382f1c79e 100644 --- a/mysql/resource_role.go +++ b/mysql/resource_role.go @@ -24,14 +24,17 @@ func resourceRole() *schema.Resource { } func CreateRole(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } roleName := d.Get("name").(string) stmtSQL := fmt.Sprintf("CREATE ROLE '%s'", roleName) log.Printf("[DEBUG] SQL: %s", stmtSQL) - _, err := db.Exec(stmtSQL) + _, err = db.Exec(stmtSQL) if err != nil { return fmt.Errorf("error creating role: %s", err) } @@ -42,12 +45,15 @@ func CreateRole(d *schema.ResourceData, meta interface{}) error { } func ReadRole(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } stmtSQL := fmt.Sprintf("SHOW GRANTS FOR '%s'", d.Id()) log.Printf("[DEBUG] SQL: %s", stmtSQL) - _, err := db.Exec(stmtSQL) + _, err = db.Exec(stmtSQL) if err != nil { log.Printf("[WARN] Role (%s) not found; removing from state", d.Id()) d.SetId("") @@ -60,12 +66,15 @@ func ReadRole(d *schema.ResourceData, meta interface{}) error { } func DeleteRole(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } stmtSQL := fmt.Sprintf("DROP ROLE '%s'", d.Get("name").(string)) log.Printf("[DEBUG] SQL: %s", stmtSQL) - _, err := db.Exec(stmtSQL) + _, err = db.Exec(stmtSQL) if err != nil { return err } diff --git a/mysql/resource_sql.go b/mysql/resource_sql.go index 3bc57875e..efb6b3ae1 100644 --- a/mysql/resource_sql.go +++ b/mysql/resource_sql.go @@ -33,13 +33,16 @@ func resourceSql() *schema.Resource { } func CreateSql(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } name := d.Get("name").(string) create_sql := d.Get("create_sql").(string) log.Println("Executing SQL", create_sql) - _, err := db.Exec(create_sql) + _, err = db.Exec(create_sql) if err != nil { return err @@ -55,12 +58,15 @@ func ReadSql(d *schema.ResourceData, meta interface{}) error { } func DeleteSql(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } delete_sql := d.Get("delete_sql").(string) log.Println("Executing SQL:", delete_sql) - _, err := db.Exec(delete_sql) + _, err = db.Exec(delete_sql) if err == nil { d.SetId("") diff --git a/mysql/resource_user.go b/mysql/resource_user.go index c2595e261..df1dac416 100644 --- a/mysql/resource_user.go +++ b/mysql/resource_user.go @@ -68,7 +68,10 @@ func resourceUser() *schema.Resource { } func CreateUser(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } var authStm string var auth string @@ -129,7 +132,10 @@ func CreateUser(d *schema.ResourceData, meta interface{}) error { } func UpdateUser(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } var auth string if v, ok := d.GetOk("auth_plugin"); ok { @@ -204,7 +210,10 @@ func UpdateUser(d *schema.ResourceData, meta interface{}) error { } func ReadUser(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } stmtSQL := fmt.Sprintf("SELECT USER FROM mysql.user WHERE USER='%s'", d.Get("user").(string)) @@ -224,7 +233,10 @@ func ReadUser(d *schema.ResourceData, meta interface{}) error { } func DeleteUser(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } stmtSQL := fmt.Sprintf("DROP USER '%s'@'%s'", d.Get("user").(string), @@ -232,7 +244,7 @@ func DeleteUser(d *schema.ResourceData, meta interface{}) error { log.Println("Executing statement:", stmtSQL) - _, err := db.Exec(stmtSQL) + _, err = db.Exec(stmtSQL) if err == nil { d.SetId("") } @@ -249,10 +261,13 @@ func ImportUser(d *schema.ResourceData, meta interface{}) ([]*schema.ResourceDat user := userHost[0] host := userHost[1] - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return nil, err + } var count int - err := db.QueryRow("SELECT COUNT(1) FROM mysql.user WHERE user = ? AND host = ?", user, host).Scan(&count) + err = db.QueryRow("SELECT COUNT(1) FROM mysql.user WHERE user = ? AND host = ?", user, host).Scan(&count) if err != nil { return nil, err diff --git a/mysql/resource_user_password.go b/mysql/resource_user_password.go index 9953fe5e3..5a1421ed3 100644 --- a/mysql/resource_user_password.go +++ b/mysql/resource_user_password.go @@ -45,7 +45,10 @@ func resourceUserPassword() *schema.Resource { } func SetUserPassword(d *schema.ResourceData, meta interface{}) error { - db := meta.(*MySQLConfiguration).Db + db, err := meta.(*MySQLConfiguration).GetDbConn() + if err != nil { + return err + } uuid, err := uuid.NewV4() if err != nil { From c5a3aada44b9b61b9ad9a8ae0b6dbb220dfd98bb Mon Sep 17 00:00:00 2001 From: Sam Fredrickson Date: Wed, 21 Apr 2021 14:01:22 -0700 Subject: [PATCH 2/2] Make 'Db' field private to prevent accidental future usage. --- mysql/provider.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql/provider.go b/mysql/provider.go index 475f784f3..2aa6529ac 100644 --- a/mysql/provider.go +++ b/mysql/provider.go @@ -27,21 +27,21 @@ const ( type MySQLConfiguration struct { Config *mysql.Config - Db *sql.DB MaxConnLifetime time.Duration MaxOpenConns int ConnectRetryTimeoutSec time.Duration + db *sql.DB } func (c *MySQLConfiguration) GetDbConn() (*sql.DB, error) { - if c.Db == nil { + if c.db == nil { db, err := connectToMySQL(c) if err != nil { return nil, err } - c.Db = db + c.db = db } - return c.Db, nil + return c.db, nil } func Provider() terraform.ResourceProvider { @@ -168,7 +168,7 @@ func providerConfigure(d *schema.ResourceData) (interface{}, error) { MaxConnLifetime: time.Duration(d.Get("max_conn_lifetime_sec").(int)) * time.Second, MaxOpenConns: d.Get("max_open_conns").(int), ConnectRetryTimeoutSec: time.Duration(d.Get("connect_retry_timeout_sec").(int)) * time.Second, - Db: nil, + db: nil, } return mysqlConf, nil