From 2fd87995a683e66c6271b58df6c6723d1e2448dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 3 Jun 2020 13:18:11 +0200 Subject: [PATCH 1/9] Added forgotten metric increase in case of 'sync conflict' on proxysql_servers --- lib/ProxySQL_Cluster.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 35c0aa5e3d..5e09bf0adb 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -637,6 +637,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_ps*10)) == 0)) { proxy_error("Cluster: detected a peer %s:%d with proxysql_servers version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %llu checks until LOAD MYSQL SERVERS TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_ps*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_proxysql_servers_share_epoch]->Increment(); } } else { if (v->diff_check && (v->diff_check % (diff_ps*10)) == 0) { From ffae79e105bf91b4f12cb0cb3bb722540291ae9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 3 Jun 2020 14:42:53 +0200 Subject: [PATCH 2/9] Added support for 'global_variables' sync in proxysql_cluster - Added several for getting the peers to be sync. - Added function to pull the target 'global_variables' to sync. - Added metrics for 'global_variables' sync: + Number of syncs done / failed. + Sync conflicts. + Sync warnings. --- include/ProxySQL_Cluster.hpp | 29 ++++ include/proxysql_admin.h | 5 + lib/ProxySQL_Admin.cpp | 32 ++++ lib/ProxySQL_Cluster.cpp | 327 ++++++++++++++++++++++++++++++++++- 4 files changed, 392 insertions(+), 1 deletion(-) diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index 1df6105928..33f514ce99 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -114,6 +114,8 @@ class ProxySQL_Cluster_Nodes { void get_peer_to_sync_mysql_query_rules(char **host, uint16_t *port); void get_peer_to_sync_mysql_servers(char **host, uint16_t *port, char **peer_checksum); void get_peer_to_sync_mysql_users(char **host, uint16_t *port); + void get_peer_to_sync_mysql_variables(char **host, uint16_t *port); + void get_peer_to_sync_admin_variables(char **host, uint16_t* port); void get_peer_to_sync_proxysql_servers(char **host, uint16_t *port); }; @@ -141,15 +143,25 @@ struct p_cluster_counter { pulled_proxysql_servers_success, pulled_proxysql_servers_failure, + pulled_mysql_variables_success, + pulled_mysql_variables_failure, + + pulled_admin_variables_success, + pulled_admin_variables_failure, + sync_conflict_mysql_query_rules_share_epoch, sync_conflict_mysql_servers_share_epoch, sync_conflict_proxysql_servers_share_epoch, sync_conflict_mysql_users_share_epoch, + sync_conflict_mysql_variables_share_epoch, + sync_conflict_admin_variables_share_epoch, sync_delayed_mysql_query_rules_version_one, sync_delayed_mysql_servers_version_one, sync_delayed_mysql_users_version_one, sync_delayed_proxysql_servers_version_one, + sync_delayed_mysql_variables_version_one, + sync_delayed_admin_variables_version_one, __size }; @@ -168,6 +180,14 @@ struct cluster_metrics_map_idx { }; }; +struct variable_type { + enum type { + mysql, + admin, + __size + }; +}; + /** * @brief Simple struct for holding a query, and three messages to report * the progress of the query execution. @@ -196,6 +216,7 @@ class ProxySQL_Cluster { pthread_mutex_t update_mysql_query_rules_mutex; pthread_mutex_t update_mysql_servers_mutex; pthread_mutex_t update_mysql_users_mutex; + pthread_mutex_t update_mysql_variables_mutex; pthread_mutex_t update_proxysql_servers_mutex; int cluster_check_interval_ms; int cluster_check_status_frequency; @@ -244,6 +265,14 @@ class ProxySQL_Cluster { void pull_mysql_query_rules_from_peer(); void pull_mysql_servers_from_peer(); void pull_mysql_users_from_peer(); + /** + * @brief Pulls from peer the specified global variables by the type parameter. + * @param type A string specifying the type of global variables to pull from the peer, supported + * values right now are: + * - 'mysql'. + * - 'admin'. + */ + void pull_global_variables_from_peer(const std::string& type); void pull_proxysql_servers_from_peer(); }; #endif /* CLASS_PROXYSQL_CLUSTER_H */ diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 01b5a61988..db1d90aafa 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -295,6 +295,11 @@ class ProxySQL_Admin { void flush_mysql_query_rules__from_disk_to_memory(); void flush_mysql_firewall__from_memory_to_disk(); void flush_mysql_firewall__from_disk_to_memory(); + void flush_mysql_variables__from_disk_to_memory(); + void flush_mysql_variables__from_memory_to_disk(); + void flush_admin_variables__from_disk_to_memory(); + void flush_admin_variables__from_memory_to_disk(); + void load_mysql_servers_to_runtime(); void save_mysql_servers_from_runtime(); char * load_mysql_query_rules_to_runtime(); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 02e7fbe59c..bf8c266ddd 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -9164,6 +9164,38 @@ void ProxySQL_Admin::flush_mysql_query_rules__from_memory_to_disk() { admindb->wrunlock(); } +void ProxySQL_Admin::flush_mysql_variables__from_disk_to_memory() { + admindb->wrlock(); + admindb->execute("PRAGMA foreign_keys = OFF"); + admindb->execute("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'mysql-%'"); + admindb->execute("PRAGMA foreign_keys = ON"); + admindb->wrunlock(); +} + +void ProxySQL_Admin::flush_mysql_variables__from_memory_to_disk() { + admindb->wrlock(); + admindb->execute("PRAGMA foreign_keys = OFF"); + admindb->execute("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'mysql-%'"); + admindb->execute("PRAGMA foreign_keys = ON"); + admindb->wrunlock(); +} + +void ProxySQL_Admin::flush_admin_variables__from_disk_to_memory() { + admindb->wrlock(); + admindb->execute("PRAGMA foreign_keys = OFF"); + admindb->execute("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'admin-%'"); + admindb->execute("PRAGMA foreign_keys = ON"); + admindb->wrunlock(); +} + +void ProxySQL_Admin::flush_admin_variables__from_memory_to_disk() { + admindb->wrlock(); + admindb->execute("PRAGMA foreign_keys = OFF"); + admindb->execute("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'admin-%'"); + admindb->execute("PRAGMA foreign_keys = ON"); + admindb->wrunlock(); +} + void ProxySQL_Admin::__attach_db(SQLite3DB *db1, SQLite3DB *db2, char *alias) { const char *a="ATTACH DATABASE '%s' AS %s"; int l=strlen(a)+strlen(db2->get_url())+strlen(alias)+5; diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 5e09bf0adb..440b6e4814 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -385,7 +385,9 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.admin_variables.checksum, row[3]); checksums_values.admin_variables.last_changed = now; checksums_values.admin_variables.diff_check = 1; + proxy_info("Cluster: detected a new checksum for admin_variables from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.admin_variables.version, checksums_values.admin_variables.epoch, checksums_values.admin_variables.checksum); } else { + proxy_info("Cluster: checksum for admin_variables from peer %s:%d matches with local checksum %s, we won't sync.\n", hostname, port, GloVars.checksums_values.admin_variables.checksum); checksums_values.admin_variables.diff_check++; } if (strcmp(checksums_values.admin_variables.checksum, GloVars.checksums_values.admin_variables.checksum) == 0) { @@ -461,6 +463,10 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { strcpy(checksums_values.mysql_variables.checksum, row[3]); checksums_values.mysql_variables.last_changed = now; checksums_values.mysql_variables.diff_check = 1; + proxy_info("Cluster: detected a new checksum for mysql_variables from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.mysql_variables.version, checksums_values.mysql_variables.epoch, checksums_values.mysql_variables.checksum); + if (strcmp(checksums_values.mysql_variables.checksum, GloVars.checksums_values.mysql_variables.checksum) == 0) { + proxy_info("Cluster: checksum for mysql_variables from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.mysql_variables.checksum); + } } else { checksums_values.mysql_variables.diff_check++; } @@ -494,6 +500,9 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { ProxySQL_Checksum_Value_2 *v = NULL; v = &checksums_values.admin_variables; v->last_updated = now; + if (strcmp(v->checksum, GloVars.checksums_values.admin_variables.checksum) == 0) { + v->diff_check = 0; + } if (v->diff_check) v->diff_check++; v = &checksums_values.mysql_query_rules; @@ -519,6 +528,9 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { v->diff_check++; v = &checksums_values.mysql_variables; v->last_updated = now; + if (strcmp(v->checksum, GloVars.checksums_values.mysql_variables.checksum) == 0) { + v->diff_check = 0; + } if (v->diff_check) v->diff_check++; v = &checksums_values.proxysql_servers; @@ -537,6 +549,8 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { unsigned int diff_ms = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync,0); unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_users_diffs_before_sync,0); unsigned int diff_ps = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_proxysql_servers_diffs_before_sync,0); + unsigned int diff_mv = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync,0); + unsigned int diff_av = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_admin_variables_diffs_before_sync,0); ProxySQL_Checksum_Value_2 *v = NULL; if (diff_mqr) { unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_query_rules.version,0); @@ -646,6 +660,62 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) { } } } + if (diff_mv) { + v = &checksums_values.mysql_variables; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.version, 0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.epoch, 0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.mysql_variables.checksum, 0); + + if (v->version > 1) { + if ( + (own_version == 1) // we just booted + || + (v->epoch > own_epoch) // epoch is newer + ) { + if (v->diff_check >= diff_mv) { + proxy_info("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + GloProxyCluster->pull_global_variables_from_peer("mysql"); + } + } + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_mv*10)) == 0)) { + proxy_error("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %llu checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_mv*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_mysql_variables_share_epoch]->Increment(); + } + } else { + if (v->diff_check && (v->diff_check % (diff_mv*10)) == 0) { + proxy_warning("Cluster: detected a peer %s:%d with mysql_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %llu checks until LOAD MYSQL VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_mv*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_mysql_variables_version_one]->Increment(); + } + } + } + if (diff_av) { + v = &checksums_values.admin_variables; + unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.version, 0); + unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.epoch, 0); + char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.admin_variables.checksum, 0); + + if (v->version > 1) { + if ( + (own_version == 1) // we just booted + || + (v->epoch > own_epoch) // epoch is newer + ) { + if (v->diff_check >= diff_av) { + proxy_info("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch); + GloProxyCluster->pull_global_variables_from_peer("admin"); + } + } + if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_av*10)) == 0)) { + proxy_error("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %llu checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_av*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_admin_variables_share_epoch]->Increment(); + } + } else { + if (v->diff_check && (v->diff_check % (diff_av*10)) == 0) { + proxy_warning("Cluster: detected a peer %s:%d with admin_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %llu checks until LOAD ADMIN VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_av*10)); + GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_admin_variables_version_one]->Increment(); + } + } + } } void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { @@ -1290,6 +1360,123 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer() { pthread_mutex_unlock(&GloProxyCluster->update_mysql_servers_mutex); } +void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_type) { + char * hostname = NULL; + uint16_t port = 0; + char* vars_type_str = nullptr; + p_cluster_counter::metric success_metric = p_cluster_counter::metric(-1); + p_cluster_counter::metric failure_metric = p_cluster_counter::metric(-1); + + if (var_type == "mysql") { + vars_type_str = const_cast("MySQL"); + success_metric = p_cluster_counter::pulled_mysql_variables_success; + failure_metric = p_cluster_counter::pulled_mysql_variables_failure; + } else if (var_type == "admin") { + vars_type_str = const_cast("Admin"); + success_metric = p_cluster_counter::pulled_admin_variables_success; + failure_metric = p_cluster_counter::pulled_admin_variables_failure; + } else { + proxy_error("Invalid parameter supplied to 'pull_global_variables_from_peer': var_type=%s", var_type.c_str()); + assert(0); + } + + pthread_mutex_lock(&GloProxyCluster->update_mysql_variables_mutex); + if (var_type == "mysql") { + nodes.get_peer_to_sync_mysql_variables(&hostname, &port); + } else { + nodes.get_peer_to_sync_admin_variables(&hostname, &port); + } + + if (hostname) { + char *username = NULL; + char *password = NULL; + MYSQL *rc_conn = nullptr; + int rc_query = 0; + int rc = 0; + MYSQL *conn = mysql_init(NULL); + + if (conn == NULL) { + proxy_error("Unable to run mysql_init()\n"); + goto __exit_pull_mysql_variables_from_peer; + } + + GloProxyCluster->get_credentials(&username, &password); + if (strlen(username)) { // do not monitor if the username is empty + unsigned int timeout = 1; + unsigned int timeout_long = 60; + mysql_options(conn, MYSQL_OPT_CONNECT_TIMEOUT, &timeout); + mysql_options(conn, MYSQL_OPT_READ_TIMEOUT, &timeout_long); + mysql_options(conn, MYSQL_OPT_WRITE_TIMEOUT, &timeout); + { unsigned char val = 1; mysql_options(conn, MYSQL_OPT_SSL_ENFORCE, &val); } + proxy_info("Cluster: Fetching %s variables from peer %s:%d started\n", vars_type_str, hostname, port); + rc_conn = mysql_real_connect(conn, hostname, username, password, NULL, port, NULL, 0); + + if (rc_conn) { + std::string s_query = ""; + string_format("SELECT * FROM runtime_global_variables WHERE variable_name LIKE '%s-%%'", s_query, var_type.c_str()); + mysql_query(conn, s_query.c_str()); + + if (rc_query == 0) { + MYSQL_RES *result = mysql_store_result(conn); + std::string d_query = ""; + string_format("DELETE FROM runtime_global_variables WHERE variable_name LIKE '%s-%%'", d_query, var_type.c_str()); + GloAdmin->admindb->execute(d_query.c_str()); + + MYSQL_ROW row; + char *q = (char *)"INSERT OR REPLACE INTO global_variables (variable_name, variable_value) VALUES (?1 , ?2)"; + sqlite3_stmt *statement1 = NULL; + rc = GloAdmin->admindb->prepare_v2(q, &statement1); + ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + + while ((row = mysql_fetch_row(result))) { + rc=(*proxy_sqlite3_bind_text)(statement1, 1, row[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // variable_name + rc=(*proxy_sqlite3_bind_text)(statement1, 2, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // variable_value + + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); + } + + mysql_free_result(result); + proxy_info("Cluster: Fetching %s Variables from peer %s:%d completed\n", vars_type_str, hostname, port); + proxy_info("Cluster: Loading to runtime %s Variables from peer %s:%d\n", vars_type_str, hostname, port); + + if (var_type == "mysql") { + GloAdmin->load_mysql_variables_to_runtime(); + + if (GloProxyCluster->cluster_mysql_variables_save_to_disk == true) { + proxy_info("Cluster: Saving to disk MySQL Variables from peer %s:%d\n", hostname, port); + GloAdmin->flush_mysql_variables__from_memory_to_disk(); + } + } else { + GloAdmin->load_admin_variables_to_runtime(); + + if (GloProxyCluster->cluster_admin_variables_save_to_disk == true) { + proxy_info("Cluster: Saving to disk Admin Variables from peer %s:%d\n", hostname, port); + GloAdmin->flush_admin_variables__from_memory_to_disk(); + } + } + metrics.p_counter_array[success_metric]->Increment(); + } else { + proxy_info("Cluster: Fetching %s Variables from peer %s:%d failed: %s\n", vars_type_str, hostname, port, mysql_error(conn)); + metrics.p_counter_array[failure_metric]->Increment(); + } + } else { + proxy_info("Cluster: Fetching %s Variables from peer %s:%d failed: %s\n", vars_type_str, hostname, port, mysql_error(conn)); + metrics.p_counter_array[failure_metric]->Increment(); + } + } +__exit_pull_mysql_variables_from_peer: + if (conn) { + if (conn->net.pvio) { + mysql_close(conn); + } + } + free(hostname); + } + pthread_mutex_unlock(&GloProxyCluster->update_mysql_variables_mutex); +} + void ProxySQL_Cluster::pull_proxysql_servers_from_peer() { char * hostname = NULL; uint16_t port = 0; @@ -1699,6 +1886,91 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_users(char **host, uint16_t } } +void ProxySQL_Cluster_Nodes::get_peer_to_sync_mysql_variables(char **host, uint16_t *port) { + unsigned long long version = 0; + unsigned long long epoch = 0; + unsigned long long max_epoch = 0; + char *hostname = NULL; + uint16_t p = 0; + unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync,0); + for (std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end();) { + ProxySQL_Node_Entry * node = it->second; + ProxySQL_Checksum_Value_2 * v = &node->checksums_values.mysql_variables; + if (v->version > 1) { + if ( v->epoch > epoch ) { + max_epoch = v->epoch; + if (v->diff_check > diff_mu) { + epoch = v->epoch; + version = v->version; + if (hostname) { + free(hostname); + } + hostname=strdup(node->get_hostname()); + p = node->get_port(); + } + } + } + it++; + } + if (epoch) { + if (max_epoch > epoch) { + proxy_warning("Cluster: detected a peer with mysql_variables epoch %llu, but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + if (hostname) { + free(hostname); + hostname = NULL; + } + } + } + if (hostname) { + *host = hostname; + *port = p; + proxy_info("Cluster: detected peer %s:%d with mysql_variables version %llu, epoch %llu\n", hostname, p, version, epoch); + } +} + + +void ProxySQL_Cluster_Nodes::get_peer_to_sync_admin_variables(char **host, uint16_t *port) { + unsigned long long version = 0; + unsigned long long epoch = 0; + unsigned long long max_epoch = 0; + char *hostname = NULL; + uint16_t p = 0; + unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_admin_variables_diffs_before_sync,0); + for (std::unordered_map::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end();) { + ProxySQL_Node_Entry * node = it->second; + ProxySQL_Checksum_Value_2 * v = &node->checksums_values.admin_variables; + if (v->version > 1) { + if ( v->epoch > epoch ) { + max_epoch = v->epoch; + if (v->diff_check > diff_mu) { + epoch = v->epoch; + version = v->version; + if (hostname) { + free(hostname); + } + hostname=strdup(node->get_hostname()); + p = node->get_port(); + } + } + } + it++; + } + if (epoch) { + if (max_epoch > epoch) { + proxy_warning("Cluster: detected a peer with admin_variables epoch %llu, but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch); + if (hostname) { + free(hostname); + hostname = NULL; + } + } + } + if (hostname) { + *host = hostname; + *port = p; + proxy_info("Cluster: detected peer %s:%d with admin_variables version %llu, epoch %llu\n", hostname, p, version, epoch); + } +} + void ProxySQL_Cluster_Nodes::get_peer_to_sync_proxysql_servers(char **host, uint16_t *port) { unsigned long long version = 0; unsigned long long epoch = 0; @@ -2052,6 +2324,34 @@ cluster_metrics_map = std::make_tuple( metric_tags { { "status", "failure" } } ), + // mysql_variables_* + std::make_tuple ( + p_cluster_counter::pulled_mysql_variables_success, + "pulled_mysql_variables", + "Number of times 'mysql_variables' have been pulled from a peer.", + metric_tags { { "status", "success" } } + ), + std::make_tuple ( + p_cluster_counter::pulled_mysql_variables_failure, + "pulled_mysql_variables", + "Number of times 'mysql_variables' have been pulled from a peer.", + metric_tags { { "status", "failure" } } + ), + + // admin_variables_* + std::make_tuple ( + p_cluster_counter::pulled_admin_variables_success, + "pulled_admin_variables", + "Number of times 'admin_variables' have been pulled from a peer.", + metric_tags { { "status", "success" } } + ), + std::make_tuple ( + p_cluster_counter::pulled_admin_variables_failure, + "pulled_admin_variables", + "Number of times 'admin_variables' have been pulled from a peer.", + metric_tags { { "status", "failure" } } + ), + // sync_conflict same epoch std::make_tuple ( p_cluster_counter::sync_conflict_mysql_query_rules_share_epoch, @@ -2077,6 +2377,18 @@ cluster_metrics_map = std::make_tuple( "Number of times 'mysql_users' has not been synced because they share the same epoch.", metric_tags { { "type", "error" } } ), + std::make_tuple ( + p_cluster_counter::sync_conflict_mysql_variables_share_epoch, + "sync_conflict_mysql_variables_share_epoch", + "Number of times 'mysql_variables' has not been synced because they share the same epoch.", + metric_tags { { "type", "error" } } + ), + std::make_tuple ( + p_cluster_counter::sync_conflict_admin_variables_share_epoch, + "sync_conflict_admin_variables_share_epoch", + "Number of times 'admin_variables' has not been synced because they share the same epoch.", + metric_tags { { "type", "error" } } + ), // sync_delayed due to version one std::make_tuple ( @@ -2102,7 +2414,20 @@ cluster_metrics_map = std::make_tuple( "sync_delayed_proxysql_servers_version_one", "Number of times 'proxysql_servers' has not been synced because version one doesn't allow sync.", metric_tags { { "type", "warning" } } - ) + ), + std::make_tuple ( + p_cluster_counter::sync_delayed_mysql_variables_version_one, + "sync_delayed_mysql_variables_version_one", + "Number of times 'mysql_variables' has not been synced because version one doesn't allow sync.", + metric_tags { { "type", "warning" } } + ), + std::make_tuple ( + p_cluster_counter::sync_delayed_admin_variables_version_one, + "sync_delayed_admin_variables_version_one", + "Number of times 'admin_variables' has not been synced because version one doesn't allow sync.", + metric_tags { { "type", "warning" } } + ), + }, cluster_gauge_vector {} ); From 698440d610328652b04e02a8a1f52e9cc541929c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 3 Jun 2020 14:47:30 +0200 Subject: [PATCH 3/9] Removed unnecesary escaping for '%' char --- lib/ProxySQL_Admin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index bf8c266ddd..933358d16f 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -5832,7 +5832,7 @@ void ProxySQL_Admin::flush_mysql_variables___database_to_runtime(SQLite3DB *db, int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'mysql-\%' ORDER BY variable_name"; + char *q=(char *)"SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'mysql-%' ORDER BY variable_name"; admindb->execute_statement(q, &error , &cols , &affected_rows , &resultset); uint64_t hash1 = resultset->raw_checksum(); uint32_t d32[2]; From d4c01507c7d44780e4572d9b7fe236ba3c7eed79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 3 Jun 2020 14:49:22 +0200 Subject: [PATCH 4/9] Fixed issue due to 'flush_mysql_variables___runtime_to_database' being called without proper locking --- lib/ProxySQL_Admin.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 933358d16f..35bc92845a 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -3108,6 +3108,7 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign if (admin) { if (dump_global_variables) { + pthread_mutex_lock(&GloVars.checksum_mutex); admindb->execute("DELETE FROM runtime_global_variables"); // extra flush_admin_variables___runtime_to_database(admindb, false, false, false, true); flush_mysql_variables___runtime_to_database(admindb, false, false, false, true); @@ -3116,6 +3117,7 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign #endif /* PROXYSQLCLICKHOUSE */ flush_sqliteserver_variables___runtime_to_database(admindb, false, false, false, true); flush_ldap_variables___runtime_to_database(admindb, false, false, false, true); + pthread_mutex_unlock(&GloVars.checksum_mutex); } if (runtime_mysql_servers) { int old_hostgroup_manager_verbose = mysql_thread___hostgroup_manager_verbose; From ea234a08e825a06930641bc1aea3b826d89976ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 3 Jun 2020 14:54:50 +0200 Subject: [PATCH 5/9] Added simple helper function to get the output from a command execution --- test/tap/tap/utils.cpp | 30 +++++++++++++++++++++++++++++- test/tap/tap/utils.h | 9 +++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/test/tap/tap/utils.cpp b/test/tap/tap/utils.cpp index 8132b73a6d..33697d1549 100644 --- a/test/tap/tap/utils.cpp +++ b/test/tap/tap/utils.cpp @@ -395,4 +395,32 @@ int execvp(const std::string& cmd, const std::vector& argv, std::st result = result_; return err; -} \ No newline at end of file +} + +int exec(const std::string& cmd, std::string& result) { + char buffer[128]; + std::string result_ = ""; + int err = 0; + + // Try to invoke the shell + FILE* pipe = popen(cmd.c_str(), "r"); + if (!pipe) { + return errno; + } + + try { + while (fgets(buffer, sizeof buffer, pipe) != NULL) { + result_ += buffer; + } + } catch (...) { + err = -1; + } + + pclose(pipe); + + if (err == 0) { + // Return the result + result = result_; + } + return err; +} diff --git a/test/tap/tap/utils.h b/test/tap/tap/utils.h index 5ed2fcae40..7f37a63317 100644 --- a/test/tap/tap/utils.h +++ b/test/tap/tap/utils.h @@ -57,4 +57,13 @@ int wexecvp(const std::string& file, const std::vector& argv, const */ int execvp(const std::string& file, const std::vector& argv, std::string& result); +/** + * @brief Executes a command using popen and returns the output in the string supplied as second parameter. + * + * @param cmd The command to be executed. + * @param result String with the output of the executed command. + * @return int The error code returned by popen. + */ +int exec(const std::string& cmd, std::string& result); + #endif // #define UTILS_H From f440a31594e6b25e06790f3fa56bd7e49931d944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 3 Jun 2020 14:56:00 +0200 Subject: [PATCH 6/9] Multiple changes to 'cluster_synct' test + Spawned proxysql instance now runs inside a docker container. + Simplified launching and logging of spawned proxysql instance. + Added more verbose logging about the tables state. + Added new tests for 'global_variables' synchronization. --- test/tap/tests/test_cluster_sync-t.cpp | 432 ++++++++++++++++-- .../test_cluster_sync-t.cnf | 7 +- 2 files changed, 408 insertions(+), 31 deletions(-) diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 49524e4217..4e1003144e 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -31,7 +31,7 @@ } while(0) const uint32_t SYNC_TIMEOUT = 10; -const uint32_t CONNECT_TIMEOUT = 20; +const uint32_t CONNECT_TIMEOUT = 60; int setup_config_file(const CommandLine& cl) { const std::string t_fmt_config_file = std::string(cl.workdir) + "test_cluster_sync_config/test_cluster_sync-t.cnf"; @@ -79,20 +79,20 @@ int setup_config_file(const CommandLine& cl) { return -1; } - // Get the datadir setting - config_setting_t* datadir = config_lookup(&cfg, "datadir"); - if (datadir == nullptr) { - fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - 'datadir' setting not found."); + // Get the current docker bridge ip address + std::string bridge_addr = ""; + int bridge_res = exec("ip -4 addr show docker0 | grep -oP '(?<=inet\\s)\\d+(\\.\\d+){3}'", bridge_addr); + if (bridge_res != 0) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Failed to get docker bridge ip - Not able to write the config file."); return -1; } - int fhost_res = config_setting_set_string(f_pserver_hostname, cl.host); + int fhost_res = config_setting_set_string(f_pserver_hostname, bridge_addr.substr(0, bridge_addr.size() - 1).c_str()); int fport_res = config_setting_set_int(f_pserver_port, cl.admin_port); int shost_res = config_setting_set_string(s_pserver_hostname, cl.host); int sport_res = config_setting_set_int(s_pserver_port, 7032); - int sdata_res = config_setting_set_string(datadir, datadir_path.c_str()); - if (fhost_res == CONFIG_FALSE || fport_res == CONFIG_FALSE || shost_res == CONFIG_FALSE || sport_res == CONFIG_FALSE || sdata_res == CONFIG_FALSE) { + if (fhost_res == CONFIG_FALSE || fport_res == CONFIG_FALSE || shost_res == CONFIG_FALSE || sport_res == CONFIG_FALSE) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - Error while trying to set the values from env variables."); return -1; } @@ -119,6 +119,7 @@ int main(int, char**) { } const std::string fmt_config_file = std::string(cl.workdir) + "test_cluster_sync_config/test_cluster_sync.cnf"; + const std::string t_debug_query = "mysql -u%s -p%s -h %s -P%d -C -e \"%s\""; MYSQL* proxysql_admin = mysql_init(NULL); MYSQL* proxysql_replica = mysql_init(NULL); @@ -144,8 +145,11 @@ int main(int, char**) { "VALUES ('%s', %d, 0, 'proxysql130'), " "('%s', %d, 0, 'proxysql131')"; + std::string bridge_addr = ""; + int bridge_res = exec("ip -4 addr show docker0 | grep -oP '(?<=inet\\s)\\d+(\\.\\d+){3}'", bridge_addr); + std::string update_proxysql_servers = ""; - string_format(t_update_proxysql_servers, update_proxysql_servers, cl.host, cl.admin_port, cl.host, 7032); + string_format(t_update_proxysql_servers, update_proxysql_servers, bridge_addr.substr(0, bridge_addr.size() - 1).c_str(), cl.admin_port, cl.host, 7032); // Setup the config file using the env variables in 'CommandLine' if (setup_config_file(cl)) { @@ -160,32 +164,24 @@ int main(int, char**) { // Launch proxysql with cluster config std::thread proxy_replica_th([&save_proxy_stderr, &cl] () { const std::string cluster_sync_node_stderr = std::string(cl.workdir) + "test_cluster_sync_config/cluster_sync_node_stderr.txt"; - const std::string fmt_config_file = std::string(cl.workdir) + "test_cluster_sync_config/test_cluster_sync.cnf"; + const std::string proxysql_db = std::string(cl.workdir) + "test_cluster_sync_config/proxysql.db"; + const std::string stats_db = std::string(cl.workdir) + "test_cluster_sync_config/proxysql_stats.db"; - std::string proxy_stdout = ""; - std::string proxy_stderr = ""; - int exec_res = wexecvp(std::string(cl.workdir) + "../../../src/proxysql", { "-f", "-M", "-c", fmt_config_file.c_str() }, NULL, proxy_stdout, proxy_stderr); + const std::string docker_command = + std::string("docker run -p 16032:6032 ") + "-v " + std::string(cl.workdir) + "../../../:/tmp/proxysql" + " ubuntu:19.10 sh -c \"./tmp/proxysql/src/proxysql -f -M -c /tmp/proxysql/test/tap/tests/test_cluster_sync_config/test_cluster_sync.cnf\" " + + std::string("> ") + cluster_sync_node_stderr + " 2>&1"; + int exec_res = system(docker_command.c_str()); ok(exec_res == 0, "proxysql cluster node should execute and shutdown nicely. 'wexecvp' result was: %d", exec_res); - // In case of error, log 'proxysql' stderr to a file - if (exec_res || save_proxy_stderr.load()) { - if (exec_res) { - diag("LOG: Proxysql cluster node execution failed, logging stderr into 'test_cluster_sync_node_stderr.txt'"); - } else { - diag("LOG: One of the tests failed to pass, logging stderr 'test_cluster_sync_node_stderr.txt'"); - } - - std::ofstream error_log_file {}; - error_log_file.open(cluster_sync_node_stderr); - error_log_file << proxy_stderr; - error_log_file.close(); - } + remove(proxysql_db.c_str()); + remove(stats_db.c_str()); }); // Waiting for proxysql to be ready uint con_waited = 0; - while (!mysql_real_connect(proxysql_replica, cl.host, cl.admin_username, cl.admin_password, NULL, 7032, NULL, 0) && con_waited < CONNECT_TIMEOUT) { + while (!mysql_real_connect(proxysql_replica, cl.host, "radmin", "radmin", NULL, 16032, NULL, 0) && con_waited < CONNECT_TIMEOUT) { mysql_close(proxysql_replica); proxysql_replica = mysql_init(NULL); @@ -206,6 +202,11 @@ int main(int, char**) { sleep(2); { + std::string print_master_galera_hostgroups = ""; + string_format(t_debug_query, print_master_galera_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_galera_hostgroups"); + std::string print_replica_galera_hostgroups = ""; + string_format(t_debug_query, print_replica_galera_hostgroups, "radmin", "radmin", cl.host, 16032, "SELECT * FROM runtime_mysql_galera_hostgroups"); + // Configure 'mysql_galera_hostgroups' and check sync with NULL comments const char* t_insert_mysql_galera_hostgroups = "INSERT INTO mysql_galera_hostgroups ( " @@ -273,6 +274,9 @@ int main(int, char**) { } MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_galera_hostgroups.c_str()); + // SYNCH CHECK // Sleep until timeout waiting for synchronization @@ -302,6 +306,8 @@ int main(int, char**) { } } + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_galera_hostgroups.c_str()); ok(not_synced_query == false, "'mysql_galera_hostgroups' with NULL comments should be synced."); // TEARDOWN CONFIG @@ -314,6 +320,11 @@ int main(int, char**) { sleep(2); { + std::string print_master_galera_hostgroups = ""; + string_format(t_debug_query, print_master_galera_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_galera_hostgroups"); + std::string print_replica_galera_hostgroups = ""; + string_format(t_debug_query, print_replica_galera_hostgroups, "radmin", "radmin", cl.host, 16032, "SELECT * FROM runtime_mysql_galera_hostgroups"); + // Configure 'mysql_galera_hostgroups' and check sync const char* t_insert_mysql_galera_hostgroups = "INSERT INTO mysql_galera_hostgroups ( " @@ -380,6 +391,8 @@ int main(int, char**) { MYSQL_QUERY__(proxysql_admin, query.c_str()); } MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_galera_hostgroups.c_str()); // Sleep until timeout waiting for synchronization uint waited = 0; @@ -408,6 +421,8 @@ int main(int, char**) { } } + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_galera_hostgroups.c_str()); ok(not_synced_query == false, "'mysql_galera_hostgroups' should be synced."); // TEARDOWN CONFIG @@ -421,6 +436,11 @@ int main(int, char**) { // Check 'mysql_group_replication_hostgroups' synchronization with NULL comments { + std::string print_master_group_replication_hostgroups = ""; + string_format(t_debug_query, print_master_group_replication_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_group_replication_hostgroups"); + std::string print_replica_group_replication_hostgroups = ""; + string_format(t_debug_query, print_replica_group_replication_hostgroups, "radmin", "radmin", cl.host, 16032, "SELECT * FROM runtime_mysql_group_replication_hostgroups"); + // Configure 'mysql_group_replication_hostgroups' and check sync const char* t_insert_mysql_group_replication_hostgroups = "INSERT INTO mysql_group_replication_hostgroups ( " @@ -487,6 +507,8 @@ int main(int, char**) { MYSQL_QUERY__(proxysql_admin, query.c_str()); } MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_group_replication_hostgroups.c_str()); // SYNCH CHECK @@ -516,6 +538,8 @@ int main(int, char**) { } } + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_group_replication_hostgroups.c_str()); ok(not_synced_query == false, "'mysql_group_replication_hostgroups' with NULL comments should be synced."); // TEARDOWN CONFIG @@ -529,6 +553,11 @@ int main(int, char**) { // Check 'mysql_group_replication_hostgroups' synchronization { + std::string print_master_group_replication_hostgroups = ""; + string_format(t_debug_query, print_master_group_replication_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_group_replication_hostgroups"); + std::string print_replica_group_replication_hostgroups = ""; + string_format(t_debug_query, print_replica_group_replication_hostgroups, "radmin", "radmin", cl.host, 16032, "SELECT * FROM runtime_mysql_group_replication_hostgroups"); + // Configure 'mysql_group_replication_hostgroups' and check sync const char* t_insert_mysql_group_replication_hostgroups = "INSERT INTO mysql_group_replication_hostgroups ( " @@ -597,6 +626,8 @@ int main(int, char**) { MYSQL_QUERY__(proxysql_admin, query.c_str()); } MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_group_replication_hostgroups.c_str()); uint waited = 0; bool not_synced_query = false; @@ -624,6 +655,8 @@ int main(int, char**) { } } + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_group_replication_hostgroups.c_str()); ok(not_synced_query == false, "'mysql_group_replication_hostgroups' should be synced."); // TEARDOWN CONFIG @@ -637,6 +670,11 @@ int main(int, char**) { // Check 'mysql_aws_aurora_hostgroups' synchronization with NULL comments { + std::string print_master_aws_aurora_hostgroups = ""; + string_format(t_debug_query, print_master_aws_aurora_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); + std::string print_replica_aws_aurora_hostgroups = ""; + string_format(t_debug_query, print_replica_aws_aurora_hostgroups, "radmin", "radmin", cl.host, 16032, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); + // Configure 'mysql_aws_aurora_hostgroups' and check sync const char* t_insert_mysql_aws_aurora_hostgroups = "INSERT INTO mysql_aws_aurora_hostgroups ( " @@ -713,6 +751,8 @@ int main(int, char**) { MYSQL_QUERY__(proxysql_admin, query.c_str()); } MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_aws_aurora_hostgroups.c_str()); // SYNCH CHECK @@ -742,6 +782,8 @@ int main(int, char**) { } } + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_aws_aurora_hostgroups.c_str()); ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' with NULL comments should be synced."); // TEARDOWN CONFIG @@ -755,6 +797,11 @@ int main(int, char**) { // Check 'mysql_aws_aurora_hostgroups' synchronization { + std::string print_master_aws_aurora_hostgroups = ""; + string_format(t_debug_query, print_master_aws_aurora_hostgroups, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); + std::string print_replica_aws_aurora_hostgroups = ""; + string_format(t_debug_query, print_replica_aws_aurora_hostgroups, "radmin", "radmin", cl.host, 16032, "SELECT * FROM runtime_mysql_aws_aurora_hostgroups"); + // Configure 'mysql_aws_aurora_hostgroups' and check sync const char* t_insert_mysql_aws_aurora_hostgroups = "INSERT INTO mysql_aws_aurora_hostgroups ( " @@ -831,6 +878,8 @@ int main(int, char**) { MYSQL_QUERY__(proxysql_admin, query.c_str()); } MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL SERVERS TO RUNTIME"); + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_aws_aurora_hostgroups.c_str()); uint waited = 0; bool not_synced_query = false; @@ -858,6 +907,8 @@ int main(int, char**) { } } + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_aws_aurora_hostgroups.c_str()); ok(not_synced_query == false, "'mysql_aws_aurora_hostgroups' should be synced."); // TEARDOWN CONFIG @@ -867,6 +918,331 @@ int main(int, char**) { MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL SERVERS TO RUNTIME"); } + sleep(2); + + // Check 'mysql_variables' synchronization + { + std::string print_master_mysql_variables = ""; + string_format(t_debug_query, print_master_mysql_variables, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_global_variables WHERE variable_name LIKE 'mysql-%'"); + std::string print_replica_mysql_variables = ""; + string_format(t_debug_query, print_replica_mysql_variables, "radmin", "radmin", cl.host, 16032, "SELECT * FROM runtime_global_variables WHERE variable_name LIKE 'mysql-%'"); + + // Configure 'mysql_mysql_variables_hostgroups' and check sync + const char* t_update_mysql_variables = + "UPDATE global_variables SET variable_value='%s' WHERE variable_name='%s'"; + std::vector> update_mysql_variables_values { + std::make_tuple("mysql-shun_on_failures" , "6" ), + std::make_tuple("mysql-shun_recovery_time_sec" , "11" ), + std::make_tuple("mysql-query_retries_on_failure" , "2" ), + std::make_tuple("mysql-client_multi_statements" , "true" ), + std::make_tuple("mysql-connect_retries_delay" , "2" ), + std::make_tuple("mysql-connection_delay_multiplex_ms" , "1" ), + std::make_tuple("mysql-connection_max_age_ms" , "1" ), + std::make_tuple("mysql-connect_timeout_server_max" , "10001" ), + std::make_tuple("mysql-eventslog_filename" , "" ), + std::make_tuple("mysql-eventslog_filesize" , "104857601" ), + std::make_tuple("mysql-eventslog_default_log" , "1" ), + std::make_tuple("mysql-eventslog_format" , "2" ), + std::make_tuple("mysql-auditlog_filename" , "" ), + std::make_tuple("mysql-auditlog_filesize" , "104857601" ), + std::make_tuple("mysql-handle_unknown_charset" , "2" ), + std::make_tuple("mysql-free_connections_pct" , "11" ), + std::make_tuple("mysql-connection_warming" , "true" ), // false + std::make_tuple("mysql-session_idle_ms" , "1001" ), + std::make_tuple("mysql-have_ssl" , "false" ), + std::make_tuple("mysql-client_found_rows" , "true" ), + std::make_tuple("mysql-monitor_enabled" , "true" ), + std::make_tuple("mysql-monitor_ping_max_failures" , "4" ), + std::make_tuple("mysql-monitor_ping_timeout" , "1001" ), + std::make_tuple("mysql-monitor_read_only_max_timeout_count" , "4" ), + std::make_tuple("mysql-monitor_replication_lag_interval" , "10001" ), + std::make_tuple("mysql-monitor_replication_lag_timeout" , "1001" ), + std::make_tuple("mysql-monitor_groupreplication_healthcheck_interval" , "5001" ), + std::make_tuple("mysql-monitor_groupreplication_healthcheck_timeout" , "801" ), + std::make_tuple("mysql-monitor_groupreplication_healthcheck_max_timeout_count" , "4" ), + std::make_tuple("mysql-monitor_groupreplication_max_transactions_behind_count" , "4" ), + std::make_tuple("mysql-monitor_galera_healthcheck_interval" , "5001" ), + std::make_tuple("mysql-monitor_galera_healthcheck_timeout" , "801" ), + std::make_tuple("mysql-monitor_galera_healthcheck_max_timeout_count" , "3" ), + std::make_tuple("mysql-monitor_replication_lag_use_percona_heartbeat" , "" ), + std::make_tuple("mysql-monitor_query_interval" , "60001" ), + std::make_tuple("mysql-monitor_query_timeout" , "101" ), + std::make_tuple("mysql-monitor_slave_lag_when_null" , "61" ), + std::make_tuple("mysql-monitor_threads_min" , "9" ), + std::make_tuple("mysql-monitor_threads_max" , "129" ), + std::make_tuple("mysql-monitor_threads_queue_maxsize" , "129" ), + std::make_tuple("mysql-monitor_wait_timeout" , "true" ), + std::make_tuple("mysql-monitor_writer_is_also_reader" , "true" ), + std::make_tuple("mysql-max_allowed_packet" , "67108864" ), + std::make_tuple("mysql-tcp_keepalive_time" , "0" ), + std::make_tuple("mysql-use_tcp_keepalive" , "0" ), + std::make_tuple("mysql-automatic_detect_sqli" , "0" ), + std::make_tuple("mysql-firewall_whitelist_enabled" , "0" ), + std::make_tuple("mysql-firewall_whitelist_errormsg" , "Firewall blocked this query"), + std::make_tuple("mysql-throttle_connections_per_sec_to_hostgroup" , "1000001" ), + std::make_tuple("mysql-max_transaction_time" , "14400001" ), + std::make_tuple("mysql-multiplexing" , "true" ), + std::make_tuple("mysql-log_unhealthy_connections" , "true" ), + std::make_tuple("mysql-forward_autocommit" , "false" ), + std::make_tuple("mysql-enforce_autocommit_on_reads" , "false" ), + std::make_tuple("mysql-autocommit_false_not_reusable" , "false" ), + std::make_tuple("mysql-autocommit_false_is_transaction" , "true" ), // false + std::make_tuple("mysql-verbose_query_error" , "true" ), // false + std::make_tuple("mysql-hostgroup_manager_verbose" , "2" ), + std::make_tuple("mysql-binlog_reader_connect_retry_msec" , "3001" ), + std::make_tuple("mysql-threshold_query_length" , "524289" ), + std::make_tuple("mysql-threshold_resultset_size" , "4194305" ), + std::make_tuple("mysql-query_digests_max_digest_length" , "2049" ), + std::make_tuple("mysql-query_digests_max_query_length" , "65001" ), + std::make_tuple("mysql-query_digests_grouping_limit" , "4" ), + std::make_tuple("mysql-wait_timeout" , "28800001" ), + std::make_tuple("mysql-throttle_max_bytes_per_second_to_client" , "1" ), + std::make_tuple("mysql-throttle_ratio_server_to_client" , "1" ), + std::make_tuple("mysql-max_stmts_per_connection" , "21" ), + std::make_tuple("mysql-max_stmts_cache" , "10001" ), + std::make_tuple("mysql-mirror_max_concurrency" , "17" ), + std::make_tuple("mysql-mirror_max_queue_length" , "32007" ), + std::make_tuple("mysql-default_max_latency_ms" , "1001" ), + std::make_tuple("mysql-query_processor_iterations" , "1" ), + std::make_tuple("mysql-query_processor_regex" , "2" ), + std::make_tuple("mysql-set_query_lock_on_hostgroup" , "0" ), // 1 + std::make_tuple("mysql-reset_connection_algorithm" , "1" ), // 2 + std::make_tuple("mysql-auto_increment_delay_multiplex" , "6" ), + std::make_tuple("mysql-long_query_time" , "1001" ), // here + std::make_tuple("mysql-query_cache_size_MB" , "256" ), + std::make_tuple("mysql-poll_timeout_on_failure" , "100" ), + std::make_tuple("mysql-keep_multiplexing_variables" , "tx_isolation,version" ), + std::make_tuple("mysql-kill_backend_connection_when_disconnect" , "true" ), + std::make_tuple("mysql-client_session_track_gtid" , "true" ), + std::make_tuple("mysql-session_idle_show_processlist" , "true" ), + std::make_tuple("mysql-show_processlist_extended" , "0" ), + std::make_tuple("mysql-query_digests" , "true" ), + std::make_tuple("mysql-query_digests_lowercase" , "false" ), + std::make_tuple("mysql-query_digests_replace_null" , "false" ), + std::make_tuple("mysql-query_digests_no_digits" , "false" ), + std::make_tuple("mysql-query_digests_normalize_digest_text" , "false" ), + std::make_tuple("mysql-query_digests_track_hostname" , "false" ), + std::make_tuple("mysql-servers_stats" , "true" ), + std::make_tuple("mysql-default_reconnect" , "true" ), + std::make_tuple("mysql-session_debug" , "true" ), + std::make_tuple("mysql-ssl_p2s_ca" , "" ), + std::make_tuple("mysql-ssl_p2s_cert" , "" ), + std::make_tuple("mysql-ssl_p2s_key" , "" ), + std::make_tuple("mysql-ssl_p2s_cipher" , "" ), + std::make_tuple("mysql-init_connect" , "" ), + std::make_tuple("mysql-ldap_user_variable" , "" ), + std::make_tuple("mysql-add_ldap_user_comment" , "" ), + std::make_tuple("mysql-default_tx_isolation" , "READ-COMMITTED" ), + std::make_tuple("mysql-default_session_track_gtids" , "OFF" ), + std::make_tuple("mysql-connpoll_reset_queue_length" , "50" ), + std::make_tuple("mysql-min_num_servers_lantency_awareness" , "1000" ), + std::make_tuple("mysql-aurora_max_lag_ms_only_read_from_replicas" , "2" ), + std::make_tuple("mysql-stats_time_backend_query" , "false" ), + std::make_tuple("mysql-stats_time_query_processor" , "false" ), + std::make_tuple("mysql-query_cache_stores_empty_result" , "true" ), + std::make_tuple("mysql-threads" , "8" ), + std::make_tuple("mysql-max_connections" , "2048" ), + // std::make_tuple("mysql-server_capabilities" , "569866" ), + }; + std::vector update_mysql_variables_queries {}; + + for (auto const& values : update_mysql_variables_values) { + std::string update_mysql_variables_query = ""; + string_format( + t_update_mysql_variables, + update_mysql_variables_query, + std::get<1>(values), + std::get<0>(values) + ); + update_mysql_variables_queries.push_back(update_mysql_variables_query); + } + + const char* t_select_mysql_variables_query = + "SELECT COUNT(*) FROM global_variables WHERE variable_name='%s' AND variable_value='%s'"; + + std::vector select_mysql_variables_queries {}; + + for (auto const& values : update_mysql_variables_values) { + std::string select_mysql_variables_query = ""; + string_format( + t_select_mysql_variables_query, + select_mysql_variables_query, + std::get<0>(values), + std::get<1>(values) + ); + select_mysql_variables_queries.push_back(select_mysql_variables_query); + } + + // Backup current table + MYSQL_QUERY__(proxysql_admin, "CREATE TABLE global_variables_sync_test_2687 AS SELECT * FROM global_variables WHERE variable_name LIKE 'mysql-%'"); + MYSQL_QUERY__(proxysql_admin, "DELETE FROM global_variables WHERE variable_name LIKE 'mysql-%'"); + + for (const auto& query : update_mysql_variables_queries) { + MYSQL_QUERY__(proxysql_admin, query.c_str()); + } + MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_mysql_variables.c_str()); + + uint waited = 0; + bool not_synced_query = false; + while (waited < SYNC_TIMEOUT) { + not_synced_query = false; + // Check that all the entries have been synced + for (const auto& query : select_mysql_variables_queries) { + MYSQL_QUERY__(proxysql_replica, query.c_str()); + MYSQL_RES* mysql_vars_res = mysql_store_result(proxysql_replica); + MYSQL_ROW row = mysql_fetch_row(mysql_vars_res); + int row_value = atoi(row[0]); + mysql_free_result(mysql_vars_res); + + if (row_value == 0) { + not_synced_query = true; + break; + } + } + + if (not_synced_query) { + waited += 1; + sleep(1); + } else { + break; + } + } + + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_mysql_variables.c_str()); + ok(not_synced_query == false, "'mysql_variables' from global_variables should be synced."); + + MYSQL_QUERY__(proxysql_admin, "INSERT OR REPLACE INTO global_variables SELECT * FROM global_variables_sync_test_2687 WHERE variable_name LIKE 'mysql-%'"); + MYSQL_QUERY__(proxysql_admin, "DROP TABLE global_variables_sync_test_2687"); + MYSQL_QUERY__(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + } + + sleep(2); + + // Check 'admin_variables' synchronization + { + std::string print_master_admin_variables = ""; + string_format(t_debug_query, print_master_admin_variables, cl.admin_username, cl.admin_password, cl.host, cl.admin_port, "SELECT * FROM runtime_global_variables WHERE variable_name LIKE 'admin-%'"); + std::string print_replica_admin_variables = ""; + string_format(t_debug_query, print_replica_admin_variables, "radmin", "radmin", cl.host, 16032, "SELECT * FROM runtime_global_variables WHERE variable_name LIKE 'admin-%'"); + + // Configure 'mysql_admin_variables_hostgroups' and check sync + const char* t_update_admin_variables = + "UPDATE global_variables SET variable_value='%s' WHERE variable_name='%s'"; + std::vector> update_admin_variables_values { + std::make_tuple("admin-admin_credentials" , "admin:admin;radmin:radmin" ), + std::make_tuple("admin-checksum_admin_variables" , "true" ), + std::make_tuple("admin-checksum_mysql_query_rules" , "true" ), + std::make_tuple("admin-checksum_mysql_servers" , "true" ), + std::make_tuple("admin-checksum_mysql_users" , "true" ), + std::make_tuple("admin-checksum_mysql_variables" , "true" ), + std::make_tuple("admin-cluster_admin_variables_diffs_before_sync" , "4" ), + std::make_tuple("admin-cluster_admin_variables_save_to_disk" , "true" ), + std::make_tuple("admin-cluster_check_interval_ms" , "1001" ), + std::make_tuple("admin-cluster_check_status_frequency" , "11" ), + std::make_tuple("admin-cluster_mysql_query_rules_diffs_before_sync", "4" ), + std::make_tuple("admin-cluster_mysql_query_rules_save_to_disk" , "true" ), + std::make_tuple("admin-cluster_mysql_servers_diffs_before_sync" , "4" ), + std::make_tuple("admin-cluster_mysql_servers_save_to_disk" , "true" ), + std::make_tuple("admin-cluster_mysql_users_diffs_before_sync" , "4" ), + std::make_tuple("admin-cluster_mysql_users_save_to_disk" , "true" ), + std::make_tuple("admin-cluster_mysql_variables_diffs_before_sync" , "4" ), + std::make_tuple("admin-cluster_mysql_variables_save_to_disk" , "true" ), + std::make_tuple("admin-cluster_password" , "" ), + std::make_tuple("admin-cluster_proxysql_servers_diffs_before_sync" , "4" ), + std::make_tuple("admin-cluster_proxysql_servers_save_to_disk" , "true" ), + std::make_tuple("admin-cluster_username" , "" ), + std::make_tuple("admin-debug" , "false" ), + std::make_tuple("admin-hash_passwords" , "true" ), + std::make_tuple("admin-mysql_ifaces" , "0.0.0.0:6032" ), + std::make_tuple("admin-prometheus_memory_metrics_interval" , "61" ), + std::make_tuple("admin-read_only" , "false" ), + std::make_tuple("admin-refresh_interval" , "2001" ), + std::make_tuple("admin-restapi_enabled" , "false" ), + std::make_tuple("admin-restapi_port" , "6071" ), + std::make_tuple("admin-stats_credentials" , "stats:stats" ), + std::make_tuple("admin-vacuum_stats" , "true" ), + std::make_tuple("admin-version" , "2.1.0-231-gbc0963e3_DEBUG" ), + std::make_tuple("admin-web_enabled" , "false" ), + std::make_tuple("admin-web_port" , "6080" ) + }; + std::vector update_admin_variables_queries {}; + + for (auto const& values : update_admin_variables_values) { + std::string update_admin_variables_query = ""; + string_format( + t_update_admin_variables, + update_admin_variables_query, + std::get<1>(values), + std::get<0>(values) + ); + update_admin_variables_queries.push_back(update_admin_variables_query); + } + + const char* t_select_admin_variables_query = + "SELECT COUNT(*) FROM global_variables WHERE variable_name='%s' AND variable_value='%s'"; + + std::vector select_admin_variables_queries {}; + + for (auto const& values : update_admin_variables_values) { + std::string select_admin_variables_query = ""; + string_format( + t_select_admin_variables_query, + select_admin_variables_query, + std::get<0>(values), + std::get<1>(values) + ); + select_admin_variables_queries.push_back(select_admin_variables_query); + } + + // Backup current table + MYSQL_QUERY__(proxysql_admin, "CREATE TABLE global_variables_sync_test_2687 AS SELECT * FROM global_variables WHERE variable_name LIKE 'admin-%'"); + MYSQL_QUERY__(proxysql_admin, "DELETE FROM global_variables WHERE variable_name LIKE 'admin-%'"); + + for (const auto& query : update_admin_variables_queries) { + MYSQL_QUERY__(proxysql_admin, query.c_str()); + } + MYSQL_QUERY__(proxysql_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; + system(print_master_admin_variables.c_str()); + + uint waited = 0; + bool not_synced_query = false; + while (waited < SYNC_TIMEOUT) { + not_synced_query = false; + // Check that all the entries have been synced + for (const auto& query : select_admin_variables_queries) { + MYSQL_QUERY__(proxysql_replica, query.c_str()); + MYSQL_RES* admin_vars_res = mysql_store_result(proxysql_replica); + MYSQL_ROW row = mysql_fetch_row(admin_vars_res); + int row_value = atoi(row[0]); + mysql_free_result(admin_vars_res); + + if (row_value == 0) { + not_synced_query = true; + break; + } + } + + if (not_synced_query) { + waited += 1; + sleep(1); + } else { + break; + } + } + + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; + system(print_replica_admin_variables.c_str()); + ok(not_synced_query == false, "'admin_variables' from global_variables should be synced."); + + MYSQL_QUERY__(proxysql_admin, "INSERT OR REPLACE INTO global_variables SELECT * FROM global_variables_sync_test_2687 WHERE variable_name LIKE 'mysql-%'"); + MYSQL_QUERY__(proxysql_admin, "DROP TABLE global_variables_sync_test_2687"); + MYSQL_QUERY__(proxysql_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + } + cleanup: // Teardown config @@ -881,7 +1257,7 @@ int main(int, char**) { mysql_query(proxysql_replica, "PROXYSQL SHUTDOWN"); proxy_replica_th.join(); - // remove(fmt_config_file.c_str()); + remove(fmt_config_file.c_str()); MYSQL_QUERY(proxysql_admin, "DELETE FROM proxysql_servers"); MYSQL_QUERY(proxysql_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); diff --git a/test/tap/tests/test_cluster_sync_config/test_cluster_sync-t.cnf b/test/tap/tests/test_cluster_sync_config/test_cluster_sync-t.cnf index fda838edf6..00a3227477 100644 --- a/test/tap/tests/test_cluster_sync_config/test_cluster_sync-t.cnf +++ b/test/tap/tests/test_cluster_sync_config/test_cluster_sync-t.cnf @@ -1,9 +1,9 @@ -datadir="" +datadir="/tmp/proxysql/test/tap/tests/test_cluster_sync_config" admin_variables= { admin_credentials="admin:admin;radmin:radmin" - mysql_ifaces="0.0.0.0:7032" + mysql_ifaces="0.0.0.0:6032" cluster_username="radmin" cluster_password="radmin" cluster_check_interval_ms=200 @@ -15,6 +15,7 @@ admin_variables= cluster_mysql_query_rules_diffs_before_sync=3 cluster_mysql_servers_diffs_before_sync=3 cluster_mysql_users_diffs_before_sync=3 + cluster_admin_variables_diffs_before_sync=3 cluster_proxysql_servers_diffs_before_sync=3 } @@ -26,7 +27,7 @@ mysql_variables= default_query_timeout=36000000 have_compress=true poll_timeout=2000 - interfaces="0.0.0.0:7033" + interfaces="0.0.0.0:6033" default_schema="information_schema" stacksize=1048576 server_version="5.5.30" From e158418b365b224bd853c4478c90718043a18a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Wed, 3 Jun 2020 16:17:13 +0200 Subject: [PATCH 7/9] Commented several variables known to not-sync and fixed port for replica 'proxysql_server' --- test/tap/tests/test_cluster_sync-t.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 4e1003144e..27020377d6 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -90,7 +90,7 @@ int setup_config_file(const CommandLine& cl) { int fhost_res = config_setting_set_string(f_pserver_hostname, bridge_addr.substr(0, bridge_addr.size() - 1).c_str()); int fport_res = config_setting_set_int(f_pserver_port, cl.admin_port); int shost_res = config_setting_set_string(s_pserver_hostname, cl.host); - int sport_res = config_setting_set_int(s_pserver_port, 7032); + int sport_res = config_setting_set_int(s_pserver_port, 6032); if (fhost_res == CONFIG_FALSE || fport_res == CONFIG_FALSE || shost_res == CONFIG_FALSE || sport_res == CONFIG_FALSE) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, "Invalid config file - Error while trying to set the values from env variables."); @@ -149,7 +149,7 @@ int main(int, char**) { int bridge_res = exec("ip -4 addr show docker0 | grep -oP '(?<=inet\\s)\\d+(\\.\\d+){3}'", bridge_addr); std::string update_proxysql_servers = ""; - string_format(t_update_proxysql_servers, update_proxysql_servers, bridge_addr.substr(0, bridge_addr.size() - 1).c_str(), cl.admin_port, cl.host, 7032); + string_format(t_update_proxysql_servers, update_proxysql_servers, bridge_addr.substr(0, bridge_addr.size() - 1).c_str(), cl.admin_port, cl.host, 6032); // Setup the config file using the env variables in 'CommandLine' if (setup_config_file(cl)) { @@ -1150,10 +1150,10 @@ int main(int, char**) { std::make_tuple("admin-cluster_mysql_users_save_to_disk" , "true" ), std::make_tuple("admin-cluster_mysql_variables_diffs_before_sync" , "4" ), std::make_tuple("admin-cluster_mysql_variables_save_to_disk" , "true" ), - std::make_tuple("admin-cluster_password" , "" ), std::make_tuple("admin-cluster_proxysql_servers_diffs_before_sync" , "4" ), std::make_tuple("admin-cluster_proxysql_servers_save_to_disk" , "true" ), - std::make_tuple("admin-cluster_username" , "" ), + // std::make_tuple("admin-cluster_username" , "" ), Known issue, can't clear + // std::make_tuple("admin-cluster_password" , "" ), Known issue, can't clear std::make_tuple("admin-debug" , "false" ), std::make_tuple("admin-hash_passwords" , "true" ), std::make_tuple("admin-mysql_ifaces" , "0.0.0.0:6032" ), @@ -1164,7 +1164,7 @@ int main(int, char**) { std::make_tuple("admin-restapi_port" , "6071" ), std::make_tuple("admin-stats_credentials" , "stats:stats" ), std::make_tuple("admin-vacuum_stats" , "true" ), - std::make_tuple("admin-version" , "2.1.0-231-gbc0963e3_DEBUG" ), + // std::make_tuple("admin-version" , "2.1.0-231-gbc0963e3_DEBUG" ), This changes at runtime, but it's not stored std::make_tuple("admin-web_enabled" , "false" ), std::make_tuple("admin-web_port" , "6080" ) }; From 7214ef47505f6c9c671c5b667ba58883a9694613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 20 Oct 2020 22:39:44 +0200 Subject: [PATCH 8/9] Removed deprecated and non-syncable variables from 'test_cluster_sync-t' --- test/tap/tests/test_cluster_sync-t.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 27020377d6..815d50d4f6 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -1024,7 +1024,7 @@ int main(int, char**) { std::make_tuple("mysql-query_digests_track_hostname" , "false" ), std::make_tuple("mysql-servers_stats" , "true" ), std::make_tuple("mysql-default_reconnect" , "true" ), - std::make_tuple("mysql-session_debug" , "true" ), + // std::make_tuple("mysql-session_debug" , "true" ), Deprecated std::make_tuple("mysql-ssl_p2s_ca" , "" ), std::make_tuple("mysql-ssl_p2s_cert" , "" ), std::make_tuple("mysql-ssl_p2s_key" , "" ), @@ -1154,7 +1154,7 @@ int main(int, char**) { std::make_tuple("admin-cluster_proxysql_servers_save_to_disk" , "true" ), // std::make_tuple("admin-cluster_username" , "" ), Known issue, can't clear // std::make_tuple("admin-cluster_password" , "" ), Known issue, can't clear - std::make_tuple("admin-debug" , "false" ), + // std::make_tuple("admin-debug" , "false" ), Should not be synced std::make_tuple("admin-hash_passwords" , "true" ), std::make_tuple("admin-mysql_ifaces" , "0.0.0.0:6032" ), std::make_tuple("admin-prometheus_memory_metrics_interval" , "61" ), From 4a5ce62cce7e9774d8c9adbc5982e91c1aad5f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 20 Oct 2020 22:43:21 +0200 Subject: [PATCH 9/9] Added extra logging for variables sync for 'test_cluster_sync-t' --- test/tap/tests/test_cluster_sync-t.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index 815d50d4f6..b485b772ff 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -1086,6 +1086,7 @@ int main(int, char**) { uint waited = 0; bool not_synced_query = false; + std::string last_failed_query {}; while (waited < SYNC_TIMEOUT) { not_synced_query = false; // Check that all the entries have been synced @@ -1097,6 +1098,7 @@ int main(int, char**) { mysql_free_result(mysql_vars_res); if (row_value == 0) { + last_failed_query = query; not_synced_query = true; break; } @@ -1110,6 +1112,10 @@ int main(int, char**) { } } + if (not_synced_query) { + std::cout << "FAILED_SYNC_CHECK: '" << last_failed_query << "'\n"; + } + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; system(print_replica_mysql_variables.c_str()); ok(not_synced_query == false, "'mysql_variables' from global_variables should be synced."); @@ -1210,6 +1216,7 @@ int main(int, char**) { uint waited = 0; bool not_synced_query = false; + std::string last_failed_query {}; while (waited < SYNC_TIMEOUT) { not_synced_query = false; // Check that all the entries have been synced @@ -1222,6 +1229,7 @@ int main(int, char**) { if (row_value == 0) { not_synced_query = true; + last_failed_query = query; break; } } @@ -1234,6 +1242,10 @@ int main(int, char**) { } } + if (not_synced_query) { + std::cout << "FAILED_SYNC_CHECK: '" << last_failed_query << "'\n"; + } + std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; system(print_replica_admin_variables.c_str()); ok(not_synced_query == false, "'admin_variables' from global_variables should be synced.");