Skip to content

Commit

Permalink
Merge pull request #4216 from sysown/v2.x-track_slow_query_vars
Browse files Browse the repository at this point in the history 
Track some slow query variables
  • Loading branch information
@renecannao
renecannao authored Jun 19, 2023
2 parents a876e27 + 73ac61b commit 86ce115
Show file tree
Hide file tree
Showing 6 changed files with 52 additions and 32 deletions.
15 changes: 11 additions & 4 deletions include/proxysql_structs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ enum mysql_variable_name {
SQL_LC_MESSAGES,
SQL_LC_TIME_NAMES,
SQL_LOCK_WAIT_TIMEOUT,
SQL_LOG_SLOW_FILTER,
SQL_LONG_QUERY_TIME,
SQL_MAX_EXECUTION_TIME,
SQL_MAX_HEAP_TABLE_SIZE,
Expand All @@ -213,6 +214,7 @@ enum mysql_variable_name {
SQL_GENERATE_INVISIBLE_PRIMARY_KEY,
SQL_SQL_LOG_BIN,
SQL_SQL_MODE,
SQL_QUOTE_SHOW_CREATE,
SQL_REQUIRE_PRIMARY_KEY,
SQL_SQL_SAFE_UPDATES,
SQL_SQL_SELECT_LIMIT,
Expand Down Expand Up @@ -549,7 +551,7 @@ struct __SQP_query_parser_t {
struct _PtrSize_t {
unsigned int size;
void *ptr;
};
};
// struct for debugging module
#ifdef DEBUG
struct _debug_level {
Expand Down Expand Up @@ -600,7 +602,7 @@ struct mysql_protocol_events {

// this struct define global variable entries, and how these are configured during startup
struct _global_variable_entry_t {
const char *group_name; // [group name] in proxysql.cnf
const char *group_name; // [group name] in proxysql.cnf
const char *key_name; // key name
int dynamic; // if dynamic > 0 , reconfigurable
//GOptionArg arg; // type of variable
Expand Down Expand Up @@ -762,7 +764,7 @@ EXTERN global_variables glovars;
#ifndef GLOBAL_DEFINED_OPTS_ENTRIES
#define GLOBAL_DEFINED_OPTS_ENTRIES
ProxySQL_GlobalVariables GloVars {};
#endif // GLOBAL_DEFINED_OPTS_ENTRIES
#endif // GLOBAL_DEFINED_OPTS_ENTRIES
#ifndef GLOBAL_DEFINED_HOSTGROUP
#define GLOBAL_DEFINED_HOSTGROUP
MySQL_HostGroups_Manager *MyHGM;
Expand Down Expand Up @@ -1155,6 +1157,10 @@ mysql_variable_st mysql_tracked_variables[] {
{ SQL_LC_MESSAGES, SETTING_VARIABLE, true, false, false, false, (char *)"lc_messages", NULL, (char *)"" , false} ,
{ SQL_LC_TIME_NAMES, SETTING_VARIABLE, true, false, false, false, (char *)"lc_time_names", NULL, (char *)"" , false} ,
{ SQL_LOCK_WAIT_TIMEOUT, SETTING_VARIABLE, false, false, true, false, (char *)"lock_wait_timeout", NULL, (char *)"" , false} ,
// log_queries_not_using_indexes is not enabled because in MySQL it is *only* a global variable, while in MariaDB is a global *and* session variable .
// We believe it is not the time to create a lot of exceptions and complex logic for conflicting backend implementations
// { SQL_LOG_QUERIES_NOT_USING_INDEXES, SETTING_VARIABLE, false, false, false, true, (char *)"log_queries_not_using_indexes", NULL, (char *)"OFF" , false} ,
{ SQL_LOG_SLOW_FILTER, SETTING_VARIABLE, true, false, false, false, (char *)"log_slow_filter", NULL, (char *)"" , false} ,
{ SQL_LONG_QUERY_TIME, SETTING_VARIABLE, false, false, true, false, (char *)"long_query_time", NULL, (char *)"" , false} ,
{ SQL_MAX_EXECUTION_TIME, SETTING_VARIABLE, false, false, true, false, (char *)"max_execution_time", NULL, (char *)"" , false} ,
{ SQL_MAX_HEAP_TABLE_SIZE, SETTING_VARIABLE, false, false, true, false, (char *)"max_heap_table_size", NULL, (char *)"18446744073709547520" , false} ,
Expand All @@ -1173,7 +1179,8 @@ mysql_variable_st mysql_tracked_variables[] {
{ SQL_GENERATE_INVISIBLE_PRIMARY_KEY, SETTING_VARIABLE, false, false, false, true, (char *)"sql_generate_invisible_primary_key", NULL, (char *)"" , false} ,
{ SQL_SQL_LOG_BIN, SETTING_VARIABLE, false, false, false, true, (char *)"sql_log_bin", NULL, (char *)"ON" , false} ,
{ SQL_SQL_MODE, SETTING_VARIABLE, true, false, false, false, (char *)"sql_mode" , NULL, (char *)"" , false} ,
{ SQL_REQUIRE_PRIMARY_KEY, SETTING_VARIABLE, false, false, false, true, (char *)"sql_require_primary_key", NULL, (char *)"" , false} ,
{ SQL_QUOTE_SHOW_CREATE, SETTING_VARIABLE, false, false, false, true, (char *)"sql_quote_show_create", NULL, (char *)"" , false} ,
{ SQL_REQUIRE_PRIMARY_KEY, SETTING_VARIABLE, false, false, false, true, (char *)"sql_require_primary_key", NULL, (char *)"" , false} ,
{ SQL_SQL_SAFE_UPDATES, SETTING_VARIABLE, true, false, false, true, (char *)"sql_safe_updates", NULL, (char *)"OFF" , false} ,
{ SQL_SQL_SELECT_LIMIT, SETTING_VARIABLE, false, false, true, false, (char *)"sql_select_limit", NULL, (char *)"DEFAULT" , false} ,
{ SQL_TIME_ZONE, SETTING_VARIABLE, true, false, false, false, (char *)"time_zone", NULL, (char *)"SYSTEM" , false} ,
Expand Down
50 changes: 26 additions & 24 deletions lib/MySQL_Session.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ static const std::set<std::string> mysql_variables_boolean = {
"sql_big_selects",
"sql_generate_invisible_primary_key",
"sql_log_bin",
"sql_quote_show_create",
"sql_require_primary_key",
"sql_safe_updates",
"unique_checks",
Expand Down Expand Up @@ -114,6 +115,7 @@ static const std::set<std::string> mysql_variables_strings = {
"group_replication_consistency",
"lc_messages",
"lc_time_names",
"log_slow_filter",
"optimizer_switch",
"wsrep_osu_method",
};
Expand Down Expand Up @@ -209,10 +211,10 @@ KillArgs::~KillArgs() {

const char* KillArgs::get_host_address() const {
const char* host_address = hostname;

if (ip_addr)
host_address = ip_addr;

return host_address;
}

Expand Down Expand Up @@ -812,7 +814,7 @@ void MySQL_Session::writeout() {
if (client_myds) client_myds->array2buffer_full();
if (mybe && mybe->server_myds && mybe->server_myds->myds_type==MYDS_BACKEND) {
if (session_type==PROXYSQL_SESSION_MYSQL) {
if (mybe->server_myds->net_failure==false) {
if (mybe->server_myds->net_failure==false) {
if (mybe->server_myds->poll_fds_idx>-1) { // NOTE: attempt to force writes
mybe->server_myds->array2buffer_full();
}
Expand Down Expand Up @@ -2819,7 +2821,7 @@ bool MySQL_Session::handler_again___status_CHANGING_SCHEMA(int *_rc) {
}


bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
//fprintf(stderr,"CONNECTING_SERVER\n");
unsigned long long curtime=monotonic_time();
thread->atomic_curtime=curtime;
Expand Down Expand Up @@ -2880,7 +2882,7 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
if (mirror) {
PROXY_TRACE();
NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA);
}
}
}

// NOTE-connect_retries_delay: This check alone is not enough for imposing
Expand Down Expand Up @@ -2966,7 +2968,7 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
}
if (mirror) {
PROXY_TRACE();
}
}
myds->destroy_MySQL_Connection_From_Pool(false);
// NOTE-connect_retries_delay: In case of failure to connect, if
// 'mysql_thread___connect_retries_delay' is set, we impose a delay in the session
Expand Down Expand Up @@ -4152,7 +4154,7 @@ int MySQL_Session::get_pkts_from_client(bool& wrong_pass, PtrSize_t& pkt) {
handler_ret = -1;
return handler_ret;
break;
}
}
break;
case FAST_FORWARD:
mybe->server_myds->PSarrayOUT->add(pkt.ptr, pkt.size);
Expand Down Expand Up @@ -5222,7 +5224,7 @@ int MySQL_Session::handler() {


__exit_DSS__STATE_NOT_INITIALIZED:


if (mybe && mybe->server_myds) {
if (mybe->server_myds->DSS > STATE_MARIADB_BEGIN && mybe->server_myds->DSS < STATE_MARIADB_END) {
Expand Down Expand Up @@ -5362,7 +5364,7 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION,8,"Session=%p , DS=%p . Returning\n", this, client_myds);
return;
}

if (
(is_encrypted == false) && // the connection was encrypted
(handshake_response_return == false) && // the authentication didn't complete
Expand All @@ -5375,16 +5377,16 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
client_myds->wbio_ssl = BIO_new(BIO_s_mem());
client_myds->ssl = GloVars.get_SSL_new();
SSL_set_fd(client_myds->ssl, client_myds->fd);
SSL_set_accept_state(client_myds->ssl);
SSL_set_accept_state(client_myds->ssl);
SSL_set_bio(client_myds->ssl, client_myds->rbio_ssl, client_myds->wbio_ssl);
l_free(pkt->size,pkt->ptr);
proxysql_keylog_attach_callback(GloVars.get_SSL_ctx());
return;
}

if (
//(client_myds->myprot.process_pkt_handshake_response((unsigned char *)pkt->ptr,pkt->size)==true)
(handshake_response_return == true)
if (
//(client_myds->myprot.process_pkt_handshake_response((unsigned char *)pkt->ptr,pkt->size)==true)
(handshake_response_return == true)
&&
(
#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP)
Expand Down Expand Up @@ -5570,7 +5572,7 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(
if (use_ssl == true && is_encrypted == false) {
*wrong_pass=true;
GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_ERR, this, NULL);

char *_a=(char *)"ProxySQL Error: Access denied for user '%s' (using password: %s). SSL is required";
char *_s=(char *)malloc(strlen(_a)+strlen(client_myds->myconn->userinfo->username)+32);
sprintf(_s, _a, client_myds->myconn->userinfo->username, (client_myds->myconn->userinfo->password ? "YES" : "NO"));
Expand Down Expand Up @@ -6321,7 +6323,7 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
}
if (mysql_variables.client_get_hash(this, idx) != var_value_int) {
const MARIADB_CHARSET_INFO *ci = NULL;
if (var == "character_set_results" || var == "character_set_connection" ||
if (var == "character_set_results" || var == "character_set_connection" ||
var == "character_set_client" || var == "character_set_database") {
ci = proxysql_find_charset_name(value1.c_str());
}
Expand Down Expand Up @@ -6832,7 +6834,7 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
l_free(pkt->size,pkt->ptr);
client_myds->setDSS_STATE_QUERY_SENT_NET();
client_myds->myprot.generate_statistics_response(true,NULL,NULL);
client_myds->DSS=STATE_SLEEP;
client_myds->DSS=STATE_SLEEP;
}

void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_CHANGE_USER(PtrSize_t *pkt, bool *wrong_pass) {
Expand Down Expand Up @@ -7317,7 +7319,7 @@ void MySQL_Session::SQLite3_to_MySQL(SQLite3_result *result, char *error, int af
myds->DSS=STATE_SLEEP;
free(l);
free(p);

} else { // no result set
if (error) {
// there was an error
Expand Down Expand Up @@ -7462,7 +7464,7 @@ unsigned long long MySQL_Session::IdleTime() {


// this is called either from RequestEnd(), or at the end of executing
// prepared statements
// prepared statements
void MySQL_Session::LogQuery(MySQL_Data_Stream *myds) {
// we need to access statistics before calling CurrentQuery.end()
// so we track the time here
Expand Down Expand Up @@ -7819,18 +7821,18 @@ bool MySQL_Session::known_query_for_locked_on_hostgroup(uint64_t digest) {
case 5915334213354374281ULL: // "SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0"
case 7837089204483965579ULL: // "SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO'"
case 4312882378746554890ULL: // "SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0"
case 4379922288366515816ULL: // "SET @rocksdb_get_is_supported = IF (@rocksdb_has_p_s_session_variables, 'SELECT COUNT(*) INTO @rocksdb_is_supported FROM performance_schema.session_variables WHERE VARIABLE_NAME...
case 4379922288366515816ULL: // "SET @rocksdb_get_is_supported = IF (@rocksdb_has_p_s_session_variables, 'SELECT COUNT(*) INTO @rocksdb_is_supported FROM performance_schema.session_variables WHERE VARIABLE_NAME...
case 12687634401278615449ULL: // "SET @rocksdb_enable_bulk_load = IF (@rocksdb_is_supported, 'SET SESSION rocksdb_bulk_load = 1', 'SET @rocksdb_dummy_bulk_load = 0')"
case 15991633859978935883ULL: // "SET @MYSQLDUMP_TEMP_LOG_BIN = @@SESSION.SQL_LOG_BIN"
case 10636751085721966716ULL: // "SET @@GLOBAL.GTID_PURGED=?"
case 15976043181199829579ULL: // "SET SQL_QUOTE_SHOW_CREATE=?"
case 12094956190640701942ULL: // "SET SESSION information_schema_stats_expiry=0"
/*
case ULL: //
case ULL: //
case ULL: //
case ULL: //
case ULL: //
case ULL: //
case ULL: //
case ULL: //
case ULL: //
case ULL: //
*/
ret = true;
break;
Expand Down
13 changes: 10 additions & 3 deletions test/tap/tests/generate_set_session_csv.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ int main() {
vars["sql_log_bin"]->add(bool_values);
vars["sql_safe_updates"] = new variable("sql_safe_updates", true, false, true);
vars["sql_safe_updates"]->add(bool_values);
vars["sql_big_selects"] = new variable("sql_safe_updates", true, false, true);
vars["sql_big_selects"] = new variable("sql_big_selects", true, false, true);
vars["sql_big_selects"]->add(bool_values);
// vars["wsrep_sync_wait"] = new variable("wsrep_sync_wait", true, false);
// vars["wsrep_sync_wait"]->add(bool_values);
Expand Down Expand Up @@ -231,8 +231,8 @@ int main() {
vars["session_track_gtids"]->add("OWN_GTID");
// vars["session_track_gtids"]->add("OFF");
// vars["session_track_gtids"]->add("ALL_GTID");


vars["optimizer_switch"] = new variable("optimizer_switch", true, false, false);
vars["optimizer_switch"]->add(std::vector<std::string> {"'materialization=off'", "`materialization=on`", "\"materialization=off\""});
vars["optimizer_switch"]->add(std::vector<std::string> {"'index_merge_union=off'", "`index_merge_union=on`", "\"index_merge_union=off\""});
Expand All @@ -252,6 +252,10 @@ int main() {
vars["lc_messages"]->add(std::vector<std::string> {"es_ES", "'es_ES'", "`es_ES`", "\"es_ES\""});
vars["lc_messages"]->add(std::vector<std::string> {"fr_FR", "'fr_FR'", "`fr_FR`", "\"fr_FR\""});

vars["log_slow_filter"] = new variable("log_slow_filter", true, false, false);
vars["log_slow_filter"]->add(std::vector<std::string> {"'not_using_index'", "`not_using_index`", "\"not_using_index\""});
vars["log_slow_filter"]->add(std::vector<std::string> {"'admin,filesort,filesort_on_disk,full_join'", "`admin,filesort,filesort_on_disk,full_join`", "\"admin,filesort,filesort_on_disk,full_join\""});

vars["time_zone"] = new variable("time_zone", true, false, false);
vars["time_zone"]->add(std::vector<std::string> {"'+01:00'", "`+02:15`", "\"+03:30\""});
vars["time_zone"]->add(std::vector<std::string> {"'+04:45'", "`+05:00`", "\"+06:10\""});
Expand Down Expand Up @@ -281,6 +285,9 @@ int main() {
vars["wsrep_osu_method"] = new variable("wsrep_osu_method", true, false, false);
add_values_and_quotes("wsrep_osu_method", {"TOI","RSU"});

vars["sql_quote_show_create"] = new variable("sql_quote_show_create", true, false, true);
vars["sql_quote_show_create"]->add(bool_values);

vars["sql_require_primary_key"] = new variable("sql_require_primary_key", true, false, true);
vars["sql_require_primary_key"]->add(bool_values);

Expand Down
2 changes: 1 addition & 1 deletion test/tap/tests/set_testing-240-t.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ char *schema=(char *)"information_schema";
int silent = 0;
int sysbench = 0;
int local=0;
int queries=3000;
int queries=4000;
int uniquequeries=0;
int histograms=-1;

Expand Down
2 changes: 2 additions & 0 deletions test/tap/tests/set_testing-240.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ const std::vector<std::string> possible_unknown_variables = {
"optimizer_use_condition_selectivity",
"max_statement_time",
"sql_generate_invisible_primary_key",
"log_slow_filter",
"sql_quote_show_create",
"sql_require_primary_key"
};

Expand Down
2 changes: 2 additions & 0 deletions test/tap/tests/set_testing.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ const std::vector<std::string> possible_unknown_variables = {
"group_replication_consistency",
"query_cache_type",
"wsrep_osu_method",
"log_slow_filter",
"sql_quote_show_create",
};

int readTestCases(const std::string& fileName) {
Expand Down

0 comments on commit 86ce115

Please sign in to comment.