diff --git a/.gitignore b/.gitignore index 4cdeaf6415..2253f1a90e 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ binaries/*deb binaries/*rpm tools/eventslog_reader_sample src/proxysql-save.cfg +src/*log* proxysql-2.0.0/ docker/images/proxysql/rhel-compliant/rpmmacros @@ -93,6 +94,8 @@ deps/prometheus-cpp/prometheus-cpp-*/ deps/re2/re2-*/ deps/sqlite3/sqlite-amalgamation-*/ deps/coredumper/coredumper-*/ +deps/postgresql/postgresql-*/ +deps/libusual/libusual-*/ test/.vagrant .DS_Store diff --git a/Makefile b/Makefile index 831da6f30f..49f65f51cf 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ O3 := -O3 -mtune=native ALL_DEBUG := $(O0) -ggdb -DDEBUG NO_DEBUG := $(O2) -ggdb DEBUG := $(ALL_DEBUG) -CURVER ?= 2.7.1 +CURVER ?= 3.0.0 #export DEBUG #export EXTRALINK export MAKE @@ -306,19 +306,19 @@ ubuntu: $(REL_ARCH)-ubuntu ; amd64-packages: amd64-centos amd64-ubuntu amd64-debian amd64-fedora amd64-opensuse amd64-almalinux amd64-almalinux: almalinux8 almalinux8-clang almalinux8-dbg almalinux9 almalinux9-clang almalinux9-dbg -amd64-centos: centos7 centos7-dbg centos8 centos8-clang centos8-dbg centos9 centos9-clang centos9-dbg +amd64-centos: centos8 centos8-clang centos8-dbg centos9 centos9-clang centos9-dbg amd64-debian: debian10 debian10-dbg debian11 debian11-clang debian11-dbg debian12 debian12-clang debian12-dbg amd64-fedora: fedora38 fedora38-clang fedora38-dbg fedora39 fedora39-clang fedora39-dbg fedora40 fedora40-clang fedora40-dbg fedora41 fedora41-clang fedora41-dbg amd64-opensuse: opensuse15 opensuse15-clang opensuse15-dbg -amd64-ubuntu: ubuntu16 ubuntu16-dbg ubuntu18 ubuntu18-dbg ubuntu20 ubuntu20-clang ubuntu20-dbg ubuntu22 ubuntu22-clang ubuntu22-dbg ubuntu24 ubuntu24-clang ubuntu24-dbg +amd64-ubuntu: ubuntu18 ubuntu18-dbg ubuntu20 ubuntu20-clang ubuntu20-dbg ubuntu22 ubuntu22-clang ubuntu22-dbg ubuntu24 ubuntu24-clang ubuntu24-dbg arm64-packages: arm64-centos arm64-debian arm64-ubuntu arm64-fedora arm64-opensuse arm64-almalinux arm64-almalinux: almalinux8 almalinux9 -arm64-centos: centos7 centos8 centos9 +arm64-centos: centos8 centos9 arm64-debian: debian10 debian11 debian12 arm64-fedora: fedora38 fedora39 fedora40 arm64-opensuse: opensuse15 -arm64-ubuntu: ubuntu16 ubuntu18 ubuntu20 ubuntu22 ubuntu24 +arm64-ubuntu: ubuntu18 ubuntu20 ubuntu22 ubuntu24 almalinux%: build-almalinux% ; centos%: build-centos% ; @@ -350,9 +350,9 @@ binaries/proxysql%: cd test/tap && ${MAKE} clean cd test/deps && ${MAKE} cleanall find . -not -path "./binaries/*" -not -path "./.git/*" -exec touch -h --date=@`git show -s --format=%ct HEAD` {} \; - @docker compose -p proxysql down -v --remove-orphans - @docker compose -p proxysql up $(IMG_NAME)$(IMG_TYPE)$(IMG_COMP)_build - @docker compose -p proxysql down -v --remove-orphans + @docker compose -p "${GIT_VERSION/./}" down -v --remove-orphans + @docker compose -p "${GIT_VERSION/./}" up $(IMG_NAME)$(IMG_TYPE)$(IMG_COMP)_build + @docker compose -p "${GIT_VERSION/./}" down -v --remove-orphans ### clean targets diff --git a/deps/Makefile b/deps/Makefile index 25bcc603a7..0361352be7 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -38,7 +38,7 @@ endif STDCPP := -std=c++$(shell echo $(CPLUSPLUS) | cut -c3-4) -DCXX$(shell echo $(CPLUSPLUS) | cut -c3-4) -targets := libconfig libdaemon jemalloc mariadb_client re2 sqlite3 pcre lz4 cityhash microhttpd curl ev libssl libhttpserver libinjection prometheus-cpp +targets := libconfig libdaemon jemalloc mariadb_client re2 sqlite3 pcre lz4 cityhash microhttpd curl ev libssl libhttpserver libinjection prometheus-cpp postgresql libusual libscram ifeq ($(SYS_KERN),Linux) targets += coredumper endif @@ -128,7 +128,8 @@ curl/curl/lib/.libs/libcurl.a: libssl/openssl/libssl.a ifeq ($(SYS_KERN),Darwin) cd curl/curl && patch configure < ../configure.patch endif - cd curl/curl && CFLAGS=-fPIC ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 --without-brotli --without-librtmp --without-libpsl --without-zstd --with-ssl=$(shell pwd)/libssl/openssl/ --enable-shared=yes && LD_LIBRARY_PATH=$(shell pwd)/libssl/openssl CC=${CC} CXX=${CXX} ${MAKE} + cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 --without-brotli --without-librtmp --without-libpsl --without-zstd --with-ssl=$(shell pwd)/libssl/openssl/ --enable-shared=yes + cd curl/curl && LD_LIBRARY_PATH=$(shell pwd)/libssl/openssl CFLAGS=-fPIC CC=${CC} CXX=${CXX} ${MAKE} curl: curl/curl/lib/.libs/libcurl.a @@ -197,7 +198,7 @@ mariadb-client-library/mariadb_client/libmariadb/libmariadbclient.a: libssl/open cd mariadb-client-library && rm -rf mariadb-connector-c-*/ || true cd mariadb-client-library && tar -zxf mariadb-connector-c-3.3.8-src.tar.gz cd mariadb-client-library/mariadb_client && patch -p0 < ../plugin_auth_CMakeLists.txt.patch - cd mariadb-client-library/mariadb_client && cmake . -Wno-dev -DOPENSSL_ROOT_DIR=$(shell pwd)/libssl/openssl/ -DOPENSSL_LIBRARIES=$(shell pwd)/libssl/openssl/ -DICONV_LIBRARIES=$(brew --prefix libiconv)/lib -DICONV_INCLUDE=$(brew --prefix libiconv)/include . + cd mariadb-client-library/mariadb_client && cmake . -Wno-dev -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOPENSSL_ROOT_DIR=$(shell pwd)/libssl/openssl/ -DOPENSSL_LIBRARIES=$(shell pwd)/libssl/openssl/ -DICONV_LIBRARIES=$(brew --prefix libiconv)/lib -DICONV_INCLUDE=$(brew --prefix libiconv)/include . ifeq ($(PROXYDEBUG),1) cd mariadb-client-library/mariadb_client && patch -p0 < ../ma_context.h.patch else ifeq ($(USEVALGRIND),1) @@ -254,7 +255,6 @@ sqlite3/sqlite3/sqlite3.o: sqlite3: sqlite3/sqlite3/sqlite3.o - libconfig/libconfig/lib/.libs/libconfig++.a: cd libconfig && rm -rf libconfig-*/ || true cd libconfig && tar -zxf libconfig-*.tar.gz @@ -304,6 +304,32 @@ pcre/pcre/.libs/libpcre.a: pcre: pcre/pcre/.libs/libpcre.a +postgresql/postgresql/src/interfaces/libpq/libpq.a : libssl/openssl/libssl.a + cd postgresql && rm -rf postgresql-*/ || true + cd postgresql && tar -zxf postgresql-*.tar.gz + cd postgresql/postgresql && patch -p0 < ../get_result_from_pgconn.patch + cd postgresql/postgresql && patch -p0 < ../handle_row_data.patch + #cd postgresql/postgresql && LD_LIBRARY_PATH="$(shell pwd)/libssl/openssl" ./configure --with-ssl=openssl --with-includes="$(shell pwd)/libssl/openssl/include/" --with-libraries="$(shell pwd)/libssl/openssl/" --without-readline --enable-debug CFLAGS="-ggdb -O0 -fno-omit-frame-pointer" CPPFLAGS="-g -O0" + cd postgresql/postgresql && LD_LIBRARY_PATH="$(shell pwd)/libssl/openssl" ./configure --with-ssl=openssl --with-includes="$(shell pwd)/libssl/openssl/include/" --with-libraries="$(shell pwd)/libssl/openssl/" --without-readline + cd postgresql/postgresql/src/interfaces/libpq && CC=${CC} CXX=${CXX} ${MAKE} MAKELEVEL=0 + #cd postgresql/postgresql && CC=${CC} CXX=${CXX} ${MAKE} -f src/interfaces/libpq/Makefile all + +postgresql: postgresql/postgresql/src/interfaces/libpq/libpq.a + +libusual/libusual/.libs/libusual.a: libssl/openssl/libssl.a + cd libusual && rm -rf libusual-*/ || true + cd libusual && tar -zxf libusual-*.tar.gz + cd libusual/libusual && ./autogen.sh + cd libusual/libusual && ./configure --with-openssl="$(shell pwd)/libssl/openssl/" --disable-shared + cd libusual/libusual && CC=${CC} CXX=${CXX} ${MAKE} + +libusual: libusual/libusual/.libs/libusual.a + +libscram/lib/libscram.a: libssl/openssl/libssl.a postgresql/postgresql/src/interfaces/libpq/libpq.a + cd libscram && rm -rf lib/* || true + cd libscram && CC=${CC} CXX=${CXX} ${MAKE} LIBOPENSSL_DIR="$(shell pwd)/libssl/openssl" POSTGRESQL_DIR="$(shell pwd)/postgresql/postgresql/" + +libscram: libscram/lib/libscram.a ### clean targets @@ -311,6 +337,7 @@ cleanpart: cd mariadb-client-library && rm -rf mariadb-connector-c-*/ || true cd jemalloc && rm -rf jemalloc-*/ || true cd sqlite3 && rm -rf sqlite-amalgamation-*/ || true + cd postgresql && rm -rf postgresql-*/ || true .PHONY: cleanpart cleanall: @@ -333,5 +360,8 @@ cleanall: cd prometheus-cpp && rm -rf prometheus-cpp-*/ || true cd cityhash && rm -rf cityhash-*/ || true cd coredumper && rm -rf coredumper-*/ || true + cd postgresql && rm -rf postgresql-*/ || true + cd libusual && rm -rf libusual-*/ || true + cd libscram && rm -rf lib/* obj/* || true .PHONY: cleanall diff --git a/deps/libscram/Makefile b/deps/libscram/Makefile new file mode 100644 index 0000000000..4be0b9d37d --- /dev/null +++ b/deps/libscram/Makefile @@ -0,0 +1,60 @@ +CC ?= gcc +CFLAGS_common = -Wall -Iinclude +CFLAGS_DEBUG = -g -O0 +CFLAGS_RELEASE = -O3 +SRC_DIR = src +OBJ_DIR_DEBUG = obj/debug +OBJ_DIR_RELEASE = obj/release +LIB_DIR = lib +TARGET_DEBUG = $(LIB_DIR)/libscram.a +TARGET_RELEASE = $(LIB_DIR)/libscram.a + +# Library directory passed as a parameter +LIBUSUAL_DIR ?= /path/to/libusual + +# OpenSSL dir +LIBOPENSSL_DIR ?= /path/to/openssl + +# PostgreSQL dir +POSTGRESQL_DIR ?= /path/to/postgresql +POSTGRESQL_INCLUDE_DIR = $(POSTGRESQL_DIR)/src/include/ + +# List all source files +SRC_FILES := $(wildcard $(SRC_DIR)/*.c) + +# Generate object file names +OBJ_FILES_DEBUG := $(patsubst $(SRC_DIR)/%.c,$(OBJ_DIR_DEBUG)/%.o,$(SRC_FILES)) +OBJ_FILES_RELEASE := $(patsubst $(SRC_DIR)/%.c,$(OBJ_DIR_RELEASE)/%.o,$(SRC_FILES)) + +# Check if ASAN is enabled +ifdef ASAN +CFLAGS_DEBUG += -fsanitize=address +endif + +# Define default target +all: $(TARGET_RELEASE) + +DEBUG: $(TARGET_DEBUG) + +RELEASE: $(TARGET_RELEASE) + +$(OBJ_DIR_DEBUG)/%.o: $(SRC_DIR)/%.c + @mkdir -p $(@D) + $(CC) $(CFLAGS_common) $(CFLAGS_DEBUG) -I$(LIBUSUAL_DIR) -I$(LIBOPENSSL_DIR) -I$(POSTGRESQL_INCLUDE_DIR) -c $< -o $@ + +$(OBJ_DIR_RELEASE)/%.o: $(SRC_DIR)/%.c + @mkdir -p $(@D) + $(CC) $(CFLAGS_common) $(CFLAGS_RELEASE) -I$(LIBUSUAL_DIR) -I$(LIBOPENSSL_DIR) -I$(POSTGRESQL_INCLUDE_DIR) -c $< -o $@ + +$(TARGET_DEBUG): $(OBJ_FILES_DEBUG) + @mkdir -p $(@D) + ar rcs $@ $^ + +$(TARGET_RELEASE): $(OBJ_FILES_RELEASE) + @mkdir -p $(@D) + ar rcs $@ $^ + +clean: + rm -rf $(OBJ_DIR_DEBUG)/* $(OBJ_DIR_RELEASE)/* $(LIB_DIR)/* + +.PHONY: all DEBUG RELEASE clean diff --git a/deps/libscram/include/scram-internal.h b/deps/libscram/include/scram-internal.h new file mode 100644 index 0000000000..a1771d250c --- /dev/null +++ b/deps/libscram/include/scram-internal.h @@ -0,0 +1,19 @@ + +/* + * Required system headers + */ +#include +#include +#include +#include +#include +#include + +#define int8 int8_t +#define uint8 uint8_t +#define uint16 uint16_t +#define uint32 uint32_t +#define uint64 uint64_t + +#define SCRAM_KEY_LEN SCRAM_SHA_256_KEY_LEN +#define SCRAM_DEFAULT_ITERATIONS SCRAM_SHA_256_DEFAULT_ITERATIONS diff --git a/deps/libscram/include/scram.h b/deps/libscram/include/scram.h new file mode 100644 index 0000000000..33c95e4119 --- /dev/null +++ b/deps/libscram/include/scram.h @@ -0,0 +1,129 @@ +/* + * PgBouncer - Lightweight connection pooler for PostgreSQL. + * + * Copyright (c) 2007-2009 Marko Kreen, Skype Technologies OÃœ + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + + /* + * Ought to match NAMEDATALEN. Some cloud services use longer user + * names, so give it some extra room. + */ +#define MAX_USERNAME 128 + + /* + * Some cloud services use very long generated passwords, so give it + * plenty of room. + */ +#define MAX_PASSWORD 2048 + +struct ScramState { + char* client_nonce; + char* client_first_message_bare; + char* client_final_message_without_proof; + char* server_nonce; + char* server_first_message; + uint8_t* SaltedPassword; + char cbind_flag; + bool adhoc; /* SCRAM data made up from plain-text password */ + int iterations; + char* salt; /* base64-encoded */ + uint8_t ClientKey[32]; /* SHA256_DIGEST_LENGTH */ + uint8_t StoredKey[32]; + uint8_t ServerKey[32]; +}; + +struct PgCredentials { + char name[MAX_USERNAME]; + char passwd[MAX_PASSWORD]; + uint8_t scram_ClientKey[32]; + uint8_t scram_ServerKey[32]; + bool has_scram_keys; /* true if the above two are valid */ + bool mock_auth; +}; + +typedef struct ScramState ScramState; +typedef struct PgCredentials PgCredentials; + +typedef enum PasswordType { + PASSWORD_TYPE_PLAINTEXT = 0, + PASSWORD_TYPE_MD5, + PASSWORD_TYPE_SCRAM_SHA_256 +} PasswordType; + +#ifdef __cplusplus +extern "C" { +#endif + + PasswordType get_password_type(const char* shadow_pass); + + // Returns the last error message + const char* scram_error(); + + // Resets the error message + void scram_reset_error(); + + // Initializes a new ScramState object + ScramState* scram_state_init(); + + // Frees the memory allocated for a ScramState object + void free_scram_state(ScramState* scram_state); + + /* + * Functions for communicating as a client with the server + */ + char *build_client_first_message(ScramState *scram_state); + + char *build_client_final_message(ScramState *scram_state, + const PgCredentials *credentials, + const char *server_nonce, + const char *salt, + int saltlen, + int iterations); + + bool read_server_first_message(ScramState* scram_state, char *input, + char **server_nonce_p, char **salt_p, int *saltlen_p, int *iterations_p); + + bool read_server_final_message(char *input, char *ServerSignature); + + bool verify_server_signature(ScramState *scram_state, const PgCredentials *credentials, const char *ServerSignature); + + + /* + * Functions for communicating as a server to the client + */ + + bool read_client_first_message(char *input, + char *cbind_flag_p, + char **client_first_message_bare_p, + char **client_nonce_p); + + bool read_client_final_message(ScramState* scram_state, const uint8_t *raw_input, char *input, + const char **client_final_nonce_p, + char **proof_p); + + char *build_server_first_message(ScramState *scram_state, + const char *username, const char *stored_secret); + + char *build_server_final_message(ScramState *scram_state); + + bool verify_final_nonce(const ScramState *scram_state, const char *client_final_nonce); + + bool verify_client_proof(ScramState *state, const char *ClientProof); + + bool scram_verify_plain_password(const char *username, const char *password, const char *secret); + +#ifdef __cplusplus +} +#endif diff --git a/deps/libscram/src/scram.c b/deps/libscram/src/scram.c new file mode 100644 index 0000000000..91189cd827 --- /dev/null +++ b/deps/libscram/src/scram.c @@ -0,0 +1,1291 @@ +/* + * PgBouncer - Lightweight connection pooler for PostgreSQL. + * + * Copyright (c) 2007-2009 Marko Kreen, Skype Technologies OÃœ + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +/* + * SCRAM support + */ + +#include "scram-internal.h" +#include "scram.h" +#include "openssl/rand.h" +#include "common/base64.h" +#include "common/saslprep.h" +#include "common/scram-common.h" +#include "common/hmac.h" + +#include + +#define MAX_ERROR_LENGTH 256 + +// Define thread-local storage for the error buffer +static __thread char errorBuffer[MAX_ERROR_LENGTH]; + +const char* scram_error() { + return errorBuffer; +} + +void scram_reset_error() { + errorBuffer[0] = '\0'; +} + +size_t strlcat(char* dst, const char* src, size_t siz) +{ + char* d = dst; + const char* s = src; + size_t n = siz; + size_t dlen; + + /* Find the end of dst and adjust bytes left but don't go past end */ + while (n-- != 0 && *d != '\0') + d++; + dlen = d - dst; + n = siz - dlen; + + if (n == 0) + return(dlen + strlen(s)); + while (*s != '\0') { + if (n != 1) { + *d++ = *s; + n--; + } + s++; + } + *d = '\0'; + + return(dlen + (s - src)); /* count does not include NUL */ +} + + +static int get_random_bytes(unsigned char* buffer, int length) { + + if (RAND_bytes(buffer, length) != 1) { + return -1; + } + return 0; +} + +static bool calculate_client_proof(ScramState *scram_state, + const PgCredentials *credentials, + const char *salt, + int saltlen, + int iterations, + const char *client_final_message_without_proof, + uint8_t *result); + + +/* + * Initialize a new SCRAM state. + * + * This function allocates memory for a new ScramState struct and initializes its fields. + * The client_nonce, client_first_message_bare, client_final_message_without_proof, + * server_nonce, server_first_message, SaltedPassword, salt, ClientKey, StoredKey, + * and ServerKey fields are set to NULL or zeroed out. + * + * Returns a pointer to the newly initialized ScramState struct, or NULL if memory allocation fails. + */ +ScramState* scram_state_init() { + ScramState* scram_state = (ScramState*)malloc(sizeof(ScramState)); + if (scram_state != NULL) { + scram_state->client_nonce = NULL; + scram_state->client_first_message_bare = NULL; + scram_state->client_final_message_without_proof = NULL; + scram_state->server_nonce = NULL; + scram_state->server_first_message = NULL; + scram_state->SaltedPassword = NULL; + scram_state->cbind_flag = '\0'; + scram_state->adhoc = false; + scram_state->iterations = 0; + scram_state->salt = NULL; + memset(scram_state->ClientKey, 0, sizeof(scram_state->ClientKey)); + memset(scram_state->StoredKey, 0, sizeof(scram_state->StoredKey)); + memset(scram_state->ServerKey, 0, sizeof(scram_state->ServerKey)); + } + return scram_state; +} + +/* + * free SCRAM state info after auth is done + */ +void free_scram_state(ScramState *scram_state) +{ + if (scram_state != NULL) { + free(scram_state->client_nonce); + free(scram_state->client_first_message_bare); + free(scram_state->client_final_message_without_proof); + free(scram_state->server_nonce); + free(scram_state->server_first_message); + free(scram_state->SaltedPassword); + free(scram_state->salt); + memset(scram_state, 0, sizeof(*scram_state)); + + free(scram_state); + } +} + +static bool is_scram_printable(char *p) +{ + /*------ + * Printable characters, as defined by SCRAM spec: (RFC 5802) + * + * printable = %x21-2B / %x2D-7E + * ;; Printable ASCII except ",". + * ;; Note that any "printable" is also + * ;; a valid "value". + *------ + */ + for (; *p; p++) + if (*p < 0x21 || *p > 0x7E || *p == 0x2C /* comma */) + return false; + + return true; +} + +static char *sanitize_char(char c) +{ + static char buf[5]; + + if (c >= 0x21 && c <= 0x7E) + snprintf(buf, sizeof(buf), "'%c'", c); + else + snprintf(buf, sizeof(buf), "0x%02x", (unsigned char) c); + return buf; +} + +/* + * Read value for an attribute part of a SCRAM message. + */ +static char *read_attr_value(char **input, char attr) +{ + char *begin = *input; + char *end; + + if (*begin != attr) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (attribute \"%c\" expected)", attr); + return NULL; + } + begin++; + + if (*begin != '=') { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (expected \"=\" after attribute \"%c\")", + attr); + return NULL; + } + begin++; + + end = begin; + while (*end && *end != ',') + end++; + + if (*end) { + *end = '\0'; + *input = end + 1; + } else { + *input = end; + } + + return begin; +} + +/* + * Read the next attribute and value in a SCRAM exchange message. + * + * Returns NULL if there is no attribute. + */ +static char *read_any_attr(char **input, char *attr_p) +{ + char *begin = *input; + char *end; + char attr = *begin; + + if (!((attr >= 'A' && attr <= 'Z') || + (attr >= 'a' && attr <= 'z'))) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (attribute expected, but found invalid character \"%s\")", + sanitize_char(attr)); + return NULL; + } + if (attr_p) + *attr_p = attr; + begin++; + + if (*begin != '=') { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (expected character \"=\" after attribute \"%c\")", + attr); + return NULL; + } + begin++; + + end = begin; + while (*end && *end != ',') + end++; + + if (*end) { + *end = '\0'; + *input = end + 1; + } else { + *input = end; + } + + return begin; +} + +/* + * Parse and validate format of given SCRAM secret. + * + * Returns true if the SCRAM secret has been parsed, and false otherwise. + */ +static bool parse_scram_secret(const char *secret, int *iterations, char **salt, + uint8_t *stored_key, uint8_t *server_key) +{ + char *s; + char *p; + char *scheme_str; + char *salt_str; + char *iterations_str; + char *storedkey_str; + char *serverkey_str; + int decoded_len; + char *decoded_salt_buf; + char *decoded_stored_buf = NULL; + char *decoded_server_buf = NULL; + + /* + * The secret is of form: + * + * SCRAM-SHA-256$:$: + */ + s = strdup(secret); + if (!s) + goto invalid_secret; + if ((scheme_str = strtok(s, "$")) == NULL) + goto invalid_secret; + if ((iterations_str = strtok(NULL, ":")) == NULL) + goto invalid_secret; + if ((salt_str = strtok(NULL, "$")) == NULL) + goto invalid_secret; + if ((storedkey_str = strtok(NULL, ":")) == NULL) + goto invalid_secret; + if ((serverkey_str = strtok(NULL, "")) == NULL) + goto invalid_secret; + + /* Parse the fields */ + if (strcmp(scheme_str, "SCRAM-SHA-256") != 0) + goto invalid_secret; + + errno = 0; + *iterations = strtol(iterations_str, &p, 10); + if (*p || errno != 0) + goto invalid_secret; + + /* + * Verify that the salt is in Base64-encoded format, by decoding it, + * although we return the encoded version to the caller. + */ + decoded_len = pg_b64_dec_len(strlen(salt_str)); + decoded_salt_buf = malloc(decoded_len); + if (!decoded_salt_buf) + goto invalid_secret; + decoded_len = pg_b64_decode(salt_str, strlen(salt_str), decoded_salt_buf, decoded_len); + free(decoded_salt_buf); + if (decoded_len < 0) + goto invalid_secret; + *salt = strdup(salt_str); + if (!*salt) + goto invalid_secret; + + /* + * Decode StoredKey and ServerKey. + */ + decoded_len = pg_b64_dec_len(strlen(storedkey_str)); + decoded_stored_buf = malloc(decoded_len); + if (!decoded_stored_buf) + goto invalid_secret; + decoded_len = pg_b64_decode(storedkey_str, strlen(storedkey_str), decoded_stored_buf, decoded_len); + if (decoded_len != SCRAM_KEY_LEN) + goto invalid_secret; + memcpy(stored_key, decoded_stored_buf, SCRAM_KEY_LEN); + + decoded_len = pg_b64_dec_len(strlen(serverkey_str)); + decoded_server_buf = malloc(decoded_len); + decoded_len = pg_b64_decode(serverkey_str, strlen(serverkey_str), + decoded_server_buf, decoded_len); + if (decoded_len != SCRAM_KEY_LEN) + goto invalid_secret; + memcpy(server_key, decoded_server_buf, SCRAM_KEY_LEN); + + free(decoded_stored_buf); + free(decoded_server_buf); + free(s); + return true; + +invalid_secret: + free(decoded_stored_buf); + free(decoded_server_buf); + free(s); + free(*salt); + *salt = NULL; + return false; +} + +#define MD5_PASSWD_CHARSET "0123456789abcdef" +#define MD5_PASSWD_LEN 35 +/* + * What kind of a password type is 'shadow_pass'? + */ +PasswordType get_password_type(const char *shadow_pass) +{ + char *encoded_salt = NULL; + int iterations; + uint8_t stored_key[SCRAM_KEY_LEN]; + uint8_t server_key[SCRAM_KEY_LEN]; + + if (strncmp(shadow_pass, "md5", 3) == 0 && + strlen(shadow_pass) == MD5_PASSWD_LEN && + strspn(shadow_pass + 3, MD5_PASSWD_CHARSET) == MD5_PASSWD_LEN - 3) + return PASSWORD_TYPE_MD5; + if (parse_scram_secret(shadow_pass, &iterations, &encoded_salt, + stored_key, server_key)) { + free(encoded_salt); + return PASSWORD_TYPE_SCRAM_SHA_256; + } + free(encoded_salt); + return PASSWORD_TYPE_PLAINTEXT; +} + +/* + * Functions for communicating as a client with the server + */ +char *build_client_first_message(ScramState *scram_state) +{ + uint8_t raw_nonce[SCRAM_RAW_NONCE_LEN + 1]; + int encoded_len; + size_t len; + char *result = NULL; + + if (get_random_bytes(raw_nonce, SCRAM_RAW_NONCE_LEN) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not generate random nonce"); + goto failed; + } + + encoded_len = pg_b64_enc_len(SCRAM_RAW_NONCE_LEN); + scram_state->client_nonce = malloc(encoded_len + 1); + if (scram_state->client_nonce == NULL) + goto failed; + encoded_len = pg_b64_encode((char *) raw_nonce, SCRAM_RAW_NONCE_LEN, scram_state->client_nonce, encoded_len); + if (encoded_len < 0) + goto failed; + scram_state->client_nonce[encoded_len] = '\0'; + + len = 8 + strlen(scram_state->client_nonce) + 1; + result = malloc(len); + if (result == NULL) + goto failed; + snprintf(result, len, "n,,n=,r=%s", scram_state->client_nonce); + + scram_state->client_first_message_bare = strdup(result + 3); + if (scram_state->client_first_message_bare == NULL) + goto failed; + + return result; + +failed: + free(result); + free(scram_state->client_nonce); + scram_state->client_nonce = NULL; + free(scram_state->client_first_message_bare); + scram_state->client_first_message_bare = NULL; + return NULL; +} + +char *build_client_final_message(ScramState *scram_state, + const PgCredentials *credentials, + const char *server_nonce, + const char *salt, + int saltlen, + int iterations) +{ + char buf[512]; + size_t len; + uint8_t client_proof[SCRAM_KEY_LEN]; + int enclen; + + snprintf(buf, sizeof(buf), "c=biws,r=%s", server_nonce); + + scram_state->client_final_message_without_proof = strdup(buf); + if (scram_state->client_final_message_without_proof == NULL) + goto failed; + + if (!calculate_client_proof(scram_state, credentials, + salt, saltlen, iterations, buf, + client_proof)) + goto failed; + + len = strlcat(buf, ",p=", sizeof(buf)); + enclen = pg_b64_enc_len(sizeof(client_proof)); + enclen = pg_b64_encode((char *) client_proof, + SCRAM_KEY_LEN, + buf + len, enclen); + if (enclen < 0) + goto failed; + len += enclen; + buf[len] = '\0'; + + return strdup(buf); +failed: + return NULL; +} + +bool read_server_first_message(ScramState* scram_state, char *input, + char **server_nonce_p, char **salt_p, int *saltlen_p, int *iterations_p) +{ + char *server_nonce; + char *encoded_salt; + char *salt = NULL; + int saltlen; + char *iterations_str; + char *endptr; + int iterations; + + scram_state->server_first_message = strdup(input); + if (scram_state->server_first_message == NULL) + goto failed; + + server_nonce = read_attr_value(&input, 'r'); + if (server_nonce == NULL) + goto failed; + + if (strlen(server_nonce) < strlen(scram_state->client_nonce) || + memcmp(server_nonce, scram_state->client_nonce, strlen(scram_state->client_nonce)) != 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "invalid SCRAM response (nonce mismatch)"); + goto failed; + } + + encoded_salt = read_attr_value(&input, 's'); + if (encoded_salt == NULL) + goto failed; + saltlen = pg_b64_dec_len(strlen(encoded_salt)); + salt = malloc(saltlen); + if (salt == NULL) + goto failed; + saltlen = pg_b64_decode(encoded_salt, + strlen(encoded_salt), + salt, saltlen); + if (saltlen < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (invalid salt)"); + goto failed; + } + + iterations_str = read_attr_value(&input, 'i'); + if (iterations_str == NULL) + goto failed; + + iterations = strtol(iterations_str, &endptr, 10); + if (*endptr != '\0' || iterations < 1) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (invalid iteration count)"); + goto failed; + } + + if (*input != '\0') { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (garbage at end of server-first-message)"); + goto failed; + } + + *server_nonce_p = server_nonce; + *salt_p = salt; + *saltlen_p = saltlen; + *iterations_p = iterations; + return true; +failed: + free(salt); + return false; +} + +bool read_server_final_message(char *input, char *ServerSignature) +{ + char *encoded_server_signature; + char *decoded_server_signature = NULL; + int server_signature_len; + + if (*input == 'e') { + char *errmsg = read_attr_value(&input, 'e'); + snprintf(errorBuffer, MAX_ERROR_LENGTH, "error received from server in SCRAM exchange: %s", + errmsg); + goto failed; + } + + encoded_server_signature = read_attr_value(&input, 'v'); + if (encoded_server_signature == NULL) + goto failed; + + if (*input != '\0') + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (garbage at end of server-final-message)"); + + server_signature_len = pg_b64_dec_len(strlen(encoded_server_signature)); + decoded_server_signature = malloc(server_signature_len); + if (!decoded_server_signature) + goto failed; + + server_signature_len = pg_b64_decode(encoded_server_signature, + strlen(encoded_server_signature), + decoded_server_signature, + server_signature_len); + if (server_signature_len != SCRAM_KEY_LEN) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (malformed server signature)"); + goto failed; + } + memcpy(ServerSignature, decoded_server_signature, SCRAM_KEY_LEN); + + free(decoded_server_signature); + return true; +failed: + free(decoded_server_signature); + return false; +} + +static bool calculate_client_proof(ScramState *scram_state, + const PgCredentials *credentials, + const char *salt, + int saltlen, + int iterations, + const char *client_final_message_without_proof, + uint8_t *result) +{ + pg_saslprep_rc rc; + char *prep_password = NULL; + uint8_t StoredKey[SCRAM_KEY_LEN]; + uint8_t ClientKey[SCRAM_KEY_LEN]; + uint8_t ClientSignature[SCRAM_KEY_LEN]; + const char* errstr = NULL; + pg_hmac_ctx* ctx = pg_hmac_create(PG_SHA256); + + assert(scram_state->SaltedPassword == NULL); + + if (credentials->has_scram_keys) { + memcpy(ClientKey, credentials->scram_ClientKey, SCRAM_KEY_LEN); + } else { + rc = pg_saslprep(credentials->passwd, &prep_password); + if (rc == SASLPREP_OOM) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "pg_saslprep out of memory"); + goto failed; + } + if (rc != SASLPREP_SUCCESS) { + prep_password = strdup(credentials->passwd); + if (!prep_password) + goto failed; + } + + scram_state->SaltedPassword = malloc(SCRAM_KEY_LEN); + if (scram_state->SaltedPassword == NULL) + goto failed; + + if (scram_SaltedPassword(prep_password, PG_SHA256, SCRAM_KEY_LEN, salt, saltlen, iterations, + scram_state->SaltedPassword, + &errstr) < 0 || + scram_ClientKey(scram_state->SaltedPassword, PG_SHA256, SCRAM_KEY_LEN, ClientKey, + &errstr) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not compute client key: %s", *errstr); + goto failed; + } + } + + /* Hash it one more time, and compare with StoredKey */ + if (scram_H(ClientKey, PG_SHA256, SCRAM_KEY_LEN, + StoredKey, &errstr) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not hash stored key: %s", *errstr); + goto failed; + } + /* + * Calculate ClientSignature. Note that we don't log directly a failure + * here even when processing the calculations as this could involve a mock + * authentication. + */ + if (pg_hmac_init(ctx, StoredKey, SCRAM_KEY_LEN) < 0 || + pg_hmac_update(ctx, + (uint8*)scram_state->client_first_message_bare, + strlen(scram_state->client_first_message_bare)) < 0 || + pg_hmac_update(ctx, (uint8*)",", 1) < 0 || + pg_hmac_update(ctx, + (uint8*)scram_state->server_first_message, + strlen(scram_state->server_first_message)) < 0 || + pg_hmac_update(ctx, (uint8*)",", 1) < 0 || + pg_hmac_update(ctx, + (uint8*)scram_state->client_final_message_without_proof, + strlen(scram_state->client_final_message_without_proof)) < 0 || + pg_hmac_final(ctx, ClientSignature, SCRAM_KEY_LEN) < 0) + { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not calculate client signature: %s", pg_hmac_error(ctx)); + goto failed; + } + + for (int i = 0; i < SCRAM_KEY_LEN; i++) + result[i] = ClientKey[i] ^ ClientSignature[i]; + + pg_hmac_free(ctx); + free(prep_password); + return true; +failed: + pg_hmac_free(ctx); + free(prep_password); + free(scram_state->SaltedPassword); + scram_state->SaltedPassword = NULL; + return false; +} + +bool verify_server_signature(ScramState *scram_state, const PgCredentials *credentials, const char *ServerSignature) +{ + uint8_t expected_ServerSignature[SCRAM_KEY_LEN]; + uint8_t ServerKey[SCRAM_KEY_LEN]; + const char* errstr = NULL; + pg_hmac_ctx* ctx = pg_hmac_create(PG_SHA256); + + if (credentials->has_scram_keys) + memcpy(ServerKey, credentials->scram_ServerKey, SCRAM_KEY_LEN); + else { + if (scram_ServerKey(scram_state->SaltedPassword, PG_SHA256, SCRAM_KEY_LEN, ServerKey, &errstr) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not compute server key: %s", *errstr); + goto failed; + } + } + + /* calculate ServerSignature */ + if (pg_hmac_init(ctx, ServerKey, SCRAM_KEY_LEN) < 0 || + pg_hmac_update(ctx, + (uint8*)scram_state->client_first_message_bare, + strlen(scram_state->client_first_message_bare)) < 0 || + pg_hmac_update(ctx, (uint8*)",", 1) < 0 || + pg_hmac_update(ctx, + (uint8*)scram_state->server_first_message, + strlen(scram_state->server_first_message)) < 0 || + pg_hmac_update(ctx, (uint8*)",", 1) < 0 || + pg_hmac_update(ctx, + (uint8*)scram_state->client_final_message_without_proof, + strlen(scram_state->client_final_message_without_proof)) < 0 || + pg_hmac_final(ctx, expected_ServerSignature, SCRAM_KEY_LEN) < 0) + { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not calculate server signature: %s", pg_hmac_error(ctx)); + goto failed; + } + + pg_hmac_free(ctx); + return (memcmp(expected_ServerSignature, ServerSignature, SCRAM_KEY_LEN) == 0); + +failed: + pg_hmac_free(ctx); + return false; +} + + +/* + * Functions for communicating as a server to the client + */ + +bool read_client_first_message(char *input, + char *cbind_flag_p, + char **client_first_message_bare_p, + char **client_nonce_p) +{ + char *client_first_message_bare = NULL; + char *client_nonce = NULL; + char *client_nonce_copy = NULL; + + *cbind_flag_p = *input; + switch (*input) { + case 'n': + /* Client does not support channel binding */ + input++; + break; + case 'y': + /* Client supports channel binding, but we're not doing it today */ + input++; + break; + case 'p': + /* Client requires channel binding. We don't support it. */ + snprintf(errorBuffer, MAX_ERROR_LENGTH, "client requires SCRAM channel binding, but it is not supported"); + goto failed; + default: + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (unexpected channel-binding flag \"%s\")", + sanitize_char(*input)); + goto failed; + } + + if (*input != ',') { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (comma expected, but found character \"%s\")", + sanitize_char(*input)); + goto failed; + } + input++; + + if (*input == 'a') { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "client uses authorization identity, but it is not supported"); + goto failed; + } + if (*input != ',') { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (unexpected attribute \"%s\" in client-first-message)", + sanitize_char(*input)); + goto failed; + } + input++; + + client_first_message_bare = strdup(input); + if (client_first_message_bare == NULL) + goto failed; + + if (*input == 'm') { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "client requires an unsupported SCRAM extension"); + goto failed; + } + + /* read and ignore user name */ + read_attr_value(&input, 'n'); + + client_nonce = read_attr_value(&input, 'r'); + if (client_nonce == NULL) + goto failed; + if (!is_scram_printable(client_nonce)) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "non-printable characters in SCRAM nonce"); + goto failed; + } + client_nonce_copy = strdup(client_nonce); + if (client_nonce_copy == NULL) + goto failed; + + /* + * There can be any number of optional extensions after this. We don't + * support any extensions, so ignore them. + */ + while (*input != '\0') { + if (!read_any_attr(&input, NULL)) + goto failed; + } + + *client_first_message_bare_p = client_first_message_bare; + *client_nonce_p = client_nonce_copy; + return true; +failed: + free(client_first_message_bare); + free(client_nonce_copy); + return false; +} + +bool read_client_final_message(ScramState* scram_state, const uint8_t *raw_input, char *input, + const char **client_final_nonce_p, + char **proof_p) +{ + const char *input_start = input; + char attr; + char *channel_binding; + char *client_final_nonce; + char *proof_start; + char *value; + char *encoded_proof; + char *proof = NULL; + int prooflen; + + /* + * Read channel-binding. We don't support channel binding, so + * it's expected to always be "biws", which is "n,,", + * base64-encoded, or "eSws", which is "y,,". We also have to + * check whether the flag is the same one that the client + * originally sent. + */ + channel_binding = read_attr_value(&input, 'c'); + if (channel_binding == NULL) + goto failed; + if (!(strcmp(channel_binding, "biws") == 0 && scram_state->cbind_flag == 'n') && + !(strcmp(channel_binding, "eSws") == 0 && scram_state->cbind_flag == 'y')) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "unexpected SCRAM channel-binding attribute in client-final-message"); + goto failed; + } + + client_final_nonce = read_attr_value(&input, 'r'); + + /* ignore optional extensions */ + do { + proof_start = input - 1; + value = read_any_attr(&input, &attr); + } while (value && attr != 'p'); + + if (!value) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not read proof"); + goto failed; + } + + encoded_proof = value; + + prooflen = pg_b64_dec_len(strlen(encoded_proof)); + proof = malloc(prooflen); + if (proof == NULL) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not decode proof"); + goto failed; + } + prooflen = pg_b64_decode(encoded_proof, + strlen(encoded_proof), + proof, prooflen); + if (prooflen != SCRAM_KEY_LEN) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (malformed proof in client-final-message)"); + goto failed; + } + + if (*input != '\0') { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "malformed SCRAM message (garbage at the end of client-final-message)"); + goto failed; + } + + scram_state->client_final_message_without_proof = malloc(proof_start - input_start + 1); + if (!scram_state->client_final_message_without_proof) + goto failed; + memcpy(scram_state->client_final_message_without_proof, raw_input, proof_start - input_start); + scram_state->client_final_message_without_proof[proof_start - input_start] = '\0'; + + *client_final_nonce_p = client_final_nonce; + *proof_p = proof; + return true; +failed: + free(proof); + return false; +} + +/* + * For doing SCRAM with a password stored in plain text, build a SCRAM + * secret on the fly. + */ +static bool build_adhoc_scram_secret(const char *plain_password, ScramState *scram_state) +{ + const char *password = NULL; + char *prep_password = NULL; + pg_saslprep_rc rc; + char saltbuf[SCRAM_DEFAULT_SALT_LEN]; + int encoded_len; + uint8_t salted_password[SCRAM_KEY_LEN]; + const char* errstr = NULL; + + assert(scram_state->salt == NULL); + + rc = pg_saslprep(plain_password, &prep_password); + if (rc == SASLPREP_OOM) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "pg_saslprep out of memory"); + goto failed; + } + else if (rc == SASLPREP_SUCCESS) + password = prep_password; + else + password = plain_password; + + if (get_random_bytes((uint8_t*)saltbuf, sizeof(saltbuf)) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not generate random nonce"); + goto failed; + } + + scram_state->adhoc = true; + scram_state->iterations = SCRAM_DEFAULT_ITERATIONS; + + encoded_len = pg_b64_enc_len(sizeof(saltbuf)); + scram_state->salt = malloc(encoded_len + 1); + if (!scram_state->salt) + goto failed; + encoded_len = pg_b64_encode(saltbuf, sizeof(saltbuf), scram_state->salt, encoded_len); + if (encoded_len < 0) + goto failed; + scram_state->salt[encoded_len] = '\0'; + + /* Calculate StoredKey and ServerKey */ + if (scram_SaltedPassword(password, PG_SHA256, SCRAM_KEY_LEN, saltbuf, sizeof(saltbuf), + scram_state->iterations, + salted_password, &errstr) < 0 || + scram_ClientKey(salted_password, PG_SHA256, SCRAM_KEY_LEN, scram_state->StoredKey, &errstr) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not compute server key: %s", *errstr); + goto failed; + } + + if (scram_H(scram_state->StoredKey, PG_SHA256, SCRAM_KEY_LEN, scram_state->StoredKey, &errstr) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not hash stored key: %s", *errstr); + goto failed; + } + + if (scram_ServerKey(salted_password, PG_SHA256, SCRAM_KEY_LEN, scram_state->ServerKey, &errstr) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not calculate server key: %s", *errstr); + goto failed; + } + + free(prep_password); + return true; +failed: + free(prep_password); + free(scram_state->salt); + scram_state->salt = NULL; + return false; +} + +/* + * Deterministically generate salt for mock authentication, using a + * SHA256 hash based on the username and an instance-level secret key. + * Target buffer needs to be of size SCRAM_DEFAULT_SALT_LEN. + */ +static void scram_mock_salt(const char *username, uint8_t *saltbuf) +{ + static uint8_t mock_auth_nonce[32]; + static bool mock_auth_nonce_initialized = false; + pg_cryptohash_ctx* ctx = NULL; + uint8_t sha_digest[PG_SHA256_DIGEST_LENGTH]; + + /* + * Generating salt using a SHA256 hash works as long as the + * required salt length is not larger than the SHA256 digest + * length. + */ + static_assert(PG_SHA256_DIGEST_LENGTH >= SCRAM_DEFAULT_SALT_LEN, + "salt length greater than SHA256 digest length"); + + if (!mock_auth_nonce_initialized) { + if (get_random_bytes(mock_auth_nonce, sizeof(mock_auth_nonce)) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not generate random nonce"); + goto failed; + } + mock_auth_nonce_initialized = true; + } + + ctx = pg_cryptohash_create(PG_SHA256); + if (pg_cryptohash_init(ctx) < 0 || + pg_cryptohash_update(ctx, (uint8*)username, strlen(username)) < 0 || + pg_cryptohash_update(ctx, (uint8*)mock_auth_nonce, sizeof(mock_auth_nonce)) < 0 || + pg_cryptohash_final(ctx, sha_digest, PG_SHA256_DIGEST_LENGTH) < 0) + { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not calculate hash"); + goto failed; + } + + pg_cryptohash_free(ctx); + memcpy(saltbuf, sha_digest, SCRAM_DEFAULT_SALT_LEN); + return; + +failed: + pg_cryptohash_free(ctx); + saltbuf[0] = '\0'; +} + +static bool build_mock_scram_secret(const char *username, ScramState *scram_state) +{ + uint8_t saltbuf[SCRAM_DEFAULT_SALT_LEN]; + int encoded_len; + + assert(scram_state->salt == NULL); + + scram_state->iterations = SCRAM_DEFAULT_ITERATIONS; + + scram_mock_salt(username, saltbuf); + encoded_len = pg_b64_enc_len(sizeof(saltbuf)); + scram_state->salt = malloc(encoded_len + 1); + if (!scram_state->salt) + goto failed; + encoded_len = pg_b64_encode((char *) saltbuf, sizeof(saltbuf), scram_state->salt, encoded_len); + if (encoded_len < 0) + goto failed; + scram_state->salt[encoded_len] = '\0'; + + return true; +failed: + free(scram_state->salt); + scram_state->salt = NULL; + return false; +} + +char *build_server_first_message(ScramState *scram_state, const char *username, const char *stored_secret) +{ + uint8_t raw_nonce[SCRAM_RAW_NONCE_LEN + 1]; + int encoded_len; + size_t len; + char *result; + + assert(scram_state->server_nonce == NULL); + assert(scram_state->server_first_message == NULL); + + if (!stored_secret) { + if (!build_mock_scram_secret(username, scram_state)) + goto failed; + } else { + switch (get_password_type(stored_secret)) { + case PASSWORD_TYPE_SCRAM_SHA_256: + if (!parse_scram_secret(stored_secret, + &scram_state->iterations, + &scram_state->salt, + scram_state->StoredKey, + scram_state->ServerKey)) + goto failed; + break; + case PASSWORD_TYPE_PLAINTEXT: + if (!build_adhoc_scram_secret(stored_secret, scram_state)) + goto failed; + break; + default: + /* shouldn't get here */ + goto failed; + } + } + + if (get_random_bytes(raw_nonce, SCRAM_RAW_NONCE_LEN) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not generate random nonce"); + goto failed; + } + + encoded_len = pg_b64_enc_len(SCRAM_RAW_NONCE_LEN); + scram_state->server_nonce = malloc(encoded_len + 1); + if (scram_state->server_nonce == NULL) + goto failed; + encoded_len = pg_b64_encode((char *) raw_nonce, SCRAM_RAW_NONCE_LEN, scram_state->server_nonce, encoded_len); + if (encoded_len < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not encode random nonce"); + goto failed; + } + scram_state->server_nonce[encoded_len] = '\0'; + + len = (2 + + strlen(scram_state->client_nonce) + + strlen(scram_state->server_nonce) + + 3 + + strlen(scram_state->salt) + + 3 + 10 + 1); + result = malloc(len); + if (!result) + goto failed; + snprintf(result, len, + "r=%s%s,s=%s,i=%u", + scram_state->client_nonce, + scram_state->server_nonce, + scram_state->salt, + scram_state->iterations); + + scram_state->server_first_message = result; + + return result; +failed: + free(scram_state->server_nonce); + free(scram_state->server_first_message); + return NULL; +} + +static char *compute_server_signature(ScramState *state) +{ + uint8_t ServerSignature[SCRAM_KEY_LEN]; + char *server_signature_base64 = NULL; + int siglen; + pg_hmac_ctx* ctx = pg_hmac_create(PG_SHA256); + + /* calculate ServerSignature */ + if (pg_hmac_init(ctx, state->ServerKey, SCRAM_KEY_LEN) < 0 || + pg_hmac_update(ctx, + (uint8*)state->client_first_message_bare, + strlen(state->client_first_message_bare)) < 0 || + pg_hmac_update(ctx, (uint8*)",", 1) < 0 || + pg_hmac_update(ctx, + (uint8*)state->server_first_message, + strlen(state->server_first_message)) < 0 || + pg_hmac_update(ctx, (uint8*)",", 1) < 0 || + pg_hmac_update(ctx, + (uint8*)state->client_final_message_without_proof, + strlen(state->client_final_message_without_proof)) < 0 || + pg_hmac_final(ctx, ServerSignature, SCRAM_KEY_LEN) < 0) + { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not calculate server signature: %s", pg_hmac_error(ctx)); + goto failed; + } + + siglen = pg_b64_enc_len(SCRAM_KEY_LEN); + server_signature_base64 = malloc(siglen + 1); + if (!server_signature_base64) + goto failed; + siglen = pg_b64_encode((const char *) ServerSignature, + SCRAM_KEY_LEN, server_signature_base64, siglen); + if (siglen < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not encode server signature"); + goto failed; + } + server_signature_base64[siglen] = '\0'; + pg_hmac_free(ctx); + return server_signature_base64; + +failed: + free(server_signature_base64); + pg_hmac_free(ctx); + return NULL; +} + +char *build_server_final_message(ScramState *scram_state) +{ + char *server_signature = NULL; + char *result = NULL; + size_t len; + + server_signature = compute_server_signature(scram_state); + if (!server_signature) + goto failed; + + len = 2 + strlen(server_signature) + 1; + + /* + * Avoid compiler warning at snprintf() below because len + * could in theory overflow snprintf() result. If this + * happened in practice, it would surely be some crazy + * corruption, so treat it as an error. + */ + if (len >= INT_MAX) + goto failed; + + result = malloc(len); + if (!result) + goto failed; + snprintf(result, len, "v=%s", server_signature); + return result; + +failed: + free(server_signature); + return NULL; +} + +bool verify_final_nonce(const ScramState *scram_state, const char *client_final_nonce) +{ + const size_t client_nonce_len = strlen(scram_state->client_nonce); + const size_t server_nonce_len = strlen(scram_state->server_nonce); + const size_t final_nonce_len = strlen(client_final_nonce); + + if (final_nonce_len != client_nonce_len + server_nonce_len) + return false; + if (memcmp(client_final_nonce, scram_state->client_nonce, client_nonce_len) != 0) + return false; + if (memcmp(client_final_nonce + client_nonce_len, scram_state->server_nonce, server_nonce_len) != 0) + return false; + + return true; +} + +bool verify_client_proof(ScramState *state, const char *ClientProof) +{ + uint8_t ClientSignature[SCRAM_KEY_LEN]; + uint8_t client_StoredKey[SCRAM_KEY_LEN]; + pg_hmac_ctx* ctx = pg_hmac_create(PG_SHA256); + int i; + const char* errstr = NULL; + + /* + * Calculate ClientSignature. Note that we don't log directly a failure + * here even when processing the calculations as this could involve a mock + * authentication. + */ + if (pg_hmac_init(ctx, state->StoredKey, SCRAM_KEY_LEN) < 0 || + pg_hmac_update(ctx, + (uint8*)state->client_first_message_bare, + strlen(state->client_first_message_bare)) < 0 || + pg_hmac_update(ctx, (uint8*)",", 1) < 0 || + pg_hmac_update(ctx, + (uint8*)state->server_first_message, + strlen(state->server_first_message)) < 0 || + pg_hmac_update(ctx, (uint8*)",", 1) < 0 || + pg_hmac_update(ctx, + (uint8*)state->client_final_message_without_proof, + strlen(state->client_final_message_without_proof)) < 0 || + pg_hmac_final(ctx, ClientSignature, SCRAM_KEY_LEN) < 0) + { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not calculate client signature: %s", pg_hmac_error(ctx)); + goto failed; + } + + /* Extract the ClientKey that the client calculated from the proof */ + for (i = 0; i < SCRAM_KEY_LEN; i++) + state->ClientKey[i] = ClientProof[i] ^ ClientSignature[i]; + + /* Hash it one more time, and compare with StoredKey */ + if (scram_H(state->ClientKey, PG_SHA256, SCRAM_KEY_LEN, client_StoredKey, &errstr) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not hash stored key: %s", *errstr); + goto failed; + } + + pg_hmac_free(ctx); + return (memcmp(client_StoredKey, state->StoredKey, SCRAM_KEY_LEN) == 0); + +failed: + pg_hmac_free(ctx); + return false; +} + +/* + * Verify a plaintext password against a SCRAM secret. This is used when + * performing plaintext password authentication for a user that has a SCRAM + * secret stored in pg_authid. + */ +bool scram_verify_plain_password(const char *username, const char *password, + const char *secret) +{ + char *encoded_salt = NULL; + char *salt = NULL; + int saltlen; + int iterations; + uint8_t salted_password[SCRAM_KEY_LEN]; + uint8_t stored_key[SCRAM_KEY_LEN]; + uint8_t server_key[SCRAM_KEY_LEN]; + uint8_t computed_key[SCRAM_KEY_LEN]; + char *prep_password = NULL; + const char* errstr = NULL; + pg_saslprep_rc rc; + + if (!parse_scram_secret(secret, &iterations, &encoded_salt, + stored_key, server_key)) { + /* The password looked like a SCRAM secret, but could not be parsed. */ + snprintf(errorBuffer, MAX_ERROR_LENGTH, "invalid SCRAM secret for user \"%s\"", username); + goto failed; + } + saltlen = pg_b64_dec_len(strlen(encoded_salt)); + salt = malloc(saltlen); + if (!salt) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "memory allocation failed for salt"); + goto failed; + } + saltlen = pg_b64_decode(encoded_salt, strlen(encoded_salt), salt, saltlen); + if (saltlen < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "invalid SCRAM secret for user \"%s\"", username); + goto failed; + } + + /* Normalize the password */ + rc = pg_saslprep(password, &prep_password); + if (rc == SASLPREP_OOM) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "pg_saslprep out of memory"); + goto failed; + } + if (rc == SASLPREP_SUCCESS) + password = prep_password; + + /* Compute Server Key based on the user-supplied plaintext password */ + if (scram_SaltedPassword(password, PG_SHA256, SCRAM_KEY_LEN, salt, saltlen, iterations, salted_password, &errstr) < 0 || + scram_ServerKey(salted_password, PG_SHA256, SCRAM_KEY_LEN, computed_key, &errstr) < 0) { + snprintf(errorBuffer, MAX_ERROR_LENGTH, "could not compute server key: %s", *errstr); + goto failed; + } + + /* + * Compare the secret's Server Key with the one computed from the + * user-supplied password. + */ + free(encoded_salt); + free(salt); + free(prep_password); + return (memcmp(computed_key, server_key, SCRAM_KEY_LEN) == 0); + +failed: + free(encoded_salt); + free(salt); + free(prep_password); + return false; +} diff --git a/deps/libusual/libusual b/deps/libusual/libusual new file mode 120000 index 0000000000..568916c77e --- /dev/null +++ b/deps/libusual/libusual @@ -0,0 +1 @@ +libusual-f8d49e2 \ No newline at end of file diff --git a/deps/libusual/libusual-f8d49e2.tar.gz b/deps/libusual/libusual-f8d49e2.tar.gz new file mode 100644 index 0000000000..952d8396f6 Binary files /dev/null and b/deps/libusual/libusual-f8d49e2.tar.gz differ diff --git a/deps/postgresql/get_result_from_pgconn.patch b/deps/postgresql/get_result_from_pgconn.patch new file mode 100644 index 0000000000..6bb37bca17 --- /dev/null +++ b/deps/postgresql/get_result_from_pgconn.patch @@ -0,0 +1,40 @@ +diff --git src/interfaces/libpq/fe-exec.c src/interfaces/libpq/fe-exec.c +index fa9d6aad..cd5cd23d 100644 +--- src/interfaces/libpq/fe-exec.c ++++ src/interfaces/libpq/fe-exec.c +@@ -4467,3 +4467,20 @@ PQunescapeBytea(const unsigned char *strtext, size_t *retbuflen) + *retbuflen = buflen; + return tmpbuf; + } ++ ++/* ++ * PQgetResultFromPGconn ++ * Get error result from PGconn ++ */ ++const PGresult * ++PQgetResultFromPGconn(PGconn *conn) ++{ ++ if (!conn) ++ return NULL; ++ ++ if (conn->asyncStatus != PGASYNC_IDLE) ++ return NULL; ++ ++ return conn->result; ++} ++ +diff --git src/interfaces/libpq/libpq-fe.h src/interfaces/libpq/libpq-fe.h +index 7476dbe0..472d0083 100644 +--- src/interfaces/libpq/libpq-fe.h ++++ src/interfaces/libpq/libpq-fe.h +@@ -668,6 +668,9 @@ extern PQsslKeyPassHook_OpenSSL_type PQgetSSLKeyPassHook_OpenSSL(void); + extern void PQsetSSLKeyPassHook_OpenSSL(PQsslKeyPassHook_OpenSSL_type hook); + extern int PQdefaultSSLKeyPassHook_OpenSSL(char *buf, int size, PGconn *conn); + ++/* Get PGresult directly from PGconn. WARNING: DO NOT RELEASE THIS RESULT */ ++extern const PGresult *PQgetResultFromPGconn(PGconn *conn); ++ + #ifdef __cplusplus + } + #endif + diff --git a/deps/postgresql/handle_row_data.patch b/deps/postgresql/handle_row_data.patch new file mode 100644 index 0000000000..dec09fe2b5 --- /dev/null +++ b/deps/postgresql/handle_row_data.patch @@ -0,0 +1,231 @@ +diff --git src/interfaces/libpq/fe-exec.c src/interfaces/libpq/fe-exec.c +index 2265ab5..56883ec 100644 +--- src/interfaces/libpq/fe-exec.c ++++ src/interfaces/libpq/fe-exec.c +@@ -4484,3 +4484,9 @@ PQgetResultFromPGconn(PGconn *conn) + return conn->result; + } + ++int PShandleRowData(PGconn *conn, PSresult* result) { ++ if (!conn || !result) ++ return 1; ++ return psHandleRowData(conn, result); ++} ++ +diff --git src/interfaces/libpq/fe-misc.c src/interfaces/libpq/fe-misc.c +index 488f7d6..65beb87 100644 +--- src/interfaces/libpq/fe-misc.c ++++ src/interfaces/libpq/fe-misc.c +@@ -1344,3 +1344,39 @@ libpq_append_conn_error(PGconn *conn, const char *fmt,...) + + appendPQExpBufferChar(&conn->errorMessage, '\n'); + } ++ ++/* ++ * psGetInt16 ++ * read 2 byte integer and convert from network byte order ++ * to local byte order ++ */ ++int ++psGetInt16(int *result, PGconn *conn) ++{ ++ uint16 tmp2; ++ ++ if (conn->inCursor + 2 > conn->inEnd) ++ return EOF; ++ memcpy(&tmp2, conn->inBuffer + conn->inCursor, 2); ++ conn->inCursor += 2; ++ *result = (int) pg_ntoh16(tmp2); ++ return 0; ++} ++ ++/* ++ * psGetInt32 ++ * read 4 byte integer and convert from network byte order ++ * to local byte order ++ */ ++int ++psGetInt32(int *result, PGconn *conn) ++{ ++ uint32 tmp4; ++ ++ if (conn->inCursor + 4 > conn->inEnd) ++ return EOF; ++ memcpy(&tmp4, conn->inBuffer + conn->inCursor, 4); ++ conn->inCursor += 4; ++ *result = (int) pg_ntoh32(tmp4); ++ return 0; ++} +diff --git src/interfaces/libpq/fe-protocol3.c src/interfaces/libpq/fe-protocol3.c +index 9c4aa7e..de0746c 100644 +--- src/interfaces/libpq/fe-protocol3.c ++++ src/interfaces/libpq/fe-protocol3.c +@@ -2299,3 +2299,105 @@ build_startup_packet(const PGconn *conn, char *packet, + + return packet_len; + } ++ ++/* ++ * psHandleRowData: Processes the incoming message from the PostgreSQL backend. ++ * This function checks if the message contains row data (indicated by 'D') ++ * and processes it accordingly. It validates the message type and length, ++ * ensures that the complete message has been received, and updates the result ++ * structure if the message contains row data. ++ * ++ * Return values: ++ * 0 -> Message processed successfully (row data handled). ++ * 1 -> Message not fully processed; the next call should be to PQisBusy. ++ * -1 -> Not enough data to process the message; the next call should be to PQconsumeInput. ++ */ ++int ++psHandleRowData(PGconn *conn, PSresult* result) ++{ ++ char id; ++ int msgLength; ++ int avail; ++ ++ if (conn->asyncStatus != PGASYNC_BUSY) ++ return 1; ++ /* ++ * Try to read a message. First get the type code and length. Return ++ * if not enough data. ++ */ ++ conn->inCursor = conn->inStart; ++ if (pqGetc(&id, conn)) ++ return EOF; ++ ++ if (id != 'D') ++ return 1; ++ ++ if (psGetInt32(&msgLength, conn)) ++ return EOF; ++ ++ /* ++ * Try to validate message type/length here. A length less than 4 is ++ * definitely broken. Large lengths should only be believed for a few ++ * message types. ++ */ ++ if (msgLength < 4) ++ return 1; ++ ++ if (msgLength > 30000 && !VALID_LONG_MESSAGE_TYPE(id)) ++ return 1; ++ ++ /* ++ * Can't process if message body isn't all here yet. ++ */ ++ msgLength -= 4; ++ avail = conn->inEnd - conn->inCursor; ++ if (avail < msgLength) { ++ if ((conn->inCursor+(size_t)msgLength) <= (size_t)conn->inBufSize) ++ return EOF; ++ return 1; ++ } ++ ++ if (conn->result != NULL && ++ conn->result->resultStatus == PGRES_TUPLES_OK) ++ { ++ PGresult *res = conn->result; ++ int nfields = res->numAttributes; ++ int tupnfields; /* # fields from tuple */ ++ int vlen; /* length of the current field value */ ++ int i; ++ ++ ++ /* Get the field count and make sure it's what we expect */ ++ if (psGetInt16(&tupnfields, conn)) ++ return 1; ++ ++ if (tupnfields != nfields) ++ return 1; ++ ++ /* Scan the fields */ ++ for (i = 0; i < nfields; i++) ++ { ++ /* get the value length */ ++ if (psGetInt32(&vlen, conn)) ++ return 1; ++ ++ /* Skip over the data value */ ++ if (vlen > 0) ++ { ++ if (pqSkipnchar(vlen, conn)) ++ return 1; ++ } ++ } ++ ++ result->id = 'D'; ++ result->len = msgLength + 5; ++ result->data = conn->inBuffer + conn->inStart; ++ conn->asyncStatus = PGASYNC_BUSY; ++ /* trust the specified message length as what to skip */ ++ conn->inStart += 5 + msgLength; ++ conn->inCursor = conn->inStart; ++ return 0; ++ } ++ return 1; ++} ++ +diff --git src/interfaces/libpq/libpq-fe.h src/interfaces/libpq/libpq-fe.h +index c5170d1..3e3cc34 100644 +--- src/interfaces/libpq/libpq-fe.h ++++ src/interfaces/libpq/libpq-fe.h +@@ -269,6 +269,18 @@ typedef struct pgresAttDesc + int atttypmod; /* type-specific modifier info */ + } PGresAttDesc; + ++/* ---------------- ++ * PSresult -- ++ * ---------------- ++ */ ++typedef struct psResult ++{ ++ char id; ++ int len; ++ const char* data; ++ //int fieldcount; ++} PSresult; ++ + /* ---------------- + * Exported functions of libpq + * ---------------- +@@ -671,6 +683,9 @@ extern int PQdefaultSSLKeyPassHook_OpenSSL(char *buf, int size, PGconn *conn); + /* Get PGresult directly from PGconn. WARNING: DO NOT RELEASE THIS RESULT */ + extern const PGresult *PQgetResultFromPGconn(PGconn *conn); + ++/* ProxySQL special handler function */ ++extern int PShandleRowData(PGconn *conn, PSresult* result); ++ + #ifdef __cplusplus + } + #endif +diff --git src/interfaces/libpq/libpq-int.h src/interfaces/libpq/libpq-int.h +index a951f49..e1df8b5 100644 +--- src/interfaces/libpq/libpq-int.h ++++ src/interfaces/libpq/libpq-int.h +@@ -727,6 +727,11 @@ extern PGresult *pqFunctionCall3(PGconn *conn, Oid fnid, + int result_is_int, + const PQArgBlock *args, int nargs); + ++ /* ++ * ProxySQL light weight routines ++ */ ++extern int psHandleRowData(PGconn *conn, PSresult* result); ++ + /* === in fe-misc.c === */ + + /* +@@ -756,6 +761,13 @@ extern int pqWaitTimed(int forRead, int forWrite, PGconn *conn, + extern int pqReadReady(PGconn *conn); + extern int pqWriteReady(PGconn *conn); + ++ /* ++ * ProxySQL light weight routines ++ */ ++extern int psGetInt16(int *result, PGconn *conn); ++extern int psGetInt32(int *result, PGconn *conn); ++ ++ + /* === in fe-secure.c === */ + + extern int pqsecure_initialize(PGconn *, bool, bool); diff --git a/deps/postgresql/postgresql b/deps/postgresql/postgresql new file mode 120000 index 0000000000..1bbe35dc8e --- /dev/null +++ b/deps/postgresql/postgresql @@ -0,0 +1 @@ +postgresql-16.3 \ No newline at end of file diff --git a/deps/postgresql/postgresql-16.3.tar.gz b/deps/postgresql/postgresql-16.3.tar.gz new file mode 100644 index 0000000000..d05e3acc5d Binary files /dev/null and b/deps/postgresql/postgresql-16.3.tar.gz differ diff --git a/docker-compose.yml b/docker-compose.yml index 1539461c6c..bc5b15f916 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.0" +#version: "3.0" services: #################################################################################################### @@ -16,31 +16,11 @@ services: command: bash -l -c /opt/entrypoint/entrypoint.bash #################################################################################################### -#################################################################################################### - centos7_build: - extends: - service: _build - image: proxysql/packaging:build-centos7-v2.6.0 - volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - - ./docker/images/proxysql/rhel-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - - ./:/opt/proxysql/ - environment: - - PKG_RELEASE=centos7 - - centos7_dbg_build: - extends: - service: centos7_build - environment: - - PKG_RELEASE=dbg-centos7 - - PROXYSQL_BUILD_TYPE=debug - #################################################################################################### centos8_build: extends: service: _build - image: proxysql/packaging:build-centos8-v2.6.0 + image: proxysql/packaging:build-centos8-v3.0.0 volumes: - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - ./docker/images/proxysql/rhel-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros @@ -53,7 +33,7 @@ services: centos8_clang_build: extends: service: centos8_build - image: proxysql/packaging:build-clang-centos8-v2.6.0 + image: proxysql/packaging:build-clang-centos8-v3.0.0 environment: - PKG_RELEASE=centos8-clang @@ -69,7 +49,7 @@ services: centos9_build: extends: service: _build - image: proxysql/packaging:build-centos9-v2.6.0 + image: proxysql/packaging:build-centos9-v3.0.0 volumes: - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - ./docker/images/proxysql/rhel-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros @@ -82,7 +62,7 @@ services: centos9_clang_build: extends: service: centos9_build - image: proxysql/packaging:build-clang-centos9-v2.6.0 + image: proxysql/packaging:build-clang-centos9-v3.0.0 environment: - PKG_RELEASE=centos9-clang @@ -98,7 +78,7 @@ services: fedora38_build: extends: service: _build - image: proxysql/packaging:build-fedora38-v2.6.0 + image: proxysql/packaging:build-fedora38-v3.0.0 volumes: - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - ./docker/images/proxysql/rhel-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros @@ -111,7 +91,7 @@ services: fedora38_clang_build: extends: service: fedora38_build - image: proxysql/packaging:build-clang-fedora38-v2.6.0 + image: proxysql/packaging:build-clang-fedora38-v3.0.0 environment: - PKG_RELEASE=fedora38-clang @@ -126,7 +106,7 @@ services: fedora39_build: extends: service: _build - image: proxysql/packaging:build-fedora39-v2.6.0 + image: proxysql/packaging:build-fedora39-v3.0.0 volumes: - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - ./docker/images/proxysql/rhel-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros @@ -139,7 +119,7 @@ services: fedora39_clang_build: extends: service: fedora39_build - image: proxysql/packaging:build-clang-fedora39-v2.6.0 + image: proxysql/packaging:build-clang-fedora39-v3.0.0 environment: - PKG_RELEASE=fedora39-clang @@ -154,7 +134,7 @@ services: fedora40_build: extends: service: _build - image: proxysql/packaging:build-fedora40-v2.6.4 + image: proxysql/packaging:build-fedora40-v3.0.0 volumes: - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - ./docker/images/proxysql/rhel-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros @@ -167,7 +147,7 @@ services: fedora40_clang_build: extends: service: fedora40_build - image: proxysql/packaging:build-clang-fedora40-v2.6.4 + image: proxysql/packaging:build-clang-fedora40-v3.0.0 environment: - PKG_RELEASE=fedora40-clang @@ -211,7 +191,7 @@ services: debian10_build: extends: service: _build - image: proxysql/packaging:build-debian10-v2.6.0 + image: proxysql/packaging:build-debian10-v3.0.0 volumes: - ./docker/images/proxysql/deb-compliant/latest-package/ctl/:/root/ctl/ - ./docker/images/proxysql/deb-compliant/entrypoint/:/opt/entrypoint/ @@ -231,7 +211,7 @@ services: debian11_build: extends: service: _build - image: proxysql/packaging:build-debian11-v2.6.0 + image: proxysql/packaging:build-debian11-v3.0.0 volumes: - ./docker/images/proxysql/deb-compliant/latest-package/ctl/:/root/ctl/ - ./docker/images/proxysql/deb-compliant/entrypoint/:/opt/entrypoint/ @@ -243,7 +223,7 @@ services: debian11_clang_build: extends: service: debian11_build - image: proxysql/packaging:build-clang-debian11-v2.6.0 + image: proxysql/packaging:build-clang-debian11-v3.0.0 environment: - PKG_RELEASE=debian11-clang @@ -258,7 +238,7 @@ services: debian12_build: extends: service: _build - image: proxysql/packaging:build-debian12-v2.6.0 + image: proxysql/packaging:build-debian12-v3.0.0 volumes: - ./docker/images/proxysql/deb-compliant/latest-package/ctl/:/root/ctl/ - ./docker/images/proxysql/deb-compliant/entrypoint/:/opt/entrypoint/ @@ -270,7 +250,7 @@ services: debian12_clang_build: extends: service: debian12_build - image: proxysql/packaging:build-clang-debian12-v2.6.0 + image: proxysql/packaging:build-clang-debian12-v3.0.0 environment: - PKG_RELEASE=debian12-clang @@ -282,30 +262,11 @@ services: - PROXYSQL_BUILD_TYPE=debug #################################################################################################### -#################################################################################################### - ubuntu16_build: - extends: - service: _build - image: proxysql/packaging:build-ubuntu16-v2.6.0 - volumes: - - ./docker/images/proxysql/deb-compliant/pre-systemd/ctl/:/root/ctl/ - - ./docker/images/proxysql/deb-compliant/entrypoint/:/opt/entrypoint/ - - ./:/opt/proxysql/ - environment: - - PKG_RELEASE=ubuntu16 - - ubuntu16_dbg_build: - extends: - service: ubuntu16_build - environment: - - PKG_RELEASE=dbg-ubuntu16 - - PROXYSQL_BUILD_TYPE=debug - #################################################################################################### ubuntu18_build: extends: service: _build - image: proxysql/packaging:build-ubuntu18-v2.6.0 + image: proxysql/packaging:build-ubuntu18-v3.0.0 volumes: - ./docker/images/proxysql/deb-compliant/latest-package/ctl/:/root/ctl/ - ./docker/images/proxysql/deb-compliant/entrypoint/:/opt/entrypoint/ @@ -325,7 +286,7 @@ services: ubuntu20_build: extends: service: _build - image: proxysql/packaging:build-ubuntu20-v2.6.0 + image: proxysql/packaging:build-ubuntu20-v3.0.0 volumes: - ./docker/images/proxysql/deb-compliant/latest-package/ctl/:/root/ctl/ - ./docker/images/proxysql/deb-compliant/entrypoint/:/opt/entrypoint/ @@ -337,7 +298,7 @@ services: ubuntu20_clang_build: extends: service: ubuntu20_build - image: proxysql/packaging:build-clang-ubuntu20-v2.6.0 + image: proxysql/packaging:build-clang-ubuntu20-v3.0.0 environment: - PKG_RELEASE=ubuntu20-clang @@ -352,7 +313,7 @@ services: ubuntu22_build: extends: service: _build - image: proxysql/packaging:build-ubuntu22-v2.6.1 + image: proxysql/packaging:build-ubuntu22-v3.0.0 volumes: - ./docker/images/proxysql/deb-compliant/latest-package/ctl/:/root/ctl/ - ./docker/images/proxysql/deb-compliant/entrypoint/:/opt/entrypoint/ @@ -364,7 +325,7 @@ services: ubuntu22_clang_build: extends: service: ubuntu22_build - image: proxysql/packaging:build-clang-ubuntu22-v2.6.1 + image: proxysql/packaging:build-clang-ubuntu22-v3.0.0 environment: - PKG_RELEASE=ubuntu22-clang @@ -379,7 +340,7 @@ services: ubuntu24_build: extends: service: _build - image: proxysql/packaging:build-ubuntu24-v2.6.4 + image: proxysql/packaging:build-ubuntu24-v3.0.0 volumes: - ./docker/images/proxysql/deb-compliant/latest-package/ctl/:/root/ctl/ - ./docker/images/proxysql/deb-compliant/entrypoint/:/opt/entrypoint/ @@ -391,7 +352,7 @@ services: ubuntu24_clang_build: extends: service: ubuntu24_build - image: proxysql/packaging:build-clang-ubuntu24-v2.6.4 + image: proxysql/packaging:build-clang-ubuntu24-v3.0.0 environment: - PKG_RELEASE=ubuntu24-clang @@ -407,7 +368,7 @@ services: opensuse15_build: extends: service: _build - image: proxysql/packaging:build-opensuse15-v2.6.0 + image: proxysql/packaging:build-opensuse15-v3.0.0 volumes: - ./docker/images/proxysql/suse-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros @@ -420,7 +381,7 @@ services: opensuse15_clang_build: extends: service: opensuse15_build - image: proxysql/packaging:build-clang-opensuse15-v2.6.0 + image: proxysql/packaging:build-clang-opensuse15-v3.0.0 environment: - PKG_RELEASE=opensuse15-clang @@ -436,7 +397,7 @@ services: almalinux8_build: extends: service: _build - image: proxysql/packaging:build-almalinux8-v2.6.0 + image: proxysql/packaging:build-almalinux8-v3.0.0 volumes: - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - ./docker/images/proxysql/rhel-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros @@ -449,7 +410,7 @@ services: almalinux8_clang_build: extends: service: almalinux8_build - image: proxysql/packaging:build-clang-almalinux8-v2.6.0 + image: proxysql/packaging:build-clang-almalinux8-v3.0.0 environment: - PKG_RELEASE=almalinux8-clang @@ -464,7 +425,7 @@ services: almalinux9_build: extends: service: _build - image: proxysql/packaging:build-almalinux9-v2.6.0 + image: proxysql/packaging:build-almalinux9-v3.0.0 volumes: - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ - ./docker/images/proxysql/rhel-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros @@ -477,7 +438,7 @@ services: almalinux9_clang_build: extends: service: almalinux9_build - image: proxysql/packaging:build-clang-almalinux9-v2.6.0 + image: proxysql/packaging:build-clang-almalinux9-v3.0.0 environment: - PKG_RELEASE=almalinux9-clang diff --git a/include/Admin_ifaces.h b/include/Admin_ifaces.h new file mode 100644 index 0000000000..7d009d0d4d --- /dev/null +++ b/include/Admin_ifaces.h @@ -0,0 +1,150 @@ +#ifndef ADMIN_IFACES_H +#define ADMIN_IFACES_H + +#define MAX_IFACES 8 +#define MAX_ADMIN_LISTENERS 16 + +typedef struct _main_args { + int nfds; + struct pollfd *fds; + int *callback_func; + volatile int *shutdown; +} main_args; + +typedef struct _ifaces_desc_t { + char **mysql_ifaces; + char **pgsql_ifaces; + char **telnet_admin_ifaces; + char **telnet_stats_ifaces; +} ifaces_desc_t; + +class ifaces_desc { + public: + PtrArray *ifaces; + ifaces_desc() { + ifaces=new PtrArray(); + } + bool add(const char *iface) { + for (unsigned int i=0; ilen; i++) { + if (strcmp((const char *)ifaces->index(i),iface)==0) { + return false; + } + } + ifaces->add(strdup(iface)); + return true; + } + ~ifaces_desc() { + while(ifaces->len) { + char *d=(char *)ifaces->remove_index_fast(0); + free(d); + } + delete ifaces; + } +}; + +class admin_main_loop_listeners { + private: + int version; +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_t rwlock; +#else + rwlock_t rwlock; +#endif + + char ** reset_ifaces(char **ifaces) { + int i; + if (ifaces) { + for (i=0; iadd(token); + i++; + } + free_tokenizer( &tok ); + version++; + wrunlock(); + } + + + bool update_ifaces(char *list, char ***_ifaces) { + wrlock(); + int i; + char **ifaces=*_ifaces; + tokenizer_t tok; + tokenizer( &tok, list, ";", TOKENIZER_NO_EMPTIES ); + const char* token; + ifaces=reset_ifaces(ifaces); + i=0; + for ( token = tokenize( &tok ) ; token && i < MAX_IFACES ; token = tokenize( &tok ) ) { + ifaces[i]=(char *)malloc(strlen(token)+1); + strcpy(ifaces[i],token); + i++; + } + free_tokenizer( &tok ); + version++; + wrunlock(); + return true; + } +}; +#endif // ADMIN_IFACES_H diff --git a/include/Base_HostGroups_Manager.h b/include/Base_HostGroups_Manager.h new file mode 100644 index 0000000000..9ff54a2a6e --- /dev/null +++ b/include/Base_HostGroups_Manager.h @@ -0,0 +1,1300 @@ +template class BaseSrvList; +template class BaseHGC; +template class Base_HostGroups_Manager; + +class MyHGC; +class PgSQL_HGC; +class MySrvC; +class PgSQL_SrvC; +class MySrvList; +class PgSQL_SrvList; + +#include "proxysql.h" +#include "cpp.h" +#include "proxysql_gtid.h" + +#include +#include +#include +#include + +// Headers for declaring Prometheus counters +#include "prometheus/counter.h" +#include "prometheus/gauge.h" + +#include "thread.h" +#include "wqueue.h" + +#include "ev.h" + +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif + +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON + +#include + +#ifndef CLASS_BASE_HOSTGROUPS_MANAGER_H +#define CLASS_BASE_HOSTGROUPS_MANAGER_H + +#ifdef DEBUG +/* */ +// Enabling STRESSTEST_POOL ProxySQL will do a lot of loops in the connection pool +// This is for internal testing ONLY!!!! +//#define STRESSTEST_POOL +#endif // DEBUG + +#if 0 + +// we have 2 versions of the same tables: with (debug) and without (no debug) checks +#ifdef DEBUG +#define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" +#else +#define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT NOT NULL DEFAULT 1 , status INT NOT NULL DEFAULT 0 , compression INT NOT NULL DEFAULT 0 , max_connections INT NOT NULL DEFAULT 1000 , max_replication_lag INT NOT NULL DEFAULT 0 , use_ssl INT NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT NOT NULL DEFAULT 0 , weight INT NOT NULL DEFAULT 1 , status INT NOT NULL DEFAULT 0 , compression INT NOT NULL DEFAULT 0 , max_connections INT NOT NULL DEFAULT 1000 , max_replication_lag INT NOT NULL DEFAULT 0 , use_ssl INT NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" +#endif /* DEBUG */ +#define MYHGM_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only','read_only|innodb_read_only','read_only&innodb_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (reader_hostgroup))" + +#define MYHGM_MYSQL_GROUP_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define MYHGM_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , " \ + "active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL DEFAULT '' , " \ + "max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , " \ + "check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , " \ + "check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , " \ + "writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , " \ + "new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , " \ + "add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , " \ + "min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , " \ + "lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , comment VARCHAR ," \ + "UNIQUE (reader_hostgroup))" + +#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port" + +#define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" + + +#define MYHGM_MYSQL_SERVERS_SSL_PARAMS "CREATE TABLE mysql_servers_ssl_params (hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , username VARCHAR NOT NULL DEFAULT '' , ssl_ca VARCHAR NOT NULL DEFAULT '' , ssl_cert VARCHAR NOT NULL DEFAULT '' , ssl_key VARCHAR NOT NULL DEFAULT '' , ssl_capath VARCHAR NOT NULL DEFAULT '' , ssl_crl VARCHAR NOT NULL DEFAULT '' , ssl_crlpath VARCHAR NOT NULL DEFAULT '' , ssl_cipher VARCHAR NOT NULL DEFAULT '' , tls_version VARCHAR NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port, username) )" + +/* + * @brief Generates the 'runtime_mysql_servers' resultset exposed to other ProxySQL cluster members. + * @details Makes 'SHUNNED' and 'SHUNNED_REPLICATION_LAG' statuses equivalent to 'ONLINE'. 'SHUNNED' states + * are by definition local transitory states, this is why a 'mysql_servers' table reconfiguration isn't + * normally performed when servers are internally imposed with these statuses. This means, that propagating + * this state to other cluster members is undesired behavior, and so it's generating a different checksum, + * due to a server having this particular state, that will result in extra unnecessary fetching operations. + * The query also filters out 'OFFLINE_HARD' servers, 'OFFLINE_HARD' is a local status which is equivalent to + * a server no longer being part of the table (DELETED state). And so, they shouldn't be propagated. + * + * For placing the query into a single line for debugging purposes: + * ``` + * sed 's/^\t\+"//g; s/"\s\\$//g; s/\\"/"/g' /tmp/select.sql | paste -sd '' + * ``` + */ +#define MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS \ + "SELECT " \ + "hostgroup_id, hostname, port, gtid_port," \ + "CASE status" \ + " WHEN 0 THEN \"ONLINE\"" \ + " WHEN 1 THEN \"ONLINE\"" \ + " WHEN 2 THEN \"OFFLINE_SOFT\"" \ + " WHEN 3 THEN \"OFFLINE_HARD\"" \ + " WHEN 4 THEN \"ONLINE\" " \ + "END status," \ + "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ + "FROM mysql_servers " \ + "WHERE status != 3 " \ + "ORDER BY hostgroup_id, hostname, port" \ + +/** + * @brief Generates the 'mysql_servers_v2' resultset exposed to other ProxySQL cluster members. + * @details The generated resultset is used for the checksum computation of the runtime ProxySQL config + * ('mysql_servers_v2' checksum), and it's also forwarded to other cluster members when querying the Admin + * interface with 'CLUSTER_QUERY_MYSQL_SERVERS_V2'. It makes 'SHUNNED' state equivalent to 'ONLINE', and also + * filters out any 'OFFLINE_HARD' entries. This is done because none of the statuses are valid configuration + * statuses, they are local, transient status that ProxySQL uses during operation. + */ +#define MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS \ + "SELECT " \ + "hostgroup_id, hostname, port, gtid_port, " \ + "CASE" \ + " WHEN status=\"SHUNNED\" THEN \"ONLINE\"" \ + " ELSE status " \ + "END AS status, " \ + "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ + "FROM main.mysql_servers " \ + "WHERE status != \"OFFLINE_HARD\" " \ + "ORDER BY hostgroup_id, hostname, port" + +typedef std::unordered_map umap_mysql_errors; + +class MySrvConnList; +class MySrvC; +class MySrvList; +class MyHGC; + +struct peer_runtime_mysql_servers_t; +struct peer_mysql_servers_v2_t; + +std::string gtid_executed_to_string(gtid_set_t& gtid_executed); +void addGtid(const gtid_t& gtid, gtid_set_t& gtid_executed); + +#include "GTID_Server_Data.h" + +/* +class GTID_Server_Data { + public: + char *address; + uint16_t port; + uint16_t mysql_port; + char *data; + size_t len; + size_t size; + size_t pos; + struct ev_io *w; + char uuid_server[64]; + unsigned long long events_read; + gtid_set_t gtid_executed; + bool active; + GTID_Server_Data(struct ev_io *_w, char *_address, uint16_t _port, uint16_t _mysql_port); + void resize(size_t _s); + ~GTID_Server_Data(); + bool readall(); + bool writeout(); + bool read_next_gtid(); + bool gtid_exists(char *gtid_uuid, uint64_t gtid_trxid); + void read_all_gtids(); + void dump(); +}; +*/ + + +class MySrvConnList { + private: + MySrvC *mysrvc; + int find_idx(MySQL_Connection *c) { + //for (unsigned int i=0; ilen; i++) { + MySQL_Connection *conn = NULL; + conn = (MySQL_Connection *)conns->index(i); + if (conn==c) { + return (unsigned int)i; + } + } + return -1; + } + public: + PtrArray *conns; + MySrvConnList(MySrvC *); + ~MySrvConnList(); + void add(MySQL_Connection *); + void remove(MySQL_Connection *c) { + int i = -1; + i = find_idx(c); + assert(i>=0); + conns->remove_index_fast((unsigned int)i); + } + MySQL_Connection *remove(int); + MySQL_Connection * get_random_MyConn(MySQL_Session *sess, bool ff); + void get_random_MyConn_inner_search(unsigned int start, unsigned int end, unsigned int& conn_found_idx, unsigned int& connection_quality_level, unsigned int& number_of_matching_session_variables, const MySQL_Connection * client_conn); + unsigned int conns_length() { return conns->len; } + void drop_all_connections(); + MySQL_Connection *index(unsigned int); +}; + + +class MySrvC { // MySQL Server Container + public: + MyHGC *myhgc; + char *address; + uint16_t port; + uint16_t gtid_port; + uint16_t flags; + int64_t weight; + unsigned int compression; + int64_t max_connections; + unsigned int aws_aurora_current_lag_us; + unsigned int max_replication_lag; + unsigned int max_connections_used; // The maximum number of connections that has been opened + unsigned int connect_OK; + unsigned int connect_ERR; + int cur_replication_lag; + unsigned int cur_replication_lag_count; + // note that these variables are in microsecond, while user defines max latency in millisecond + unsigned int current_latency_us; + unsigned int max_latency_us; + time_t time_last_detected_error; + unsigned int connect_ERR_at_time_last_detected_error; + unsigned long long queries_sent; + unsigned long long queries_gtid_sync; + unsigned long long bytes_sent; + unsigned long long bytes_recv; + bool shunned_automatic; + bool shunned_and_kill_all_connections; // if a serious failure is detected, this will cause all connections to die even if the server is just shunned + int32_t use_ssl; + char *comment; + MySrvConnList *ConnectionsUsed; + MySrvConnList *ConnectionsFree; + /** + * @brief Constructs a new MySQL Server Container. + * @details For 'server_defaults' parameters, if '-1' is supplied, they try to be obtained from + * 'servers_defaults' entry from 'mysql_hostgroup_attributes' when adding the server to it's target + * hostgroup(via 'MySQL_HostGroups_Manager::add'), if not found, value is set with 'mysql_servers' + * defaults. + * @param addr Address of the server, specified either by IP or hostname. + * @param port Server port. + * @param gitd_port If non-zero, enables GTID tracking for the server. + * @param _weight Server weight. 'server_defaults' param, check @details. + * @param _status Initial server status. + * @param _compression Enables compression for server connections. + * @param _max_connections Max server connections. 'server_defaults' param, check @details. + * @param _max_replication_lag If non-zero, enables replication lag checks. + * @param _use_ssl Enables SSL for server connections. 'servers_defaults' param, check @details. + * @param _max_latency_ms Max ping server latency. When exceeded, server gets excluded from conn-pool. + * @param _comment User defined comment. + */ + MySrvC( + char* addr, uint16_t port, uint16_t gitd_port, int64_t _weight, enum MySerStatus _status, unsigned int _compression, + int64_t _max_connections, unsigned int _max_replication_lag, int32_t _use_ssl, unsigned int _max_latency_ms, + char* _comment + ); + ~MySrvC(); + void connect_error(int, bool get_mutex=true); + void shun_and_killall(); + /** + * @brief Update the maximum number of used connections + * @return The maximum number of used connections + */ + unsigned int update_max_connections_used() + { + unsigned int connections_used = ConnectionsUsed->conns_length(); + if (max_connections_used < connections_used) + max_connections_used = connections_used; + return max_connections_used; + } + void set_status(MySerStatus _status); + inline + MySerStatus get_status() const { return status; } +private: + enum MySerStatus status; +}; + +#endif // 0 + +template +class BaseSrvList { // MySQL Server List + private: + HGC *myhgc; + using TypeSrvC = typename std::conditional< + std::is_same_v, MySrvC, PgSQL_SrvC + >::type; + int find_idx(TypeSrvC *); + public: + PtrArray *servers; + unsigned int cnt() { return servers->len; } + BaseSrvList(HGC *); + ~BaseSrvList(); + void add(TypeSrvC *); + void remove(TypeSrvC *); + TypeSrvC * idx(unsigned int i) {return (TypeSrvC *)servers->index(i); } + + friend class PgSQL_SrvList; + friend class PgSQL_HGC; + +}; + + +template +class BaseHGC { // MySQL Host Group Container + public: + unsigned int hid; + std::atomic num_online_servers; + time_t last_log_time_num_online_servers; + unsigned long long current_time_now; + uint32_t new_connections_now; + using TypeSrvList = typename std::conditional< + std::is_same_v, MySrvList, PgSQL_SrvList + >::type; + BaseSrvList *mysrvs; + struct { // this is a series of attributes specific for each hostgroup + char * init_connect; + char * comment; + char * ignore_session_variables_text; // this is the original version (text format) of ignore_session_variables + uint32_t max_num_online_servers; + uint32_t throttle_connections_per_sec; + int32_t monitor_slave_lag_when_null; + int8_t autocommit; + int8_t free_connections_pct; + int8_t handle_warnings; + bool multiplex; + bool connection_warming; + bool configured; // this variable controls if attributes are configured or not. If not configured, they do not apply + bool initialized; // this variable controls if attributes were ever configured or not. Used by reset_attributes() + nlohmann::json * ignore_session_variables_json = NULL; // the JSON format of ignore_session_variables + } attributes; + struct { + int64_t weight; + int64_t max_connections; + int32_t use_ssl; + } servers_defaults; + void reset_attributes(); + inline + bool handle_warnings_enabled() const { + return attributes.configured == true && attributes.handle_warnings != -1 ? attributes.handle_warnings : mysql_thread___handle_warnings; + } + inline + int32_t get_monitor_slave_lag_when_null() const { + return attributes.configured == true && attributes.monitor_slave_lag_when_null != -1 ? attributes.monitor_slave_lag_when_null : mysql_thread___monitor_slave_lag_when_null; + } + BaseHGC(int); + virtual ~BaseHGC(); + using TypeSrvC = typename std::conditional< + std::is_same_v, MySrvC, PgSQL_SrvC + >::type; + using TypeSess = typename std::conditional< + std::is_same_v, MySQL_Session, PgSQL_Session + >::type; + TypeSess *get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms, TypeSess *sess); + void refresh_online_server_count(); + void log_num_online_server_count_error(); + inline + bool online_servers_within_threshold() const { + if (num_online_servers.load(std::memory_order_relaxed) <= attributes.max_num_online_servers) return true; + return false; + } +}; + +#if 0 +class Group_Replication_Info { + public: + int writer_hostgroup; + int backup_writer_hostgroup; + int reader_hostgroup; + int offline_hostgroup; + int max_writers; + int max_transactions_behind; + char *comment; + bool active; + int writer_is_also_reader; + bool __active; + bool need_converge; // this is set to true on LOAD MYSQL SERVERS TO RUNTIME . This ensure that checks wil take an action + int current_num_writers; + int current_num_backup_writers; + int current_num_readers; + int current_num_offline; + Group_Replication_Info(int w, int b, int r, int o, int mw, int mtb, bool _a, int _w, char *c); + bool update(int b, int r, int o, int mw, int mtb, bool _a, int _w, char *c); + ~Group_Replication_Info(); +}; + +class Galera_Info { + public: + int writer_hostgroup; + int backup_writer_hostgroup; + int reader_hostgroup; + int offline_hostgroup; + int max_writers; + int max_transactions_behind; + char *comment; + bool active; + int writer_is_also_reader; + bool __active; + bool need_converge; // this is set to true on LOAD MYSQL SERVERS TO RUNTIME . This ensure that checks wil take an action + int current_num_writers; + int current_num_backup_writers; + int current_num_readers; + int current_num_offline; + Galera_Info(int w, int b, int r, int o, int mw, int mtb, bool _a, int _w, char *c); + bool update(int b, int r, int o, int mw, int mtb, bool _a, int _w, char *c); + ~Galera_Info(); +}; + +class AWS_Aurora_Info { + public: + int writer_hostgroup; + int reader_hostgroup; + int aurora_port; + int max_lag_ms; + int add_lag_ms; + int min_lag_ms; + int lag_num_checks; + int check_interval_ms; + int check_timeout_ms; + int writer_is_also_reader; + int new_reader_weight; + // TODO + // add intermediary status value, for example the last check time + char * domain_name; + char * comment; + bool active; + bool __active; + AWS_Aurora_Info(int w, int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, char *c); + bool update(int r, int _port, char *_end_addr, int maxl, int al, int minl, int lnc, int ci, int ct, bool _a, int wiar, int nrw, char *c); + ~AWS_Aurora_Info(); +}; + +class MySQLServers_SslParams { + public: + string hostname; + int port; + string username; + string ssl_ca; + string ssl_cert; + string ssl_key; + string ssl_capath; + string ssl_crl; + string ssl_crlpath; + string ssl_cipher; + string tls_version; + string comment; + string MapKey; + MySQLServers_SslParams(string _h, int _p, string _u, + string ca, string cert, string key, string capath, + string crl, string crlpath, string cipher, string tls, + string c) { + hostname = _h; + port = _p; + username = _u; + ssl_ca = ca; + ssl_cert = cert; + ssl_key = key; + ssl_capath = capath; + ssl_crl = crl; + ssl_crlpath = crlpath; + ssl_cipher = cipher; + tls_version = tls; + comment = c; + MapKey = ""; + } + MySQLServers_SslParams(char * _h, int _p, char * _u, + char * ca, char * cert, char * key, char * capath, + char * crl, char * crlpath, char * cipher, char * tls, + char * c) { + hostname = string(_h); + port = _p; + username = string(_u); + ssl_ca = string(ca); + ssl_cert = string(cert); + ssl_key = string(key); + ssl_capath = string(capath); + ssl_crl = string(crl); + ssl_crlpath = string(crlpath); + ssl_cipher = string(cipher); + tls_version = string(tls); + comment = string(c); + MapKey = ""; + } + MySQLServers_SslParams(string _h, int _p, string _u) { + MySQLServers_SslParams(_h, _p, _u, "", "", "", "", "", "", "", "", ""); + } + string getMapKey(const char *del) { + if (MapKey == "") { + MapKey = hostname + string(del) + to_string(port) + string(del) + username; + } + return MapKey; + } +}; + +struct p_hg_counter { + enum metric { + servers_table_version = 0, + server_connections_created, + server_connections_delayed, + server_connections_aborted, + client_connections_created, + client_connections_aborted, + com_autocommit, + com_autocommit_filtered, + com_rollback, + com_rollback_filtered, + com_backend_change_user, + com_backend_init_db, + // TODO: https://github.com/sysown/proxysql/issues/2690 + com_backend_set_names, + com_frontend_init_db, + com_frontend_set_names, + com_frontend_use_db, + com_commit_cnt, + com_commit_cnt_filtered, + selects_for_update__autocommit0, + access_denied_wrong_password, + access_denied_max_connections, + access_denied_max_user_connections, + myhgm_myconnpool_get, + myhgm_myconnpool_get_ok, + myhgm_myconnpool_get_ping, + myhgm_myconnpool_push, + myhgm_myconnpool_reset, + myhgm_myconnpool_destroy, + auto_increment_delay_multiplex, + __size + }; +}; + +struct p_hg_gauge { + enum metric { + server_connections_connected = 0, + client_connections_connected, + __size + }; +}; + +struct p_hg_dyn_counter { + enum metric { + conn_pool_bytes_data_recv = 0, + conn_pool_bytes_data_sent, + connection_pool_conn_err, + connection_pool_conn_ok, + connection_pool_queries, + gtid_executed, + proxysql_mysql_error, + mysql_error, + __size + }; +}; + +enum class p_mysql_error_type { + mysql, + proxysql +}; + +struct p_hg_dyn_gauge { + enum metric { + connection_pool_conn_free = 0, + connection_pool_conn_used, + connection_pool_latency_us, + connection_pool_status, + __size + }; +}; + +struct hg_metrics_map_idx { + enum index { + counters = 0, + gauges, + dyn_counters, + dyn_gauges, + }; +}; + +/** + * @brief Required server info for the read_only Monitoring actions and replication_lag Monitoring actions. + */ +using hostgroupid_t = int; +using hostname_t = std::string; +using address_t = std::string; +using port_t = unsigned int; +using read_only_t = int; +using current_replication_lag = int; +using override_replication_lag = bool; + +using read_only_server_t = std::tuple; +using replication_lag_server_t = std::tuple; + +enum READ_ONLY_SERVER_T { + ROS_HOSTNAME = 0, + ROS_PORT, + ROS_READONLY, + ROS__SIZE +}; + +enum REPLICATION_LAG_SERVER_T { + RLS_HOSTGROUP_ID = 0, + RLS_ADDRESS, + RLS_PORT, + RLS_CURRENT_REPLICATION_LAG, + RLS_OVERRIDE_REPLICATION_LAG, + RLS__SIZE +}; + +/** + * @brief Contains the minimal info for server creation. + */ +struct srv_info_t { + /* @brief Server address */ + string addr; + /* @brief Server port */ + uint16_t port; + /* @brief Server type identifier, used for logging, e.g: 'Aurora AWS', 'GR', etc... */ + string kind; +}; + +/** + * @brief Contains options to be specified during server creation. + */ +struct srv_opts_t { + int64_t weigth; + int64_t max_conns; + int32_t use_ssl; +}; +#endif // 0 + +template +class Base_HostGroups_Manager { + private: + SQLite3DB *admindb; + SQLite3DB *mydb; + pthread_mutex_t readonly_mutex; + std::set read_only_set1; + std::set read_only_set2; + pthread_mutex_t lock; + + PtrArray *MyHostGroups; + std::unordered_mapMyHostGroups_map; + + HGC * MyHGC_find(unsigned int); + HGC * MyHGC_create(unsigned int); + + public: + Base_HostGroups_Manager(); + HGC * MyHGC_lookup(unsigned int); + SQLite3_result * execute_query(char *query, char **error); + + void wrlock(); + void wrunlock(); +#ifdef DEBUG + bool is_locked = false; +#endif + + friend class MySQL_HostGroups_Manager; + friend class PgSQL_HostGroups_Manager; + +}; + +#if 0 +class MySQL_HostGroups_Manager { + private: + SQLite3DB *admindb; + SQLite3DB *mydb; + pthread_mutex_t readonly_mutex; + std::set read_only_set1; + std::set read_only_set2; + pthread_mutex_t lock; + + enum HGM_TABLES { + MYSQL_SERVERS_V2 = 0, + MYSQL_REPLICATION_HOSTGROUPS, + MYSQL_GROUP_REPLICATION_HOSTGROUPS, + MYSQL_GALERA_HOSTGROUPS, + MYSQL_AWS_AURORA_HOSTGROUPS, + MYSQL_HOSTGROUP_ATTRIBUTES, + MYSQL_SERVERS_SSL_PARAMS, + MYSQL_SERVERS, + + __HGM_TABLES_SIZE + }; + + std::array table_resultset_checksum { {0} }; + + class HostGroup_Server_Mapping { + public: + enum Type { + WRITER = 0, + READER = 1, + + __TYPE_SIZE + }; + + struct Node { + MySrvC* srv = NULL; + unsigned int reader_hostgroup_id = -1; + unsigned int writer_hostgroup_id = -1; + //MySerStatus server_status = MYSQL_SERVER_STATUS_OFFLINE_HARD; + }; + + HostGroup_Server_Mapping(MySQL_HostGroups_Manager* hgm) : readonly_flag(1), myHGM(hgm) { } + ~HostGroup_Server_Mapping() = default; + + /** + * @brief Copies all unique nodes from source vector to destination vector. + * @details Copies all unique nodes from source vector to destination vector. The source and destination + * vectors are identified by an input enumeration type, which can be either a reader or a writer. + * During the copying process, the function also adds servers to the HostGroup connection container. + * @param dest_type Input Can be reader or writer + * @param src_type Input Can be reader or writer + */ + void copy_if_not_exists(Type dest_type, Type src_type); + + /** + * @brief Removes node located at the specified index. + * @details Node is removed from vector located at the specified index identified by an input enumeration type. + * Node that was removed is marked as offline in the HostGroup connection container. + * @param dest_type Input Can be reader or writer + * @param index Input Index of node to be removed + */ + void remove(Type type, size_t index); + + /** + * @brief Removes all nodes. + * @details All nodes are removed from vector, identified by an input enumeration type. + * Nodes that are removed is marked as offline in the HostGroup connection container. + * @param type Input Can be reader or writer + */ + void clear(Type type); + + inline + const std::vector& get(Type type) const { + return mapping[type]; + } + + inline + void add(Type type, Node& node) { + mapping[type].push_back(node); + } + + inline + void set_readonly_flag(int val) { + readonly_flag = val; + } + + inline + int get_readonly_flag() const { + return readonly_flag; + } + + private: + unsigned int get_hostgroup_id(Type type, const Node& node) const; + MySrvC* insert_HGM(unsigned int hostgroup_id, const MySrvC* srv); + void remove_HGM(MySrvC* srv); + + std::array, __TYPE_SIZE> mapping; // index 0 contains reader and 1 contains writer hostgroups + int readonly_flag; + MySQL_HostGroups_Manager* myHGM; + }; + + /** + * @brief Used by 'MySQL_Monitor::read_only' to hold a mapping between servers and hostgroups. + * @details The hostgroup mapping holds the MySrvC for each of the hostgroups in which the servers is + * present, distinguishing between 'READER' and 'WRITER' hostgroups. + */ + std::unordered_map> hostgroup_server_mapping; + /** + * @brief Holds the previous computed checksum for 'mysql_servers'. + * @details Used to check if the servers checksums has changed during 'commit', if a change is detected, + * the member 'hostgroup_server_mapping' is required to be regenerated. + * + * This is only updated during 'read_only_action_v2', since the action itself modifies + * 'hostgroup_server_mapping' in case any actions needs to be performed against the servers. + */ + uint64_t hgsm_mysql_servers_checksum = 0; + /** + * @brief Holds the previous checksum for the 'MYSQL_REPLICATION_HOSTGROUPS'. + * @details Used during 'commit' to determine if config has changed for 'MYSQL_REPLICATION_HOSTGROUPS', + * and 'hostgroup_server_mapping' should be rebuild. + */ + uint64_t hgsm_mysql_replication_hostgroups_checksum = 0; + + + PtrArray *MyHostGroups; + std::unordered_mapMyHostGroups_map; + + std::mutex Servers_SSL_Params_map_mutex; + std::unordered_map Servers_SSL_Params_map; + + MyHGC * MyHGC_find(unsigned int); + MyHGC * MyHGC_create(unsigned int); + + void add(MySrvC *, unsigned int); + void purge_mysql_servers_table(); + void generate_mysql_servers_table(int *_onlyhg=NULL); + void generate_mysql_replication_hostgroups_table(); + Galera_Info *get_galera_node_info(int hostgroup); + + /** + * @brief This resultset holds the current values for 'runtime_mysql_servers' computed by either latest + * 'commit' or fetched from another Cluster node. It's also used by ProxySQL_Admin to respond to the + * intercepted query 'CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS'. + * @details This resultset can't right now just contain the value for 'incoming_mysql_servers' as with the + * rest of the intercepted resultset. This is due to 'runtime_mysql_servers' reconfigurations that can be + * triggered by monitoring actions like 'Galera' currently performs. These actions not only trigger status + * changes in the servers, but also re-generate the servers table via 'commit', thus generating a new + * checksum in the process. Because of this potential mismatch, the fetching server wouldn't be able to + * compute the proper checksum for the fetched 'runtime_mysql_servers' config. + * + * As previously stated, these reconfigurations are monitoring actions, they can't be packed or performed + * in a single action, since monitoring data is required, which may not be already present. This makes + * this a convergent, but iterative process, that can't be compressed into a single action. Using other + * nodes 'runtime_mysql_servers' while fetching represents a best effort for avoiding these + * reconfigurations in nodes that already holds the same monitoring conditions. If monitoring + * conditions are not the same, circular fetching is still possible due to the previously described + * scenario. + */ + SQLite3_result* runtime_mysql_servers; + /** + * @brief These resultset holds the latest values for 'incoming_*' tables used to promoted servers to runtime. + * @details All these resultsets are used by 'Cluster' to fetch and promote the same configuration used in the + * node across the whole cluster. For these, the queries: + * - 'CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS' + * - 'CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS' + * - 'CLUSTER_QUERY_MYSQL_GALERA' + * - 'CLUSTER_QUERY_MYSQL_AWS_AURORA' + * - 'CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES' + * Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets. + */ + SQLite3_result *incoming_replication_hostgroups; + + void generate_mysql_group_replication_hostgroups_table(); + /** + * @brief Regenerates the resultset used by 'MySQL_Monitor' containing the servers to be monitored. + * @details This function is required to be called after any action that results in the addition of a new + * server that 'MySQL_Monitor' should be aware of for 'group_replication', i.e. a server added to the + * hostgroups present in any entry of 'mysql_group_replication_hostgroups'. E.g: + * - Inside 'generate_mysql_group_replication_hostgroups_table'. + * - Autodiscovery. + * + * NOTE: This is a common pattern for all the clusters monitoring. + */ + void generate_mysql_group_replication_hostgroups_monitor_resultset(); + SQLite3_result *incoming_group_replication_hostgroups; + + pthread_mutex_t Group_Replication_Info_mutex; + std::map Group_Replication_Info_Map; + + void generate_mysql_galera_hostgroups_table(); + SQLite3_result *incoming_galera_hostgroups; + + pthread_mutex_t Galera_Info_mutex; + std::map Galera_Info_Map; + + void generate_mysql_aws_aurora_hostgroups_table(); + SQLite3_result *incoming_aws_aurora_hostgroups; + + pthread_mutex_t AWS_Aurora_Info_mutex; + std::map AWS_Aurora_Info_Map; + + void generate_mysql_hostgroup_attributes_table(); + SQLite3_result *incoming_hostgroup_attributes; + + void generate_mysql_servers_ssl_params_table(); + SQLite3_result *incoming_mysql_servers_ssl_params; + + SQLite3_result* incoming_mysql_servers_v2; + + std::thread *HGCU_thread; + + std::thread *GTID_syncer_thread; + //pthread_t GTID_syncer_thread_id; + //pthread_t HGCU_thread_id; + + char rand_del[8]; + pthread_mutex_t mysql_errors_mutex; + umap_mysql_errors mysql_errors_umap; + + /** + * @brief Update the prometheus "connection_pool" counters. + */ + void p_update_connection_pool(); + /** + * @brief Update the "stats_mysql_gtid_executed" counters. + */ + void p_update_mysql_gtid_executed(); + + void p_update_connection_pool_update_counter( + const std::string& endpoint_id, const std::map& labels, + std::map& m_map, unsigned long long value, p_hg_dyn_counter::metric idx + ); + void p_update_connection_pool_update_gauge( + const std::string& endpoint_id, const std::map& labels, + std::map& m_map, unsigned long long value, p_hg_dyn_gauge::metric idx + ); + + void group_replication_lag_action_set_server_status(MyHGC* myhgc, char* address, int port, int lag_count, bool enable); + + public: + std::mutex galera_set_writer_mutex; + /** + * @brief Mutex used to guard 'mysql_servers_to_monitor' resulset. + */ + std::mutex mysql_servers_to_monitor_mutex; + /** + * @brief Resulset containing the latest 'mysql_servers' present in 'mydb'. + * @details This resulset should be updated via 'update_table_mysql_servers_for_monitor' each time actions + * that modify the 'mysql_servers' table are performed. + */ + SQLite3_result* mysql_servers_to_monitor; + + pthread_rwlock_t gtid_rwlock; + std::unordered_map gtid_map; + struct ev_async * gtid_ev_async; + struct ev_loop * gtid_ev_loop; + struct ev_timer * gtid_ev_timer; + bool gtid_missing_nodes; + struct { + unsigned int servers_table_version; + pthread_mutex_t servers_table_version_lock; + pthread_cond_t servers_table_version_cond; + unsigned long client_connections_aborted; + unsigned long client_connections_created; + int client_connections; + unsigned long server_connections_aborted; + unsigned long server_connections_created; + unsigned long server_connections_delayed; + unsigned long server_connections_connected; + unsigned long myconnpoll_get; + unsigned long myconnpoll_get_ok; + unsigned long myconnpoll_get_ping; + unsigned long myconnpoll_push; + unsigned long myconnpoll_reset; + unsigned long myconnpoll_destroy; + unsigned long long autocommit_cnt; + unsigned long long commit_cnt; + unsigned long long rollback_cnt; + unsigned long long autocommit_cnt_filtered; + unsigned long long commit_cnt_filtered; + unsigned long long rollback_cnt_filtered; + unsigned long long backend_change_user; + unsigned long long backend_init_db; + unsigned long long backend_set_names; + unsigned long long frontend_init_db; + unsigned long long frontend_set_names; + unsigned long long frontend_use_db; + unsigned long long access_denied_wrong_password; + unsigned long long access_denied_max_connections; + unsigned long long access_denied_max_user_connections; + unsigned long long select_for_update_or_equivalent; + unsigned long long auto_increment_delay_multiplex; + + ////////////////////////////////////////////////////// + /// Prometheus Metrics /// + ////////////////////////////////////////////////////// + + /// Prometheus metrics arrays + std::array p_counter_array {}; + std::array p_gauge_array {}; + + // Prometheus dyn_metrics families arrays + std::array*, p_hg_dyn_counter::__size> p_dyn_counter_array {}; + std::array*, p_hg_dyn_gauge::__size> p_dyn_gauge_array {}; + + /// Prometheus connection_pool metrics + std::map p_conn_pool_bytes_data_recv_map {}; + std::map p_conn_pool_bytes_data_sent_map {}; + std::map p_connection_pool_conn_err_map {}; + std::map p_connection_pool_conn_free_map {}; + std::map p_connection_pool_conn_ok_map {}; + std::map p_connection_pool_conn_used_map {}; + std::map p_connection_pool_latency_us_map {}; + std::map p_connection_pool_queries_map {}; + std::map p_connection_pool_status_map {}; + + /// Prometheus gtid_executed metrics + std::map p_gtid_executed_map {}; + + /// Prometheus mysql_error metrics + std::map p_mysql_errors_map {}; + + ////////////////////////////////////////////////////// + } status; + /** + * @brief Update the module prometheus metrics. + */ + void p_update_metrics(); + /** + * @brief Updates the 'mysql_error' counter identified by the 'm_id' parameter, + * or creates a new one in case of not existing. + * + * @param hid The hostgroup identifier. + * @param address The connection address that triggered the error. + * @param port The port of the connection that triggered the error. + * @param errno The error code itself. + */ + void p_update_mysql_error_counter(p_mysql_error_type err_type, unsigned int hid, char* address, uint16_t port, unsigned int code); + + wqueue queue; + // has_gtid_port is set to true if *any* of the servers in mysql_servers has gtid_port enabled + // it is configured during commit() + // NOTE: this variable is currently NOT used, but in future will be able + // to deprecate mysql-default_session_track_gtids because proxysql will + // be automatically able to determine when to enable GTID tracking + std::atomic has_gtid_port; + MySQL_HostGroups_Manager(); + ~MySQL_HostGroups_Manager(); + void init(); + void wrlock(); + void wrunlock(); +#ifdef DEBUG + bool is_locked = false; +#endif + int servers_add(SQLite3_result *resultset); + /** + * @brief Generates a new global checksum for module 'mysql_servers_v2' using the provided hash. + * @param servers_v2_hash The 'raw_checksum' from 'MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS' or peer node. + * @return Checksum computed using the provided hash, and 'mysql_servers' config tables hashes. + */ + std::string gen_global_mysql_servers_v2_checksum(uint64_t servers_v2_hash); + bool commit(); + bool commit( + const peer_runtime_mysql_servers_t& peer_runtime_mysql_servers, + const peer_mysql_servers_v2_t& peer_mysql_servers_v2, + bool only_commit_runtime_mysql_servers = true, + bool update_version = false + ); + /** + * @brief Extracted from 'commit'. Performs the following actions: + * 1. Re-generates the 'myhgm.mysql_servers' table. + * 2. If supplied 'runtime_mysql_servers' is 'nullptr': + * 1. Gets the contents of the table via 'MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS'. + * 2. Save the resultset into 'this->runtime_mysql_servers'. + * 3. If supplied 'runtime_mysql_servers' isn't 'nullptr': + * 1. Updates the 'this->runtime_mysql_servers' with it. + * 4. Updates 'HGM_TABLES::MYSQL_SERVERS' with raw checksum from 'this->runtime_mysql_servers'. + * @param runtime_mysql_servers If not 'nullptr', used to update 'this->runtime_mysql_servers'. + * @return The updated 'MySQL_HostGroups_Manager::runtime_mysql_servers'. + */ + uint64_t commit_update_checksum_from_mysql_servers(SQLite3_result* runtime_mysql_servers = nullptr); + /** + * @brief Analogous to 'commit_generate_mysql_servers_table' but for 'incoming_mysql_servers_v2'. + */ + uint64_t commit_update_checksum_from_mysql_servers_v2(SQLite3_result* incoming_mysql_servers_v2 = nullptr); + /** + * @brief Update all HGM_TABLES checksums and uses them to update the supplied SpookyHash. + * @details Checksums are the checksums for the following tables: + * - mysql_replication_hostgroups + * - mysql_group_replication_hostgroups + * - mysql_galera_hostgroups + * - mysql_aws_aurora_hostgroups + * - mysql_hostgroup_attributes + * + * These checksums are used to compute the global checksum for 'mysql_servers_v2'. + * @param myhash SpookyHash to be updated with all the computed checksums. + * @param init Indicates if the SpookyHash checksum is initialized. + */ + void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init); + /** + * @brief Performs the following actions: + * 1. Gets the current contents of table 'myhgm.TableName', using 'ColumnName' ordering. + * 2. Computes the checksum for that resultset. + * 3. Updates the supplied 'raw_checksum' and the supplied 'SpookyHash' with it. + * @details Stands for 'commit_update_checksum_from_table_1'. + * @param myhash Hash to be updated with the resultset checksum from the selected table. + * @param init If the supplied 'SpookyHash' has already being initialized. + * @param TableName The tablename from which to obtain the resultset for the 'raw_checksum' computation. + * @param ColumnName A column name to use for ordering in the supplied 'TableName'. + * @param raw_checksum A 'raw_checksum' to be updated with the obtained resultset. + */ + void CUCFT1( + SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum + ); + /** + * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime. + * The store configuration is later used by Cluster to propagate current config. + * @param The resulset to be stored replacing the current one. + */ + void save_runtime_mysql_servers(SQLite3_result *); + + /** + * @brief Store the resultset for the 'mysql_servers_v2' table. + * The store configuration is later used by Cluster to propagate current config. + * @param The resulset to be stored replacing the current one. + */ + void save_mysql_servers_v2(SQLite3_result* s); + + /** + * @brief These setters/getter functions store and retrieve the currently hold resultset for the + * 'incoming_*' table set that have been loaded to runtime. The store configuration is later used by + * Cluster to propagate current config. + * @param The resulset to be stored replacing the current one. + */ + + void save_incoming_mysql_table(SQLite3_result *, const string&); + SQLite3_result* get_current_mysql_table(const string& name); + + SQLite3_result * execute_query(char *query, char **error); + + + /** + * @brief Creates a resultset with the current full content of the target table. + * @param string The target table. Valid values are: + * - "mysql_aws_aurora_hostgroups" + * - "mysql_galera_hostgroups" + * - "mysql_group_replication_hostgroups" + * - "mysql_replication_hostgroups" + * - "mysql_hostgroup_attributes" + * - "mysql_servers" + * - "cluster_mysql_servers" + * When targeting 'mysql_servers' table is purged and regenerated. + * @return The generated resultset. + */ + SQLite3_result* dump_table_mysql(const string&); + + /** + * @brief Update the public member resulset 'mysql_servers_to_monitor'. This resulset should contain the latest + * 'mysql_servers' present in 'MySQL_HostGroups_Manager' db, which are not 'OFFLINE_HARD'. The resulset + * fields match the definition of 'monitor_internal.mysql_servers' table. + * @details Several details: + * - Function assumes that 'mysql_servers' table from 'MySQL_HostGroups_Manager' db is ready + * to be consumed, because of this it doesn't perform any of the following operations: + * - Purging 'mysql_servers' table. + * - Regenerating 'mysql_servers' table. + * - Function locks on 'mysql_servers_to_monitor_mutex'. + * @param lock When supplied the function calls 'wrlock()' and 'wrunlock()' functions for accessing the db. + */ + void update_table_mysql_servers_for_monitor(bool lock=false); + + void MyConn_add_to_pool(MySQL_Connection *); + /** + * @brief Creates a new server in the target hostgroup if isn't already present. + * @details If the server is found already in the target hostgroup, no action is taken, unless its status + * is 'OFFLINE_HARD'. In case of finding it as 'OFFLINE_HARD': + * 1. Server hostgroup attributes are reset to known values, so they can be updated. + * 2. Server attributes are updated to either table definition values, or hostgroup 'servers_defaults'. + * 3. Server is bring back as 'ONLINE'. + * @param hid The hostgroup in which the server is to be created (or to bring it back as 'ONLINE'). + * @param srv_info Basic server info to be used during creation. + * @param srv_opts Server creation options. + * @return 0 in case of success, -1 in case of failure. + */ + int create_new_server_in_hg(uint32_t hid, const srv_info_t& srv_info, const srv_opts_t& srv_opts); + /** + * @brief Completely removes server from the target hostgroup if found. + * @details Several actions are taken if server is found: + * - Set the server as 'OFFLINE_HARD'. + * - Drop all current FREE connections to the server. + * - Delete the server from the 'myhgm.mysql_servers' table. + * + * This later step is not required if the caller is already going to perform a full deletion of the + * servers in the target hostgroup. Which is a common operation during table regeneration. + * @param hid Target hostgroup id. + * @param addr Target server address. + * @param port Target server port. + * @return 0 in case of success, -1 in case of failure. + */ + int remove_server_in_hg(uint32_t hid, const string& addr, uint16_t port); + + MySQL_Connection * get_MyConn_from_pool(unsigned int hid, MySQL_Session *sess, bool ff, char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms); + + void drop_all_idle_connections(); + int get_multiple_idle_connections(int, unsigned long long, MySQL_Connection **, int); + SQLite3_result * SQL3_Connection_Pool(bool _reset, int *hid = NULL); + SQLite3_result * SQL3_Free_Connections(); + + void push_MyConn_to_pool(MySQL_Connection *, bool _lock=true); + void push_MyConn_to_pool_array(MySQL_Connection **, unsigned int); + void destroy_MyConn_from_pool(MySQL_Connection *, bool _lock=true); + + void replication_lag_action_inner(MyHGC *, const char*, unsigned int, int, bool); + void replication_lag_action(const std::list& mysql_servers); + void read_only_action(char *hostname, int port, int read_only); + void read_only_action_v2(const std::list& mysql_servers); + unsigned int get_servers_table_version(); + void wait_servers_table_version(unsigned, unsigned); + bool shun_and_killall(char *hostname, int port); + void set_server_current_latency_us(char *hostname, int port, unsigned int _current_latency_us); + unsigned long long Get_Memory_Stats(); + + void add_discovered_servers_to_mysql_servers_and_replication_hostgroups(const vector>& new_servers); + + void update_group_replication_set_offline(char *_hostname, int _port, int _writer_hostgroup, char *error); + void update_group_replication_set_read_only(char *_hostname, int _port, int _writer_hostgroup, char *error); + void update_group_replication_set_writer(char *_hostname, int _port, int _writer_hostgroup); + /** + * @brief Tries to add a new server found during GR autodiscovery to the supplied hostgroup. + * @details For adding the new server, several actions are performed: + * 1. Lookup the target server in the corresponding MyHGC for the supplied hostgroup. + * 2. If server is found, and it's status isn't 'OFFLINE_HARD' do nothing. Otherwise: + * - If server is found as 'OFFLINE_HARD', reset the internal values corresponding to + * 'servers_defaults' values to '-1', update the defaulted values to the ones in its 'MyHGC', lastly + * re-enable the server and log the action. + * - If server isn't found, create it in the corresponding reader hostgroup of the supplied writer + * hostgroup, setting all 'servers_defaults' params as '-1', log the action. + * - After any of the two previous actions, always regenerate servers data structures. + * + * NOTE: Server data structures regeneration requires: + * 1. Purging the 'mysql_servers_table' (Lazy removal of 'OFFLINE_HARD' servers.) + * 2. Regenerate the actual 'myhgm::mysql_servers' table from memory structures. + * 3. Update the 'mysql_servers' resultset used for monitoring. This resultset is used for general + * monitoring actions like 'ping', 'connect'. + * 4. Regenerate the specific resultset for 'Group Replication' monitoring. This resultset is the way to + * communicate back to the main monitoring thread that servers config has changed, and a new thread + * shall be created with the new servers config. This same principle is used for Aurora. + * + * @param _host Server address. + * @param _port Server port. + * @param _wr_hg Writer hostgroup of the cluster being monitored. Autodiscovered servers are always added + * to the reader hostgroup by default, later monitoring actions will re-position the server is required. + */ + void update_group_replication_add_autodiscovered(const std::string& _host, int _port, int _wr_hg); + void converge_group_replication_config(int _writer_hostgroup); + /** + * @brief Set the supplied server as SHUNNED, this function shall be called + * to 'SHUNNED' those servers which replication lag is bigger than: + * - `mysql_thread___monitor_groupreplication_max_transactions_behind_count` + * + * @details The function automatically handles the appropriate operation to + * perform on the supplied server, based on the supplied 'enable' flag and + * in 'monitor_groupreplication_max_transaction_behind_for_read_only' + * variable. In case the value of the variable is: + * + * * '0' or '2': It's required to search the writer hostgroup for + * finding the supplied server. + * * '1' or '2': It's required to search the reader hostgroup for + * finding the supplied server. + * + * @param _hid The writer hostgroup. + * @param address The server address. + * @param port The server port. + * @param lag_counts The computed lag for the sever. + * @param read_only Boolean specifying the read_only flag value of the server. + * @param enable Boolean specifying if the server needs to be disabled / enabled, + * 'true' for enabling the server if it's 'SHUNNED', 'false' for disabling it. + */ + void group_replication_lag_action(int _hid, char *address, unsigned int port, int lag_counts, bool read_only, bool enable); + void update_galera_set_offline(char *_hostname, int _port, int _writer_hostgroup, char *error, bool soft=false); + void update_galera_set_read_only(char *_hostname, int _port, int _writer_hostgroup, char *error); + void update_galera_set_writer(char *_hostname, int _port, int _writer_hostgroup); + void converge_galera_config(int _writer_hostgroup); + + // FIXME : add action functions for AWS Aurora + //void aws_aurora_replication_lag_action(int _whid, int _rhid, char *address, unsigned int port, float current_replication_lag, bool enable, bool verbose=true); + //bool aws_aurora_replication_lag_action(int _whid, int _rhid, char *address, unsigned int port, unsigned int current_replication_lag_us, bool enable, bool is_writer, bool verbose=true); + //void update_aws_aurora_set_writer(int _whid, int _rhid, char *address, unsigned int port, bool verbose=true); + //void update_aws_aurora_set_reader(int _whid, int _rhid, char *_hostname, int _port); + bool aws_aurora_replication_lag_action(int _whid, int _rhid, char *server_id, float current_replication_lag_ms, bool enable, bool is_writer, bool verbose=true); + void update_aws_aurora_set_writer(int _whid, int _rhid, char *server_id, bool verbose=true); + void update_aws_aurora_set_reader(int _whid, int _rhid, char *server_id); + /** + * @brief Updates the resultset and corresponding checksum used by Monitor for AWS Aurora. + * @details This is required to be called when: + * - The 'mysql_aws_aurora_hostgroups' table is regenerated (via 'commit'). + * - When new servers are discovered, and created in already monitored Aurora clusters. + * + * The resultset holds the servers that are present in 'mysql_servers' table, and share hostgroups with + * the **active** clusters specified in 'mysql_aws_aurora_hostgroups'. See query + * 'SELECT_AWS_AURORA_SERVERS_FOR_MONITOR'. + * @param lock Wether if both 'AWS_Aurora_Info_mutex' and 'MySQL_Monitor::aws_aurora_mutex' mutexes should + * be taken or not. + */ + void update_aws_aurora_hosts_monitor_resultset(bool lock=false); + + SQLite3_result * get_stats_mysql_gtid_executed(); + void generate_mysql_gtid_executed_tables(); + bool gtid_exists(MySrvC *mysrvc, char * gtid_uuid, uint64_t gtid_trxid); + + SQLite3_result *SQL3_Get_ConnPool_Stats(); + void increase_reset_counter(); + + void add_mysql_errors(int hostgroup, char *hostname, int port, char *username, char *address, char *schemaname, int err_no, char *last_error); + SQLite3_result *get_mysql_errors(bool); + + void shutdown(); + void unshun_server_all_hostgroups(const char * address, uint16_t port, time_t t, int max_wait_sec, unsigned int *skip_hid); + MySrvC* find_server_in_hg(unsigned int _hid, const std::string& addr, int port); + + MySQLServers_SslParams * get_Server_SSL_Params(char *hostname, int port, char *username); + +private: + void update_hostgroup_manager_mappings(); + uint64_t get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers = nullptr); + uint64_t get_mysql_servers_v2_checksum(SQLite3_result* incoming_mysql_servers_v2 = nullptr); +}; + +#endif // 0 +#endif // CLASS_BASE_HOSTGROUPS_MANAGER_H diff --git a/include/Base_Session.h b/include/Base_Session.h new file mode 100644 index 0000000000..52474d8dc9 --- /dev/null +++ b/include/Base_Session.h @@ -0,0 +1,146 @@ +template class Base_Session; + +//// avoid loading definition of MySQL_Session and PgSQL_Session +//#define __CLASS_MYSQL_SESSION_H +//#define __CLASS_PGSQL_SESSION_H + +#include "proxysql.h" +#include "cpp.h" + +#ifndef CLASS_BASE_SESSION_H +#define CLASS_BASE_SESSION_H + +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON + +class MySQL_STMTs_meta; +class StmtLongDataHandler; +class MySQL_Session; +class PgSQL_Session; + +template +class Base_Session { + public: + Base_Session(); + virtual ~Base_Session(); + + // uint64_t + unsigned long long start_time; + unsigned long long pause_until; + + unsigned long long idle_since; + unsigned long long transaction_started_at; + + T * thread; + B *mybe; + PtrArray *mybes; + DS * client_myds; + /* + * @brief Store the hostgroups that hold connections that have been flagged as 'expired' by the + * maintenance thread. These values will be used to release the retained connections in the specific + * hostgroups in housekeeping operations, before client packet processing. Currently 'housekeeping_before_pkts'. + */ + std::vector hgs_expired_conns {}; + char * default_schema; + char * user_attributes; + + //this pointer is always initialized inside handler(). + // it is an attempt to start simplifying the complexing of handler() + uint32_t thread_session_id; + unsigned long long last_insert_id; + int last_HG_affected_rows; + enum session_status status; + int healthy; + int user_max_connections; + int current_hostgroup; + int default_hostgroup; + int previous_hostgroup; + /** + * @brief Charset directly specified by the client. Supplied and updated via 'HandshakeResponse' + * and 'COM_CHANGE_USER' packets. + * @details Used when session needs to be restored via 'COM_RESET_CONNECTION'. + */ + int default_charset; + int locked_on_hostgroup; + int next_query_flagIN; + int mirror_hostgroup; + int mirror_flagOUT; + unsigned int active_transactions; + int autocommit_on_hostgroup; + int transaction_persistent_hostgroup; + int to_process; + int pending_connect; + enum proxysql_session_type session_type; + int warning_in_hg; + + // bool + bool autocommit; + bool autocommit_handled; + bool sending_set_autocommit; + bool killed; + bool locked_on_hostgroup_and_all_variables_set; + //bool admin; + bool max_connections_reached; + bool client_authenticated; + bool connections_handler; + bool mirror; + //bool stats; + bool schema_locked; + bool transaction_persistent; + bool session_fast_forward; + bool started_sending_data_to_client; // this status variable tracks if some result set was sent to the client, or if proxysql is still buffering everything + bool use_ssl; + MySQL_STMTs_meta *sess_STMTs_meta; + StmtLongDataHandler *SLDH; + + + + void init(); + //template B * find_backend(int hostgroup_id); + //template B * create_backend(int, DS * _myds = NULL); + //template B * find_or_create_backend(int, DS * _myds = NULL); + B * find_backend(int hostgroup_id); + B * create_backend(int, DS * _myds = NULL); + B * find_or_create_backend(int, DS * _myds = NULL); + void writeout(); + void return_proxysql_internal(PtrSize_t* pkt); + virtual void generate_proxysql_internal_session_json(nlohmann::json &) = 0; + virtual void RequestEnd(DS *) = 0; + virtual void SQLite3_to_MySQL(SQLite3_result*, char*, int, MySQL_Protocol*, bool in_transaction = false, bool deprecate_eof_active = false) = 0; + bool has_any_backend(); + void reset_all_backends(); + bool handler_special_queries_STATUS(PtrSize_t*); + /** + * @brief Performs the required housekeeping operations over the session and its connections before + * performing any processing on received client packets. + */ + void housekeeping_before_pkts(); + virtual void create_new_session_and_reset_connection(DS *_myds) = 0; + + using TypeConn = typename std::conditional< + std::is_same_v, MySQL_Connection, PgSQL_Connection + >::type; + void update_expired_conns(const std::vector>&); + + void set_unhealthy(); + unsigned int NumActiveTransactions(bool check_savpoint=false); + bool HasOfflineBackends(); + bool SetEventInOfflineBackends(); + /** + * @brief Finds one active transaction in the current backend connections. + * @details Since only one connection is returned, if the session holds multiple backend connections with + * potential transactions, the priority is: + * 1. Connections flagged with 'SERVER_STATUS_IN_TRANS', or 'autocommit=0' in combination with + * 'autocommit_false_is_transaction'. + * 2. Connections with 'autocommit=0' holding a 'SAVEPOINT'. + * 3. Connections with 'unknown transaction status', e.g: connections with errors. + * @param check_savepoint Used to also check for connections holding savepoints. See MySQL bug + * https://bugs.mysql.com/bug.php?id=107875. + * @returns The hostgroup in which the connection was found, -1 in case no connection is found. + */ + int FindOneActiveTransaction(bool check_savepoint=false); +}; + +#endif // CLASS_BASE_SESSION_H diff --git a/include/Base_Thread.h b/include/Base_Thread.h new file mode 100644 index 0000000000..1b628f8112 --- /dev/null +++ b/include/Base_Thread.h @@ -0,0 +1,95 @@ +#ifndef CLASS_BASE_THREAD_H +#define CLASS_BASE_THREAD_H + +#include "proxysql.h" + +typedef struct _thr_id_username_t { + uint32_t id; + char *username; +} thr_id_usr; + +typedef struct _kill_queue_t { + pthread_mutex_t m; + std::vector conn_ids; + std::vector query_ids; +} kill_queue_t; + +/** + * @class Session_Regex + * @brief Encapsulates regex operations for session handling. + * + * This class is used for matching patterns in SQL queries, specifically for + * settings like sql_log_bin, sql_mode, and time_zone. + * See issues #509 , #815 and #816 + */ +class Session_Regex { +private: + void* opt; + void* re; + char* s; +public: + Session_Regex(char* p); + ~Session_Regex(); + bool match(char* m); +}; + +class MySQL_Thread; +class PgSQL_Thread; + +class Base_Thread { + private: + bool maintenance_loop; + public: + unsigned long long curtime; + unsigned long long last_move_to_idle_thread_time; + bool epoll_thread; + int shutdown; + PtrArray *mysql_sessions; + Session_Regex **match_regexes; + Base_Thread(); + ~Base_Thread(); + template + S create_new_session_and_client_data_stream(int _fd); + template + void register_session(T, S, bool up_start = true); + template + void check_timing_out_session(unsigned int n); + template + void check_for_invalid_fd(unsigned int n); + template + void ProcessAllSessions_SortingSessions(); + template + void ProcessAllMyDS_AfterPoll(); + template + void read_one_byte_from_pipe(unsigned int n); + template + void tune_timeout_for_myds_needs_pause(DS * myds); + template + void tune_timeout_for_session_needs_pause(DS * myds); + template + void configure_pollout(DS * myds, unsigned int n); + template + bool set_backend_to_be_skipped_if_frontend_is_slow(DS * myds, unsigned int n); +#ifdef IDLE_THREADS + template bool move_session_to_idle_mysql_sessions(DS * myds, unsigned int n); +#endif // IDLE_THREADS + template unsigned int find_session_idx_in_mysql_sessions(S * sess); + template void ProcessAllMyDS_BeforePoll(); + template void run_SetAllSession_ToProcess0(); + + +#if ENABLE_TIMER + // for now this is not accessible via Admin/Prometheus , thus useful only with gdb + struct { + TimerCount Sessions_Handlers; + TimerCount Connections_Handlers; + } Timers; +#endif // ENABLE_TIMER + + friend class MySQL_Thread; + friend class PgSQL_Thread; +}; + +std::string proxysql_session_type_str(enum proxysql_session_type session_type); + +#endif // CLASS_BASE_THREAD_H diff --git a/include/ClickHouse_Authentication.hpp b/include/ClickHouse_Authentication.hpp index 1faf18078e..76e4e4a0d7 100644 --- a/include/ClickHouse_Authentication.hpp +++ b/include/ClickHouse_Authentication.hpp @@ -25,6 +25,11 @@ typedef struct _ch_account_details_t { bool __active; } ch_account_details_t; +struct ch_dup_account_details_t { + bool default_schema; + bool sha1_pass; +}; + typedef std::map ch_umap_auth; #endif // CH_ACCOUNT_DETAILS_T @@ -72,6 +77,7 @@ class ClickHouse_Authentication { bool *use_ssl, int *default_hostgroup, char **default_schema, bool *schema_locked, bool *transaction_persistent, bool *fast_forward, int *max_connections, void **sha1_pass); + ch_account_details_t lookup(char* username, enum cred_username_type usertype, const ch_dup_account_details_t& dup_details); int dump_all_users(ch_account_details_t ***, bool _complete = true); int increase_frontend_user_connections(char *username, int *mc = NULL); void decrease_frontend_user_connections(char *username); diff --git a/include/ClickHouse_Server.h b/include/ClickHouse_Server.h index b0a5fcc873..c7eaebe829 100644 --- a/include/ClickHouse_Server.h +++ b/include/ClickHouse_Server.h @@ -6,7 +6,12 @@ #include "cpp.h" #include -#include "clickhouse/client.h" +//#include "clickhouse/client.h" + + +namespace clickhouse { + class Client; +} class ClickHouse_Session { public: @@ -18,7 +23,6 @@ class ClickHouse_Session { bool init(); bool connected; ~ClickHouse_Session(); - clickhouse::ClientOptions co; clickhouse::Client *client; }; diff --git a/include/Client_Session.h b/include/Client_Session.h new file mode 100644 index 0000000000..1e6cdd401e --- /dev/null +++ b/include/Client_Session.h @@ -0,0 +1,123 @@ +#ifndef __CLASS_CLIENT_SESSION_H +#define __CLASS_CLIENT_SESSION_H + +#include +#include + +#include "proxysql.h" +#include "cpp.h" +#include "MySQL_Variables.h" + +//#include "../deps/json/json.hpp" +//using json = nlohmann::json; + +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON + +class MySQL_Session; +class PgSQL_Session; + +#if 0 +// this code was moved into Base_Session.h +/** + * @class Session_Regex + * @brief Encapsulates regex operations for session handling. + * + * This class is used for matching patterns in SQL queries, specifically for + * settings like sql_log_bin, sql_mode, and time_zone. + * See issues #509 , #815 and #816 + */ +class Session_Regex { +private: + void* opt; + void* re; + char* s; +public: + Session_Regex(char* p); + ~Session_Regex(); + bool match(char* m); +}; +#endif // 0 + +template +class TypeSelector { +}; + +template +class TypeSelector { +public: + TypeSelector(T* _ptr) : ptr(_ptr) { } + ~TypeSelector() {} + TypeSelector& operator=(T* _ptr) { + ptr = _ptr; + return *this; + } + + T* operator->() { + return ptr; + } + bool operator==(T* _ptr) { return (ptr == _ptr); } + T& operator*() const noexcept { return *ptr; } + explicit operator bool() { return ptr != nullptr; } + operator T* () { + return ptr; + } + +private: + T* ptr; +}; + +/* Issues with forward class declaration +template +using Client_Session = TypeSelector; +*/ +template +class Client_Session : public TypeSelector { +public: + //Client_Session(const Client_Session&) = default; + //Client_Session(Client_Session&&) = default; + //Client_Session& operator=(const Client_Session&) = default; + //Client_Session& operator=(Client_Session&&) = default; + //~Client_Session() = default; + using TypeSelector::TypeSelector; +}; +#define TO_CLIENT_SESSION(sess) Client_Session(sess) + +/* Issues with forward class declaration +template +using Query_Info_T = TypeSelector; +*/ +template +class Query_Info_T : public TypeSelector { +public: + using TypeSelector::TypeSelector; +}; +#define TO_QUERY_INFO(query_info) Query_Info_T(query_info) + +/* Issues with forward class declaration +template +using Data_Stream_T = TypeSelector; +*/ +template +class Data_Stream_T : public TypeSelector { +public: + using TypeSelector::TypeSelector; +}; +#define TO_DATA_STREAM(data_stream) Data_Stream_T(data_stream) + +/* Issues with forward class declaration +template +using Connection_Info_T = TypeSelector; +*/ +template +class Connection_Info_T : public TypeSelector { +public: + using TypeSelector::TypeSelector; +}; +#define TO_CONNECTION_INFO(connection_info) Connection_Info_T(connection_info) + +std::string proxysql_session_type_str(enum proxysql_session_type session_type); + +#endif /* __CLASS_CLIENT_SESSION_H */ diff --git a/include/Command_Counter.h b/include/Command_Counter.h new file mode 100644 index 0000000000..9b790e6977 --- /dev/null +++ b/include/Command_Counter.h @@ -0,0 +1,64 @@ +#ifndef __CLASS_COMMAND_COUNTER_H +#define __CLASS_COMMAND_COUNTER_H + +class Command_Counter { +public: + Command_Counter(int cmd_idx, int col_count, char** cmd_desc) : _counters{}, _total_time(0), _cmd_idx(cmd_idx), + _col_count(col_count), _cmd_desc(cmd_desc) { + + //memset(_counters, 0, sizeof(_counters)); + } + void add_and_reset(Command_Counter* cc) { + for (int j = 0; j < static_cast(sizeof(_counters)/sizeof(_counters[0])); j++) { + if (cc->_counters[j]) { + __sync_fetch_and_add(&_counters[j], cc->_counters[j]); + cc->_counters[j] = 0; + } + } + if (cc->_total_time) + __sync_fetch_and_add(&_total_time, cc->_total_time); + cc->_total_time = 0; + } + unsigned long long add_time(unsigned long long t) { + _total_time += t; + _counters[0]++; + int i = _add_idx(t); + _counters[i + 1]++; + return _total_time; + } + char** get_row() { + char** pta = (char**)malloc(sizeof(char*) * _col_count); + pta[0] = _cmd_desc[_cmd_idx]; + itostr(pta[1], _total_time); + for (int i = 0; i < static_cast(sizeof(_counters)/sizeof(_counters[0])); i++) itostr(pta[i + 2], _counters[i]); + return pta; + } + void free_row(char** pta) { + for (int i = 1; i < _col_count; i++) free(pta[i]); + free(pta); + } + +private: + unsigned long long _counters[13]; + unsigned long long _total_time; + const int _cmd_idx; + const int _col_count; + char** _cmd_desc; + + int _add_idx(unsigned long long t) { + if (t <= 100) return 0; + if (t <= 500) return 1; + if (t <= 1000) return 2; + if (t <= 5000) return 3; + if (t <= 10000) return 4; + if (t <= 50000) return 5; + if (t <= 100000) return 6; + if (t <= 500000) return 7; + if (t <= 1000000) return 8; + if (t <= 5000000) return 9; + if (t <= 10000000) return 10; + return 11; + } +}; + +#endif /* __CLASS_COMMAND_COUNTER_H */ diff --git a/include/MySQL_Authentication.hpp b/include/MySQL_Authentication.hpp index fbd70bed03..1da4d56506 100644 --- a/include/MySQL_Authentication.hpp +++ b/include/MySQL_Authentication.hpp @@ -9,11 +9,11 @@ #ifndef ACCOUNT_DETAILS_T #define ACCOUNT_DETAILS_T typedef struct _account_details_t { - char *username; - char *password; - void *sha1_pass; - char *clear_text_password; - char *default_schema; + char *username = nullptr; + char *password = nullptr; + void *sha1_pass = nullptr; + char *clear_text_password[2] = { nullptr }; + char *default_schema = nullptr; int default_hostgroup; bool use_ssl; bool schema_locked; @@ -21,13 +21,24 @@ typedef struct _account_details_t { bool fast_forward; int max_connections; int num_connections_used; - bool __frontend; // this is used only during the dump - bool __backend; // this is used only during the dump + int num_connections_used_addl_pass; + bool __frontend; // this is used only during the dump + bool __backend; // this is used only during the dump bool __active; - char *attributes; - char *comment; + char *attributes = nullptr; + char *comment = nullptr; } account_details_t; +/** + * @brief Free all resources from an 'account_details_t' object. + */ +void free_account_details(account_details_t&); + +struct dup_account_details_t { + bool default_schema; + bool sha1_pass; + bool attributes; +}; typedef std::map umap_auth; #endif // ACCOUNT_DETAILS_T @@ -73,14 +84,14 @@ class MySQL_Authentication { bool reset(); void print_version(); bool exists(char *username); - char * lookup(char *username, enum cred_username_type usertype, bool *use_ssl, int *default_hostgroup, char **default_schema, bool *schema_locked, bool *transaction_persistent, bool *fast_forward, int *max_connections, void **sha1_pass, char **attributes); + account_details_t lookup(char* username, enum cred_username_type usertype, const dup_account_details_t& dup_details); int dump_all_users(account_details_t ***, bool _complete=true); - int increase_frontend_user_connections(char *username, int *mc=NULL); - void decrease_frontend_user_connections(char *username); + int increase_frontend_user_connections(char *username, PASSWORD_TYPE::E passtype, int *mc=NULL); + void decrease_frontend_user_connections(char *username, PASSWORD_TYPE::E passtype); void set_all_inactive(enum cred_username_type usertype); void remove_inactives(enum cred_username_type usertype); bool set_SHA1(char *username, enum cred_username_type usertype, void *sha_pass); - bool set_clear_text_password(char *username, enum cred_username_type usertype, const char *clear_text_password); + bool set_clear_text_password(char* username, enum cred_username_type usertype, const char* clear_text_password, PASSWORD_TYPE::E passtype); unsigned int memory_usage(); uint64_t get_runtime_checksum(); /** diff --git a/include/MySQL_Data_Stream.h b/include/MySQL_Data_Stream.h index 53eafe8ec3..c234b6da5b 100644 --- a/include/MySQL_Data_Stream.h +++ b/include/MySQL_Data_Stream.h @@ -7,6 +7,11 @@ #include "MySQL_Protocol.h" #include "proxy_protocol_info.h" +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON + #ifndef uchar typedef unsigned char uchar; #endif @@ -120,7 +125,7 @@ class MySQL_Data_Stream PtrSizeArray *resultset; unsigned int resultset_length; - ProxySQL_Poll *mypolls; + ProxySQL_Poll *mypolls; //int listener; MySQL_Connection *myconn; MySQL_Session *sess; // pointer to the session using this data stream @@ -179,8 +184,7 @@ class MySQL_Data_Stream char *com_field_wild; MySQL_Data_Stream(); - ~MySQL_Data_Stream(); - + virtual ~MySQL_Data_Stream(); int array2buffer_full(); void init(); // initialize the data stream void init(enum MySQL_DS_type, MySQL_Session *, int); // initialize with arguments @@ -275,6 +279,8 @@ class MySQL_Data_Stream bool data_in_rbio(); - void get_client_myds_info_json(json&); + void reset_connection(); + + void get_client_myds_info_json(nlohmann::json&); }; #endif /* __CLASS_MYSQL_DATA_STREAM_H */ diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 383d8bc38d..ecb641cc91 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -23,8 +23,10 @@ #define SPOOKYV2 #endif -#include "../deps/json/json.hpp" -using json = nlohmann::json; +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON #ifdef DEBUG /* */ @@ -33,8 +35,8 @@ using json = nlohmann::json; //#define STRESSTEST_POOL #endif // DEBUG -#define MHM_PTHREAD_MUTEX +#include "Base_HostGroups_Manager.h" // we have 2 versions of the same tables: with (debug) and without (no debug) checks #ifdef DEBUG @@ -117,7 +119,7 @@ using json = nlohmann::json; "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ "FROM main.mysql_servers " \ "WHERE status != \"OFFLINE_HARD\" " \ - "ORDER BY hostgroup_id, hostname, port" \ + "ORDER BY hostgroup_id, hostname, port" typedef std::unordered_map umap_mysql_errors; @@ -126,6 +128,8 @@ class MySrvC; class MySrvList; class MyHGC; +struct peer_runtime_mysql_servers_t; +struct peer_mysql_servers_v2_t; std::string gtid_executed_to_string(gtid_set_t& gtid_executed); void addGtid(const gtid_t& gtid, gtid_set_t& gtid_executed); @@ -269,68 +273,16 @@ class MySrvC { // MySQL Server Container enum MySerStatus status; }; -class MySrvList { // MySQL Server List - private: - MyHGC *myhgc; - int find_idx(MySrvC *); +class MySrvList: public BaseSrvList { // MySQL Server List public: - PtrArray *servers; - unsigned int cnt() { return servers->len; } - MySrvList(MyHGC *); - ~MySrvList(); - void add(MySrvC *); - void remove(MySrvC *); - MySrvC * idx(unsigned int i) {return (MySrvC *)servers->index(i); } + MySrvList(MyHGC* hgc) : BaseSrvList(hgc) {} }; -class MyHGC { // MySQL Host Group Container + +class MyHGC: public BaseHGC { public: - unsigned int hid; - std::atomic num_online_servers; - time_t last_log_time_num_online_servers; - unsigned long long current_time_now; - uint32_t new_connections_now; - MySrvList *mysrvs; - struct { // this is a series of attributes specific for each hostgroup - char * init_connect; - char * comment; - char * ignore_session_variables_text; // this is the original version (text format) of ignore_session_variables - uint32_t max_num_online_servers; - uint32_t throttle_connections_per_sec; - int32_t monitor_slave_lag_when_null; - int8_t autocommit; - int8_t free_connections_pct; - int8_t handle_warnings; - bool multiplex; - bool connection_warming; - bool configured; // this variable controls if attributes are configured or not. If not configured, they do not apply - bool initialized; // this variable controls if attributes were ever configured or not. Used by reset_attributes() - json ignore_session_variables_json; // the JSON format of ignore_session_variables - } attributes; - struct { - int64_t weight; - int64_t max_connections; - int32_t use_ssl; - } servers_defaults; - void reset_attributes(); - inline - bool handle_warnings_enabled() const { - return attributes.configured == true && attributes.handle_warnings != -1 ? attributes.handle_warnings : mysql_thread___handle_warnings; - } - inline - int32_t get_monitor_slave_lag_when_null() const { - return attributes.configured == true && attributes.monitor_slave_lag_when_null != -1 ? attributes.monitor_slave_lag_when_null : mysql_thread___monitor_slave_lag_when_null; - } - MyHGC(int); - ~MyHGC(); + MyHGC(int _hid) : BaseHGC(_hid) {} MySrvC *get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms, MySQL_Session *sess); - void refresh_online_server_count(); - void log_num_online_server_count_error(); - inline - bool online_servers_within_threshold() const { - if (num_online_servers.load(std::memory_order_relaxed) <= attributes.max_num_online_servers) return true; - return false; - } }; class Group_Replication_Info { @@ -471,6 +423,7 @@ struct p_hg_counter { server_connections_aborted, client_connections_created, client_connections_aborted, + client_connections_sha2cached, com_autocommit, com_autocommit_filtered, com_rollback, @@ -503,6 +456,8 @@ struct p_hg_gauge { enum metric { server_connections_connected = 0, client_connections_connected, + client_connections_connected_prim, + client_connections_connected_addl, __size }; }; @@ -596,19 +551,16 @@ struct srv_opts_t { int32_t use_ssl; }; -class MySQL_HostGroups_Manager { +class MySQL_HostGroups_Manager : public Base_HostGroups_Manager { private: +#if 0 SQLite3DB *admindb; SQLite3DB *mydb; pthread_mutex_t readonly_mutex; std::set read_only_set1; std::set read_only_set2; -#ifdef MHM_PTHREAD_MUTEX pthread_mutex_t lock; -#else - rwlock_t rwlock; -#endif - +#endif // 0 enum HGM_TABLES { MYSQL_SERVERS_V2 = 0, MYSQL_REPLICATION_HOSTGROUPS, @@ -723,15 +675,16 @@ class MySQL_HostGroups_Manager { uint64_t hgsm_mysql_replication_hostgroups_checksum = 0; +#if 0 PtrArray *MyHostGroups; std::unordered_mapMyHostGroups_map; - +#endif // 0 std::mutex Servers_SSL_Params_map_mutex; std::unordered_map Servers_SSL_Params_map; - +#if 0 MyHGC * MyHGC_find(unsigned int); MyHGC * MyHGC_create(unsigned int); - +#endif // 0 void add(MySrvC *, unsigned int); void purge_mysql_servers_table(); void generate_mysql_servers_table(int *_onlyhg=NULL); @@ -863,7 +816,10 @@ class MySQL_HostGroups_Manager { pthread_cond_t servers_table_version_cond; unsigned long client_connections_aborted; unsigned long client_connections_created; + unsigned long client_connections_sha2cached; int client_connections; + int client_connections_prim_pass; + int client_connections_addl_pass; unsigned long server_connections_aborted; unsigned long server_connections_created; unsigned long server_connections_delayed; @@ -948,11 +904,13 @@ class MySQL_HostGroups_Manager { MySQL_HostGroups_Manager(); ~MySQL_HostGroups_Manager(); void init(); +#if 0 void wrlock(); void wrunlock(); #ifdef DEBUG bool is_locked = false; #endif +#endif // 0 int servers_add(SQLite3_result *resultset); /** * @brief Generates a new global checksum for module 'mysql_servers_v2' using the provided hash. @@ -960,9 +918,10 @@ class MySQL_HostGroups_Manager { * @return Checksum computed using the provided hash, and 'mysql_servers' config tables hashes. */ std::string gen_global_mysql_servers_v2_checksum(uint64_t servers_v2_hash); + bool commit(); bool commit( - const peer_runtime_mysql_servers_t& peer_runtime_mysql_servers = {}, - const peer_mysql_servers_v2_t& peer_mysql_servers_v2 = {}, + const peer_runtime_mysql_servers_t& peer_runtime_mysql_servers, + const peer_mysql_servers_v2_t& peer_mysql_servers_v2, bool only_commit_runtime_mysql_servers = true, bool update_version = false ); @@ -1036,7 +995,7 @@ class MySQL_HostGroups_Manager { void save_incoming_mysql_table(SQLite3_result *, const string&); SQLite3_result* get_current_mysql_table(const string& name); - SQLite3_result * execute_query(char *query, char **error); + //SQLite3_result * execute_query(char *query, char **error); /** * @brief Creates a resultset with the current full content of the target table. * @param string The target table. Valid values are: @@ -1065,7 +1024,6 @@ class MySQL_HostGroups_Manager { * @param lock When supplied the function calls 'wrlock()' and 'wrunlock()' functions for accessing the db. */ void update_table_mysql_servers_for_monitor(bool lock=false); - MyHGC * MyHGC_lookup(unsigned int); void MyConn_add_to_pool(MySQL_Connection *); /** @@ -1224,47 +1182,5 @@ class MySQL_HostGroups_Manager { uint64_t get_mysql_servers_v2_checksum(SQLite3_result* incoming_mysql_servers_v2 = nullptr); }; -/** - * @brief Helper function used to try to extract a value from the JSON field 'servers_defaults'. - * - * @param j JSON object constructed from 'servers_defaults' field. - * @param hid Hostgroup for which the 'servers_defaults' is defined in 'mysql_hostgroup_attributes'. Used for - * error logging. - * @param key The key for the value to be extracted. - * @param val_check A validation function, checks if the value is within a expected range. - * - * @return The value extracted from the supplied JSON. In case of error '-1', and error cause is logged. - */ -template ::value, bool>::type = true> -T j_get_srv_default_int_val( - const json& j, uint32_t hid, const string& key, const function& val_check -) { - if (j.find(key) != j.end()) { - const json::value_t val_type = j[key].type(); - const char* type_name = j[key].type_name(); - - if (val_type == json::value_t::number_integer || val_type == json::value_t::number_unsigned) { - T val = j[key].get(); - - if (val_check(val)) { - return val; - } else { - proxy_error( - "Invalid value %ld supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." - " Value NOT UPDATED.\n", - static_cast(val), key.c_str(), hid - ); - } - } else { - proxy_error( - "Invalid type '%s'(%hhu) supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." - " Value NOT UPDATED.\n", - type_name, static_cast(val_type), key.c_str(), hid - ); - } - } - - return static_cast(-1); -} #endif /* __CLASS_MYSQL_HOSTGROUPS_MANAGER_H */ diff --git a/include/MySQL_LDAP_Authentication.hpp b/include/MySQL_LDAP_Authentication.hpp index 970d4cb43a..95114f4877 100644 --- a/include/MySQL_LDAP_Authentication.hpp +++ b/include/MySQL_LDAP_Authentication.hpp @@ -33,6 +33,7 @@ class MySQL_LDAP_Authentication { virtual void load_mysql_ldap_mapping(SQLite3_result *result) {}; virtual SQLite3_result * dump_table_mysql_ldap_mapping() { return NULL; }; + virtual SQLite3_result * dump_table_pgsql_ldap_mapping() { return NULL; }; virtual uint64_t get_ldap_mapping_runtime_checksum() { return 0; }; virtual SQLite3_result * SQL3_getStats() { return NULL; } diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 513d043dca..08901f5616 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -398,9 +398,7 @@ struct DNS_Cache_Record { class DNS_Cache { public: - // By default, the DNS cache is disabled. - // This handles the case when ProxySQL is executed with the -M/--no-monitor option. - DNS_Cache() : enabled(false) { + DNS_Cache() : enabled(true) { int rc = pthread_rwlock_init(&rwlock_, NULL); assert(rc == 0); } @@ -447,7 +445,6 @@ class MySQL_Monitor { static std::string dns_lookup(const std::string& hostname, bool return_hostname_if_lookup_fails = true, size_t* ip_count = NULL); static std::string dns_lookup(const char* hostname, bool return_hostname_if_lookup_fails = true, size_t* ip_count = NULL); static bool update_dns_cache_from_mysql_conn(const MYSQL* mysql); - static void remove_dns_record_from_dns_cache(const std::string& hostname); static void trigger_dns_cache_update(); void process_discovered_topology(const std::string& originating_server_hostname, const vector& discovered_servers, int reader_hostgroup); @@ -460,7 +457,6 @@ class MySQL_Monitor { void drop_tables_defs(std::vector *tables_defs); void check_and_build_standard_tables(SQLite3DB *db, std::vector *tables_defs); static bool _dns_cache_update(const std::string& hostname, std::vector&& ip_address); - static void _remove_dns_record_from_dns_cache(const std::string& hostname); public: pthread_mutex_t group_replication_mutex; // for simplicity, a mutex instead of a rwlock diff --git a/include/MySQL_Protocol.h b/include/MySQL_Protocol.h index a95251f8bf..7b74c68391 100644 --- a/include/MySQL_Protocol.h +++ b/include/MySQL_Protocol.h @@ -10,6 +10,9 @@ extern MySQL_Variables mysql_variables; +/** @brief Forward declaration. */ +typedef struct _account_details_t account_details_t; + /* The default mariadb-connecter 3.1.4 does not yet implement CLIENT_DEPRECATE_EOF * flag. */ @@ -96,7 +99,8 @@ class MyProt_tmp_auth_vars { char *db = NULL; char *db_tmp = NULL; unsigned char *pass = NULL; - char *password = NULL; + char* password { nullptr }; + PASSWORD_TYPE::E passtype = PASSWORD_TYPE::PRIMARY; unsigned char *auth_plugin = NULL; void *sha1_pass=NULL; unsigned char *_ptr = NULL;; @@ -165,7 +169,7 @@ class MySQL_Protocol { bool generate_pkt_field(bool send, void **ptr, unsigned int *len, uint8_t sequence_id, char *schema, char *table, char *org_table, char *name, char *org_name, uint16_t charset, uint32_t column_length, uint8_t type, uint16_t flags, uint8_t decimals, bool field_list, uint64_t defvalue_length, char *defvalue, MySQL_ResultSet *myrs=NULL); bool generate_pkt_row(bool send, void **ptr, unsigned int *len, uint8_t sequence_id, int colnums, unsigned long *fieldslen, char **fieldstxt); uint8_t generate_pkt_row3(MySQL_ResultSet *myrs, unsigned int *len, uint8_t sequence_id, int colnums, unsigned long *fieldslen, char **fieldstxt, unsigned long rl); - bool generate_pkt_initial_handshake(bool send, void **ptr, unsigned int *len, uint32_t *thread_id, bool deprecate_eof_active); + virtual bool generate_pkt_initial_handshake(bool send, void **ptr, unsigned int *len, uint32_t *thread_id, bool deprecate_eof_active); // bool generate_statistics_response(MySQL_Data_Stream *myds, bool send, void **ptr, unsigned int *len); bool generate_statistics_response(bool send, void **ptr, unsigned int *len); @@ -181,14 +185,18 @@ class MySQL_Protocol { void PPHR_3(MyProt_tmp_auth_vars& vars1); bool PPHR_4auth0(unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1); bool PPHR_4auth1(unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1); - void PPHR_5passwordTrue(unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, MyProt_tmp_auth_attrs& attr1); - void PPHR_5passwordFalse_0(unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, MyProt_tmp_auth_attrs& attr1); - void PPHR_5passwordFalse_auth2(unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, MyProt_tmp_auth_attrs& attr1 , void *& sha1_pass); + void PPHR_5passwordTrue(bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, account_details_t& attr1); + void PPHR_5passwordFalse_0(bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, account_details_t& attr1); + void PPHR_5passwordFalse_auth2(bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, account_details_t& attr1); void PPHR_6auth2(bool& ret, MyProt_tmp_auth_vars& vars1); - void PPHR_sha2full(bool& ret, MyProt_tmp_auth_vars& vars1, enum proxysql_auth_plugins passformat); - void PPHR_7auth1(unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, MyProt_tmp_auth_attrs& attr1 , void *& sha1_pass); - void PPHR_7auth2(unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, MyProt_tmp_auth_attrs& attr1 , void *& sha1_pass); - void PPHR_SetConnAttrs(MyProt_tmp_auth_vars& vars1, MyProt_tmp_auth_attrs& attr1); + bool PPHR_verify_sha2(MyProt_tmp_auth_vars& vars1, enum proxysql_auth_plugins passformat, PASSWORD_TYPE::E passtype); + void PPHR_sha2full(bool& ret, MyProt_tmp_auth_vars& vars1, enum proxysql_auth_plugins passformat, PASSWORD_TYPE::E passtype); + void PPHR_7auth1(bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, account_details_t& attr1); + void PPHR_7auth2(bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, account_details_t& attr1); + void PPHR_next_auth_stage(MyProt_tmp_auth_vars& vars1, PASSWORD_TYPE::E passtype); + void PPHR_SetConnAttrs(MyProt_tmp_auth_vars& vars1, account_details_t& attr1); + bool PPHR_verify_password(MyProt_tmp_auth_vars& vars1, account_details_t& account_details); + bool PPHR_verify_password_2(MyProt_tmp_auth_vars& vars1, account_details_t& account_details); void generate_one_byte_pkt(unsigned char b); diff --git a/include/MySQL_Query_Processor.h b/include/MySQL_Query_Processor.h new file mode 100644 index 0000000000..cb45fe9289 --- /dev/null +++ b/include/MySQL_Query_Processor.h @@ -0,0 +1,93 @@ +#ifndef __CLASS_MYSQL_QUERY_PROCESSOR_H +#define __CLASS_MYSQL_QUERY_PROCESSOR_H +#include "proxysql.h" +#include "cpp.h" +#include "QP_rule_text.h" +#include "query_processor.h" + +class Command_Counter; +typedef struct _MySQL_Query_processor_Rule_t : public QP_rule_t { + int gtid_from_hostgroup; +} MySQL_Query_Processor_Rule_t; + +class MySQL_Query_Processor_Output : public Query_Processor_Output { +public: + MySQL_Query_Processor_Output() = default; + ~MySQL_Query_Processor_Output() = default; + + void init() { + Query_Processor_Output::init(); + min_gtid = NULL; + gtid_from_hostgroup = -1; + } + void destroy() { + Query_Processor_Output::destroy(); + if (min_gtid) { + free(min_gtid); + min_gtid = NULL; + } + } + + char* min_gtid; + int gtid_from_hostgroup; +}; + +class MySQL_Rule_Text : public QP_rule_text { +public: + MySQL_Rule_Text(const MySQL_Query_Processor_Rule_t* mqr); + ~MySQL_Rule_Text() = default; +}; + +class MySQL_Query_Processor : public Query_Processor { +public: + MySQL_Query_Processor(); + ~MySQL_Query_Processor(); + + void init_thread(); + void end_thread(); + void update_query_processor_stats(); + SQLite3_result* get_current_query_rules(); + SQLite3_result* get_stats_commands_counters(); + MySQL_Query_Processor_Output* process_query(MySQL_Session* sess, void* ptr, unsigned int size, Query_Info* qi); + unsigned long long query_parser_update_counters(MySQL_Session* sess, enum MYSQL_COM_QUERY_command c, SQP_par_t* qp, unsigned long long t); + static enum MYSQL_COM_QUERY_command query_parser_command_type(SQP_par_t* qp); + static MySQL_Query_Processor_Rule_t* new_query_rule(int rule_id, bool active, const char* username, const char* schemaname, int flagIN, const char* client_addr, + const char* proxy_addr, int proxy_port, const char* digest, const char* match_digest, const char* match_pattern, bool negate_match_pattern, + const char* re_modifiers, int flagOUT, const char* replace_pattern, int destination_hostgroup, int cache_ttl, int cache_empty_result, + int cache_timeout, int reconnect, int timeout, int retries, int delay, int next_query_flagIN, int mirror_hostgroup, + int mirror_flagOUT, const char* error_msg, const char* OK_msg, int sticky_conn, int multiplex, int gtid_from_hostgroup, int log, + bool apply, const char* attributes, const char* comment); + +private: + Command_Counter* commands_counters[MYSQL_COM_QUERY___NONE]; + static bool _is_valid_gtid(char* gtid, size_t gtid_len); + static MySQL_Query_Processor_Rule_t* new_query_rule(const MySQL_Query_Processor_Rule_t* mqr); + + inline + void process_query_extended(MySQL_Query_Processor_Output* ret, const MySQL_Query_Processor_Rule_t* mqr) { + if (mqr->gtid_from_hostgroup >= 0) { + // Note: negative gtid_from_hostgroup means this rule doesn't change the gtid_from_hostgroup + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set gtid from hostgroup: %d. A new session will be created\n", mqr->rule_id, mqr->gtid_from_hostgroup); + ret->gtid_from_hostgroup = mqr->gtid_from_hostgroup; + } + } + + inline + void query_parser_first_comment_extended(const char* key, const char* value, MySQL_Query_Processor_Output* qpo) { + if (!strcasecmp(key, "min_gtid")) { + size_t l = strlen(value); + if (_is_valid_gtid((char*)value, l)) { + char* buf = (char*)malloc(l + 1); + strncpy(buf, value, l); + buf[l + 1] = '\0'; + qpo->min_gtid = buf; + } else { + proxy_warning("Invalid gtid value=%s\n", value); + } + } + } + + friend class Query_Processor; +}; + +#endif /* __CLASS_MYSQL_QUERY_PROCESSOR_H */ diff --git a/include/MySQL_Session.h b/include/MySQL_Session.h index fc7327b69c..541ce2105b 100644 --- a/include/MySQL_Session.h +++ b/include/MySQL_Session.h @@ -3,7 +3,7 @@ * @brief Declaration of the MySQL_Session class and associated types and enums. */ - +#ifdef CLASS_BASE_SESSION_H #ifndef __CLASS_MYSQL_SESSION_H #define __CLASS_MYSQL_SESSION_H @@ -13,9 +13,12 @@ #include "proxysql.h" #include "cpp.h" #include "MySQL_Variables.h" +#include "Base_Session.h" -#include "../deps/json/json.hpp" -using json = nlohmann::json; +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON extern class MySQL_Variables mysql_variables; @@ -23,6 +26,7 @@ extern class MySQL_Variables mysql_variables; * @enum proxysql_session_type * @brief Defines the types of ProxySQL sessions. */ +/* enum proxysql_session_type { PROXYSQL_SESSION_MYSQL, PROXYSQL_SESSION_ADMIN, @@ -33,6 +37,7 @@ enum proxysql_session_type { PROXYSQL_SESSION_NONE }; +*/ /** * @enum ps_type @@ -44,26 +49,9 @@ enum ps_type : uint8_t { ps_type_execute_stmt = 0x2 }; -std::string proxysql_session_type_str(enum proxysql_session_type session_type); -/** - * @class Session_Regex - * @brief Encapsulates regex operations for session handling. - * - * This class is used for matching patterns in SQL queries, specifically for - * settings like sql_log_bin, sql_mode, and time_zone. - * See issues #509 , #815 and #816 - */ -class Session_Regex { - private: - void *opt; - void *re; - char *s; - public: - Session_Regex(char *p); - ~Session_Regex(); - bool match(char *m); -}; + +//std::string proxysql_session_type_str(enum proxysql_session_type session_type); /** * @class Query_Info @@ -117,7 +105,7 @@ class Query_Info { * This class is central to ProxySQL's handling of client connections. It manages the lifecycle * of a session, processes queries, and communicates with backend MySQL servers. */ -class MySQL_Session +class MySQL_Session: public Base_Session { private: //int handler_ret; @@ -165,9 +153,9 @@ class MySQL_Session void handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED__get_connection(); - void return_proxysql_internal(PtrSize_t *); + //void return_proxysql_internal(PtrSize_t *); bool handler_special_queries(PtrSize_t *); - bool handler_special_queries_STATUS(PtrSize_t *); + //bool handler_special_queries_STATUS(PtrSize_t *); /** * @brief Handles 'COMMIT|ROLLBACK' commands. * @details Forwarding the packet is required when there are active transactions. Since we are limited to @@ -197,7 +185,7 @@ class MySQL_Session * @param myds If not null, should point to a MySQL_Data_Stream (backend connection) which connection status * should be updated, and previous query resources cleanup. */ - void RequestEnd(MySQL_Data_Stream *); + void RequestEnd(MySQL_Data_Stream *) override; void LogQuery(MySQL_Data_Stream *); void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___create_mirror_session(); @@ -226,15 +214,17 @@ class MySQL_Session bool handler_again___status_SETTING_MULTI_STMT(int *_rc); bool handler_again___multiple_statuses(int *rc); - void init(); + //void init(); void reset(); void add_ldap_comment_to_pkt(PtrSize_t *); + +#if 0 /** * @brief Performs the required housekeeping operations over the session and its connections before * performing any processing on received client packets. */ void housekeeping_before_pkts(); - +#endif // 0 int get_pkts_from_client(bool&, PtrSize_t&); // GPFC_ functions are subfunctions of get_pkts_from_client() @@ -277,19 +267,17 @@ class MySQL_Session void handler_WCD_SS_MCQ_qpo_OK_msg(PtrSize_t *pkt); void handler_WCD_SS_MCQ_qpo_error_msg(PtrSize_t *pkt); void handler_WCD_SS_MCQ_qpo_LargePacket(PtrSize_t *pkt); -// int handler_WCD_SS_MCQ_qpo_Parse_SQL_LOG_BIN(PtrSize_t *pkt, bool *lock_hostgroup, unsigned int nTrx, string& nq); public: bool handler_again___status_SETTING_GENERIC_VARIABLE(int *_rc, const char *var_name, const char *var_value, bool no_quote=false, bool set_transaction=false); bool handler_again___status_SETTING_SQL_LOG_BIN(int *); std::stack previous_status; - void * operator new(size_t); - void operator delete(void *); Query_Info CurrentQuery; PtrSize_t mirrorPkt; PtrSize_t pkt; +#if 0 // uint64_t unsigned long long start_time; unsigned long long pause_until; @@ -299,12 +287,16 @@ class MySQL_Session // pointers MySQL_Thread *thread; - Query_Processor_Output *qpo; +#endif // 0 + MySQL_Query_Processor_Output *qpo; StatCounters *command_counters; +#if 0 MySQL_Backend *mybe; PtrArray *mybes; MySQL_Data_Stream *client_myds; +#endif // 0 MySQL_Data_Stream *server_myds; +#if 0 /* * @brief Store the hostgroups that hold connections that have been flagged as 'expired' by the * maintenance thread. These values will be used to release the retained connections in the specific @@ -361,6 +353,7 @@ class MySQL_Session bool session_fast_forward; bool started_sending_data_to_client; // this status variable tracks if some result set was sent to the client, or if proxysql is still buffering everything bool use_ssl; +#endif // 0 /** * @brief This status variable tracks whether the session is performing an * 'Auth Switch' due to a 'COM_CHANGE_USER' packet. @@ -378,8 +371,8 @@ class MySQL_Session //uint64_t gtid_trxid; int gtid_hid; - MySQL_STMTs_meta *sess_STMTs_meta; - StmtLongDataHandler *SLDH; +// MySQL_STMTs_meta *sess_STMTs_meta; +// StmtLongDataHandler *SLDH; Session_Regex **match_regexes; @@ -392,22 +385,22 @@ class MySQL_Session MySQL_Session(); ~MySQL_Session(); - void set_unhealthy(); + //void set_unhealthy(); void set_status(enum session_status e); int handler(); - void (*handler_function) (MySQL_Session *arg, void *, PtrSize_t *pkt); - MySQL_Backend * find_backend(int); - MySQL_Backend * create_backend(int, MySQL_Data_Stream *_myds=NULL); - MySQL_Backend * find_or_create_backend(int, MySQL_Data_Stream *_myds=NULL); + void (*handler_function) (MySQL_Session* sess, void *, PtrSize_t *pkt); + //MySQL_Backend * find_backend(int); + //MySQL_Backend * create_backend(int, MySQL_Data_Stream *_myds=NULL); + //MySQL_Backend * find_or_create_backend(int, MySQL_Data_Stream *_myds=NULL); - void SQLite3_to_MySQL(SQLite3_result *, char *, int , MySQL_Protocol *, bool in_transaction=false, bool deprecate_eof_active=false); + void SQLite3_to_MySQL(SQLite3_result *, char *, int , MySQL_Protocol *, bool in_transaction=false, bool deprecate_eof_active=false) override; void MySQL_Result_to_MySQL_wire(MYSQL *mysql, MySQL_ResultSet *MyRS, unsigned int warning_count, MySQL_Data_Stream *_myds=NULL); void MySQL_Stmt_Result_to_MySQL_wire(MYSQL_STMT *stmt, MySQL_Connection *myconn); - unsigned int NumActiveTransactions(bool check_savpoint=false); - bool HasOfflineBackends(); - bool SetEventInOfflineBackends(); + //unsigned int NumActiveTransactions(bool check_savpoint=false); + //bool HasOfflineBackends(); + //bool SetEventInOfflineBackends(); /** * @brief Finds one active transaction in the current backend connections. * @details Since only one connection is returned, if the session holds multiple backend connections with @@ -420,15 +413,15 @@ class MySQL_Session * https://bugs.mysql.com/bug.php?id=107875. * @returns The hostgroup in which the connection was found, -1 in case no connection is found. */ - int FindOneActiveTransaction(bool check_savepoint=false); + //int FindOneActiveTransaction(bool check_savepoint=false); unsigned long long IdleTime(); - void reset_all_backends(); - void writeout(); + //void reset_all_backends(); + //void writeout(); void Memory_Stats(); - void create_new_session_and_reset_connection(MySQL_Data_Stream *_myds); + void create_new_session_and_reset_connection(MySQL_Data_Stream *_myds) override; bool handle_command_query_kill(PtrSize_t *); - void update_expired_conns(const std::vector>&); + //void update_expired_conns(const std::vector>&); /** * @brief Performs the final operations after current query has finished to be executed. It updates the session * 'transaction_persistent_hostgroup', and updates the 'MySQL_Data_Stream' and 'MySQL_Connection' before @@ -442,16 +435,16 @@ class MySQL_Session * params. */ void finishQuery(MySQL_Data_Stream *myds, MySQL_Connection *myconn, bool); - void generate_proxysql_internal_session_json(json &); + void generate_proxysql_internal_session_json(nlohmann::json &) override; bool known_query_for_locked_on_hostgroup(uint64_t); void unable_to_parse_set_statement(bool *); - bool has_any_backend(); + //bool has_any_backend(); void detected_broken_connection(const char *file, unsigned int line, const char *func, const char *action, MySQL_Connection *myconn, int myerr, const char *message, bool verbose=false); void generate_status_one_hostgroup(int hid, std::string& s); void reset_warning_hostgroup_flag_and_release_connection(); - friend void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt); - void set_previous_status_mode3(bool allow_execute=true); + + friend void SQLite3_Server_session_handler(MySQL_Session*, void *_pa, PtrSize_t *pkt); }; #define KILL_QUERY 1 @@ -473,8 +466,6 @@ class KillArgs { KillArgs(char *u, char *p, char *h, unsigned int P, unsigned int _hid, unsigned long i, int kt, int _use_ssl, MySQL_Thread* _mt, char *ip); ~KillArgs(); const char* get_host_address() const; - void resolve_hostname(); - void remove_dns_record(); private: char* ip_addr; @@ -483,3 +474,4 @@ class KillArgs { void * kill_query_thread(void *arg); #endif /* __CLASS_MYSQL_SESSION_ H */ +#endif // CLASS_BASE_SESSION_H diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index b238915b48..b3d1b83649 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -16,9 +16,11 @@ #include "set_parser.h" +/* #define MIN_POLL_LEN 8 #define MIN_POLL_DELETE_RATIO 8 #define MY_EPOLL_THREAD_MAXEVENTS 128 +*/ #define ADMIN_HOSTGROUP -2 #define STATS_HOSTGROUP -3 @@ -41,17 +43,6 @@ typedef struct __attribute__((aligned(64))) _conn_exchange_t { } conn_exchange_t; #endif // IDLE_THREADS -typedef struct _thr_id_username_t { - uint32_t id; - char *username; -} thr_id_usr; - -typedef struct _kill_queue_t { - pthread_mutex_t m; - std::vector conn_ids; - std::vector query_ids; -} kill_queue_t; - enum MySQL_Thread_status_variable { st_var_backend_stmt_prepare, st_var_backend_stmt_execute, @@ -93,20 +84,21 @@ enum MySQL_Thread_status_variable { st_var_hostgroup_locked_queries, st_var_aws_aurora_replicas_skipped_during_query, st_var_automatic_detected_sqli, - st_var_whitelisted_sqli_fingerprint, + st_var_mysql_whitelisted_sqli_fingerprint, st_var_client_host_error_killed_connections, - st_var_END + MY_st_var_END }; -class __attribute__((aligned(64))) MySQL_Thread +class __attribute__((aligned(64))) MySQL_Thread : public Base_Thread { + friend class PgSQL_Thread; private: unsigned int servers_table_version_previous; unsigned int servers_table_version_current; unsigned long long last_processing_idles; MySQL_Connection **my_idle_conns; bool processing_idles; - bool maintenance_loop; + //bool maintenance_loop; bool retrieve_gtids_required; // if any of the servers has gtid_port enabled, this needs to be turned on too PtrArray *cached_connections; @@ -118,7 +110,7 @@ class __attribute__((aligned(64))) MySQL_Thread std::map sessmap; #endif // IDLE_THREADS - Session_Regex **match_regexes; + //Session_Regex **match_regexes; #ifdef IDLE_THREADS void worker_thread_assigns_sessions_to_idle_thread(MySQL_Thread *thr); @@ -128,26 +120,26 @@ class __attribute__((aligned(64))) MySQL_Thread void idle_thread_check_if_worker_thread_has_unprocess_resumed_sessions_and_signal_it(MySQL_Thread *thr); void idle_thread_prepares_session_to_send_to_worker_thread(int i); void idle_thread_to_kill_idle_sessions(); - bool move_session_to_idle_mysql_sessions(MySQL_Data_Stream *myds, unsigned int n); + //bool move_session_to_idle_mysql_sessions(MySQL_Data_Stream *myds, unsigned int n); void run_Handle_epoll_wait(int); #endif // IDLE_THREADS - unsigned int find_session_idx_in_mysql_sessions(MySQL_Session *sess); - bool set_backend_to_be_skipped_if_frontend_is_slow(MySQL_Data_Stream *myds, unsigned int n); + //unsigned int find_session_idx_in_mysql_sessions(MySQL_Session *sess); + //bool set_backend_to_be_skipped_if_frontend_is_slow(MySQL_Data_Stream *myds, unsigned int n); void handle_mirror_queue_mysql_sessions(); void handle_kill_queues(); - void check_timing_out_session(unsigned int n); - void check_for_invalid_fd(unsigned int n); - void read_one_byte_from_pipe(unsigned int n); - void tune_timeout_for_myds_needs_pause(MySQL_Data_Stream *myds); - void tune_timeout_for_session_needs_pause(MySQL_Data_Stream *myds); - void configure_pollout(MySQL_Data_Stream *myds, unsigned int n); + //void check_timing_out_session(unsigned int n); + //void check_for_invalid_fd(unsigned int n); + //void read_one_byte_from_pipe(unsigned int n); + //void tune_timeout_for_myds_needs_pause(MySQL_Data_Stream *myds); + //void tune_timeout_for_session_needs_pause(MySQL_Data_Stream *myds); + //void configure_pollout(MySQL_Data_Stream *myds, unsigned int n); void run_MoveSessionsBetweenThreads(); void run_BootstrapListener(); int run_ComputePollTimeout(); void run_StopListener(); - void run_SetAllSession_ToProcess0(); + //void run_SetAllSession_ToProcess0(); protected: @@ -157,14 +149,14 @@ class __attribute__((aligned(64))) MySQL_Thread void *gen_args; // this is a generic pointer to create any sort of structure - ProxySQL_Poll mypolls; + ProxySQL_Poll mypolls; pthread_t thread_id; - unsigned long long curtime; +// unsigned long long curtime; unsigned long long pre_poll_time; unsigned long long last_maintenance_time; - unsigned long long last_move_to_idle_thread_time; + //unsigned long long last_move_to_idle_thread_time; std::atomic atomic_curtime; - PtrArray *mysql_sessions; + //PtrArray *mysql_sessions; PtrArray *mirror_queue_mysql_sessions; PtrArray *mirror_queue_mysql_sessions_cache; #ifdef IDLE_THREADS @@ -175,17 +167,17 @@ class __attribute__((aligned(64))) MySQL_Thread #endif // IDLE_THREADS int pipefd[2]; - int shutdown; +// int shutdown; kill_queue_t kq; - bool epoll_thread; + //bool epoll_thread; bool poll_timeout_bool; // status variables are per thread only // in this way, there is no need for atomic operation and there is no cache miss // when it is needed a total, all threads are checked struct { - unsigned long long stvar[st_var_END]; + unsigned long long stvar[MY_st_var_END]; unsigned int active_transactions; } status_variables; @@ -204,20 +196,20 @@ class __attribute__((aligned(64))) MySQL_Thread MySQL_Thread(); ~MySQL_Thread(); - MySQL_Session * create_new_session_and_client_data_stream(int _fd); + //MySQL_Session * create_new_session_and_client_data_stream(int _fd); bool init(); void run___get_multiple_idle_connections(int& num_idles); void run___cleanup_mirror_queue(); - void ProcessAllMyDS_BeforePoll(); - void ProcessAllMyDS_AfterPoll(); + //void ProcessAllMyDS_BeforePoll(); + //void ProcessAllMyDS_AfterPoll(); void run(); void poll_listener_add(int sock); void poll_listener_del(int sock); - void register_session(MySQL_Session*, bool up_start=true); + //void register_session(MySQL_Session *, bool up_start=true); void unregister_session(int); struct pollfd * get_pollfd(unsigned int i); bool process_data_on_data_stream(MySQL_Data_Stream *myds, unsigned int n); - void ProcessAllSessions_SortingSessions(); + //void ProcessAllSessions_SortingSessions(); void ProcessAllSessions_CompletedMirrorSession(unsigned int& n, MySQL_Session *sess); void ProcessAllSessions_MaintenanceLoop(MySQL_Session *sess, unsigned long long sess_time, unsigned int& total_active_transactions_); void ProcessAllSessions_Healthy0(MySQL_Session *sess, unsigned int& n); @@ -287,7 +279,7 @@ struct p_th_counter { mysql_unexpected_frontend_packets, aws_aurora_replicas_skipped_during_query, automatic_detected_sql_injection, - whitelisted_sqli_fingerprint, + mysql_whitelisted_sqli_fingerprint, mysql_killed_backend_connections, mysql_killed_backend_queries, client_host_error_killed_connections, diff --git a/include/PgSQL_Authentication.h b/include/PgSQL_Authentication.h new file mode 100644 index 0000000000..9b053c714e --- /dev/null +++ b/include/PgSQL_Authentication.h @@ -0,0 +1,116 @@ +#ifndef __CLASS_PGSQL_AUTHENTICATION_H +#define __CLASS_PGSQL_AUTHENTICATION_H + +#include "proxysql.h" +#include "cpp.h" + +#define PROXYSQL_AUTH_PTHREAD_MUTEX + +#ifndef PGSQL_ACCOUNT_DETAILS_T +#define PGSQL_ACCOUNT_DETAILS_T + +struct _scram_keys { + uint8_t scram_ClientKey[32]; + uint8_t scram_ServerKey[32]; +}; + +typedef struct _pgsql_account_details_t { + char* username; + char* password; + char* attributes; + char* comment; + void* sha1_pass; + // TODO POSGRESQL: add client and server scram keys + //_scram_keys* scram_keys; + int default_hostgroup; + int max_connections; + int num_connections_used; + bool use_ssl; + bool transaction_persistent; + bool fast_forward; + bool __frontend; // this is used only during the dump + bool __backend; // this is used only during the dump + bool __active; + +} pgsql_account_details_t; + +typedef std::map umap_pgauth; +#endif // PGSQL_ACCOUNT_DETAILS_T + +#ifdef DEBUG +#define DEB "_DEBUG" +#else +#define DEB "" +#endif /* DEBUG */ +#define PGSQL_AUTHENTICATION_VERSION "0.1.0001" DEB + + +class PtrArray; + +#ifndef CREDS_GROUPS_T +#define CREDS_GROUPS_T +typedef struct _creds_group_t { +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_t lock; +#else + rwlock_t lock; +#endif + umap_pgauth bt_map; + PtrArray *cred_array; +} creds_group_t; +#endif // CREDS_GROUPS_T + +class PgSQL_Authentication { + private: + /** + * @brief Holds the current value for 'runtime_pgsql_users' used by 'ProxySQL_Admin' to reply to + * 'CLUSTER_QUERY_PGSQL_USERS'. + */ + std::unique_ptr pgsql_users_resultset { nullptr }; + creds_group_t creds_backends; + creds_group_t creds_frontends; + bool _reset(enum cred_username_type usertype); + uint64_t _get_runtime_checksum(enum cred_username_type usertype); + public: + PgSQL_Authentication(); + ~PgSQL_Authentication(); + bool add(char *username, char *password, enum cred_username_type usertype, bool use_ssl, int default_hostgroup, bool transaction_persistent, bool fast_forward, int max_connections, char* attributes, char *comment); + bool del(char *username, enum cred_username_type usertype, bool set_lock=true); + bool reset(); + void print_version(); + bool exists(char *username); + char * lookup(char *username, enum cred_username_type usertype, bool *use_ssl, int *default_hostgroup, bool *transaction_persistent, bool *fast_forward, int *max_connections, void **sha1_pass, char **attributes); + int dump_all_users(pgsql_account_details_t***, bool _complete=true); + int increase_frontend_user_connections(char *username, int *mc=NULL); + void decrease_frontend_user_connections(char *username); + void set_all_inactive(enum cred_username_type usertype); + void remove_inactives(enum cred_username_type usertype); + bool set_SHA1(char *username, enum cred_username_type usertype, void *sha_pass); + unsigned int memory_usage(); + uint64_t get_runtime_checksum(); + /** + * @brief Computes the checksum for the 'pgsql_users' table contained in the supplied resultset. + * It's UNSAFE to call this function with another resultset than the specified in @param doc. + * @param resultset Assumed to be the result of hte following query against the Admin interface: + * - '"SELECT username, password, active, use_ssl, default_hostgroup, + * transaction_persistent, fast_forward, backend, frontend, max_connections, + * attributes, comment FROM runtime_pgsql_users"' + * The order isn't relevant in the query itself because ordering is performed while processing. + * @param pgsql_users A 'unique_ptr' to be filled with the 'frontend' and 'backend' users found in the + * provided resulset. + * @return The computed hash for the provided resultset. + */ + uint64_t get_runtime_checksum(MYSQL_RES* resultset, unique_ptr& pgsql_users); + /** + * @brief Takes ownership of the supplied resultset and stores it in 'pgsql_users_resultset' field. + * @param users Holds the current value for 'runtime_pgsql_users'. + */ + void save_pgsql_users(std::unique_ptr&& users); + /** + * @brief Return a pointer to internally managed 'pgsql_users_resultset' field. DO NOT FREE. + * @return A pointer to the internally managed 'pgsql_users_resultset'. + */ + SQLite3_result* get_current_pgsql_users(); +}; + +#endif /* __CLASS_PGSQL_AUTHENTICATION_H */ diff --git a/include/PgSQL_Backend.h b/include/PgSQL_Backend.h new file mode 100644 index 0000000000..15d75697fe --- /dev/null +++ b/include/PgSQL_Backend.h @@ -0,0 +1,27 @@ +#ifndef __CLASS_PGSQL_BACKEND_H +#define __CLASS_PGSQL_BACKEND_H +#include "proxysql.h" +#include "cpp.h" + +/* + * @brief A backend class handling connections and data streams for PostgreSQL clients. + */ +class PgSQL_Backend +{ + public: + void * operator new(size_t); + void operator delete(void *); + int hostgroup_id; //< The ID of the host group this connection belongs to. Set to -1 if uninitialized + char gtid_uuid[128]; //< An array to store a unique identifier for each transaction : for now unused + uint64_t gtid_trxid; //< The ID of the current transaction : for now unused + PgSQL_Data_Stream *server_myds; + // mysql_cp_entry_t *server_mycpe; + bytes_stats_t server_bytes_at_cmd; //< A structure storing the number of bytes received and sent + //MySQL_Hostgroup_Entry *mshge; + //MySQL_Connection *myconn; + PgSQL_Backend(); + ~PgSQL_Backend(); + void reset(); //< A method that resets and releases resources associated with this backend instance +}; + +#endif /* __CLASS_PGSQLL_BACKEND_H */ diff --git a/include/PgSQL_Connection.h b/include/PgSQL_Connection.h new file mode 100644 index 0000000000..9bcf0dba60 --- /dev/null +++ b/include/PgSQL_Connection.h @@ -0,0 +1,651 @@ +#ifndef __CLASS_PGSQL_CONNECTION_H +#define __CLASS_PGSQL_CONNECTION_H +#include "libpq-fe.h" +#include "proxysql.h" +#include "cpp.h" +#include "PgSQL_Error_Helper.h" + +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON + +class PgSQL_SrvC; +class PgSQL_Query_Result; +//#define STATUS_MYSQL_CONNECTION_TRANSACTION 0x00000001 // DEPRECATED +#define STATUS_MYSQL_CONNECTION_COMPRESSION 0x00000002 +#define STATUS_MYSQL_CONNECTION_USER_VARIABLE 0x00000004 +#define STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT 0x00000008 +#define STATUS_MYSQL_CONNECTION_LOCK_TABLES 0x00000010 +#define STATUS_MYSQL_CONNECTION_TEMPORARY_TABLE 0x00000020 +#define STATUS_MYSQL_CONNECTION_GET_LOCK 0x00000040 +#define STATUS_MYSQL_CONNECTION_NO_MULTIPLEX 0x00000080 +#define STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0 0x00000100 +#define STATUS_MYSQL_CONNECTION_FOUND_ROWS 0x00000200 +#define STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG 0x00000400 +#define STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT 0x00000800 +#define STATUS_MYSQL_CONNECTION_HAS_WARNINGS 0x00001000 + + +enum PgSQL_Param_Name { + PG_HOST = 0, // Name of host to connect to + PG_HOSTADDR, // Numeric IP address of host to connect to + PG_PORT, // Port number to connect to at the server host + PG_DATABASE, // The database name + PG_USER, // PgSQL user name to connect as + PG_PASSWORD, // Password to be used if the server demands password authentication + PG_PASSFILE, // Specifies the name of the file used to store passwords + PG_REQUIRE_AUTH, // Specifies the authentication method that the client requires from the server + PG_CHANNEL_BINDING, // Controls the client's use of channel binding + PG_CONNECT_TIMEOUT, // Maximum time to wait while connecting, in seconds + PG_CLIENT_ENCODING, // Sets the client_encoding configuration parameter for this connection + PG_OPTIONS, // Specifies command-line options to send to the server at connection start + PG_APPLICATION_NAME, // Specifies a value for the application_name configuration parameter + PG_FALLBACK_APPLICATION_NAME, // Specifies a fallback value for the application_name configuration parameter + PG_KEEPALIVES, // Controls whether client-side TCP keepalives are used + PG_KEEPALIVES_IDLE, // Controls the number of seconds of inactivity after which TCP should send a keepalive message to the server + PG_KEEPALIVES_INTERVAL, // Controls the number of seconds after which a TCP keepalive message that is not acknowledged by the server should be retransmitted + PG_KEEPALIVES_COUNT, // Controls the number of TCP keepalives that can be lost before the client's connection to the server is considered dead + PG_TCP_USER_TIMEOUT, // Controls the number of milliseconds that transmitted data may remain unacknowledged before a connection is forcibly closed + PG_REPLICATION, // Determines whether the connection should use the replication protocol instead of the normal protocol + PG_GSSENCMODE, // Determines whether a secure GSS TCP/IP connection will be negotiated with the server + PG_SSLMODE, // Determines whether a secure SSL TCP/IP connection will be negotiated with the server + PG_REQUIRESSL, // Requires an SSL connection to the server + PG_SSLCOMPRESSION, // If set, data sent over SSL connections will be compressed + PG_SSLCERT, // Specifies the file name of the client SSL certificate + PG_SSLKEY, // Specifies the location for the secret key used for the client certificate + PG_SSLPASSWORD, // Specifies the password for the secret key specified in sslkey + PG_SSLCERTMODE, // Determines whether a client certificate may be sent to the server + PG_SSLROOTCERT, // Specifies the name of a file containing SSL certificate authority (CA) certificate(s) + PG_SSLCRL, // Specifies the file name of the SSL server certificate revocation list (CRL) + PG_SSLCRLDIR, // Specifies the directory name of the SSL server certificate revocation list (CRL) + PG_SSLSNI, // Sets the TLS extension “Server Name Indication” (SNI) on SSL-enabled connections + PG_REQUIREPEER, // Specifies the operating-system user name of the server + PG_SSL_MIN_PROTOCOL_VERSION, // Specifies the minimum SSL/TLS protocol version to allow for the connection + PG_SSL_MAX_PROTOCOL_VERSION, // Specifies the maximum SSL/TLS protocol version to allow for the connection + PG_KRBSRVNAME, // Kerberos service name to use when authenticating with GSSAPI + PG_GSSLIB, // GSS library to use for GSSAPI authentication + PG_GSSDELEGATION, // Forward (delegate) GSS credentials to the server + PG_SERVICE, // Service name to use for additional parameters + PG_TARGET_SESSION_ATTRS, // Determines whether the session must have certain properties to be acceptable + PG_LOAD_BALANCE_HOSTS, // Controls the order in which the client tries to connect to the available hosts and addresses + + PG_PARAM_SIZE +}; + +static const char* PgSQL_Param_Name_Str[] = { + "host", + "hostaddr", + "port", + "database", + "user", + "password", + "passfile", + "require_auth", + "channel_binding", + "connect_timeout", + "client_encoding", + "options", + "application_name", + "fallback_application_name", + "keepalives", + "keepalives_idle", + "keepalives_interval", + "keepalives_count", + "tcp_user_timeout", + "replication", + "gsseencmode", + "sslmode", + "requiressl", + "sslcompression", + "sslcert", + "sslkey", + "sslpassword", + "sslcertmode", + "sslrootcert", + "sslcrl", + "sslcrldir", + "sslsni", + "requirepeer", + "ssl_min_protocol_version", + "ssl_max_protocol_version", + "krbsrvname", + "gsslib", + "gssdelegation", + "service", + "target_session_attrs", + "load_balance_hosts" +}; + +struct Param_Name_Validation { + const char** accepted_values; + int default_value_idx; + +}; + +static const Param_Name_Validation require_auth {(const char*[]){"password","md5","gss","sspi","scram-sha-256","none",nullptr},-1}; +static const Param_Name_Validation replication {(const char*[]){"true","on","yes","1","database","false","off","no","0",nullptr},-1}; +static const Param_Name_Validation gsseencmode {(const char*[]){"disable","prefer","require",nullptr},1}; +static const Param_Name_Validation sslmode {(const char*[]){"disable","allow","prefer","require","verify-ca","verify-full",nullptr},2}; +static const Param_Name_Validation sslcertmode {(const char*[]){"disable","allow","require",nullptr},1}; +static const Param_Name_Validation target_session_attrs {(const char*[]){"any","read-write","read-only","primary","standby","prefer-standby",nullptr},0 }; +static const Param_Name_Validation load_balance_hosts {(const char*[]){"disable","random",nullptr},-1}; + +static const Param_Name_Validation* PgSQL_Param_Name_Accepted_Values[PG_PARAM_SIZE] = { + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + &require_auth, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + &replication, + &gsseencmode, + &sslmode, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + &sslcertmode, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + &target_session_attrs, + &load_balance_hosts +}; + +#define PG_EVENT_NONE 0x00 +#define PG_EVENT_READ 0x01 +#define PG_EVENT_WRITE 0x02 +#define PG_EVENT_EXCEPT 0x04 +#define PG_EVENT_TIMEOUT 0x08 + +class PgSQL_Conn_Param { +private: + bool validate(PgSQL_Param_Name key, const char* val) { + assert(val); + const Param_Name_Validation* validation = PgSQL_Param_Name_Accepted_Values[key]; + + if (validation != nullptr && validation->accepted_values) { + const char** accepted_value = validation->accepted_values; + while (accepted_value != nullptr) { + if (strcmp(val, *accepted_value) == 0) { + return true; + } + } + } else { + return true; + } + + return false; + } + +public: + PgSQL_Conn_Param() {} + ~PgSQL_Conn_Param() { + for (int i = 0; i < PG_PARAM_SIZE; i++) { + if (param_value[i]) + free(param_value[i]); + } + } + + bool set_value(PgSQL_Param_Name key, const char* val) { + if (validate(key, val)) { + if (param_value[key]) { + free(param_value[key]); + } + param_value[key] = strdup(val); + param_set.push_back(key); + return true; + } + return false; + } + + bool set_value(const char* key, const char* val) { + return set_value((PgSQL_Param_Name)get_param_name(key), val); + } + + void reset_value(PgSQL_Param_Name key) { + if (param_value[key]) { + free(param_value[key]); + } + param_value[key] = nullptr; + + // this has O(n) complexity. need to fix it.... + param_set.erase(param_set.begin() + static_cast(key)); + } + + const char* get_value(PgSQL_Param_Name key) const { + return param_value[key]; + } + + int get_param_name(const char* name) { + int key = -1; + + for (int i = 0; i < PG_PARAM_SIZE; i++) { + if (strcmp(name, PgSQL_Param_Name_Str[i]) == 0) { + key = i; + break; + } + } + + assert(key != -1); + return key; + } + + std::vector param_set; + char* param_value[PG_PARAM_SIZE]{}; +}; + +class PgSQL_Variable { +public: + char *value = (char*)""; + void fill_server_internal_session(nlohmann::json &j, int conn_num, int idx); + void fill_client_internal_session(nlohmann::json &j, int idx); +}; + +enum pgsql_charset_action { + POSTGRESQL_CHARSET_ACTION_UNKNOWN, + POSTGRESQL_CHARSET_ACTION_NAMES, + POSTGRESQL_CHARSET_ACTION_CHARSET, + POSTGRESQL_CHARSET_ACTION_CONNECT_START +}; + +class PgSQL_Connection_userinfo { + private: + uint64_t compute_hash(); + public: + uint64_t hash; + char *username; + char *password; + union { + char* dbname; + char* schemaname; // temporary fix. To avoid changes in Base_Session and Query_Processor + }; + char *sha1_pass; + char *fe_username; + // TODO POSGRESQL: add client and server scram keys + PgSQL_Connection_userinfo(); + ~PgSQL_Connection_userinfo(); + void set(char *, char *, char *, char *); + void set(PgSQL_Connection_userinfo *); + bool set_dbname(const char *); +}; + +class PgSQL_Connection_Placeholder { + private: + void update_warning_count_from_connection(); + void update_warning_count_from_statement(); + bool is_expired(unsigned long long timeout); + unsigned long long inserted_into_pool; + public: + struct { + char *server_version; + uint32_t session_track_gtids_int; + uint32_t max_allowed_pkt; + uint32_t server_capabilities; + uint32_t client_flag; + unsigned int compression_min_length; + char *init_connect; + bool init_connect_sent; + char * session_track_gtids; + char *ldap_user_variable; + char *ldap_user_variable_value; + bool session_track_gtids_sent; + bool ldap_user_variable_sent; + uint8_t protocol_version; + int8_t last_set_autocommit; + bool autocommit; + bool no_backslash_escapes; + } options; + + PgSQL_Conn_Param conn_params; + + PgSQL_Variable variables[SQL_NAME_LAST_HIGH_WM]; + uint32_t var_hash[SQL_NAME_LAST_HIGH_WM]; + // for now we store possibly missing variables in the lower range + // we may need to fix that, but this will cost performance + bool var_absent[SQL_NAME_LAST_HIGH_WM] = {false}; + + std::vector dynamic_variables_idx; + unsigned int reorder_dynamic_variables_idx(); + + struct { + unsigned long length; + char *ptr; + MYSQL_STMT *stmt; + MYSQL_RES *stmt_result; + stmt_execute_metadata_t *stmt_meta; + } query; + char scramble_buff[40]; + unsigned long long creation_time; + unsigned long long last_time_used; + unsigned long long timeout; + int auto_increment_delay_token; + int fd; + MySQL_STMTs_local_v14 *local_stmts; // local view of prepared statements + MYSQL *pgsql; + MYSQL *ret_mysql; + MYSQL_RES *mysql_result; + MYSQL_ROW mysql_row; + PgSQL_SrvC *parent; + PgSQL_Connection_userinfo *userinfo; + PgSQL_Data_Stream *myds; + + struct { + char* hostname; + char* ip; + } connected_host_details; + /** + * @brief Keeps tracks of the 'server_status'. Do not confuse with the 'server_status' from the + * 'MYSQL' connection itself. This flag keeps track of the configured server status from the + * parent 'MySrvC'. + */ + enum MySerStatus server_status; // this to solve a side effect of #774 + + bytes_stats_t bytes_info; // bytes statistics + struct { + unsigned long long questions; + unsigned long long pgconnpoll_get; + unsigned long long pgconnpoll_put; + } statuses; + + unsigned long largest_query_length; + unsigned int warning_count; + /** + * @brief This represents the internal knowledge of ProxySQL about the connection. It keeps track of those + * states which *are not reflected* into 'server_status', but are relevant for connection handling. + */ + uint32_t status_flags; + int async_exit_status; // exit status of MariaDB Client Library Non blocking API + int interr; // integer return + PG_ASYNC_ST async_state_machine; // Async state machine + short wait_events; + uint8_t compression_pkt_id; + my_bool ret_bool; + bool async_fetch_row_start; + bool send_quit; + bool reusable; + bool processing_multi_statement; + bool multiplex_delayed; + bool unknown_transaction_status; + void compute_unknown_transaction_status(); + char gtid_uuid[128]; + PgSQL_Connection_Placeholder(); + ~PgSQL_Connection_Placeholder(); + bool set_autocommit(bool); + bool set_no_backslash_escapes(bool); + unsigned int set_charset(unsigned int, enum pgsql_charset_action); + + void set_status(bool set, uint32_t status_flag); + bool get_status(uint32_t status_flag); +#if 0 + void set_status_sql_log_bin0(bool); + bool get_status_sql_log_bin0(); + void set_autocommit_start(); + void set_autocommit_cont(short event); +#endif // 0 + void set_names_start(); + void set_names_cont(short event); +#ifndef PROXYSQL_USE_RESULT + void store_result_start(); + void store_result_cont(short event); +#endif // PROXYSQL_USE_RESULT +#if 0 + void initdb_start(); + void initdb_cont(short event); + void set_option_start(); + void set_option_cont(short event); +#endif // 0 + void set_query(char *stmt, unsigned long length); + + int async_set_autocommit(short event, bool); + int async_set_names(short event, unsigned int nr); + int async_send_simple_command(short event, char *stmt, unsigned long length); // no result set expected + + int async_set_option(short event, bool mask); + + void stmt_prepare_start(); + void stmt_prepare_cont(short event); + void stmt_execute_start(); + void stmt_execute_cont(short event); + void stmt_execute_store_result_start(); + void stmt_execute_store_result_cont(short event); + +#if 0 + /** + * @brief Process the rows returned by 'async_stmt_execute_store_result'. Extracts all the received + * rows from 'query.stmt->result.data' but the last one, adds them to 'MyRS', frees the buffer + * used by 'query.stmt' and allocates a new one with the last row, leaving it ready for being filled + * with the new rows to be received. + * @param processed_bytes Reference to the already processed bytes to be updated with the rows + * that are being read and added to 'MyRS'. + */ + void process_rows_in_ASYNC_STMT_EXECUTE_STORE_RESULT_CONT(unsigned long long& processed_bytes); +#endif // 0 + + void async_free_result(); + + + bool IsAutoCommit(); + bool AutocommitFalse_AndSavepoint(); + bool MultiplexDisabled(bool check_delay_token = true); + bool IsKeepMultiplexEnabledVariables(char *query_digest_text); + void ProcessQueryAndSetStatusFlags(char *query_digest_text); + void optimize(); + void close_mysql(); + + void set_is_client(); // used for local_stmts + + void reset(); + + bool get_gtid(char *buff, uint64_t *trx_id); + void reduce_auto_increment_delay_token() { if (auto_increment_delay_token) auto_increment_delay_token--; }; + + bool match_tracked_options(const PgSQL_Connection *c); + unsigned int number_of_matching_session_variables(const PgSQL_Connection *client_conn, unsigned int& not_matching); + unsigned long get_mysql_thread_id() { return pgsql ? pgsql->thread_id : 0; } + + + /********* These will be removed **********/ + MySQL_ResultSet* MyRS; + MySQL_ResultSet* MyRS_reuse; + + // these method should not be called from this class + int async_select_db(short event) { assert(0); return -1; } + bool IsServerOffline() { assert(0); return false; } + bool IsKnownActiveTransaction() { assert(0); return false; } + bool IsActiveTransaction() { assert(0); return false; } + PG_ASYNC_ST handler(short event) { assert(0); return ASYNC_IDLE; } + /********* End of remove ******************/ +}; + +class PgSQL_Connection : public PgSQL_Connection_Placeholder { +public: + PgSQL_Connection(); + ~PgSQL_Connection(); + + PG_ASYNC_ST handler(short event); + void connect_start(); + void connect_cont(short event); + void query_start(); + void query_cont(short event); + void fetch_result_start(); + void fetch_result_cont(short event); + void reset_session_start(); + void reset_session_cont(short event); + + int async_connect(short event); +#if 0 + int async_set_autocommit(short event, bool ac); +#endif // 0 + int async_query(short event, char* stmt, unsigned long length, MYSQL_STMT** _stmt = NULL, stmt_execute_metadata_t* _stmt_meta = NULL); + int async_ping(short event); + int async_reset_session(short event); + + void next_event(PG_ASYNC_ST new_st); + bool IsAutoCommit(); + bool is_connected() const; + void compute_unknown_transaction_status(); + void async_free_result(); + void flush(); + bool IsActiveTransaction(); + bool IsKnownActiveTransaction(); + bool IsServerOffline(); + + bool is_connection_in_reusable_state() const; + + bool requires_RESETTING_CONNECTION(const PgSQL_Connection* client_conn); + + bool has_same_connection_options(const PgSQL_Connection* c); + + int get_server_version() { + return PQserverVersion(pgsql_conn); + } + + int get_protocol_version() { + return PQprotocolVersion(pgsql_conn); + } + + inline + bool is_error_present() const { + if (error_info.severity == PGSQL_ERROR_SEVERITY::ERRSEVERITY_FATAL || + error_info.severity == PGSQL_ERROR_SEVERITY::ERRSEVERITY_ERROR || + error_info.severity == PGSQL_ERROR_SEVERITY::ERRSEVERITY_PANIC) { + return true; + } + return false; + } + + inline + PGSQL_ERROR_SEVERITY get_error_severity() const { + return error_info.severity; + } + + inline + PGSQL_ERROR_CATEGORY get_error_category() const { + return error_info.category; + } + + inline + const std::string& get_error_message() const { + return error_info.message; + } + + inline + const char* get_error_code_str() const { + return error_info.sqlstate; + } + + inline + PGSQL_ERROR_CODES get_error_code() const { + return error_info.code; + } + + inline + std::string get_error_code_with_message() const { + return ("[" + std::string(error_info.sqlstate) + "] " + error_info.message); + } + + void set_error(const char* code, const char* message, bool is_fatal) { + PgSQL_Error_Helper::fill_error_info(error_info, code, message, is_fatal ? "FATAL" : "ERROR"); + } + + void set_error(PGSQL_ERROR_CODES code, const char* message, bool is_fatal) { + PgSQL_Error_Helper::fill_error_info(error_info, code, message, is_fatal ? + PGSQL_ERROR_SEVERITY::ERRSEVERITY_FATAL : PGSQL_ERROR_SEVERITY::ERRSEVERITY_ERROR); + } + + // safety check. Sometimes libpq return garbage result when connection is lost with the backend + bool is_error_result_valid(const PGresult* result) const { + if (result == nullptr) + return false; + return (PQresultErrorField(result, PG_DIAG_SQLSTATE) != nullptr); + } + + void set_error_from_result(const PGresult* result, uint16_t ext_fields = 0) { + if (is_error_result_valid(result)) { + PgSQL_Error_Helper::fill_error_info(error_info, result, ext_fields); + } else { + const char* errmsg = PQerrorMessage(pgsql_conn); + set_error(PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION, errmsg ? errmsg : "Unknown error", true); + //PgSQL_Error_Helper::fill_error_info_from_error_message(error_info, errmsg); + } + } + + void reset_error() { reset_error_info(error_info, false); } + + bool reset_session_in_txn = false; + + PGresult* get_result(); + void next_multi_statement_result(PGresult* result); + bool set_single_row_mode(); + void optimize() {} + void update_bytes_recv(uint64_t bytes_recv); + void update_bytes_sent(uint64_t bytes_sent); + + inline const PGconn* get_pg_connection() const { return pgsql_conn; } + inline int get_pg_server_version() { return PQserverVersion(pgsql_conn); } + inline int get_pg_protocol_version() { return PQprotocolVersion(pgsql_conn); } + inline const char* get_pg_host() { return PQhost(pgsql_conn); } + inline const char* get_pg_hostaddr() { return PQhostaddr(pgsql_conn); } + inline const char* get_pg_port() { return PQport(pgsql_conn); } + inline const char* get_pg_dbname() { return PQdb(pgsql_conn); } + inline const char* get_pg_user() { return PQuser(pgsql_conn); } + inline const char* get_pg_password() { return PQpass(pgsql_conn); } + inline const char* get_pg_options() { return PQoptions(pgsql_conn); } + inline int get_pg_socket_fd() { return PQsocket(pgsql_conn); } + inline int get_pg_backend_pid() { return PQbackendPID(pgsql_conn); } + inline int get_pg_connection_needs_password() { return PQconnectionNeedsPassword(pgsql_conn); } + inline int get_pg_connection_used_password() { return PQconnectionUsedPassword(pgsql_conn); } + inline int get_pg_connection_used_gssapi() { return PQconnectionUsedGSSAPI(pgsql_conn); } + inline int get_pg_client_encoding() { return PQclientEncoding(pgsql_conn); } + inline int get_pg_ssl_in_use() { return PQsslInUse(pgsql_conn); } + inline ConnStatusType get_pg_connection_status() { return PQstatus(pgsql_conn); } + inline PGTransactionStatusType get_pg_transaction_status() { return PQtransactionStatus(pgsql_conn); } + inline int get_pg_is_nonblocking() { return PQisnonblocking(pgsql_conn); } + inline int get_pg_is_threadsafe() { return PQisthreadsafe(); } + inline const char* get_pg_error_message() { return PQerrorMessage(pgsql_conn); } + const char* get_pg_server_version_str(char* buff, int buff_size); + const char* get_pg_connection_status_str(); + const char* get_pg_transaction_status_str(); + + unsigned int get_memory_usage() const; + + //PgSQL_Conn_Param conn_params; + PgSQL_ErrorInfo error_info; + PGconn* pgsql_conn; + uint8_t result_type; + PGresult* pgsql_result; + PSresult ps_result; + PgSQL_Query_Result* query_result; + PgSQL_Query_Result* query_result_reuse; + bool new_result; + //PgSQL_SrvC* parent; + //PgSQL_Connection_userinfo* userinfo; + //PgSQL_Data_Stream* myds; + //int fd; +}; + +#endif /* __CLASS_PGSQL_CONNECTION_H */ diff --git a/include/PgSQL_Data_Stream.h b/include/PgSQL_Data_Stream.h new file mode 100644 index 0000000000..d830fbfda5 --- /dev/null +++ b/include/PgSQL_Data_Stream.h @@ -0,0 +1,272 @@ +#ifndef __CLASS_PGSQL_DATA_STREAM_H +#define __CLASS_PGSQL_DATA_STREAM_H + +#include "proxysql.h" +#include "cpp.h" + +#include "PgSQL_Protocol.h" +#include "scram.h" + +#ifndef uchar +typedef unsigned char uchar; +#endif + +#include "ma_pvio.h" + +#define QUEUE_T_DEFAULT_SIZE 32768 +#define MY_SSL_BUFFER 8192 + +typedef struct _pgsql_queue_t { + void* buffer; + unsigned int size; + unsigned int head; + unsigned int tail; + unsigned int partial; + PtrSize_t pkt; + mysql_hdr hdr; +} pgsql_queue_t; + +// this class avoid copying data +class PgSQL_MyDS_real_query { +public: + PtrSize_t pkt; // packet coming from the client + char* QueryPtr; // pointer to beginning of the query + unsigned int QuerySize; // size of the query + void init(PtrSize_t* _pkt) { + /* + assert(QueryPtr==NULL); + assert(QuerySize==0); + assert(pkt.ptr==NULL); + assert(pkt.size==0); + */ + pkt.ptr = _pkt->ptr; + pkt.size = _pkt->size; + QuerySize = pkt.size - 5; + if (QuerySize == 0) { + QueryPtr = const_cast(""); + } + else { + QueryPtr = (char*)pkt.ptr + 5; + } + } + void end() { + l_free(pkt.size, pkt.ptr); + pkt.size = 0; + QuerySize = 0; + pkt.ptr = NULL; + QueryPtr = NULL; + } +}; + +enum pgsql_sslstatus { PGSQL_SSLSTATUS_OK, PGSQL_SSLSTATUS_WANT_IO, PGSQL_SSLSTATUS_FAIL }; + +class PgSQL_Data_Stream +{ +private: + int array2buffer(); + int buffer2array(); + void generate_compressed_packet(); + enum pgsql_sslstatus do_ssl_handshake(); + void queue_encrypted_bytes(const char* buf, size_t len); +public: + void* operator new(size_t); + void operator delete(void*); + + pgsql_queue_t queueIN; + uint64_t pkts_recv; // counter of received packets + pgsql_queue_t queueOUT; + uint64_t pkts_sent; // counter of sent packets + + struct { + PtrSize_t pkt; + unsigned int partial; + } CompPktIN; + struct { + PtrSize_t pkt; + unsigned int partial; + } CompPktOUT; + + PgSQL_Protocol myprot; + PgSQL_MyDS_real_query mysql_real_query; + bytes_stats_t bytes_info; // bytes statistics + + PtrSize_t multi_pkt; + + unsigned long long pause_until; + unsigned long long wait_until; + unsigned long long killed_at; + unsigned long long max_connect_time; + + struct { + unsigned long long questions; + unsigned long long pgconnpoll_get; + unsigned long long pgconnpoll_put; + } statuses; + + PtrSizeArray* PSarrayIN; + PtrSizeArray* PSarrayOUT; + FixedSizeQueue data_packets_history_IN; + FixedSizeQueue data_packets_history_OUT; + //PtrSizeArray *PSarrayOUTpending; + PtrSizeArray* resultset; + unsigned int resultset_length; + + ProxySQL_Poll* mypolls; + //int listener; + PgSQL_Connection* myconn; + PgSQL_Session* sess; // pointer to the session using this data stream + PgSQL_Backend* mybe; // if this is a connection to a mysql server, this points to a backend structure + char* x509_subject_alt_name; + SSL* ssl; + BIO* rbio_ssl; + BIO* wbio_ssl; + char* ssl_write_buf; + size_t ssl_write_len; + struct sockaddr* client_addr; + + struct { + char* addr; + int port; + } addr; + struct { + char* addr; + int port; + } proxy_addr; + + AUTHENTICATION_METHOD auth_method = AUTHENTICATION_METHOD::NO_PASSWORD; + uint32_t auth_next_pkt_type = 0; + bool auth_received_startup = false; + + ScramState* scram_state; + + unsigned int connect_tries; + int query_retries_on_failure; + int connect_retries_on_failure; + enum mysql_data_stream_status DSS; + enum MySQL_DS_type myds_type; + + socklen_t client_addrlen; + + int fd; // file descriptor + int poll_fds_idx; + + + int active_transaction; // 1 if there is an active transaction + int active; // data stream is active. If not, shutdown+close needs to be called + int status; // status . FIXME: make it a ORable variable + + int switching_auth_stage; + int switching_auth_type; + unsigned int tmp_charset; + + short revents; + + char kill_type; + + bool encrypted; + bool net_failure; + + uint8_t pkt_sid; + + bool com_field_list; + char* com_field_wild; + + PgSQL_Data_Stream(); + virtual ~PgSQL_Data_Stream(); + int array2buffer_full(); + void init(); // initialize the data stream + void init(enum MySQL_DS_type, PgSQL_Session*, int); // initialize with arguments + void shut_soft(); + void shut_hard(); + int read_from_net(); + int write_to_net(); + int write_to_net_poll(); + bool available_data_out(); + void remove_pollout(); + void set_pollout(); + void mysql_free(); + + void set_net_failure(); + void setDSS_STATE_QUERY_SENT_NET(); + + void setDSS(enum mysql_data_stream_status dss) { + DSS = dss; + } + + int read_pkts(); + int write_pkts(); + + void unplug_backend(); + + void check_data_flow(); + int assign_fd_from_mysql_conn(); + + unsigned char* resultset2buffer(bool); + void buffer2resultset(unsigned char*, unsigned int); + + // safe way to attach a PgSQL Connection + void attach_connection(PgSQL_Connection* mc) { + statuses.pgconnpoll_get++; + myconn = mc; + myconn->statuses.pgconnpoll_get++; + mc->myds = this; + encrypted = false; // this is the default + // PMC-10005 + // we handle encryption for backend + // + // we have a similar code in MySQL_Connection + // in case of ASYNC_CONNECT_SUCCESSFUL + if (sess != NULL && sess->session_fast_forward == true) { + // if frontend and backend connection use SSL we will set + // encrypted = true and we will start using the SSL structure + // directly from P_MARIADB_TLS structure. + // + // For futher details: + // - without ssl: we use the file descriptor from pgsql connection + // - with ssl: we use the SSL structure from pgsql connection + if (myconn->pgsql && myconn->ret_mysql) { + if (myconn->pgsql->options.use_ssl == 1) { + encrypted = true; + if (ssl == NULL) { + // check the definition of P_MARIADB_TLS +// P_MARIADB_TLS* matls = (P_MARIADB_TLS*)myconn->pgsql->net.pvio->ctls; +// ssl = (SSL*)matls->ssl; +// rbio_ssl = BIO_new(BIO_s_mem()); +// wbio_ssl = BIO_new(BIO_s_mem()); +// SSL_set_bio(ssl, rbio_ssl, wbio_ssl); + } + } + } + } + } + + // safe way to detach a MySQL Connection + void detach_connection() { + assert(myconn); + myconn->statuses.pgconnpoll_put++; + statuses.pgconnpoll_put++; + myconn->myds = NULL; + myconn = NULL; + if (encrypted == true) { + if (sess != NULL && sess->session_fast_forward == true) { + // it seems we are a connection with SSL on a fast_forward session. + // See attach_connection() for more details . + // We now disable SSL metadata from the Data Stream + encrypted = false; + ssl = NULL; + } + } + } + + void return_MySQL_Connection_To_Pool(); + + void destroy_MySQL_Connection_From_Pool(bool sq); + void free_mysql_real_query(); + void reinit_queues(); + void destroy_queues(); + + bool data_in_rbio(); + + void reset_connection(); +}; +#endif /* __CLASS_PGSQL_DATA_STREAM_H */ diff --git a/include/PgSQL_Error_Helper.h b/include/PgSQL_Error_Helper.h new file mode 100644 index 0000000000..5ae88da5f8 --- /dev/null +++ b/include/PgSQL_Error_Helper.h @@ -0,0 +1,672 @@ +#ifndef __CLASS_PGSQL_ERROR_HELPER_H +#define __CLASS_PGSQL_ERROR_HELPER_H + +#include + +#define PGSQL_ERROR_FIELD_TEXT 0x0001 +#define PGSQL_ERROR_FIELD_DETAIL 0x0002 +#define PGSQL_ERROR_FIELD_HINT 0x0004 +#define PGSQL_ERROR_FIELD_POSITION 0x0008 +#define PGSQL_ERROR_FIELD_INTERNAL_POSITION 0x0010 +#define PGSQL_ERROR_FIELD_INTERNAL_QUERY 0x0020 +#define PGSQL_ERROR_FIELD_CONTEXT 0x0040 +#define PGSQL_ERROR_FIELD_SCHEMA_NAME 0x0080 +#define PGSQL_ERROR_FIELD_TABLE_NAME 0x0100 +#define PGSQL_ERROR_FIELD_COLUMN_NAME 0x0200 +#define PGSQL_ERROR_FIELD_DATA_TYPE_NAME 0x0400 +#define PGSQL_ERROR_FIELD_CONSTRAINT_NAME 0x0800 +#define PGSQL_ERROR_FIELD_FILE 0x1000 +#define PGSQL_ERROR_FIELD_LINE 0x2000 +#define PGSQL_ERROR_FIELD_ROUTINE 0x4000 +#define PGSQL_ERROR_FIELD_ALL 0xFFFF + +// these are standard SQLSTATES +enum class PGSQL_ERROR_CODES : uint8_t { + ERRCODE_SUCCESSFUL_COMPLETION, + ERRCODE_WARNING, + ERRCODE_DYNAMIC_RESULT_SETS_RETURNED, + ERRCODE_IMPLICIT_ZERO_BIT_PADDING, + ERRCODE_NULL_VALUE_ELIMINATED_IN_SET_FUNCTION, + ERRCODE_PRIVILEGE_NOT_GRANTED, + ERRCODE_PRIVILEGE_NOT_REVOKED, + ERRCODE_STRING_DATA_RIGHT_TRUNCATION, + ERRCODE_DEPRECATED_FEATURE, + ERRCODE_NO_DATA, + ERRCODE_NO_ADDITIONAL_DYNAMIC_RESULT_SETS_RETURNED, + ERRCODE_SQL_STATEMENT_NOT_YET_COMPLETE, + ERRCODE_CONNECTION_EXCEPTION, + ERRCODE_CONNECTION_DOES_NOT_EXIST, + ERRCODE_CONNECTION_FAILURE, + ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION, + ERRCODE_SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION, + ERRCODE_TRANSACTION_RESOLUTION_UNKNOWN, + ERRCODE_PROTOCOL_VIOLATION, + ERRCODE_TRIGGERED_ACTION_EXCEPTION, + ERRCODE_FEATURE_NOT_SUPPORTED, + ERRCODE_INVALID_TRANSACTION_INITIATION, + ERRCODE_LOCATOR_EXCEPTION, + ERRCODE_INVALID_LOCATOR_SPECIFICATION, + ERRCODE_INVALID_GRANTOR, + ERRCODE_INVALID_GRANT_OPERATION, + ERRCODE_INVALID_ROLE_SPECIFICATION, + ERRCODE_CARDINALITY_VIOLATION, + ERRCODE_DATA_EXCEPTION, + ERRCODE_ARRAY_ELEMENT_ERROR, + ERRCODE_ARRAY_SUBSCRIPT_ERROR, + ERRCODE_CHARACTER_NOT_IN_REPERTOIRE, + ERRCODE_DATETIME_FIELD_OVERFLOW, + ERRCODE_DIVISION_BY_ZERO, + ERRCODE_ERROR_IN_ASSIGNMENT, + ERRCODE_ESCAPE_CHARACTER_CONFLICT, + ERRCODE_INDICATOR_OVERFLOW, + ERRCODE_INTERVAL_FIELD_OVERFLOW, + ERRCODE_INVALID_ARGUMENT_FOR_LOG, + ERRCODE_INVALID_ARGUMENT_FOR_POWER_FUNCTION, + ERRCODE_INVALID_ARGUMENT_FOR_WIDTH_BUCKET_FUNCTION, + ERRCODE_INVALID_CHARACTER_VALUE_FOR_CAST, + ERRCODE_INVALID_DATETIME_FORMAT, + ERRCODE_INVALID_ESCAPE_CHARACTER, + ERRCODE_INVALID_ESCAPE_OCTET, + ERRCODE_INVALID_ESCAPE_SEQUENCE, + ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER, + ERRCODE_INVALID_INDICATOR_PARAMETER_VALUE, + ERRCODE_INVALID_PARAMETER_VALUE, + ERRCODE_INVALID_REGULAR_EXPRESSION, + ERRCODE_INVALID_ROW_COUNT_IN_LIMIT_CLAUSE, + ERRCODE_INVALID_ROW_COUNT_IN_RESULT_OFFSET_CLAUSE, + ERRCODE_INVALID_TABLESAMPLE_ARGUMENT, + ERRCODE_INVALID_TABLESAMPLE_REPEAT, + ERRCODE_INVALID_TIME_ZONE_DISPLACEMENT_VALUE, + ERRCODE_INVALID_USE_OF_ESCAPE_CHARACTER, + ERRCODE_MOST_SPECIFIC_TYPE_MISMATCH, + ERRCODE_NULL_VALUE_NOT_ALLOWED, + ERRCODE_NULL_VALUE_NO_INDICATOR_PARAMETER, + ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE, + ERRCODE_STRING_DATA_LENGTH_MISMATCH, + ERRCODE_SUBSTRING_ERROR, + ERRCODE_TRIM_ERROR, + ERRCODE_UNTERMINATED_C_STRING, + ERRCODE_ZERO_LENGTH_CHARACTER_STRING, + ERRCODE_FLOATING_POINT_EXCEPTION, + ERRCODE_INVALID_TEXT_REPRESENTATION, + ERRCODE_INVALID_BINARY_REPRESENTATION, + ERRCODE_BAD_COPY_FILE_FORMAT, + ERRCODE_UNTRANSLATABLE_CHARACTER, + ERRCODE_NOT_AN_XML_DOCUMENT, + ERRCODE_INVALID_XML_DOCUMENT, + ERRCODE_INVALID_XML_CONTENT, + ERRCODE_INVALID_XML_COMMENT, + ERRCODE_INVALID_XML_PROCESSING_INSTRUCTION, + ERRCODE_INTEGRITY_CONSTRAINT_VIOLATION, + ERRCODE_RESTRICT_VIOLATION, + ERRCODE_NOT_NULL_VIOLATION, + ERRCODE_FOREIGN_KEY_VIOLATION, + ERRCODE_UNIQUE_VIOLATION, + ERRCODE_CHECK_VIOLATION, + ERRCODE_EXCLUSION_VIOLATION, + ERRCODE_INVALID_CURSOR_STATE, + ERRCODE_INVALID_TRANSACTION_STATE, + ERRCODE_ACTIVE_SQL_TRANSACTION, + ERRCODE_BRANCH_TRANSACTION_ALREADY_ACTIVE, + ERRCODE_HELD_CURSOR_REQUIRES_SAME_ISOLATION_LEVEL, + ERRCODE_INAPPROPRIATE_ACCESS_MODE_FOR_BRANCH_TRANSACTION, + ERRCODE_INAPPROPRIATE_ISOLATION_LEVEL_FOR_BRANCH_TRANSACTION, + ERRCODE_NO_ACTIVE_SQL_TRANSACTION_FOR_BRANCH_TRANSACTION, + ERRCODE_READ_ONLY_SQL_TRANSACTION, + ERRCODE_SCHEMA_AND_DATA_STATEMENT_MIXING_NOT_SUPPORTED, + ERRCODE_NO_ACTIVE_SQL_TRANSACTION, + ERRCODE_IN_FAILED_SQL_TRANSACTION, + ERRCODE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT, + ERRCODE_INVALID_SQL_STATEMENT_NAME, + ERRCODE_TRIGGERED_DATA_CHANGE_VIOLATION, + ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION, + ERRCODE_INVALID_PASSWORD, + ERRCODE_DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST, + ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST, + ERRCODE_INVALID_TRANSACTION_TERMINATION, + ERRCODE_SQL_ROUTINE_EXCEPTION, + ERRCODE_S_R_E_FUNCTION_EXECUTED_NO_RETURN_STATEMENT, + ERRCODE_S_R_E_MODIFYING_SQL_DATA_NOT_PERMITTED, + ERRCODE_S_R_E_PROHIBITED_SQL_STATEMENT_ATTEMPTED, + ERRCODE_S_R_E_READING_SQL_DATA_NOT_PERMITTED, + ERRCODE_INVALID_CURSOR_NAME, + ERRCODE_EXTERNAL_ROUTINE_EXCEPTION, + ERRCODE_E_R_E_CONTAINING_SQL_NOT_PERMITTED, + ERRCODE_E_R_E_MODIFYING_SQL_DATA_NOT_PERMITTED, + ERRCODE_E_R_E_PROHIBITED_SQL_STATEMENT_ATTEMPTED, + ERRCODE_E_R_E_READING_SQL_DATA_NOT_PERMITTED, + ERRCODE_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION, + ERRCODE_E_R_I_E_INVALID_SQLSTATE_RETURNED, + ERRCODE_E_R_I_E_NULL_VALUE_NOT_ALLOWED, + ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED, + ERRCODE_E_R_I_E_SRF_PROTOCOL_VIOLATED, + ERRCODE_E_R_I_E_EVENT_TRIGGER_PROTOCOL_VIOLATED, + ERRCODE_SAVEPOINT_EXCEPTION, + ERRCODE_S_E_INVALID_SPECIFICATION, + ERRCODE_INVALID_CATALOG_NAME, + ERRCODE_INVALID_SCHEMA_NAME, + ERRCODE_TRANSACTION_ROLLBACK, + ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION, + ERRCODE_T_R_SERIALIZATION_FAILURE, + ERRCODE_T_R_STATEMENT_COMPLETION_UNKNOWN, + ERRCODE_T_R_DEADLOCK_DETECTED, + ERRCODE_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION, + ERRCODE_SYNTAX_ERROR, + ERRCODE_INSUFFICIENT_PRIVILEGE, + ERRCODE_CANNOT_COERCE, + ERRCODE_GROUPING_ERROR, + ERRCODE_WINDOWING_ERROR, + ERRCODE_INVALID_RECURSION, + ERRCODE_INVALID_FOREIGN_KEY, + ERRCODE_INVALID_NAME, + ERRCODE_NAME_TOO_LONG, + ERRCODE_RESERVED_NAME, + ERRCODE_DATATYPE_MISMATCH, + ERRCODE_INDETERMINATE_DATATYPE, + ERRCODE_COLLATION_MISMATCH, + ERRCODE_INDETERMINATE_COLLATION, + ERRCODE_WRONG_OBJECT_TYPE, + ERRCODE_GENERATED_ALWAYS, + ERRCODE_UNDEFINED_COLUMN, + ERRCODE_UNDEFINED_CURSOR, + ERRCODE_UNDEFINED_DATABASE, + ERRCODE_UNDEFINED_FUNCTION, + ERRCODE_UNDEFINED_PSTATEMENT, + ERRCODE_UNDEFINED_SCHEMA, + ERRCODE_UNDEFINED_TABLE, + ERRCODE_UNDEFINED_PARAMETER, + ERRCODE_UNDEFINED_OBJECT, + ERRCODE_DUPLICATE_COLUMN, + ERRCODE_DUPLICATE_CURSOR, + ERRCODE_DUPLICATE_DATABASE, + ERRCODE_DUPLICATE_FUNCTION, + ERRCODE_DUPLICATE_PSTATEMENT, + ERRCODE_DUPLICATE_SCHEMA, + ERRCODE_DUPLICATE_TABLE, + ERRCODE_DUPLICATE_ALIAS, + ERRCODE_DUPLICATE_OBJECT, + ERRCODE_AMBIGUOUS_COLUMN, + ERRCODE_AMBIGUOUS_FUNCTION, + ERRCODE_AMBIGUOUS_PARAMETER, + ERRCODE_AMBIGUOUS_ALIAS, + ERRCODE_INVALID_COLUMN_REFERENCE, + ERRCODE_INVALID_COLUMN_DEFINITION, + ERRCODE_INVALID_CURSOR_DEFINITION, + ERRCODE_INVALID_DATABASE_DEFINITION, + ERRCODE_INVALID_FUNCTION_DEFINITION, + ERRCODE_INVALID_PSTATEMENT_DEFINITION, + ERRCODE_INVALID_SCHEMA_DEFINITION, + ERRCODE_INVALID_TABLE_DEFINITION, + ERRCODE_INVALID_OBJECT_DEFINITION, + ERRCODE_WITH_CHECK_OPTION_VIOLATION, + ERRCODE_INSUFFICIENT_RESOURCES, + ERRCODE_DISK_FULL, + ERRCODE_OUT_OF_MEMORY, + ERRCODE_TOO_MANY_CONNECTIONS, + ERRCODE_CONFIGURATION_LIMIT_EXCEEDED, + ERRCODE_PROGRAM_LIMIT_EXCEEDED, + ERRCODE_STATEMENT_TOO_COMPLEX, + ERRCODE_TOO_MANY_COLUMNS, + ERRCODE_TOO_MANY_ARGUMENTS, + ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE, + ERRCODE_OBJECT_IN_USE, + ERRCODE_CANT_CHANGE_RUNTIME_PARAM, + ERRCODE_LOCK_NOT_AVAILABLE, + ERRCODE_UNSAFE_NEW_ENUM_VALUE_USAGE, + ERRCODE_OPERATOR_INTERVENTION, + ERRCODE_QUERY_CANCELED, + ERRCODE_ADMIN_SHUTDOWN, + ERRCODE_CRASH_SHUTDOWN, + ERRCODE_CANNOT_CONNECT_NOW, + ERRCODE_DATABASE_DROPPED, + ERRCODE_IDLE_SESSION_TIMEOUT, + ERRCODE_SYSTEM_ERROR, + ERRCODE_IO_ERROR, + ERRCODE_UNDEFINED_FILE, + ERRCODE_DUPLICATE_FILE, + ERRCODE_CONFIG_FILE_ERROR, + ERRCODE_LOCK_FILE_EXISTS, + ERRCODE_FDW_ERROR, + ERRCODE_FDW_COLUMN_NAME_NOT_FOUND, + ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED, + ERRCODE_FDW_FUNCTION_SEQUENCE_ERROR, + ERRCODE_FDW_INCONSISTENT_DESCRIPTOR_INFORMATION, + ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE, + ERRCODE_FDW_INVALID_COLUMN_NAME, + ERRCODE_FDW_INVALID_COLUMN_NUMBER, + ERRCODE_FDW_INVALID_DATA_TYPE, + ERRCODE_FDW_INVALID_DATA_TYPE_DESCRIPTORS, + ERRCODE_FDW_INVALID_DESCRIPTOR_FIELD_IDENTIFIER, + ERRCODE_FDW_INVALID_HANDLE, + ERRCODE_FDW_INVALID_OPTION_INDEX, + ERRCODE_FDW_INVALID_OPTION_NAME, + ERRCODE_FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH, + ERRCODE_FDW_INVALID_STRING_FORMAT, + ERRCODE_FDW_INVALID_USE_OF_NULL_POINTER, + ERRCODE_FDW_TOO_MANY_HANDLES, + ERRCODE_FDW_OUT_OF_MEMORY, + ERRCODE_FDW_NO_SCHEMAS, + ERRCODE_FDW_OPTION_NAME_NOT_FOUND, + ERRCODE_FDW_REPLY_HANDLE, + ERRCODE_FDW_SCHEMA_NOT_FOUND, + ERRCODE_FDW_TABLE_NOT_FOUND, + ERRCODE_FDW_UNABLE_TO_CREATE_EXECUTION, + ERRCODE_FDW_UNABLE_TO_CREATE_REPLY, + ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION, + ERRCODE_PLPGSQL_ERROR, + ERRCODE_RAISE_EXCEPTION, + ERRCODE_ASSERT_FAILURE, + ERRCODE_INTERNAL_ERROR, + ERRCODE_DATA_CORRUPTED, + ERRCODE_INDEX_CORRUPTED, + // Add more error codes here if needed. Make sure to update error_code_str also. + ERRCODE_UNKNOWN, + PGSQL_ERROR_CODES_COUNT // This should always be the last entry +}; + +// Enum to represent different error types +enum class PGSQL_ERROR_CLASS : uint8_t { + ERRCLASS_UNKNOWN_ERROR, + ERRCLASS_SUCCESS, + ERRCLASS_WARNING, + ERRCLASS_NO_DATA, + ERRCLASS_SQL_STATEMENT_NOT_YET_COMPLETE, + ERRCLASS_CONNECTION_EXCEPTION, + ERRCLASS_TRIGGERED_ACTION_EXCEPTION, + ERRCLASS_FEATURE_NOT_SUPPORTED, + ERRCLASS_INVALID_TRANSACTION_INITIATION, + ERRCLASS_LOCATOR_EXCEPTION, + ERRCLASS_INVALID_GRANTOR, + ERRCLASS_INVALID_ROLE_SPECIFICATION, + ERRCLASS_DIAGNOSTICS_EXCEPTION, + ERRCLASS_CASE_NOT_FOUND, + ERRCLASS_CARDINALITY_VIOLATION, + ERRCLASS_DATA_EXCEPTION, + ERRCLASS_INTEGRITY_CONSTRAINT_VIOLATION, + ERRCLASS_INVALID_CURSOR_STATE, + ERRCLASS_INVALID_TRANSACTION_STATE, + ERRCLASS_INVALID_SQL_STATEMENT_NAME, + ERRCLASS_TRIGGERED_DATA_CHANGE_VIOLATION, + ERRCLASS_INVALID_AUTHORIZATION_SPECIFICATION, + ERRCLASS_DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST, + ERRCLASS_INVALID_TRANSACTION_TERMINATION, + ERRCLASS_SQL_ROUTINE_EXCEPTION, + ERRCLASS_INVALID_CURSOR_NAME, + ERRCLASS_EXTERNAL_ROUTINE_EXCEPTION, + ERRCLASS_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION, + ERRCLASS_SAVEPOINT_EXCEPTION, + ERRCLASS_INVALID_CATALOG_NAME, + ERRCLASS_INVALID_SCHEMA_NAME, + ERRCLASS_TRANSACTION_ROLLBACK, + ERRCLASS_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION, + ERRCLASS_WITH_CHECK_OPTION_VIOLATION, + ERRCLASS_INSUFFICIENT_RESOURCES, + ERRCLASS_PROGRAM_LIMIT_EXCEEDED, + ERRCLASS_OBJECT_NOT_IN_PREREQUISITE_STATE, + ERRCLASS_OPERATOR_INTERVENTION, + ERRCLASS_SYSTEM_ERROR_UNSPECIFIED, + ERRCLASS_CONFIG_FILE_ERROR, + ERRCLASS_CRASH_SHUTDOWN, + ERRCLASS_PROTOCOL_VIOLATION, + ERRCLASS_FOREIGN_DATA_WRAPPER_ERROR, + ERRCLASS_PLPGSQL_ERROR, + ERRCLASS_INTERNAL_ERROR, + ERRCLASS_MYSQL_SPECIFIC_ERROR +}; + +// Enum to represent error categories +enum class PGSQL_ERROR_CATEGORY : uint8_t { + ERRCATEGORY_UNKNOWN_CATEGORY, + ERRCATEGORY_STATUS, // For success and warnings, which are not errors. + ERRCATEGORY_CLIENT_ERROR, + ERRCATEGORY_CONNECTION_ERROR, + ERRCATEGORY_AUTHORIZATION_ERROR, + ERRCATEGORY_RESOURCE_ERROR, + ERRCATEGORY_CONFIGURATION_ERROR, + ERRCATEGORY_SYNTAX_ERROR, + ERRCATEGORY_FEATURE_NOT_SUPPORTED, + ERRCATEGORY_TRANSACTION_ERROR, + ERRCATEGORY_DATA_ERROR, + ERRCATEGORY_ROUTINE_ERROR, + ERRCATEGORY_CURSOR_ERROR, + ERRCATEGORY_EXTERNAL_ROUTINE_ERROR, + ERRCATEGORY_RESOURCE_LIMIT_ERROR, + ERRCATEGORY_OBJECT_STATE_ERROR, + ERRCATEGORY_OPERATOR_INTERVENTION_ERROR, + ERRCATEGORY_FDW_ERROR, + ERRCATEGORY_PLPGSQL_ERROR, + ERRCATEGORY_INTERNAL_ERROR_CATEGORY +}; + +enum class PGSQL_ERROR_SEVERITY : uint8_t { + ERRSEVERITY_UNKNOWN_SEVERITY, + ERRSEVERITY_FATAL, + ERRSEVERITY_PANIC, + ERRSEVERITY_ERROR, + ERRSEVERITY_WARNING, + ERRSEVERITY_NOTICE, + ERRSEVERITY_DEBUG, + ERRSEVERITY_INFO, + ERRSEVERITY_LOG, + PGSQL_ERROR_SEVERITY_COUNT +}; + +struct PgSQL_ErrorInfo_Ext { + // bitmap fields present in the error message + PGSQL_ERROR_SEVERITY text; + std::string detail; + std::string hint; + std::string position; + std::string internal_position; + std::string internal_query; + std::string context; + std::string schema_name; + std::string table_name; + std::string column_name; + std::string datatype_name; + std::string constraint_name; + std::string source_file; + std::string source_line; + std::string source_function; + void reset(); +}; + +struct pg_result; +typedef struct pg_result PGresult; + +struct PgSQL_ErrorInfo { + PGSQL_ERROR_SEVERITY severity = PGSQL_ERROR_SEVERITY::ERRSEVERITY_UNKNOWN_SEVERITY; + PGSQL_ERROR_CODES code = PGSQL_ERROR_CODES::ERRCODE_SUCCESSFUL_COMPLETION; + PGSQL_ERROR_CLASS type = PGSQL_ERROR_CLASS::ERRCLASS_UNKNOWN_ERROR; + PGSQL_ERROR_CATEGORY category = PGSQL_ERROR_CATEGORY::ERRCATEGORY_UNKNOWN_CATEGORY; + char sqlstate[5 + 1] = {}; // 5 bytes for SQLSTATE + 1 for null terminator + PgSQL_ErrorInfo_Ext* ext_info = NULL; + std::string message; +}; + +void reset_error_info(PgSQL_ErrorInfo& err_info, bool release_extented); + +class PgSQL_Error_Helper { +public: + static constexpr const char* get_error_code(PGSQL_ERROR_CODES code) { + return error_code_str[static_cast(code)]; + } + + static constexpr const char* get_severity(PGSQL_ERROR_SEVERITY severity) { + return severity_str[static_cast(severity)]; + } + + static void fill_error_info(PgSQL_ErrorInfo& err_info, const PGresult* result, uint16_t ext_fields); + static void fill_error_info(PgSQL_ErrorInfo& err_info, const char* code, const char* msg, const char* severity); + static void fill_error_info(PgSQL_ErrorInfo& err_info, PGSQL_ERROR_CODES code, const char* msg, PGSQL_ERROR_SEVERITY severity); + //static void fill_error_info_from_error_message(PgSQL_ErrorInfo& err_info, const char* error_msg); + +private: + static PGSQL_ERROR_CODES identify_error_code(const char* code); + static PGSQL_ERROR_CLASS identify_error_class(const char* code); + static PGSQL_ERROR_CATEGORY categorize_error_class(PGSQL_ERROR_CLASS err_class); + static PGSQL_ERROR_SEVERITY identify_error_severity(const char* severity); + static void fill_extended_error_info(PgSQL_ErrorInfo& err_info, const PGresult* result, uint16_t ext_fields); + + /* All the error codes from https://www.postgresql.org/docs/current/errcodes-appendix.html */ + static constexpr const char* error_code_str[] = { + "00000", // ERRCODE_SUCCESSFUL_COMPLETION + "01000", // ERRCODE_WARNING + "0100C", // ERRCODE_DYNAMIC_RESULT_SETS_RETURNED + "01008", // ERRCODE_IMPLICIT_ZERO_BIT_PADDING + "01003", // ERRCODE_NULL_VALUE_ELIMINATED_IN_SET_FUNCTION + "01007", // ERRCODE_PRIVILEGE_NOT_GRANTED + "01006", // ERRCODE_PRIVILEGE_NOT_REVOKED + "01004", // ERRCODE_STRING_DATA_RIGHT_TRUNCATION + "01P01", // ERRCODE_DEPRECATED_FEATURE + "02000", // ERRCODE_NO_DATA + "02001", // ERRCODE_NO_ADDITIONAL_DYNAMIC_RESULT_SETS_RETURNED + "03000", // ERRCODE_SQL_STATEMENT_NOT_YET_COMPLETE + "08000", // ERRCODE_CONNECTION_EXCEPTION + "08003", // ERRCODE_CONNECTION_DOES_NOT_EXIST + "08006", // ERRCODE_CONNECTION_FAILURE + "08001", // ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION + "08004", // ERRCODE_SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION + "08007", // ERRCODE_TRANSACTION_RESOLUTION_UNKNOWN + "08P01", // ERRCODE_PROTOCOL_VIOLATION + "09000", // ERRCODE_TRIGGERED_ACTION_EXCEPTION + "0A000", // ERRCODE_FEATURE_NOT_SUPPORTED + "0B000", // ERRCODE_INVALID_TRANSACTION_INITIATION + "0F000", // ERRCODE_LOCATOR_EXCEPTION + "0F001", // ERRCODE_INVALID_LOCATOR_SPECIFICATION + "0L000", // ERRCODE_INVALID_GRANTOR + "0LP01", // ERRCODE_INVALID_GRANT_OPERATION + "0P000", // ERRCODE_INVALID_ROLE_SPECIFICATION + "21000", // ERRCODE_CARDINALITY_VIOLATION + "22000", // ERRCODE_DATA_EXCEPTION + "2202E", // ERRCODE_ARRAY_ELEMENT_ERROR + "2202E", // ERRCODE_ARRAY_SUBSCRIPT_ERROR + "22021", // ERRCODE_CHARACTER_NOT_IN_REPERTOIRE + "22008", // ERRCODE_DATETIME_FIELD_OVERFLOW + "22012", // ERRCODE_DIVISION_BY_ZERO + "22005", // ERRCODE_ERROR_IN_ASSIGNMENT + "2200B", // ERRCODE_ESCAPE_CHARACTER_CONFLICT + "22022", // ERRCODE_INDICATOR_OVERFLOW + "22015", // ERRCODE_INTERVAL_FIELD_OVERFLOW + "2201E", // ERRCODE_INVALID_ARGUMENT_FOR_LOG + "2201F", // ERRCODE_INVALID_ARGUMENT_FOR_POWER_FUNCTION + "2201G", // ERRCODE_INVALID_ARGUMENT_FOR_WIDTH_BUCKET_FUNCTION + "22018", // ERRCODE_INVALID_CHARACTER_VALUE_FOR_CAST + "22007", // ERRCODE_INVALID_DATETIME_FORMAT + "22019", // ERRCODE_INVALID_ESCAPE_CHARACTER + "2200D", // ERRCODE_INVALID_ESCAPE_OCTET + "22025", // ERRCODE_INVALID_ESCAPE_SEQUENCE + "22P06", // ERRCODE_NONSTANDARD_USE_OF_ESCAPE_CHARACTER + "22010", // ERRCODE_INVALID_INDICATOR_PARAMETER_VALUE + "22023", // ERRCODE_INVALID_PARAMETER_VALUE + "2201B", // ERRCODE_INVALID_REGULAR_EXPRESSION + "2201W", // ERRCODE_INVALID_ROW_COUNT_IN_LIMIT_CLAUSE + "2201X", // ERRCODE_INVALID_ROW_COUNT_IN_RESULT_OFFSET_CLAUSE + "2202H", // ERRCODE_INVALID_TABLESAMPLE_ARGUMENT + "2202G", // ERRCODE_INVALID_TABLESAMPLE_REPEAT + "22009", // ERRCODE_INVALID_TIME_ZONE_DISPLACEMENT_VALUE + "2200C", // ERRCODE_INVALID_USE_OF_ESCAPE_CHARACTER + "2200G", // ERRCODE_MOST_SPECIFIC_TYPE_MISMATCH + "22004", // ERRCODE_NULL_VALUE_NOT_ALLOWED + "22002", // ERRCODE_NULL_VALUE_NO_INDICATOR_PARAMETER + "22003", // ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE + "22026", // ERRCODE_STRING_DATA_LENGTH_MISMATCH + "22011", // ERRCODE_SUBSTRING_ERROR + "22027", // ERRCODE_TRIM_ERROR + "22024", // ERRCODE_UNTERMINATED_C_STRING + "2200F", // ERRCODE_ZERO_LENGTH_CHARACTER_STRING + "22P01", // ERRCODE_FLOATING_POINT_EXCEPTION + "22P02", // ERRCODE_INVALID_TEXT_REPRESENTATION + "22P03", // ERRCODE_INVALID_BINARY_REPRESENTATION + "22P04", // ERRCODE_BAD_COPY_FILE_FORMAT + "22P05", // ERRCODE_UNTRANSLATABLE_CHARACTER + "2200L", // ERRCODE_NOT_AN_XML_DOCUMENT + "2200M", // ERRCODE_INVALID_XML_DOCUMENT + "2200N", // ERRCODE_INVALID_XML_CONTENT + "2200S", // ERRCODE_INVALID_XML_COMMENT + "2200T", // ERRCODE_INVALID_XML_PROCESSING_INSTRUCTION + "23000", // ERRCODE_INTEGRITY_CONSTRAINT_VIOLATION + "23001", // ERRCODE_RESTRICT_VIOLATION + "23502", // ERRCODE_NOT_NULL_VIOLATION + "23503", // ERRCODE_FOREIGN_KEY_VIOLATION + "23505", // ERRCODE_UNIQUE_VIOLATION + "23514", // ERRCODE_CHECK_VIOLATION + "23P01", // ERRCODE_EXCLUSION_VIOLATION + "24000", // ERRCODE_INVALID_CURSOR_STATE + "25000", // ERRCODE_INVALID_TRANSACTION_STATE + "25001", // ERRCODE_ACTIVE_SQL_TRANSACTION + "25002", // ERRCODE_BRANCH_TRANSACTION_ALREADY_ACTIVE + "25008", // ERRCODE_HELD_CURSOR_REQUIRES_SAME_ISOLATION_LEVEL + "25003", // ERRCODE_INAPPROPRIATE_ACCESS_MODE_FOR_BRANCH_TRANSACTION + "25004", // ERRCODE_INAPPROPRIATE_ISOLATION_LEVEL_FOR_BRANCH_TRANSACTION + "25005", // ERRCODE_NO_ACTIVE_SQL_TRANSACTION_FOR_BRANCH_TRANSACTION + "25006", // ERRCODE_READ_ONLY_SQL_TRANSACTION + "25007", // ERRCODE_SCHEMA_AND_DATA_STATEMENT_MIXING_NOT_SUPPORTED + "25P01", // ERRCODE_NO_ACTIVE_SQL_TRANSACTION + "25P02", // ERRCODE_IN_FAILED_SQL_TRANSACTION + "25P03", // ERRCODE_IDLE_IN_TRANSACTION_SESSION_TIMEOUT + "26000", // ERRCODE_INVALID_SQL_STATEMENT_NAME + "27000", // ERRCODE_TRIGGERED_DATA_CHANGE_VIOLATION + "28000", // ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION + "28P01", // ERRCODE_INVALID_PASSWORD + "2B000", // ERRCODE_DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST + "2BP01", // ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST + "2D000", // ERRCODE_INVALID_TRANSACTION_TERMINATION + "2F000", // ERRCODE_SQL_ROUTINE_EXCEPTION + "2F005", // ERRCODE_S_R_E_FUNCTION_EXECUTED_NO_RETURN_STATEMENT + "2F002", // ERRCODE_S_R_E_MODIFYING_SQL_DATA_NOT_PERMITTED + "2F003", // ERRCODE_S_R_E_PROHIBITED_SQL_STATEMENT_ATTEMPTED + "2F004", // ERRCODE_S_R_E_READING_SQL_DATA_NOT_PERMITTED + "34000", // ERRCODE_INVALID_CURSOR_NAME + "38000", // ERRCODE_EXTERNAL_ROUTINE_EXCEPTION + "38001", // ERRCODE_E_R_E_CONTAINING_SQL_NOT_PERMITTED + "38002", // ERRCODE_E_R_E_MODIFYING_SQL_DATA_NOT_PERMITTED + "38003", // ERRCODE_E_R_E_PROHIBITED_SQL_STATEMENT_ATTEMPTED + "38004", // ERRCODE_E_R_E_READING_SQL_DATA_NOT_PERMITTED + "39000", // ERRCODE_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION + "39001", // ERRCODE_E_R_I_E_INVALID_SQLSTATE_RETURNED + "39004", // ERRCODE_E_R_I_E_NULL_VALUE_NOT_ALLOWED + "39P01", // ERRCODE_E_R_I_E_TRIGGER_PROTOCOL_VIOLATED + "39P02", // ERRCODE_E_R_I_E_SRF_PROTOCOL_VIOLATED + "39P03", // ERRCODE_E_R_I_E_EVENT_TRIGGER_PROTOCOL_VIOLATED + "3B000", // ERRCODE_SAVEPOINT_EXCEPTION + "3B001", // ERRCODE_S_E_INVALID_SPECIFICATION + "3D000", // ERRCODE_INVALID_CATALOG_NAME + "3F000", // ERRCODE_INVALID_SCHEMA_NAME + "40000", // ERRCODE_TRANSACTION_ROLLBACK + "40002", // ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION + "40001", // ERRCODE_T_R_SERIALIZATION_FAILURE + "40003", // ERRCODE_T_R_STATEMENT_COMPLETION_UNKNOWN + "40P01", // ERRCODE_T_R_DEADLOCK_DETECTED + "42000", // ERRCODE_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION + "42601", // ERRCODE_SYNTAX_ERROR + "42501", // ERRCODE_INSUFFICIENT_PRIVILEGE + "42846", // ERRCODE_CANNOT_COERCE + "42803", // ERRCODE_GROUPING_ERROR + "42P20", // ERRCODE_WINDOWING_ERROR + "42P19", // ERRCODE_INVALID_RECURSION + "42830", // ERRCODE_INVALID_FOREIGN_KEY + "42602", // ERRCODE_INVALID_NAME + "42622", // ERRCODE_NAME_TOO_LONG + "42939", // ERRCODE_RESERVED_NAME + "42804", // ERRCODE_DATATYPE_MISMATCH + "42P18", // ERRCODE_INDETERMINATE_DATATYPE + "42P21", // ERRCODE_COLLATION_MISMATCH + "42P22", // ERRCODE_INDETERMINATE_COLLATION + "42809", // ERRCODE_WRONG_OBJECT_TYPE + "428C9", // ERRCODE_GENERATED_ALWAYS + "42703", // ERRCODE_UNDEFINED_COLUMN + "34000", // ERRCODE_UNDEFINED_CURSOR + "3D000", // ERRCODE_UNDEFINED_DATABASE + "42883", // ERRCODE_UNDEFINED_FUNCTION + "26000", // ERRCODE_UNDEFINED_PSTATEMENT + "3F000", // ERRCODE_UNDEFINED_SCHEMA + "42P01", // ERRCODE_UNDEFINED_TABLE + "42P02", // ERRCODE_UNDEFINED_PARAMETER + "42704", // ERRCODE_UNDEFINED_OBJECT + "42701", // ERRCODE_DUPLICATE_COLUMN + "42P03", // ERRCODE_DUPLICATE_CURSOR + "42P04", // ERRCODE_DUPLICATE_DATABASE + "42723", // ERRCODE_DUPLICATE_FUNCTION + "42P05", // ERRCODE_DUPLICATE_PSTATEMENT + "42P06", // ERRCODE_DUPLICATE_SCHEMA + "42P07", // ERRCODE_DUPLICATE_TABLE + "42712", // ERRCODE_DUPLICATE_ALIAS + "42710", // ERRCODE_DUPLICATE_OBJECT + "42702", // ERRCODE_AMBIGUOUS_COLUMN + "42725", // ERRCODE_AMBIGUOUS_FUNCTION + "42P08", // ERRCODE_AMBIGUOUS_PARAMETER + "42P09", // ERRCODE_AMBIGUOUS_ALIAS + "42P10", // ERRCODE_INVALID_COLUMN_REFERENCE + "42611", // ERRCODE_INVALID_COLUMN_DEFINITION + "42P11", // ERRCODE_INVALID_CURSOR_DEFINITION + "42P12", // ERRCODE_INVALID_DATABASE_DEFINITION + "42P13", // ERRCODE_INVALID_FUNCTION_DEFINITION + "42P14", // ERRCODE_INVALID_PSTATEMENT_DEFINITION + "42P15", // ERRCODE_INVALID_SCHEMA_DEFINITION + "42P16", // ERRCODE_INVALID_TABLE_DEFINITION + "42P17", // ERRCODE_INVALID_OBJECT_DEFINITION + "44000", // ERRCODE_WITH_CHECK_OPTION_VIOLATION + "53000", // ERRCODE_INSUFFICIENT_RESOURCES + "53100", // ERRCODE_DISK_FULL + "53200", // ERRCODE_OUT_OF_MEMORY + "53300", // ERRCODE_TOO_MANY_CONNECTIONS + "53400", // ERRCODE_CONFIGURATION_LIMIT_EXCEEDED + "54000", // ERRCODE_PROGRAM_LIMIT_EXCEEDED + "54001", // ERRCODE_STATEMENT_TOO_COMPLEX + "54011", // ERRCODE_TOO_MANY_COLUMNS + "54023", // ERRCODE_TOO_MANY_ARGUMENTS + "55000", // ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE + "55006", // ERRCODE_OBJECT_IN_USE + "55P02", // ERRCODE_CANT_CHANGE_RUNTIME_PARAM + "55P03", // ERRCODE_LOCK_NOT_AVAILABLE + "55P04", // ERRCODE_UNSAFE_NEW_ENUM_VALUE_USAGE + "57000", // ERRCODE_OPERATOR_INTERVENTION + "57014", // ERRCODE_QUERY_CANCELED + "57P01", // ERRCODE_ADMIN_SHUTDOWN + "57P02", // ERRCODE_CRASH_SHUTDOWN + "57P03", // ERRCODE_CANNOT_CONNECT_NOW + "57P04", // ERRCODE_DATABASE_DROPPED + "57P05", // ERRCODE_IDLE_SESSION_TIMEOUT + "58000", // ERRCODE_SYSTEM_ERROR + "58030", // ERRCODE_IO_ERROR + "58P01", // ERRCODE_UNDEFINED_FILE + "58P02", // ERRCODE_DUPLICATE_FILE + "F0000", // ERRCODE_CONFIG_FILE_ERROR + "F0001", // ERRCODE_LOCK_FILE_EXISTS + "HV000", // ERRCODE_FDW_ERROR + "HV005", // ERRCODE_FDW_COLUMN_NAME_NOT_FOUND + "HV002", // ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED + "HV010", // ERRCODE_FDW_FUNCTION_SEQUENCE_ERROR + "HV021", // ERRCODE_FDW_INCONSISTENT_DESCRIPTOR_INFORMATION + "HV024", // ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE + "HV007", // ERRCODE_FDW_INVALID_COLUMN_NAME + "HV008", // ERRCODE_FDW_INVALID_COLUMN_NUMBER + "HV004", // ERRCODE_FDW_INVALID_DATA_TYPE + "HV006", // ERRCODE_FDW_INVALID_DATA_TYPE_DESCRIPTORS + "HV091", // ERRCODE_FDW_INVALID_DESCRIPTOR_FIELD_IDENTIFIER + "HV00B", // ERRCODE_FDW_INVALID_HANDLE + "HV00C", // ERRCODE_FDW_INVALID_OPTION_INDEX + "HV00D", // ERRCODE_FDW_INVALID_OPTION_NAME + "HV090", // ERRCODE_FDW_INVALID_STRING_LENGTH_OR_BUFFER_LENGTH + "HV00A", // ERRCODE_FDW_INVALID_STRING_FORMAT + "HV009", // ERRCODE_FDW_INVALID_USE_OF_NULL_POINTER + "HV014", // ERRCODE_FDW_TOO_MANY_HANDLES + "HV001", // ERRCODE_FDW_OUT_OF_MEMORY + "HV00P", // ERRCODE_FDW_NO_SCHEMAS + "HV00J", // ERRCODE_FDW_OPTION_NAME_NOT_FOUND + "HV00K", // ERRCODE_FDW_REPLY_HANDLE + "HV00Q", // ERRCODE_FDW_SCHEMA_NOT_FOUND + "HV00R", // ERRCODE_FDW_TABLE_NOT_FOUND + "HV00L", // ERRCODE_FDW_UNABLE_TO_CREATE_EXECUTION + "HV00M", // ERRCODE_FDW_UNABLE_TO_CREATE_REPLY + "HV00N", // ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION + "P0000", // ERRCODE_PLPGSQL_ERROR + "P0001", // ERRCODE_RAISE_EXCEPTION + "P0002", // ERRCODE_ASSERT_FAILURE + "XX000", // ERRCODE_INTERNAL_ERROR + "XX001", // ERRCODE_DATA_CORRUPTED + "XX002", // ERRCODE_INDEX_CORRUPTED + // Add more error codes here if needed. Make sure to update PGSQL_ERROR_CODES also. + "XXXXX", // ERRCODE_UNKNOWN + }; + + static_assert(static_cast(PGSQL_ERROR_CODES::PGSQL_ERROR_CODES_COUNT) == sizeof(error_code_str) / sizeof(char*), "Mismatch between PGSQL_ERROR_CODES_COUNT and error_code_str array size"); + + static constexpr const char* severity_str[] = { + "UNKNOWN", + "FATAL", + "PANIC", + "ERROR", + "WARNING", + "NOTICE", + "DEBUG", + "INFO", + "LOG" + }; + static_assert(static_cast(PGSQL_ERROR_SEVERITY::PGSQL_ERROR_SEVERITY_COUNT) == sizeof(severity_str) / sizeof(char*), "Mismatch between PGSQL_ERROR_SEVERITY_COUNT and severity_str array size"); +}; + +#define PGSQL_GET_ERROR_CODE_STR(ENUM_CODE) PgSQL_Error_Helper::get_error_code(PGSQL_ERROR_CODES::ENUM_CODE) + +#endif /* __CLASS_PGSQL_ERROR_HELPER_H */ diff --git a/include/PgSQL_HostGroups_Manager.h b/include/PgSQL_HostGroups_Manager.h new file mode 100644 index 0000000000..8a8e94f043 --- /dev/null +++ b/include/PgSQL_HostGroups_Manager.h @@ -0,0 +1,866 @@ +#ifndef __CLASS_PGSQL_HOSTGROUPS_MANAGER_H +#define __CLASS_PGSQL_HOSTGROUPS_MANAGER_H +#include "proxysql.h" +#include "cpp.h" +#include "proxysql_gtid.h" +#include "proxysql_admin.h" +#include +#include +#include +#include + +// Headers for declaring Prometheus counters +#include "prometheus/counter.h" +#include "prometheus/gauge.h" + +#include "thread.h" +#include "wqueue.h" + +#include "ev.h" + +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif + +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON + +#ifdef DEBUG +/* */ +// Enabling STRESSTEST_POOL ProxySQL will do a lot of loops in the connection pool +// This is for internal testing ONLY!!!! +//#define STRESSTEST_POOL +#endif // DEBUG + + +#include "Base_HostGroups_Manager.h" + +// we have 2 versions of the same tables: with (debug) and without (no debug) checks +#ifdef DEBUG +#define MYHGM_PgSQL_SERVERS "CREATE TABLE pgsql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 5432 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define MYHGM_PgSQL_SERVERS_INCOMING "CREATE TABLE pgsql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 5432 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" +#else +#define MYHGM_PgSQL_SERVERS "CREATE TABLE pgsql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 5432 , weight INT NOT NULL DEFAULT 1 , status INT NOT NULL DEFAULT 0 , compression INT NOT NULL DEFAULT 0 , max_connections INT NOT NULL DEFAULT 1000 , max_replication_lag INT NOT NULL DEFAULT 0 , use_ssl INT NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define MYHGM_PgSQL_SERVERS_INCOMING "CREATE TABLE pgsql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 5432 , weight INT NOT NULL DEFAULT 1 , status INT NOT NULL DEFAULT 0 , compression INT NOT NULL DEFAULT 0 , max_connections INT NOT NULL DEFAULT 1000 , max_replication_lag INT NOT NULL DEFAULT 0 , use_ssl INT NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" +#endif /* DEBUG */ +#define MYHGM_PgSQL_REPLICATION_HOSTGROUPS "CREATE TABLE pgsql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only','read_only|innodb_read_only','read_only&innodb_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (reader_hostgroup))" + +#define PGHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM pgsql_servers ORDER BY hostgroup_id, hostname, port" + +#define MYHGM_PgSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE pgsql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" + +/* + * @brief Generates the 'runtime_pgsql_servers' resultset exposed to other ProxySQL cluster members. + * @details Makes 'SHUNNED' and 'SHUNNED_REPLICATION_LAG' statuses equivalent to 'ONLINE'. 'SHUNNED' states + * are by definition local transitory states, this is why a 'pgsql_servers' table reconfiguration isn't + * normally performed when servers are internally imposed with these statuses. This means, that propagating + * this state to other cluster members is undesired behavior, and so it's generating a different checksum, + * due to a server having this particular state, that will result in extra unnecessary fetching operations. + * The query also filters out 'OFFLINE_HARD' servers, 'OFFLINE_HARD' is a local status which is equivalent to + * a server no longer being part of the table (DELETED state). And so, they shouldn't be propagated. + * + * For placing the query into a single line for debugging purposes: + * ``` + * sed 's/^\t\+"//g; s/"\s\\$//g; s/\\"/"/g' /tmp/select.sql | paste -sd '' + * ``` + */ +#define PGHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS \ + "SELECT " \ + "hostgroup_id, hostname, port, " \ + "CASE status" \ + " WHEN 0 THEN \"ONLINE\"" \ + " WHEN 1 THEN \"ONLINE\"" \ + " WHEN 2 THEN \"OFFLINE_SOFT\"" \ + " WHEN 3 THEN \"OFFLINE_HARD\"" \ + " WHEN 4 THEN \"ONLINE\" " \ + "END status," \ + "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ + "FROM pgsql_servers " \ + "WHERE status != 3 " \ + "ORDER BY hostgroup_id, hostname, port" \ + +/** + * @brief Generates the 'pgsql_servers_v2' resultset exposed to other ProxySQL cluster members. + * @details The generated resultset is used for the checksum computation of the runtime ProxySQL config + * ('pgsql_servers_v2' checksum), and it's also forwarded to other cluster members when querying the Admin + * interface with 'CLUSTER_QUERY_PgSQL_SERVERS_V2'. It makes 'SHUNNED' state equivalent to 'ONLINE', and also + * filters out any 'OFFLINE_HARD' entries. This is done because none of the statuses are valid configuration + * statuses, they are local, transient status that ProxySQL uses during operation. + */ +#define PGHGM_GEN_CLUSTER_ADMIN_PGSQL_SERVERS \ + "SELECT " \ + "hostgroup_id, hostname, port, " \ + "CASE" \ + " WHEN status=\"SHUNNED\" THEN \"ONLINE\"" \ + " ELSE status " \ + "END AS status, " \ + "weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment " \ + "FROM main.pgsql_servers " \ + "WHERE status != \"OFFLINE_HARD\" " \ + "ORDER BY hostgroup_id, hostname, port" + +class PgSQL_SrvConnList; +class PgSQL_SrvC; +class PgSQL_SrvList; +class PgSQL_HGC; +class PgSQL_Errors_stats; + +typedef std::unordered_map umap_pgsql_errors; + +class PgSQL_GTID_Server_Data { + public: + char *address; + uint16_t port; + uint16_t pgsql_port; + char *data; + size_t len; + size_t size; + size_t pos; + struct ev_io *w; + char uuid_server[64]; + unsigned long long events_read; + gtid_set_t gtid_executed; + bool active; + PgSQL_GTID_Server_Data(struct ev_io *_w, char *_address, uint16_t _port, uint16_t _pgsql_port); + void resize(size_t _s); + ~PgSQL_GTID_Server_Data(); + bool readall(); + bool writeout(); + bool read_next_gtid(); + bool gtid_exists(char *gtid_uuid, uint64_t gtid_trxid); + void read_all_gtids(); + void dump(); +}; + + + +class PgSQL_SrvConnList { + private: + PgSQL_SrvC *mysrvc; + int find_idx(PgSQL_Connection *c) { + //for (unsigned int i=0; ilen; i++) { + PgSQL_Connection *conn = NULL; + conn = (PgSQL_Connection *)conns->index(i); + if (conn==c) { + return (unsigned int)i; + } + } + return -1; + } + public: + PtrArray *conns; + PgSQL_SrvConnList(PgSQL_SrvC *); + ~PgSQL_SrvConnList(); + void add(PgSQL_Connection *); + void remove(PgSQL_Connection *c) { + int i = -1; + i = find_idx(c); + assert(i>=0); + conns->remove_index_fast((unsigned int)i); + } + PgSQL_Connection *remove(int); + PgSQL_Connection * get_random_MyConn(PgSQL_Session *sess, bool ff); + void get_random_MyConn_inner_search(unsigned int start, unsigned int end, unsigned int& conn_found_idx, unsigned int& connection_quality_level, unsigned int& number_of_matching_session_variables, const PgSQL_Connection * client_conn); + unsigned int conns_length() { return conns->len; } + void drop_all_connections(); + PgSQL_Connection *index(unsigned int); +}; + +class PgSQL_SrvC { // MySQL Server Container + public: + PgSQL_HGC *myhgc; + char *address; + uint16_t port; + uint16_t flags; + int64_t weight; + enum MySerStatus status; + unsigned int compression; + int64_t max_connections; + unsigned int aws_aurora_current_lag_us; + unsigned int max_replication_lag; + unsigned int max_connections_used; // The maximum number of connections that has been opened + unsigned int connect_OK; + unsigned int connect_ERR; + unsigned int cur_replication_lag_count; + // note that these variables are in microsecond, while user defines max latency in millisecond + unsigned int current_latency_us; + unsigned int max_latency_us; + time_t time_last_detected_error; + unsigned int connect_ERR_at_time_last_detected_error; + unsigned long long queries_sent; + unsigned long long bytes_sent; + unsigned long long bytes_recv; + bool shunned_automatic; + bool shunned_and_kill_all_connections; // if a serious failure is detected, this will cause all connections to die even if the server is just shunned + int32_t use_ssl; + char *comment; + PgSQL_SrvConnList *ConnectionsUsed; + PgSQL_SrvConnList *ConnectionsFree; + /** + * @brief Constructs a new MySQL Server Container. + * @details For 'server_defaults' parameters, if '-1' is supplied, they try to be obtained from + * 'servers_defaults' entry from 'pgsql_hostgroup_attributes' when adding the server to it's target + * hostgroup(via 'PgSQL_HostGroups_Manager::add'), if not found, value is set with 'pgsql_servers' + * defaults. + * @param addr Address of the server, specified either by IP or hostname. + * @param port Server port. + * @param gitd_port If non-zero, enables GTID tracking for the server. + * @param _weight Server weight. 'server_defaults' param, check @details. + * @param _status Initial server status. + * @param _compression Enables compression for server connections. + * @param _max_connections Max server connections. 'server_defaults' param, check @details. + * @param _max_replication_lag If non-zero, enables replication lag checks. + * @param _use_ssl Enables SSL for server connections. 'servers_defaults' param, check @details. + * @param _max_latency_ms Max ping server latency. When exceeded, server gets excluded from conn-pool. + * @param _comment User defined comment. + */ + PgSQL_SrvC( + char* addr, uint16_t port, int64_t _weight, enum MySerStatus _status, unsigned int _compression, + int64_t _max_connections, unsigned int _max_replication_lag, int32_t _use_ssl, unsigned int _max_latency_ms, + char* _comment + ); + ~PgSQL_SrvC(); + void connect_error(int, bool get_mutex=true); + void shun_and_killall(); + /** + * @brief Update the maximum number of used connections + * @return The maximum number of used connections + */ + unsigned int update_max_connections_used() + { + unsigned int connections_used = ConnectionsUsed->conns_length(); + if (max_connections_used < connections_used) + max_connections_used = connections_used; + return max_connections_used; + } +}; + +class PgSQL_SrvList: public BaseSrvList { + public: + PgSQL_SrvList(PgSQL_HGC* hgc) : BaseSrvList(hgc) {} + friend class PgSQL_HGC; +}; + + +class PgSQL_HGC: public BaseHGC { + public: + PgSQL_HGC(int _hid) : BaseHGC(_hid) {} + PgSQL_SrvC *get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms, PgSQL_Session *sess); +}; + +struct PgSQL_p_hg_counter { + enum metric { + servers_table_version = 0, + server_connections_created, + server_connections_delayed, + server_connections_aborted, + client_connections_created, + client_connections_aborted, + //com_autocommit, + //com_autocommit_filtered, + com_rollback, + com_rollback_filtered, + com_backend_reset_connection, + //com_backend_init_db, + // TODO: https://github.com/sysown/proxysql/issues/2690 + com_backend_set_client_encoding, + //com_frontend_init_db, + com_frontend_set_client_encoding, + //com_frontend_use_db, + com_commit_cnt, + com_commit_cnt_filtered, + selects_for_update__autocommit0, + access_denied_wrong_password, + access_denied_max_connections, + access_denied_max_user_connections, + pghgm_pgconnpool_get, + pghgm_pgconnpool_get_ok, + pghgm_pgconnpool_get_ping, + pghgm_pgconnpool_push, + pghgm_pgconnpool_reset, + pghgm_pgconnpool_destroy, + auto_increment_delay_multiplex, + __size + }; +}; + +struct PgSQL_p_hg_gauge { + enum metric { + server_connections_connected = 0, + client_connections_connected, + __size + }; +}; + +struct PgSQL_p_hg_dyn_counter { + enum metric { + conn_pool_bytes_data_recv = 0, + conn_pool_bytes_data_sent, + connection_pool_conn_err, + connection_pool_conn_ok, + connection_pool_queries, + gtid_executed, + proxysql_pgsql_error, + pgsql_error, + __size + }; +}; + +enum class p_pgsql_error_type { + pgsql, + proxysql +}; + +struct PgSQL_p_hg_dyn_gauge { + enum metric { + connection_pool_conn_free = 0, + connection_pool_conn_used, + connection_pool_latency_us, + connection_pool_status, + __size + }; +}; + +struct PgSQL_hg_metrics_map_idx { + enum index { + counters = 0, + gauges, + dyn_counters, + dyn_gauges, + }; +}; + +/** + * @brief Required server info for the read_only Monitoring actions and replication_lag Monitoring actions. + */ +using hostgroupid_t = int; +using hostname_t = std::string; +using address_t = std::string; +using port_t = unsigned int; +using read_only_t = int; +using current_replication_lag = int; +using override_replication_lag = bool; + +using read_only_server_t = std::tuple; +using replication_lag_server_t = std::tuple; + +enum PgSQL_READ_ONLY_SERVER_T { + PG_ROS_HOSTNAME = 0, + PG_ROS_PORT, + PG_ROS_READONLY, + PG_ROS__SIZE +}; + +enum PgSQL_REPLICATION_LAG_SERVER_T { + PG_RLS_HOSTGROUP_ID = 0, + PG_RLS_ADDRESS, + PG_RLS_PORT, + PG_RLS_CURRENT_REPLICATION_LAG, + PG_RLS__SIZE +}; + +/** + * @brief Contains the minimal info for server creation. + */ +struct PgSQL_srv_info_t { + /* @brief Server address */ + string addr; + /* @brief Server port */ + uint16_t port; + /* @brief Server type identifier, used for logging, e.g: 'Aurora AWS', 'GR', etc... */ + string kind; +}; + +/** + * @brief Contains options to be specified during server creation. + */ +struct PgSQL_srv_opts_t { + int64_t weigth; + int64_t max_conns; + int32_t use_ssl; +}; + +class PgSQL_HostGroups_Manager : public Base_HostGroups_Manager { +#if 0 + SQLite3DB *admindb; + SQLite3DB *mydb; + pthread_mutex_t readonly_mutex; + std::set read_only_set1; + std::set read_only_set2; + pthread_mutex_t lock; +#endif // 0 + private: + enum HGM_TABLES { + PgSQL_SERVERS_V2 = 0, + PgSQL_REPLICATION_HOSTGROUPS, + PgSQL_GROUP_REPLICATION_HOSTGROUPS, + PgSQL_GALERA_HOSTGROUPS, + PgSQL_AWS_AURORA_HOSTGROUPS, + PgSQL_HOSTGROUP_ATTRIBUTES, + PgSQL_SERVERS, + + __HGM_TABLES_SIZE + }; + + std::array table_resultset_checksum { {0} }; + + class HostGroup_Server_Mapping { + public: + enum Type { + WRITER = 0, + READER = 1, + + __TYPE_SIZE + }; + + struct Node { + PgSQL_SrvC* srv = NULL; + unsigned int reader_hostgroup_id = -1; + unsigned int writer_hostgroup_id = -1; + //MySerStatus server_status = PgSQL_SERVER_STATUS_OFFLINE_HARD; + }; + + HostGroup_Server_Mapping(PgSQL_HostGroups_Manager* hgm) : readonly_flag(1), myHGM(hgm) { } + ~HostGroup_Server_Mapping() = default; + + /** + * @brief Copies all unique nodes from source vector to destination vector. + * @details Copies all unique nodes from source vector to destination vector. The source and destination + * vectors are identified by an input enumeration type, which can be either a reader or a writer. + * During the copying process, the function also adds servers to the HostGroup connection container. + * @param dest_type Input Can be reader or writer + * @param src_type Input Can be reader or writer + */ + void copy_if_not_exists(Type dest_type, Type src_type); + + /** + * @brief Removes node located at the specified index. + * @details Node is removed from vector located at the specified index identified by an input enumeration type. + * Node that was removed is marked as offline in the HostGroup connection container. + * @param dest_type Input Can be reader or writer + * @param index Input Index of node to be removed + */ + void remove(Type type, size_t index); + + /** + * @brief Removes all nodes. + * @details All nodes are removed from vector, identified by an input enumeration type. + * Nodes that are removed is marked as offline in the HostGroup connection container. + * @param type Input Can be reader or writer + */ + void clear(Type type); + + inline + const std::vector& get(Type type) const { + return mapping[type]; + } + + inline + void add(Type type, Node& node) { + mapping[type].push_back(node); + } + + inline + void set_readonly_flag(int val) { + readonly_flag = val; + } + + inline + int get_readonly_flag() const { + return readonly_flag; + } + + private: + unsigned int get_hostgroup_id(Type type, const Node& node) const; + PgSQL_SrvC* insert_HGM(unsigned int hostgroup_id, const PgSQL_SrvC* srv); + void remove_HGM(PgSQL_SrvC* srv); + + std::array, __TYPE_SIZE> mapping; // index 0 contains reader and 1 contains writer hostgroups + int readonly_flag; + PgSQL_HostGroups_Manager* myHGM; + }; + + /** + * @brief Used by 'MySQL_Monitor::read_only' to hold a mapping between servers and hostgroups. + * @details The hostgroup mapping holds the PgSQL_SrvC for each of the hostgroups in which the servers is + * present, distinguishing between 'READER' and 'WRITER' hostgroups. + */ + std::unordered_map> hostgroup_server_mapping; + /** + * @brief Holds the previous computed checksum for 'pgsql_servers'. + * @details Used to check if the servers checksums has changed during 'commit', if a change is detected, + * the member 'hostgroup_server_mapping' is required to be regenerated. + * + * This is only updated during 'read_only_action_v2', since the action itself modifies + * 'hostgroup_server_mapping' in case any actions needs to be performed against the servers. + */ + uint64_t hgsm_pgsql_servers_checksum = 0; + /** + * @brief Holds the previous checksum for the 'PgSQL_REPLICATION_HOSTGROUPS'. + * @details Used during 'commit' to determine if config has changed for 'PgSQL_REPLICATION_HOSTGROUPS', + * and 'hostgroup_server_mapping' should be rebuild. + */ + uint64_t hgsm_pgsql_replication_hostgroups_checksum = 0; + +#if 0 + PtrArray *MyHostGroups; + std::unordered_mapMyHostGroups_map; + + PgSQL_HGC * MyHGC_find(unsigned int); + PgSQL_HGC * MyHGC_create(unsigned int); +#endif // 0 + + void add(PgSQL_SrvC *, unsigned int); + void purge_pgsql_servers_table(); + void generate_pgsql_servers_table(int *_onlyhg=NULL); + void generate_pgsql_replication_hostgroups_table(); + + /** + * @brief This resultset holds the current values for 'runtime_pgsql_servers' computed by either latest + * 'commit' or fetched from another Cluster node. It's also used by ProxySQL_Admin to respond to the + * intercepted query 'CLUSTER_QUERY_RUNTIME_PgSQL_SERVERS'. + * @details This resultset can't right now just contain the value for 'incoming_pgsql_servers' as with the + * rest of the intercepted resultset. This is due to 'runtime_pgsql_servers' reconfigurations that can be + * triggered by monitoring actions like 'Galera' currently performs. These actions not only trigger status + * changes in the servers, but also re-generate the servers table via 'commit', thus generating a new + * checksum in the process. Because of this potential mismatch, the fetching server wouldn't be able to + * compute the proper checksum for the fetched 'runtime_pgsql_servers' config. + * + * As previously stated, these reconfigurations are monitoring actions, they can't be packed or performed + * in a single action, since monitoring data is required, which may not be already present. This makes + * this a convergent, but iterative process, that can't be compressed into a single action. Using other + * nodes 'runtime_pgsql_servers' while fetching represents a best effort for avoiding these + * reconfigurations in nodes that already holds the same monitoring conditions. If monitoring + * conditions are not the same, circular fetching is still possible due to the previously described + * scenario. + */ + SQLite3_result* runtime_pgsql_servers; + /** + * @brief These resultset holds the latest values for 'incoming_*' tables used to promoted servers to runtime. + * @details All these resultsets are used by 'Cluster' to fetch and promote the same configuration used in the + * node across the whole cluster. For these, the queries: + * - 'CLUSTER_QUERY_PgSQL_REPLICATION_HOSTGROUPS' + * - 'CLUSTER_QUERY_PgSQL_GROUP_REPLICATION_HOSTGROUPS' + * - 'CLUSTER_QUERY_PgSQL_GALERA' + * - 'CLUSTER_QUERY_PgSQL_AWS_AURORA' + * - 'CLUSTER_QUERY_PgSQL_HOSTGROUP_ATTRIBUTES' + * Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets. + */ + SQLite3_result *incoming_replication_hostgroups; + + void generate_pgsql_hostgroup_attributes_table(); + SQLite3_result *incoming_hostgroup_attributes; + + SQLite3_result* incoming_pgsql_servers_v2; + + std::thread *HGCU_thread; + + std::thread *GTID_syncer_thread; + //pthread_t GTID_syncer_thread_id; + //pthread_t HGCU_thread_id; + + char rand_del[8]; + pthread_mutex_t pgsql_errors_mutex; + umap_pgsql_errors pgsql_errors_umap; + + /** + * @brief Update the prometheus "connection_pool" counters. + */ + void p_update_connection_pool(); + + void p_update_connection_pool_update_counter( + const std::string& endpoint_id, const std::map& labels, + std::map& m_map, unsigned long long value, PgSQL_p_hg_dyn_counter::metric idx + ); + void p_update_connection_pool_update_gauge( + const std::string& endpoint_id, const std::map& labels, + std::map& m_map, unsigned long long value, PgSQL_p_hg_dyn_gauge::metric idx + ); + + public: + /** + * @brief Mutex used to guard 'pgsql_servers_to_monitor' resulset. + */ + std::mutex pgsql_servers_to_monitor_mutex; + /** + * @brief Resulset containing the latest 'pgsql_servers' present in 'mydb'. + * @details This resulset should be updated via 'update_table_pgsql_servers_for_monitor' each time actions + * that modify the 'pgsql_servers' table are performed. + */ + SQLite3_result* pgsql_servers_to_monitor; + + pthread_rwlock_t gtid_rwlock; + std::unordered_map gtid_map; + struct ev_async * gtid_ev_async; + struct ev_loop * gtid_ev_loop; + struct ev_timer * gtid_ev_timer; + bool gtid_missing_nodes; + struct { + unsigned int servers_table_version; + pthread_mutex_t servers_table_version_lock; + pthread_cond_t servers_table_version_cond; + unsigned long client_connections_aborted; + unsigned long client_connections_created; + int client_connections; + unsigned long server_connections_aborted; + unsigned long server_connections_created; + unsigned long server_connections_delayed; + unsigned long server_connections_connected; + unsigned long pgconnpoll_get; + unsigned long pgconnpoll_get_ok; + unsigned long pgconnpoll_get_ping; + unsigned long pgconnpoll_push; + unsigned long pgconnpoll_reset; + unsigned long pgconnpoll_destroy; + unsigned long long autocommit_cnt; + unsigned long long commit_cnt; + unsigned long long rollback_cnt; + unsigned long long autocommit_cnt_filtered; + unsigned long long commit_cnt_filtered; + unsigned long long rollback_cnt_filtered; + unsigned long long backend_reset_connection; + //unsigned long long backend_init_db; + unsigned long long backend_set_client_encoding; + //unsigned long long frontend_init_db; + unsigned long long frontend_set_client_encoding; + //unsigned long long frontend_use_db; + unsigned long long access_denied_wrong_password; + unsigned long long access_denied_max_connections; + unsigned long long access_denied_max_user_connections; + unsigned long long select_for_update_or_equivalent; + unsigned long long auto_increment_delay_multiplex; + + ////////////////////////////////////////////////////// + /// Prometheus Metrics /// + ////////////////////////////////////////////////////// + + /// Prometheus metrics arrays + std::array p_counter_array {}; + std::array p_gauge_array {}; + + // Prometheus dyn_metrics families arrays + std::array*, PgSQL_p_hg_dyn_counter::__size> p_dyn_counter_array {}; + std::array*, PgSQL_p_hg_dyn_gauge::__size> p_dyn_gauge_array {}; + + /// Prometheus connection_pool metrics + std::map p_conn_pool_bytes_data_recv_map {}; + std::map p_conn_pool_bytes_data_sent_map {}; + std::map p_connection_pool_conn_err_map {}; + std::map p_connection_pool_conn_free_map {}; + std::map p_connection_pool_conn_ok_map {}; + std::map p_connection_pool_conn_used_map {}; + std::map p_connection_pool_latency_us_map {}; + std::map p_connection_pool_queries_map {}; + std::map p_connection_pool_status_map {}; + + /// Prometheus gtid_executed metrics + std::map p_gtid_executed_map {}; + + /// Prometheus pgsql_error metrics + std::map p_pgsql_errors_map {}; + + ////////////////////////////////////////////////////// + } status; + /** + * @brief Update the module prometheus metrics. + */ + void p_update_metrics(); + /** + * @brief Updates the 'pgsql_error' counter identified by the 'm_id' parameter, + * or creates a new one in case of not existing. + * + * @param hid The hostgroup identifier. + * @param address The connection address that triggered the error. + * @param port The port of the connection that triggered the error. + * @param errno The error code itself. + */ + void p_update_pgsql_error_counter(p_pgsql_error_type err_type, unsigned int hid, char* address, uint16_t port, unsigned int code); + + wqueue queue; + + PgSQL_HostGroups_Manager(); + ~PgSQL_HostGroups_Manager(); + void init(); + //void wrlock(); + //void wrunlock(); + int servers_add(SQLite3_result *resultset); + /** + * @brief Generates a new global checksum for module 'pgsql_servers_v2' using the provided hash. + * @param servers_v2_hash The 'raw_checksum' from 'PGHGM_GEN_CLUSTER_ADMIN_PGSQL_SERVERS' or peer node. + * @return Checksum computed using the provided hash, and 'pgsql_servers' config tables hashes. + */ + std::string gen_global_pgsql_servers_v2_checksum(uint64_t servers_v2_hash); + bool commit( + const peer_runtime_pgsql_servers_t& peer_runtime_pgsql_servers = {}, + const peer_pgsql_servers_v2_t& peer_pgsql_servers_v2 = {}, + bool only_commit_runtime_pgsql_servers = true, + bool update_version = false + ); + /** + * @brief Extracted from 'commit'. Performs the following actions: + * 1. Re-generates the 'myhgm.pgsql_servers' table. + * 2. If supplied 'runtime_pgsql_servers' is 'nullptr': + * 1. Gets the contents of the table via 'PGHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS'. + * 2. Save the resultset into 'this->runtime_pgsql_servers'. + * 3. If supplied 'runtime_pgsql_servers' isn't 'nullptr': + * 1. Updates the 'this->runtime_pgsql_servers' with it. + * 4. Updates 'HGM_TABLES::PgSQL_SERVERS' with raw checksum from 'this->runtime_pgsql_servers'. + * @param runtime_pgsql_servers If not 'nullptr', used to update 'this->runtime_pgsql_servers'. + * @return The updated 'PgSQL_HostGroups_Manager::runtime_pgsql_servers'. + */ + uint64_t commit_update_checksum_from_pgsql_servers(SQLite3_result* runtime_pgsql_servers = nullptr); + /** + * @brief Analogous to 'commit_generate_pgsql_servers_table' but for 'incoming_pgsql_servers_v2'. + */ + uint64_t commit_update_checksum_from_pgsql_servers_v2(SQLite3_result* incoming_pgsql_servers_v2 = nullptr); + /** + * @brief Update all HGM_TABLES checksums and uses them to update the supplied SpookyHash. + * @details Checksums are the checksums for the following tables: + * - pgsql_replication_hostgroups + * - pgsql_hostgroup_attributes + * + * These checksums are used to compute the global checksum for 'pgsql_servers_v2'. + * @param myhash SpookyHash to be updated with all the computed checksums. + * @param init Indicates if the SpookyHash checksum is initialized. + */ + void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init); + /** + * @brief Performs the following actions: + * 1. Gets the current contents of table 'myhgm.TableName', using 'ColumnName' ordering. + * 2. Computes the checksum for that resultset. + * 3. Updates the supplied 'raw_checksum' and the supplied 'SpookyHash' with it. + * @details Stands for 'commit_update_checksum_from_table_1'. + * @param myhash Hash to be updated with the resultset checksum from the selected table. + * @param init If the supplied 'SpookyHash' has already being initialized. + * @param TableName The tablename from which to obtain the resultset for the 'raw_checksum' computation. + * @param ColumnName A column name to use for ordering in the supplied 'TableName'. + * @param raw_checksum A 'raw_checksum' to be updated with the obtained resultset. + */ + void CUCFT1( + SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum + ); + /** + * @brief Store the resultset for the 'runtime_pgsql_servers' table set that have been loaded to runtime. + * The store configuration is later used by Cluster to propagate current config. + * @param The resulset to be stored replacing the current one. + */ + void save_runtime_pgsql_servers(SQLite3_result *); + + /** + * @brief Store the resultset for the 'pgsql_servers_v2' table. + * The store configuration is later used by Cluster to propagate current config. + * @param The resulset to be stored replacing the current one. + */ + void save_pgsql_servers_v2(SQLite3_result* s); + + /** + * @brief These setters/getter functions store and retrieve the currently hold resultset for the + * 'incoming_*' table set that have been loaded to runtime. The store configuration is later used by + * Cluster to propagate current config. + * @param The resulset to be stored replacing the current one. + */ + + void save_incoming_pgsql_table(SQLite3_result *, const string&); + SQLite3_result* get_current_pgsql_table(const string& name); + + //SQLite3_result * execute_query(char *query, char **error); + /** + * @brief Creates a resultset with the current full content of the target table. + * @param string The target table. Valid values are: + * - "pgsql_replication_hostgroups" + * - "pgsql_hostgroup_attributes" + * - "pgsql_servers" + * - "cluster_pgsql_servers" + * When targeting 'pgsql_servers' table is purged and regenerated. + * @return The generated resultset. + */ + SQLite3_result* dump_table_pgsql(const string&); + + /** + * @brief Update the public member resulset 'pgsql_servers_to_monitor'. This resulset should contain the latest + * 'pgsql_servers' present in 'PgSQL_HostGroups_Manager' db, which are not 'OFFLINE_HARD'. The resulset + * fields match the definition of 'monitor_internal.pgsql_servers' table. + * @details Several details: + * - Function assumes that 'pgsql_servers' table from 'PgSQL_HostGroups_Manager' db is ready + * to be consumed, because of this it doesn't perform any of the following operations: + * - Purging 'pgsql_servers' table. + * - Regenerating 'pgsql_servers' table. + * - Function locks on 'pgsql_servers_to_monitor_mutex'. + * @param lock When supplied the function calls 'wrlock()' and 'wrunlock()' functions for accessing the db. + */ + void update_table_pgsql_servers_for_monitor(bool lock=false); + + void MyConn_add_to_pool(PgSQL_Connection *); + /** + * @brief Creates a new server in the target hostgroup if isn't already present. + * @details If the server is found already in the target hostgroup, no action is taken, unless its status + * is 'OFFLINE_HARD'. In case of finding it as 'OFFLINE_HARD': + * 1. Server hostgroup attributes are reset to known values, so they can be updated. + * 2. Server attributes are updated to either table definition values, or hostgroup 'servers_defaults'. + * 3. Server is bring back as 'ONLINE'. + * @param hid The hostgroup in which the server is to be created (or to bring it back as 'ONLINE'). + * @param srv_info Basic server info to be used during creation. + * @param srv_opts Server creation options. + * @return 0 in case of success, -1 in case of failure. + */ + int create_new_server_in_hg(uint32_t hid, const PgSQL_srv_info_t& srv_info, const PgSQL_srv_opts_t& srv_opts); + /** + * @brief Completely removes server from the target hostgroup if found. + * @details Several actions are taken if server is found: + * - Set the server as 'OFFLINE_HARD'. + * - Drop all current FREE connections to the server. + * - Delete the server from the 'myhgm.pgsql_servers' table. + * + * This later step is not required if the caller is already going to perform a full deletion of the + * servers in the target hostgroup. Which is a common operation during table regeneration. + * @param hid Target hostgroup id. + * @param addr Target server address. + * @param port Target server port. + * @return 0 in case of success, -1 in case of failure. + */ + int remove_server_in_hg(uint32_t hid, const string& addr, uint16_t port); + + PgSQL_Connection * get_MyConn_from_pool(unsigned int hid, PgSQL_Session *sess, bool ff, char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms); + + void drop_all_idle_connections(); + int get_multiple_idle_connections(int, unsigned long long, PgSQL_Connection **, int); + SQLite3_result * SQL3_Connection_Pool(bool _reset, int *hid = NULL); + SQLite3_result * SQL3_Free_Connections(); + + void push_MyConn_to_pool(PgSQL_Connection *, bool _lock=true); + void push_MyConn_to_pool_array(PgSQL_Connection **, unsigned int); + void destroy_MyConn_from_pool(PgSQL_Connection *, bool _lock=true); + + void replication_lag_action_inner(PgSQL_HGC *, const char*, unsigned int, int); + void replication_lag_action(const std::list& pgsql_servers); + void read_only_action(char *hostname, int port, int read_only); + void read_only_action_v2(const std::list& pgsql_servers); + unsigned int get_servers_table_version(); + void wait_servers_table_version(unsigned, unsigned); + bool shun_and_killall(char *hostname, int port); + void set_server_current_latency_us(char *hostname, int port, unsigned int _current_latency_us); + unsigned long long Get_Memory_Stats(); + + SQLite3_result *SQL3_Get_ConnPool_Stats(); + void increase_reset_counter(); + + void add_pgsql_errors(int hostgroup, const char* hostname, int port, const char* username, const char* address, + const char* dbname, const char* sqlstate, const char* errmsg); + SQLite3_result *get_pgsql_errors(bool); + + void shutdown(); + void unshun_server_all_hostgroups(const char * address, uint16_t port, time_t t, int max_wait_sec, unsigned int *skip_hid); + PgSQL_SrvC* find_server_in_hg(unsigned int _hid, const std::string& addr, int port); + +private: + void update_hostgroup_manager_mappings(); + uint64_t get_pgsql_servers_checksum(SQLite3_result* runtime_pgsql_servers = nullptr); + uint64_t get_pgsql_servers_v2_checksum(SQLite3_result* incoming_pgsql_servers_v2 = nullptr); +}; + + +#endif /* __CLASS_PGSQL_HOSTGROUPS_MANAGER_H */ diff --git a/include/PgSQL_Logger.hpp b/include/PgSQL_Logger.hpp new file mode 100644 index 0000000000..f24416e862 --- /dev/null +++ b/include/PgSQL_Logger.hpp @@ -0,0 +1,98 @@ +#ifndef __CLASS_PGSQL_LOGGER_H +#define __CLASS_PGSQL_LOGGER_H +#include "proxysql.h" +#include "cpp.h" + +#define PROXYSQL_LOGGER_PTHREAD_MUTEX + +class PgSQL_Event { + private: + uint32_t thread_id; + char *username; + char *schemaname; + size_t username_len; + size_t schemaname_len; + uint64_t start_time; + uint64_t end_time; + uint64_t query_digest; + char *query_ptr; + size_t query_len; + char *server; + char *client; + size_t server_len; + size_t client_len; + //uint64_t total_length; + unsigned char buf[10]; + enum log_event_type et; + uint64_t hid; + char *extra_info; + bool have_affected_rows; + bool have_rows_sent; + + uint64_t affected_rows; + uint64_t last_insert_id; + uint64_t rows_sent; + uint32_t client_stmt_id; + + public: + PgSQL_Event(log_event_type _et, uint32_t _thread_id, char * _username, char * _schemaname , uint64_t _start_time , uint64_t _end_time , uint64_t _query_digest, char *_client, size_t _client_len); + uint64_t write(std::fstream *f, PgSQL_Session *sess); + uint64_t write_query_format_1(std::fstream *f); + uint64_t write_query_format_2_json(std::fstream *f); + void write_auth(std::fstream *f, PgSQL_Session *sess); + void set_client_stmt_id(uint32_t client_stmt_id); + void set_query(const char *ptr, int len); + void set_server(int _hid, const char *ptr, int len); + void set_extra_info(char *); + void set_affected_rows(uint64_t ar, uint64_t lid); + void set_rows_sent(uint64_t rs); +}; + +class PgSQL_Logger { + private: + struct { + bool enabled; + char *base_filename; + char *datadir; + unsigned int log_file_id; + unsigned int max_log_file_size; + std::fstream *logfile; + } events; + struct { + bool enabled; + char *base_filename; + char *datadir; + unsigned int log_file_id; + unsigned int max_log_file_size; + std::fstream *logfile; + } audit; +#ifdef PROXYSQL_LOGGER_PTHREAD_MUTEX + pthread_mutex_t wmutex; +#else + rwlock_t rwlock; +#endif + void events_close_log_unlocked(); + void events_open_log_unlocked(); + void audit_close_log_unlocked(); + void audit_open_log_unlocked(); + unsigned int events_find_next_id(); + unsigned int audit_find_next_id(); + public: + PgSQL_Logger(); + ~PgSQL_Logger(); + void print_version(); + void flush_log(); + void events_flush_log_unlocked(); + void audit_flush_log_unlocked(); + void events_set_datadir(char *); + void events_set_base_filename(); + void audit_set_datadir(char *); + void audit_set_base_filename(); + void log_request(PgSQL_Session *, PgSQL_Data_Stream *); + void log_audit_entry(log_event_type, PgSQL_Session *, PgSQL_Data_Stream *, char *e = NULL); + void flush(); + void wrlock(); + void wrunlock(); +}; + +#endif /* __CLASS_PGSQL_LOGGER_H */ diff --git a/include/PgSQL_Protocol.h b/include/PgSQL_Protocol.h new file mode 100644 index 0000000000..fdc095f52d --- /dev/null +++ b/include/PgSQL_Protocol.h @@ -0,0 +1,287 @@ +#ifndef __POSTGRES_PROTOCOL_H +#define __POSTGRES_PROTOCOL_H + +#include "proxysql.h" +#include "gen_utils.h" +#include "MySQL_Protocol.h" + + +/* no-auth modes */ +#define PG_PKT_AUTH_ANY -1 /* same as trust but without username check */ +#define PG_PKT_AUTH_TRUST AUTH_OK + +/* protocol codes in Authentication* 'R' messages from server */ +#define PG_PKT_AUTH_OK 0 +#define PG_PKT_AUTH_KRB4 1 /* not supported */ +#define PG_PKT_AUTH_KRB5 2 /* not supported */ +#define PG_PKT_AUTH_PLAIN 3 +#define PG_PKT_AUTH_CRYPT 4 /* not supported */ +#define PG_PKT_AUTH_MD5 5 +#define PG_PKT_AUTH_SCM_CREDS 6 /* not supported */ +#define PG_PKT_AUTH_GSS 7 /* not supported */ +#define PG_PKT_AUTH_GSS_CONT 8 /* not supported */ +#define PG_PKT_AUTH_SSPI 9 /* not supported */ +#define PG_PKT_AUTH_SASL 10 +#define PG_PKT_AUTH_SASL_CONT 11 +#define PG_PKT_AUTH_SASL_FIN 12 + +/* internal codes */ +#define AUTH_CERT 107 +#define AUTH_PEER 108 +#define AUTH_HBA 109 +#define AUTH_REJECT 110 +#define AUTH_PAM 111 +#define AUTH_SCRAM_SHA_256 112 + +#define PG_PKT_STARTUP_V2 0x20000 +#define PG_PKT_STARTUP 0x30000 +#define PG_PKT_CANCEL 80877102 +#define PG_PKT_SSLREQ 80877103 +#define PG_PKT_GSSENCREQ 80877104 + +#define PG_PKT_DEFAULT_SIZE 64 + + +/* old style V2 header: len:4b code:4b */ +#define OLD_HEADER_LEN 8 +/* new style V3 packet header len - type:1b, len:4b */ +#define NEW_HEADER_LEN 5 + +#define PGSQL_RESULTSET_BUFLEN (16 * 1024) + +class ProxySQL_Admin; +struct PgCredentials; +struct ScramState; + +enum class EXECUTION_STATE { + FAILED = 0, + SUCCESSFUL, + PENDING +}; + +struct pgsql_hdr { + uint32_t type; + uint32_t len; + PtrSize_t data; +}; + +struct PG_Field { + char* name; + uint32_t tbl_oid; + uint16_t col_idx; + uint32_t type_oid; + uint16_t col_len; + uint32_t type_mod; + uint16_t fmt; +}; + +using PG_Fields = std::vector; + +class PG_pkt +{ +public: + PG_pkt(unsigned c = PG_PKT_DEFAULT_SIZE) { + ownership = true; + capacity = l_near_pow_2(c); + size = 0; + ptr = (char*)malloc(capacity); + multiple_pkt_mode = false; + } + PG_pkt(void* _ptr, unsigned int _capacity) { + ownership = false; + ptr = (char*)_ptr; + capacity = _capacity; + size = 0; + } + ~PG_pkt() { + reset(); + } + + void reset() { + if (ptr) { + if (ownership == true) + free(ptr); + else + assert(size == capacity); // just to check if we are not passing buffer boundaries + } + ptr = nullptr; + size = 0; + capacity = 0; + multiple_pkt_mode = false; + pkt_offset.clear(); + } + + std::pair detach() { + std::pair result(ptr, size); + ptr = nullptr; + size = 0; + capacity = 0; + multiple_pkt_mode = false; + pkt_offset.clear(); + return result; + } + + PtrSize_t* get_PtrSize(unsigned c = PG_PKT_DEFAULT_SIZE); + void to_PtrSizeArray(PtrSizeArray* psa, unsigned c = PG_PKT_DEFAULT_SIZE); + + void set_multi_pkt_mode(bool mode) { + multiple_pkt_mode = mode; + + if (mode == false) + pkt_offset.clear(); + } + void make_space(unsigned int len); + void put_char(char val); + void put_uint16(uint16_t val); + void put_uint32(uint32_t val); + void put_uint64(uint64_t val); + void put_bytes(const void* data, int len); + void put_string(const char* str); + + void write_generic(int type, const char* pktdesc, ...); + + void write_ParameterStatus(const char* key, const char* val) { + write_generic('S', "ss", key, val); + } + void write_AuthenticationOk() { + write_generic('R', "i", 0); + } + void write_AuthenticationRequest(uint32_t auth_type, const uint8_t* data, int len) { + write_generic('R', "ib", auth_type, data, len); + } + void write_ReadyForQuery(char txn_state = 'I') { + write_generic('Z', "c", txn_state); + } + void write_CommandComplete(const char* desc) { + write_generic('C', "s", desc); + } + void write_BackendKeyData(const uint8_t* key) { + write_generic('K', "b", key, 8); + } + void write_StartupMessage(const char* user, const char* parms, int parms_len) { + write_generic(PG_PKT_STARTUP, "bsss", parms, parms_len, "user", user, ""); + } + void write_PasswordMessage(const char* psw) { + write_generic('p', "s", psw); + } + + void write_RowDescription(const char* tupdesc, ...); + void write_DataRow(const char* tupdesc, ...); + +private: + void start_packet(int type); + void finish_packet(); + + char* ptr; + unsigned int size; + unsigned int capacity; + + // currently for debug only. will replace this with a single variable that will contain last pkt offset + std::vector pkt_offset; + bool multiple_pkt_mode = false; + bool ownership = true; + friend void SQLite3_to_Postgres(PtrSizeArray* psa, SQLite3_result* result, char* error, int affected_rows, const char* query_type); +}; + +class PgSQL_Protocol; + +#define PGSQL_QUERY_RESULT_NO_DATA 0x00 +#define PGSQL_QUERY_RESULT_TUPLE 0x01 +#define PGSQL_QUERY_RESULT_COMMAND 0x02 +#define PGSQL_QUERY_RESULT_READY 0x04 +#define PGSQL_QUERY_RESULT_ERROR 0x08 +#define PGSQL_QUERY_RESULT_EMPTY 0x10 + +class PgSQL_Query_Result { +public: + PgSQL_Query_Result(); + ~PgSQL_Query_Result(); + + void init(PgSQL_Protocol* _proto, PgSQL_Data_Stream* _myds, PgSQL_Connection* _conn); + unsigned int add_row_description(const PGresult* result); + unsigned int add_row(const PGresult* result); + unsigned int add_row(const PSresult* result); + unsigned int add_command_completion(const PGresult* result, bool extract_affected_rows = true); + unsigned int add_error(const PGresult* result); + unsigned int add_empty_query_response(const PGresult* result); + unsigned int add_ready_status(PGTransactionStatusType txn_status); + bool get_resultset(PtrSizeArray* PSarrayFinal); // this also calls reset + + unsigned long long current_size(); + inline bool is_transfer_started() const { return transfer_started; } + inline unsigned long long get_num_rows() const { return num_rows; } + inline unsigned long long get_affected_rows() const { return affected_rows; } + inline unsigned int get_num_fields() const { return num_fields; } + inline unsigned long long get_resultset_size() const { return resultset_size; } + inline uint8_t get_result_packet_type() const { return result_packet_type; } + +private: + void buffer_init(); + inline unsigned int buffer_available_capacity() const { return (PGSQL_RESULTSET_BUFLEN - buffer_used); } + unsigned char* buffer_reserve_space(unsigned int size); + void buffer_to_PSarrayOut(); + void reset(); + + PtrSizeArray PSarrayOUT; + unsigned long long resultset_size; + unsigned long long num_rows; + unsigned long long pkt_count; + unsigned long long affected_rows; + unsigned int num_fields; + unsigned int buffer_used; + unsigned char* buffer; + PgSQL_Protocol* proto; + PgSQL_Data_Stream* myds; + PgSQL_Connection* conn; + bool transfer_started; + uint8_t result_packet_type; + + friend class PgSQL_Protocol; + friend class PgSQL_Connection; +}; + +class PgSQL_Protocol : public MySQL_Protocol { +public: + void init(PgSQL_Data_Stream** __myds, PgSQL_Connection_userinfo* __userinfo, PgSQL_Session* __sess) { + myds = __myds; + userinfo = __userinfo; + sess = __sess; + current_PreStmt = NULL; + } + PgSQL_Data_Stream* get_myds() { return *myds; } + + bool generate_pkt_initial_handshake(bool send, void** ptr, unsigned int* len, uint32_t* thread_id, bool deprecate_eof_active) override; + bool process_startup_packet(unsigned char* pkt, unsigned int len, bool& ssl_request); + EXECUTION_STATE process_handshake_response_packet(unsigned char* pkt, unsigned int len); + void welcome_client(); + + void generate_error_packet(bool send, bool ready, const char* msg, PGSQL_ERROR_CODES code, bool fatal, bool track = false, PtrSize_t* _ptr = NULL); + bool generate_ok_packet(bool send, bool ready, const char* msg, int rows, const char* query, PtrSize_t* _ptr = NULL); + + //bool generate_row_description(bool send, PgSQL_Query_Result* rs, const PG_Fields& fields, unsigned int size); + + unsigned int copy_row_description_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result); + unsigned int copy_row_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result); + unsigned int copy_command_completion_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result, bool extract_affected_rows); + unsigned int copy_error_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result); + unsigned int copy_empty_query_response_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result); + unsigned int copy_ready_status_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, PGTransactionStatusType txn_status); + unsigned int copy_buffer_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PSresult* result); + +private: + bool get_header(unsigned char* pkt, unsigned int len, pgsql_hdr* hdr); + void load_conn_parameters(pgsql_hdr* pkt, bool startup); + bool scram_handle_client_first(ScramState* scram_state, PgCredentials* user, const unsigned char* data, uint32_t datalen); + bool scram_handle_client_final(ScramState* scram_state, PgCredentials* user, const unsigned char* data, uint32_t datalen); + + PgSQL_Data_Stream** myds; + PgSQL_Connection_userinfo* userinfo; + PgSQL_Session* sess; + + template + friend void admin_session_handler(S* sess, void* _pa, PtrSize_t* pkt); +}; + +void SQLite3_to_Postgres(PtrSizeArray* psa, SQLite3_result* result, char* error, int affected_rows, const char* query_type); + +#endif // __POSTGRES_PROTOCOL_H diff --git a/include/PgSQL_Query_Processor.h b/include/PgSQL_Query_Processor.h new file mode 100644 index 0000000000..1387abf4b7 --- /dev/null +++ b/include/PgSQL_Query_Processor.h @@ -0,0 +1,45 @@ +#ifndef __CLASS_PGSQL_QUERY_PROCESSOR_H +#define __CLASS_PGSQL_QUERY_PROCESSOR_H +#include "proxysql.h" +#include "cpp.h" +#include "QP_rule_text.h" +#include "query_processor.h" + +class Command_Counter; +struct PgSQL_Query_Processor_Rule_t : public QP_rule_t {}; +class PgSQL_Query_Processor_Output : public Query_Processor_Output {}; + +class PgSQL_Rule_Text : public QP_rule_text { +public: + PgSQL_Rule_Text(const PgSQL_Query_Processor_Rule_t* pqr); + ~PgSQL_Rule_Text() = default; +}; + +class PgSQL_Query_Processor : public Query_Processor { +public: + PgSQL_Query_Processor(); + ~PgSQL_Query_Processor(); + + void init_thread(); + void end_thread(); + void update_query_processor_stats(); + SQLite3_result* get_stats_commands_counters(); + SQLite3_result* get_current_query_rules(); + PgSQL_Query_Processor_Output* process_query(PgSQL_Session* sess, void* ptr, unsigned int size, PgSQL_Query_Info* qi); + unsigned long long query_parser_update_counters(PgSQL_Session* sess, enum PGSQL_QUERY_command c, SQP_par_t* qp, unsigned long long t); + static enum PGSQL_QUERY_command query_parser_command_type(SQP_par_t* qp); + static PgSQL_Query_Processor_Rule_t* new_query_rule(int rule_id, bool active, const char* username, const char* schemaname, int flagIN, const char* client_addr, + const char* proxy_addr, int proxy_port, const char* digest, const char* match_digest, const char* match_pattern, bool negate_match_pattern, + const char* re_modifiers, int flagOUT, const char* replace_pattern, int destination_hostgroup, int cache_ttl, int cache_empty_result, + int cache_timeout, int reconnect, int timeout, int retries, int delay, int next_query_flagIN, int mirror_hostgroup, + int mirror_flagOUT, const char* error_msg, const char* OK_msg, int sticky_conn, int multiplex, int log, + bool apply, const char* attributes, const char* comment); + +private: + Command_Counter* commands_counters[PGSQL_QUERY___NONE]; + static PgSQL_Query_Processor_Rule_t* new_query_rule(const PgSQL_Query_Processor_Rule_t* mqr); + + friend class Query_Processor; +}; + +#endif /* __CLASS_PGSQL_QUERY_PROCESSOR_H */ diff --git a/include/PgSQL_Session.h b/include/PgSQL_Session.h new file mode 100644 index 0000000000..6ffe982f66 --- /dev/null +++ b/include/PgSQL_Session.h @@ -0,0 +1,459 @@ +#ifdef CLASS_BASE_SESSION_H + +#ifndef __CLASS_PGSQL_SESSION_H +#define __CLASS_PGSQL_SESSION_H + +#include +#include + +#include "proxysql.h" +#include "Base_Session.h" +#include "cpp.h" +#include "PgSQL_Variables.h" +#include "Base_Session.h" + + +class PgSQL_Query_Result; +//#include "../deps/json/json.hpp" +//using json = nlohmann::json; + +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON + +extern class PgSQL_Variables pgsql_variables; + +/* +enum proxysql_session_type { + PROXYSQL_SESSION_MYSQL, + PROXYSQL_SESSION_ADMIN, + PROXYSQL_SESSION_STATS, + PROXYSQL_SESSION_SQLITE, + PROXYSQL_SESSION_CLICKHOUSE, + PROXYSQL_SESSION_MYSQL_EMU, + + PROXYSQL_SESSION_NONE +}; +*/ + +enum PgSQL_ps_type : uint8_t { + PgSQL_ps_type_not_set = 0x0, + PgSQL_ps_type_prepare_stmt = 0x1, + PgSQL_ps_type_execute_stmt = 0x2 +}; + + + +//std::string proxysql_session_type_str(enum proxysql_session_type session_type); + +// these structs will be used for various regex hardcoded +// their initial use will be for sql_log_bin , sql_mode and time_zone +// issues #509 , #815 and #816 +class PgSQL_Session_Regex { +private: + void* opt; + void* re; + char* s; +public: + PgSQL_Session_Regex(char* p); + ~PgSQL_Session_Regex(); + bool match(char* m); +}; + + +class PgSQL_Query_Info { +public: + SQP_par_t QueryParserArgs; + PgSQL_Session* sess; + unsigned char* QueryPointer; + unsigned long long start_time; + unsigned long long end_time; + + MYSQL_STMT* mysql_stmt; + stmt_execute_metadata_t* stmt_meta; + uint64_t stmt_global_id; + uint64_t stmt_client_id; + MySQL_STMT_Global_info* stmt_info; + + int QueryLength; + enum PGSQL_QUERY_command PgQueryCmd; + bool bool_is_select_NOT_for_update; + bool bool_is_select_NOT_for_update_computed; + bool have_affected_rows; // if affected rows is set, last_insert_id is set too + uint64_t affected_rows; + uint64_t last_insert_id; + uint64_t rows_sent; + uint64_t waiting_since; + std::string show_warnings_prev_query_digest; + + PgSQL_Query_Info(); + ~PgSQL_Query_Info(); + void init(unsigned char* _p, int len, bool mysql_header = false); + void query_parser_init(); + enum PGSQL_QUERY_command query_parser_command_type(); + void query_parser_free(); + unsigned long long query_parser_update_counters(); + void begin(unsigned char* _p, int len, bool mysql_header = false); + void end(); + char* get_digest_text(); + bool is_select_NOT_for_update(); +}; + +class PgSQL_Session : public Base_Session { +private: + //int handler_ret; + void handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(PtrSize_t*, bool*); + + // void handler___status_CHANGING_USER_CLIENT___STATE_CLIENT_HANDSHAKE(PtrSize_t *, bool *); + + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_FIELD_LIST(PtrSize_t*); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_INIT_DB(PtrSize_t*); + /** + * @brief Handles 'COM_QUERIES' holding 'USE DB' statements. + * + * @param pkt The packet being processed. + * @param query_digest The query digest returned by the 'QueryProcessor' + * holding the 'USE' statement without the initial comment. + * + * @details NOTE: This function used to be called from 'handler_special_queries'. + * But since it was change for handling 'USE' statements which are preceded by + * comments, it's called after 'QueryProcessor' has processed the query. + */ + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_USE_DB(PtrSize_t* pkt); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_PING(PtrSize_t*); + + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_CHANGE_USER(PtrSize_t*, bool*); + /** + * @brief Handles the command 'COM_RESET_CONNECTION'. + * @param pkt Pointer to packet received holding the 'COM_RESET_CONNECTION'. + * @details 'COM_RESET_CONNECTION' command is currently supported only for 'sesssion_types': + * - 'PROXYSQL_SESSION_MYSQL'. + * - 'PROXYSQL_SESSION_SQLITE'. + * If the command is received for other sessions, the an error packet with error '1047' is sent to the + * client. If the session is supported, it performs the following operations over the current session: + * 1. Store the current relevent session variables to be recovered after the 'RESET'. + * 2. Perform a reset and initialization of current session. + * 3. Recover the relevant session variables and other initial state associated with the current session + * user. + * 4. Respond to client with 'OK' packet. + */ + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_RESET_CONNECTION(PtrSize_t* pkt); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_SET_OPTION(PtrSize_t*); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STATISTICS(PtrSize_t*); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_PROCESS_KILL(PtrSize_t*); + bool handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(PtrSize_t*, bool* lock_hostgroup, PgSQL_ps_type prepare_stmt_type = PgSQL_ps_type_not_set); + + void handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED__get_connection(); + + //void return_proxysql_internal(PtrSize_t*); + bool handler_special_queries(PtrSize_t*); + //bool handler_special_queries_STATUS(PtrSize_t*); + /** + * @brief Handles 'COMMIT|ROLLBACK' commands. + * @details Forwarding the packet is required when there are active transactions. Since we are limited to + * forwarding just one 'COMMIT|ROLLBACK', we work under the assumption that we only have one active + * transaction. If more transactions are simultaneously open for the session, more 'COMMIT|ROLLBACK'. + * commands are required to be issued by the client, so they could be forwarded to the corresponding + * backend connections. + * @param The received packet to be handled. + * @return 'true' if the packet is intercepted and never forwarded to the client, 'false' otherwise. + */ + bool handler_CommitRollback(PtrSize_t*); + //bool handler_SetAutocommit(PtrSize_t*); + /** + * @brief Should execute most of the commands executed when a request is finalized. + * @details Cleanup of current session state, and required operations to the supplied 'PgSQL_Data_Stream' + * for further queries processing. Takes care of the following actions: + * - Update the status of the backend connection (if supplied), with previous query actions. + * - Log the query for the required statuses. + * - Cleanup the previous Query_Processor output. + * - Free the resources of the backend connection (if supplied). + * - Reset all the required session status flags. E.g: + * + status + * + client_myds::DSS + * + started_sending_data_to_client + * + previous_hostgroup + * NOTE: Should become the place to hook other functions. + * @param myds If not null, should point to a PgSQL_Data_Stream (backend connection) which connection status + * should be updated, and previous query resources cleanup. + */ + void RequestEnd(PgSQL_Data_Stream*) override; + void LogQuery(PgSQL_Data_Stream*); + + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___create_mirror_session(); + int handler_again___status_PINGING_SERVER(); + int handler_again___status_RESETTING_CONNECTION(); + void handler_again___new_thread_to_kill_connection(); + + bool handler_again___verify_init_connect(); +#if 0 + bool handler_again___verify_ldap_user_variable(); + bool handler_again___verify_backend_autocommit(); + bool handler_again___verify_backend_session_track_gtids(); + bool handler_again___verify_backend_multi_statement(); +#endif // 0 + bool handler_again___verify_backend_user_db(); + bool handler_again___status_SETTING_INIT_CONNECT(int*); +#if 0 + bool handler_again___status_SETTING_LDAP_USER_VARIABLE(int*); + bool handler_again___status_SETTING_SQL_MODE(int*); + bool handler_again___status_SETTING_SESSION_TRACK_GTIDS(int*); +#endif // 0 + bool handler_again___status_CHANGING_CHARSET(int* _rc); +#if 0 + bool handler_again___status_CHANGING_SCHEMA(int*); +#endif // 0 + bool handler_again___status_CONNECTING_SERVER(int*); + bool handler_again___status_RESETTING_CONNECTION(int*); + //bool handler_again___status_CHANGING_AUTOCOMMIT(int*); +#if 0 + bool handler_again___status_SETTING_MULTI_STMT(int* _rc); +#endif // 0 + bool handler_again___multiple_statuses(int* rc); + //void init(); + void reset(); +#if 0 + void add_ldap_comment_to_pkt(PtrSize_t*); + /** + * @brief Performs the required housekeeping operations over the session and its connections before + * performing any processing on received client packets. + */ + void housekeeping_before_pkts(); +#endif // 0 + int get_pkts_from_client(bool&, PtrSize_t&); +#if 0 + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_RESET(PtrSize_t&); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_CLOSE(PtrSize_t&); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_SEND_LONG_DATA(PtrSize_t&); +#endif // 0 + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_PREPARE(PtrSize_t& pkt); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_EXECUTE(PtrSize_t& pkt); + + // these functions have code that used to be inline, and split into functions for readibility + int handler_ProcessingQueryError_CheckBackendConnectionStatus(PgSQL_Data_Stream* myds); + void SetQueryTimeout(); + bool handler_rc0_PROCESSING_STMT_PREPARE(enum session_status& st, PgSQL_Data_Stream* myds, bool& prepared_stmt_with_no_params); + void handler_rc0_PROCESSING_STMT_EXECUTE(PgSQL_Data_Stream* myds); + bool handler_minus1_ClientLibraryError(PgSQL_Data_Stream* myds); + void handler_minus1_LogErrorDuringQuery(PgSQL_Connection* myconn); + bool handler_minus1_HandleErrorCodes(PgSQL_Data_Stream* myds, int& handler_ret); + void handler_minus1_GenerateErrorMessage(PgSQL_Data_Stream* myds, bool& wrong_pass); + void handler_minus1_HandleBackendConnection(PgSQL_Data_Stream* myds); + int RunQuery(PgSQL_Data_Stream* myds, PgSQL_Connection* myconn); + void handler___status_WAITING_CLIENT_DATA(); + void handler_rc0_Process_GTID(PgSQL_Connection* myconn); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_INIT_DB_replace_CLICKHOUSE(PtrSize_t& pkt); + void handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___not_mysql(PtrSize_t& pkt); + bool handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_detect_SQLi(); + bool handler___status_WAITING_CLIENT_DATA___STATE_SLEEP_MULTI_PACKET(PtrSize_t& pkt); + bool handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM__various(PtrSize_t* pkt, bool* wrong_pass); + void handler___status_WAITING_CLIENT_DATA___default(); + void handler___status_NONE_or_default(PtrSize_t& pkt); + + void handler_WCD_SS_MCQ_qpo_QueryRewrite(PtrSize_t* pkt); + void handler_WCD_SS_MCQ_qpo_OK_msg(PtrSize_t* pkt); + void handler_WCD_SS_MCQ_qpo_error_msg(PtrSize_t* pkt); + void handler_WCD_SS_MCQ_qpo_LargePacket(PtrSize_t* pkt); + +public: + bool handler_again___status_SETTING_GENERIC_VARIABLE(int* _rc, const char* var_name, const char* var_value, bool no_quote = false, bool set_transaction = false); +#if 0 + bool handler_again___status_SETTING_SQL_LOG_BIN(int*); +#endif // 0 + std::stack previous_status; + + PgSQL_Query_Info CurrentQuery; + PtrSize_t mirrorPkt; + PtrSize_t pkt; + +#if 0 + // uint64_t + unsigned long long start_time; + unsigned long long pause_until; + + unsigned long long idle_since; + unsigned long long transaction_started_at; + + // pointers + PgSQL_Thread* thread; +#endif // 0 + PgSQL_Query_Processor_Output* qpo; + StatCounters* command_counters; +#if 0 + PgSQL_Backend* mybe; + PtrArray* mybes; + PgSQL_Data_Stream* client_myds; +#endif // 0 + PgSQL_Data_Stream* server_myds; +#if 0 + /* + * @brief Store the hostgroups that hold connections that have been flagged as 'expired' by the + * maintenance thread. These values will be used to release the retained connections in the specific + * hostgroups in housekeeping operations, before client packet processing. Currently 'housekeeping_before_pkts'. + */ + std::vector hgs_expired_conns{}; + char* default_schema; + char* user_attributes; + + //this pointer is always initialized inside handler(). + // it is an attempt to start simplifying the complexing of handler() + + uint32_t thread_session_id; + unsigned long long last_insert_id; + int last_HG_affected_rows; + enum session_status status; + int healthy; + int user_max_connections; + int current_hostgroup; + int default_hostgroup; + int previous_hostgroup; + /** + * @brief Charset directly specified by the client. Supplied and updated via 'HandshakeResponse' + * and 'COM_CHANGE_USER' packets. + * @details Used when session needs to be restored via 'COM_RESET_CONNECTION'. + */ + int default_charset; + int locked_on_hostgroup; + int next_query_flagIN; + int mirror_hostgroup; + int mirror_flagOUT; + unsigned int active_transactions; + int autocommit_on_hostgroup; + int transaction_persistent_hostgroup; + int to_process; + int pending_connect; + enum proxysql_session_type session_type; + int warning_in_hg; + + // bool + bool autocommit; + bool autocommit_handled; + bool sending_set_autocommit; + bool killed; + bool locked_on_hostgroup_and_all_variables_set; + //bool admin; + bool max_connections_reached; + bool client_authenticated; + bool connections_handler; + bool mirror; + //bool stats; + bool schema_locked; + bool transaction_persistent; + bool session_fast_forward; + bool started_sending_data_to_client; // this status variable tracks if some result set was sent to the client, or if proxysql is still buffering everything + bool use_ssl; +#endif // 0 + /** + * @brief This status variable tracks whether the session is performing an + * 'Auth Switch' due to a 'COM_CHANGE_USER' packet. + * @details It becomes 'true' when the packet is detected and processed by: + * - 'MySQL_Protocol::process_pkt_COM_CHANGE_USER' + * It's reset before sending the final response for 'Auth Switch' to the client by: + * - 'PgSQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE' + * This flag was introduced for issue #3504. + */ + bool change_user_auth_switch; + +// MySQL_STMTs_meta* sess_STMTs_meta; +// StmtLongDataHandler* SLDH; + + Session_Regex** match_regexes; + + ProxySQL_Node_Address* proxysql_node_address; // this is used ONLY for Admin, and only if the other party is another proxysql instance part of a cluster + bool use_ldap_auth; + + // this variable is relevant only if status == SETTING_VARIABLE + enum mysql_variable_name changing_variable_idx; + + PgSQL_Session(); + ~PgSQL_Session(); + + //void set_unhealthy(); + + void set_status(enum session_status e); + int handler(); + + void (*handler_function) (PgSQL_Session* sess, void*, PtrSize_t* pkt); + //PgSQL_Backend* find_backend(int); + //PgSQL_Backend* create_backend(int, PgSQL_Data_Stream* _myds = NULL); + //PgSQL_Backend* find_or_create_backend(int, PgSQL_Data_Stream* _myds = NULL); + + void SQLite3_to_MySQL(SQLite3_result*, char*, int, MySQL_Protocol*, bool in_transaction = false, bool deprecate_eof_active = false) override; + void PgSQL_Result_to_PgSQL_wire(PgSQL_Connection* conn, PgSQL_Data_Stream* _myds = NULL); + void MySQL_Stmt_Result_to_MySQL_wire(MYSQL_STMT* stmt, PgSQL_Connection* myconn); + //unsigned int NumActiveTransactions(bool check_savpoint = false); + //bool HasOfflineBackends(); + //bool SetEventInOfflineBackends(); + /** + * @brief Finds one active transaction in the current backend connections. + * @details Since only one connection is returned, if the session holds multiple backend connections with + * potential transactions, the priority is: + * 1. Connections flagged with 'SERVER_STATUS_IN_TRANS', or 'autocommit=0' in combination with + * 'autocommit_false_is_transaction'. + * 2. Connections with 'autocommit=0' holding a 'SAVEPOINT'. + * 3. Connections with 'unknown transaction status', e.g: connections with errors. + * @param check_savepoint Used to also check for connections holding savepoints. See MySQL bug + * https://bugs.mysql.com/bug.php?id=107875. + * @returns The hostgroup in which the connection was found, -1 in case no connection is found. + */ + //int FindOneActiveTransaction(bool check_savepoint = false); + unsigned long long IdleTime(); + + //void reset_all_backends(); + //void writeout(); + void Memory_Stats(); + void create_new_session_and_reset_connection(PgSQL_Data_Stream* _myds) override; + bool handle_command_query_kill(PtrSize_t*); + //void update_expired_conns(const std::vector>&); + /** + * @brief Performs the final operations after current query has finished to be executed. It updates the session + * 'transaction_persistent_hostgroup', and updates the 'PgSQL_Data_Stream' and 'PgSQL_Connection' before + * returning the connection back to the connection pool. After this operation the session should be ready + * for handling new client connections. + * + * @param myds The 'PgSQL_Data_Stream' which status should be updated. + * @param myconn The 'PgSQL_Connection' which status should be updated, and which should be returned to + * the connection pool. + * @param prepared_stmt_with_no_params specifies if the processed query was a prepared statement with no + * params. + */ + void finishQuery(PgSQL_Data_Stream* myds, PgSQL_Connection* myconn, bool); + void generate_proxysql_internal_session_json(nlohmann::json&) override; + bool known_query_for_locked_on_hostgroup(uint64_t); + void unable_to_parse_set_statement(bool*); + //bool has_any_backend(); + void detected_broken_connection(const char* file, unsigned int line, const char* func, const char* action, PgSQL_Connection* myconn, bool verbose = false); + void generate_status_one_hostgroup(int hid, std::string& s); + void reset_warning_hostgroup_flag_and_release_connection(); + void set_previous_status_mode3(bool allow_execute = true); +}; + +#define PgSQL_KILL_QUERY 1 +#define PgSQL_KILL_CONNECTION 2 + +class PgSQL_KillArgs { +public: + PgSQL_Thread* mt; + char* username; + char* password; + char* hostname; + unsigned int port; + unsigned long id; + int kill_type; + unsigned int hid; + int use_ssl; + + PgSQL_KillArgs(char* u, char* p, char* h, unsigned int P, unsigned int _hid, unsigned long i, int kt, int _use_ssl, PgSQL_Thread* _mt); + PgSQL_KillArgs(char* u, char* p, char* h, unsigned int P, unsigned int _hid, unsigned long i, int kt, int _use_ssl, PgSQL_Thread* _mt, char* ip); + ~PgSQL_KillArgs(); + const char* get_host_address() const; + +private: + char* ip_addr; +}; + +void* PgSQL_kill_query_thread(void* arg); + +#endif /* __CLASS_PGSQL_SESSION_H */ +#endif // CLASS_BASE_SESSION_H diff --git a/include/PgSQL_Thread.h b/include/PgSQL_Thread.h new file mode 100644 index 0000000000..094e98634c --- /dev/null +++ b/include/PgSQL_Thread.h @@ -0,0 +1,711 @@ +#ifndef __CLASS_PGSQL_THREAD_H +#define __CLASS_PGSQL_THREAD_H +#define ____CLASS_STANDARD_PGSQL_THREAD_H +#include +#include + +#include "proxysql.h" +#include "Base_Thread.h" +#include "cpp.h" +#include "ProxySQL_Poll.h" +#include "PgSQL_Variables.h" +#ifdef IDLE_THREADS +#include +#endif // IDLE_THREADS +#include + +#include "prometheus_helpers.h" + +#include "set_parser.h" + +enum class AUTHENTICATION_METHOD { + NO_PASSWORD, + CLEAR_TEXT_PASSWORD, + MD5_PASSWORD, + SASL_SCRAM_SHA_256, + SASL_SCRAM_SHA_256_PLUS +}; + +constexpr const char* AUTHENTICATION_METHOD_STR[] = { + "NO_PASSWORD", + "CLEAR_TEXT_PASSWORD", + "MD5_PASSWORD", + "SASL_SCRAM_SHA_256", + "SASL_SCRAM_SHA_256_PLUS" +}; + +/* +#define MIN_POLL_LEN 8 +#define MIN_POLL_DELETE_RATIO 8 +#define MY_EPOLL_THREAD_MAXEVENTS 128 +*/ + +#define ADMIN_HOSTGROUP -2 +#define STATS_HOSTGROUP -3 +#define SQLITE_HOSTGROUP -4 + + +#define MYSQL_DEFAULT_SESSION_TRACK_GTIDS "OFF" +#define MYSQL_DEFAULT_COLLATION_CONNECTION "" +#define MYSQL_DEFAULT_NET_WRITE_TIMEOUT "60" +#define MYSQL_DEFAULT_MAX_JOIN_SIZE "18446744073709551615" + +extern class PgSQL_Variables pgsql_variables; + +#ifdef IDLE_THREADS +typedef struct __attribute__((aligned(64))) _pgsql_conn_exchange_t { + pthread_mutex_t mutex_idles; + PtrArray* idle_mysql_sessions; + pthread_mutex_t mutex_resumes; + PtrArray* resume_mysql_sessions; +} pgsql_conn_exchange_t; +#endif // IDLE_THREADS + +typedef struct _pgsql_thr_id_username_t { + uint32_t id; + char* username; +} pgsql_thr_id_username; + +typedef struct _pgsql_kill_queue_t { + pthread_mutex_t m; + std::vector conn_ids; + std::vector query_ids; +} pgsql_kill_queue; + +enum PgSQL_Thread_status_variable { + /*st_var_backend_stmt_prepare, + st_var_backend_stmt_execute, + st_var_backend_stmt_close, + st_var_frontend_stmt_prepare, + st_var_frontend_stmt_execute, + st_var_frontend_stmt_close, + st_var_queries, + st_var_queries_slow, + st_var_queries_gtid, + st_var_queries_with_max_lag_ms, + st_var_queries_with_max_lag_ms__delayed, + st_var_queries_with_max_lag_ms__total_wait_time_us, + st_var_queries_backends_bytes_sent, + st_var_queries_backends_bytes_recv, + st_var_queries_frontends_bytes_sent, + st_var_queries_frontends_bytes_recv, + st_var_query_processor_time, + st_var_backend_query_time, + st_var_mysql_backend_buffers_bytes, + st_var_mysql_frontend_buffers_bytes, + st_var_mysql_session_internal_bytes, + st_var_ConnPool_get_conn_immediate, + st_var_ConnPool_get_conn_success, + st_var_ConnPool_get_conn_failure, + st_var_ConnPool_get_conn_latency_awareness, + st_var_gtid_binlog_collected, + st_var_gtid_session_collected, + st_var_generated_pkt_err, + st_var_max_connect_timeout_err, + st_var_backend_lagging_during_query, + st_var_backend_offline_during_query, + st_var_unexpected_com_quit, + st_var_unexpected_packet, + st_var_killed_connections, + st_var_killed_queries, + st_var_hostgroup_locked, + st_var_hostgroup_locked_set_cmds, + st_var_hostgroup_locked_queries, + st_var_aws_aurora_replicas_skipped_during_query, + st_var_automatic_detected_sqli, + st_var_mysql_whitelisted_sqli_fingerprint, + st_var_client_host_error_killed_connections, + */ + PG_st_var_END = 42 // to avoid ASAN complaining. TO FIX +}; + +class __attribute__((aligned(64))) PgSQL_Thread : public Base_Thread +{ +private: + unsigned int servers_table_version_previous; + unsigned int servers_table_version_current; + unsigned long long last_processing_idles; + PgSQL_Connection** my_idle_conns; + bool processing_idles; + //bool maintenance_loop; + + PtrArray* cached_connections; + +#ifdef IDLE_THREADS + struct epoll_event events[MY_EPOLL_THREAD_MAXEVENTS]; + int efd; + unsigned int mysess_idx; + std::map sessmap; +#endif // IDLE_THREADS + + //Session_Regex** match_regexes; + +#ifdef IDLE_THREADS + void worker_thread_assigns_sessions_to_idle_thread(PgSQL_Thread * thr); + void worker_thread_gets_sessions_from_idle_thread(); + void idle_thread_gets_sessions_from_worker_thread(); + void idle_thread_assigns_sessions_to_worker_thread(PgSQL_Thread * thr); + void idle_thread_check_if_worker_thread_has_unprocess_resumed_sessions_and_signal_it(PgSQL_Thread * thr); + void idle_thread_prepares_session_to_send_to_worker_thread(int i); + void idle_thread_to_kill_idle_sessions(); + //bool move_session_to_idle_mysql_sessions(PgSQL_Data_Stream * myds, unsigned int n); +#endif // IDLE_THREADS + + //unsigned int find_session_idx_in_mysql_sessions(PgSQL_Session * sess); + //bool set_backend_to_be_skipped_if_frontend_is_slow(PgSQL_Data_Stream * myds, unsigned int n); + void handle_mirror_queue_mysql_sessions(); + void handle_kill_queues(); + //void check_timing_out_session(unsigned int n); + //void check_for_invalid_fd(unsigned int n); + //void read_one_byte_from_pipe(unsigned int n); + //void tune_timeout_for_myds_needs_pause(PgSQL_Data_Stream * myds); + //void tune_timeout_for_session_needs_pause(PgSQL_Data_Stream * myds); + //void configure_pollout(PgSQL_Data_Stream * myds, unsigned int n); + +protected: + int nfds; + +public: + + void* gen_args; // this is a generic pointer to create any sort of structure + + ProxySQL_Poll mypolls; + pthread_t thread_id; + unsigned long long pre_poll_time; + unsigned long long last_maintenance_time; + //unsigned long long last_move_to_idle_thread_time; + std::atomic atomic_curtime; + //PtrArray* mysql_sessions; + PtrArray* mirror_queue_mysql_sessions; + PtrArray* mirror_queue_mysql_sessions_cache; +#ifdef IDLE_THREADS + PtrArray* idle_mysql_sessions; + PtrArray* resume_mysql_sessions; + + pgsql_conn_exchange_t myexchange; +#endif // IDLE_THREADS + + int pipefd[2]; + kill_queue_t kq; + + //bool epoll_thread; + bool poll_timeout_bool; + + // status variables are per thread only + // in this way, there is no need for atomic operation and there is no cache miss + // when it is needed a total, all threads are checked + struct { + unsigned long long stvar[PG_st_var_END]; + unsigned int active_transactions; + } status_variables; + + struct { + int min_num_servers_lantency_awareness; + int aurora_max_lag_ms_only_read_from_replicas; + bool stats_time_backend_query; + bool stats_time_query_processor; + bool query_cache_stores_empty_result; + } variables; + + pthread_mutex_t thread_mutex; + + // if set_parser_algorithm == 2 , a single thr_SetParser is used + SetParser* thr_SetParser; + + PgSQL_Thread(); + ~PgSQL_Thread(); +// PgSQL_Session* create_new_session_and_client_data_stream(int _fd); + bool init(); + void run___get_multiple_idle_connections(int& num_idles); + void run___cleanup_mirror_queue(); + //void ProcessAllMyDS_BeforePoll(); + //void ProcessAllMyDS_AfterPoll(); + void run(); + void poll_listener_add(int sock); + void poll_listener_del(int sock); + //void register_session(PgSQL_Session*, bool up_start = true); + void unregister_session(int); + struct pollfd* get_pollfd(unsigned int i); + bool process_data_on_data_stream(PgSQL_Data_Stream * myds, unsigned int n); + //void ProcessAllSessions_SortingSessions(); + void ProcessAllSessions_CompletedMirrorSession(unsigned int& n, PgSQL_Session * sess); + void ProcessAllSessions_MaintenanceLoop(PgSQL_Session * sess, unsigned long long sess_time, unsigned int& total_active_transactions_); + void process_all_sessions(); + void refresh_variables(); + void register_session_connection_handler(PgSQL_Session * _sess, bool _new = false); + void unregister_session_connection_handler(int idx, bool _new = false); + void listener_handle_new_connection(PgSQL_Data_Stream * myds, unsigned int n); + void Get_Memory_Stats(); + PgSQL_Connection* get_MyConn_local(unsigned int, PgSQL_Session * sess, char* gtid_uuid, uint64_t gtid_trxid, int max_lag_ms); + void push_MyConn_local(PgSQL_Connection*); + void return_local_connections(); + void Scan_Sessions_to_Kill(PtrArray * mysess); + void Scan_Sessions_to_Kill_All(); +}; + + +typedef PgSQL_Thread* create_PgSQL_Thread_t(); +typedef void destroy_PgSQL_Thread_t(PgSQL_Thread*); + +class PgSQL_Listeners_Manager { +private: + PtrArray* ifaces; +public: + PgSQL_Listeners_Manager(); + ~PgSQL_Listeners_Manager(); + int add(const char* iface, unsigned int num_threads, int** perthrsocks); + int find_idx(const char* iface); + int find_idx(const char* address, int port); + iface_info* find_iface_from_fd(int fd); + int get_fd(unsigned int idx); + void del(unsigned int idx); +}; + +/*struct p_th_counter { + enum metric { + queries_backends_bytes_sent = 0, + queries_backends_bytes_recv, + queries_frontends_bytes_sent, + queries_frontends_bytes_recv, + query_processor_time_nsec, + backend_query_time_nsec, + com_backend_stmt_prepare, + com_backend_stmt_execute, + com_backend_stmt_close, + com_frontend_stmt_prepare, + com_frontend_stmt_execute, + com_frontend_stmt_close, + questions, + slow_queries, + gtid_consistent_queries, + gtid_session_collected, + connpool_get_conn_latency_awareness, + connpool_get_conn_immediate, + connpool_get_conn_success, + connpool_get_conn_failure, + generated_error_packets, + max_connect_timeouts, + backend_lagging_during_query, + backend_offline_during_query, + queries_with_max_lag_ms, + queries_with_max_lag_ms__delayed, + queries_with_max_lag_ms__total_wait_time_us, + mysql_unexpected_frontend_com_quit, + hostgroup_locked_set_cmds, + hostgroup_locked_queries, + mysql_unexpected_frontend_packets, + aws_aurora_replicas_skipped_during_query, + automatic_detected_sql_injection, + mysql_whitelisted_sqli_fingerprint, + mysql_killed_backend_connections, + mysql_killed_backend_queries, + client_host_error_killed_connections, + __size + }; +}; + +struct p_th_gauge { + enum metric { + active_transactions = 0, + client_connections_non_idle, + client_connections_hostgroup_locked, + mysql_backend_buffers_bytes, + mysql_frontend_buffers_bytes, + mysql_session_internal_bytes, + mirror_concurrency, + mirror_queue_lengths, + mysql_thread_workers, + // global_variables + mysql_wait_timeout, + mysql_max_connections, + mysql_monitor_enabled, + mysql_monitor_ping_interval, + mysql_monitor_ping_timeout, + mysql_monitor_ping_max_failures, + mysql_monitor_aws_rds_topology_discovery_interval, + mysql_monitor_read_only_interval, + mysql_monitor_read_only_timeout, + mysql_monitor_writer_is_also_reader, + mysql_monitor_replication_lag_group_by_host, + mysql_monitor_replication_lag_interval, + mysql_monitor_replication_lag_timeout, + mysql_monitor_history, + __size + }; +}; + +struct th_metrics_map_idx { + enum index { + counters = 0, + gauges + }; +}; +*/ +/** + * @brief Structure holding the data for a Client_Host_Cache entry. + */ +typedef struct _PgSQL_Client_Host_Cache_Entry { + /** + * @brief Last time the entry was updated. + */ + uint64_t updated_at; + /** + * @brief Error count associated with the entry. + */ + uint32_t error_count; +} PgSQL_Client_Host_Cache_Entry; + +class PgSQL_Threads_Handler +{ +private: + int shutdown_; + size_t stacksize; + pthread_attr_t attr; + pthread_rwlock_t rwlock; + PtrArray* bind_fds; + PgSQL_Listeners_Manager* MLM; + // VariablesPointers_int stores: + // key: variable name + // tuple: + // variable address + // min value + // max value + // special variable : if true, min and max values are ignored, and further input validation is required + std::unordered_map> VariablesPointers_int; + // VariablesPointers_bool stores: + // key: variable name + // tuple: + // variable address + // special variable : if true, further input validation is required + std::unordered_map> VariablesPointers_bool; + /** + * @brief Holds the clients host cache. It keeps track of the number of + * errors associated to a specific client: + * - Key: client identifier, based on 'clientaddr'. + * - Value: Structure of type 'PgSQL_Client_Host_Cache_Entry' holding + * the last time the entry was updated and the error count associated + * with the client. + */ + std::unordered_map client_host_cache; + /** + * @brief Holds the mutex for accessing 'client_host_cache', since every + * access can potentially perform 'read/write' operations, a regular mutex + * is enough. + */ + pthread_mutex_t mutex_client_host_cache; + +public: + struct { + int authentication_method; + char* server_version; + + int monitor_history; + int monitor_connect_interval; + int monitor_connect_timeout; + //! Monitor ping interval. Unit: 'ms'. + int monitor_ping_interval; + int monitor_ping_max_failures; + //! Monitor ping timeout. Unit: 'ms'. + int monitor_ping_timeout; + //! Monitor aws rds topology discovery interval. Unit: 'one discovery check per X monitor_read_only checks'. + int monitor_aws_rds_topology_discovery_interval; + //! Monitor read only timeout. Unit: 'ms'. + int monitor_read_only_interval; + //! Monitor read only timeout. Unit: 'ms'. + int monitor_read_only_timeout; + int monitor_read_only_max_timeout_count; + bool monitor_enabled; + //! ProxySQL session wait timeout. Unit: 'ms'. + bool monitor_wait_timeout; + bool monitor_writer_is_also_reader; + bool monitor_replication_lag_group_by_host; + //! How frequently a replication lag check is performed. Unit: 'ms'. + int monitor_replication_lag_interval; + //! Read only check timeout. Unit: 'ms'. + int monitor_replication_lag_timeout; + int monitor_replication_lag_count; + int monitor_groupreplication_healthcheck_interval; + int monitor_groupreplication_healthcheck_timeout; + int monitor_groupreplication_healthcheck_max_timeout_count; + int monitor_groupreplication_max_transactions_behind_count; + int monitor_groupreplication_max_transactions_behind_for_read_only; + int monitor_galera_healthcheck_interval; + int monitor_galera_healthcheck_timeout; + int monitor_galera_healthcheck_max_timeout_count; + int monitor_query_interval; + int monitor_query_timeout; + int monitor_slave_lag_when_null; + int monitor_threads_min; + int monitor_threads_max; + int monitor_threads_queue_maxsize; + int monitor_local_dns_cache_ttl; + int monitor_local_dns_cache_refresh_interval; + int monitor_local_dns_resolver_queue_maxsize; + char* monitor_username; + char* monitor_password; + char* monitor_replication_lag_use_percona_heartbeat; + int ping_interval_server_msec; + int ping_timeout_server; + int shun_on_failures; + int shun_recovery_time_sec; + int unshun_algorithm; + int query_retries_on_failure; + bool connection_warming; + int client_host_cache_size; + int client_host_error_counts; + int connect_retries_on_failure; + int connect_retries_delay; + int connection_delay_multiplex_ms; + int connection_max_age_ms; + int connect_timeout_client; + int connect_timeout_server; + int connect_timeout_server_max; + int free_connections_pct; + int show_processlist_extended; +#ifdef IDLE_THREADS + int session_idle_ms; + bool session_idle_show_processlist; +#endif // IDLE_THREADS + bool sessions_sort; + char* default_schema; + char* interfaces; + char* keep_multiplexing_variables; + char* default_client_encoding; + //unsigned int default_charset; // removed in 2.0.13 . Obsoleted previously using PgSQL_Variables instead + int handle_unknown_charset; + bool servers_stats; + bool commands_stats; + bool query_digests; + bool query_digests_lowercase; + bool query_digests_replace_null; + bool query_digests_no_digits; + bool query_digests_normalize_digest_text; + bool query_digests_track_hostname; + bool query_digests_keep_comment; + int query_digests_grouping_limit; + int query_digests_groups_grouping_limit; + bool parse_failure_logs_digest; + bool default_reconnect; + bool have_compress; + bool have_ssl; + bool multiplexing; + // bool stmt_multiplexing; + bool log_unhealthy_connections; + bool enforce_autocommit_on_reads; + bool autocommit_false_not_reusable; + bool autocommit_false_is_transaction; + bool verbose_query_error; + int max_allowed_packet; + bool automatic_detect_sqli; + bool firewall_whitelist_enabled; + bool use_tcp_keepalive; + int tcp_keepalive_time; + int throttle_connections_per_sec_to_hostgroup; + int max_transaction_idle_time; + int max_transaction_time; + int threshold_query_length; + int threshold_resultset_size; + int query_digests_max_digest_length; + int query_digests_max_query_length; + int query_rules_fast_routing_algorithm; + int wait_timeout; + int throttle_max_bytes_per_second_to_client; + int throttle_ratio_server_to_client; + int max_connections; + int max_stmts_per_connection; + int max_stmts_cache; + int mirror_max_concurrency; + int mirror_max_queue_length; + int default_max_latency_ms; + int default_query_delay; + int default_query_timeout; + int query_processor_iterations; + int query_processor_regex; + int set_query_lock_on_hostgroup; + int set_parser_algorithm; + int auto_increment_delay_multiplex; + int auto_increment_delay_multiplex_timeout_ms; + int long_query_time; + int hostgroup_manager_verbose; + int binlog_reader_connect_retry_msec; + char* init_connect; + char* ldap_user_variable; + char* add_ldap_user_comment; + char* default_session_track_gtids; + char* default_variables[SQL_NAME_LAST_LOW_WM]; + char* firewall_whitelist_errormsg; +#ifdef DEBUG + bool session_debug; +#endif /* DEBUG */ + uint32_t server_capabilities; + int poll_timeout; + int poll_timeout_on_failure; + char* eventslog_filename; + int eventslog_filesize; + int eventslog_default_log; + int eventslog_format; + char* auditlog_filename; + int auditlog_filesize; + // SSL related, proxy to server + char* ssl_p2s_ca; + char* ssl_p2s_capath; + char* ssl_p2s_cert; + char* ssl_p2s_key; + char* ssl_p2s_cipher; + char* ssl_p2s_crl; + char* ssl_p2s_crlpath; + int query_cache_size_MB; + int query_cache_soft_ttl_pct; + int query_cache_handle_warnings; + int min_num_servers_lantency_awareness; + int aurora_max_lag_ms_only_read_from_replicas; + bool stats_time_backend_query; + bool stats_time_query_processor; + bool query_cache_stores_empty_result; + bool kill_backend_connection_when_disconnect; + bool client_session_track_gtid; + bool enable_client_deprecate_eof; + bool enable_server_deprecate_eof; + bool enable_load_data_local_infile; + bool log_mysql_warnings_enabled; + int data_packets_history_size; + int handle_warnings; + } variables; + struct { + unsigned int mirror_sessions_current; + int threads_initialized = 0; + /// Prometheus metrics arrays + //std::array p_counter_array{}; + //std::array p_gauge_array{}; + } status_variables; + + std::atomic bootstrapping_listeners; + + /** + * @brief Update the client host cache with the supplied 'client_sockaddr', + * and the supplied 'error' parameter specifying if there was a connection + * error or not. + * + * NOTE: This function is not safe, the supplied 'client_sockaddr' should + * have been initialized by 'accept' or 'getpeername'. NULL checks are not + * performed. + * + * @details The 'client_sockaddr' parameter is inspected, and the + * 'client_host_cache' map is only updated in case of: + * - 'address_family' is either 'AF_INET' or 'AF_INET6'. + * - The address obtained from it isn't '127.0.0.1'. + * + * In case 'client_sockaddr' matches the previous description, the update + * of the client host cache is performed in the following way: + * 1. If the cache is full, the oldest element in the cache is searched. + * In case the oldest element address doesn't match the supplied + * address, the oldest element is removed. + * 2. The cache is searched looking for the supplied address, in case of + * being found, the entry is updated, otherwise the entry is inserted in + * the cache. + * + * @param client_sockaddr A 'sockaddr' holding the required client information + * to update the 'client_host_cache_map'. + * @param error 'true' if there was an error in the connection that should be + * register, 'false' otherwise. + */ + void update_client_host_cache(struct sockaddr* client_sockaddr, bool error); + /** + * @brief Retrieves the entry of the underlying 'client_host_cache' map for + * the supplied 'client_sockaddr' in case of existing. In case it doesn't + * exist or the supplied 'client_sockaddr' doesn't met the requirements + * for being registered in the map, and zeroed 'PgSQL_Client_Host_Cache_Entry' + * is returned. + * + * NOTE: This function is not safe, the supplied 'client_sockaddr' should + * have been initialized by 'accept' or 'getpeername'. NULL checks are not + * performed. + * + * @details The 'client_sockaddr' parameter is inspected, and the + * 'client_host_cache' map is only searched in case of: + * - 'address_family' is either 'AF_INET' or 'AF_INET6'. + * - The address obtained from it isn't '127.0.0.1'. + * + * @param client_sockaddr A 'sockaddr' holding the required client information + * to update the 'client_host_cache_map'. + * @return If found, the corresponding entry for the supplied 'client_sockaddr', + * a zeroed 'PgSQL_Client_Host_Cache_Entry' otherwise. + */ + PgSQL_Client_Host_Cache_Entry find_client_host_cache(struct sockaddr* client_sockaddr); + /** + * @brief Delete all the entries in the 'client_host_cache' internal map. + */ + void flush_client_host_cache(); + /** + * @brief Returns the current entries of 'client_host_cache' in a + * 'SQLite3_result'. In case the param 'reset' is specified, the structure + * is cleaned after being queried. + * + * @param reset If 'true' the entries of the internal structure + * 'client_host_cache' will be cleaned after scrapping. + * + * @return SQLite3_result holding the current entries of the + * 'client_host_cache'. In the following format: + * + * [ 'client_address', 'error_num', 'last_updated' ] + * + * Where 'last_updated' is the last updated time expressed in 'ns'. + */ + SQLite3_result* get_client_host_cache(bool reset); + /** + * @brief Callback to update the metrics. + */ + void p_update_metrics(); + unsigned int num_threads; + proxysql_pgsql_thread_t* pgsql_threads; +#ifdef IDLE_THREADS + proxysql_pgsql_thread_t* pgsql_threads_idles; +#endif // IDLE_THREADS + unsigned int get_global_version(); + void wrlock(); + void wrunlock(); + void commit(); + char* get_variable(char* name); + bool set_variable(char* name, const char* value); + char** get_variables_list(); + bool has_variable(const char* name); + + PgSQL_Threads_Handler(); + ~PgSQL_Threads_Handler(); + + char* get_variable_string(char* name); + uint16_t get_variable_uint16(char* name); + int get_variable_int(const char* name); + void print_version(); + void init(unsigned int num = 0, size_t stack = 0); + proxysql_pgsql_thread_t* create_thread(unsigned int tn, void* (*start_routine) (void*), bool); + void shutdown_threads(); + int listener_add(const char* iface); + int listener_add(const char* address, int port); + int listener_del(const char* iface); + int listener_del(const char* address, int port); + void start_listeners(); + void stop_listeners(); + void signal_all_threads(unsigned char _c = 0); + SQLite3_result* SQL3_Processlist(); + SQLite3_result* SQL3_GlobalStatus(bool _memory); + bool kill_session(uint32_t _thread_session_id); + unsigned long long get_total_mirror_queue(); + //unsigned long long get_status_variable(enum PgSQL_Thread_status_variable v_idx, p_th_counter::metric m_idx, unsigned long long conv = 0); + //unsigned long long get_status_variable(enum PgSQL_Thread_status_variable v_idx, p_th_gauge::metric m_idx, unsigned long long conv = 0); + unsigned int get_active_transations(); +#ifdef IDLE_THREADS + unsigned int get_non_idle_client_connections(); +#endif // IDLE_THREADS + unsigned long long get_pgsql_backend_buffers_bytes(); + unsigned long long get_pgsql_frontend_buffers_bytes(); + unsigned long long get_pgsql_session_internal_bytes(); + iface_info* MLM_find_iface_from_fd(int fd) { + return MLM->find_iface_from_fd(fd); + } + void Get_Memory_Stats(); + void kill_connection_or_query(uint32_t _thread_session_id, bool query, char* username); +}; + + +#endif /* __CLASS_PGSQL_THREAD_H */ diff --git a/include/PgSQL_Variables.h b/include/PgSQL_Variables.h new file mode 100644 index 0000000000..d8a5ccbef2 --- /dev/null +++ b/include/PgSQL_Variables.h @@ -0,0 +1,57 @@ +#ifndef PGSQL_VARIABLES_H +#define PGSQL_VARIABLES_H + +#include "proxysql.h" +#include "cpp.h" + +#include +#include +#include + +class PgSQL_Session; + +extern void print_backtrace(void); + +typedef bool (*pgsql_verify_var)(PgSQL_Session* session, int idx, uint32_t client_hash, uint32_t server_hash); +typedef bool (*pgsql_update_var)(PgSQL_Session* session, int idx, int &_rc); + +bool validate_charset(PgSQL_Session* session, int idx, int &_rc); +bool update_server_variable(PgSQL_Session* session, int idx, int &_rc); +bool verify_server_variable(PgSQL_Session* session, int idx, uint32_t client_hash, uint32_t server_hash); +bool verify_set_names(PgSQL_Session* session); +#if 0 +bool logbin_update_server_variable(PgSQL_Session* session, int idx, int &_rc); +#endif // 0 + +class PgSQL_Variables { + static pgsql_verify_var verifiers[SQL_NAME_LAST_HIGH_WM]; + static pgsql_update_var updaters[SQL_NAME_LAST_HIGH_WM]; + +public: + std::string variables_regexp; + // ignore_vars is a list of all variables that proxysql will parse but ignore its value + std::vector ignore_vars; +public: + PgSQL_Variables(); + ~PgSQL_Variables(); + + bool client_set_value(PgSQL_Session* session, int idx, const std::string& value); + bool client_set_hash_and_value(PgSQL_Session* session, int idx, const std::string& value, uint32_t hash); + void client_reset_value(PgSQL_Session* session, int idx); + const char* client_get_value(PgSQL_Session* session, int idx) const; + uint32_t client_get_hash(PgSQL_Session* session, int idx) const; + + void server_set_value(PgSQL_Session* session, int idx, const char* value); + void server_set_hash_and_value(PgSQL_Session* session, int idx, const char* value, uint32_t hash); + void server_reset_value(PgSQL_Session* session, int idx); + const char* server_get_value(PgSQL_Session* session, int idx) const; + inline uint32_t server_get_hash(PgSQL_Session* session, int idx) const; + + bool verify_variable(PgSQL_Session* session, int idx) const; + bool update_variable(PgSQL_Session* session, session_status status, int &_rc); + bool parse_variable_boolean(PgSQL_Session*sess, int idx, std::string &value1, bool* lock_hostgroup); + bool parse_variable_number(PgSQL_Session*sess, int idx, std::string &value1, bool* lock_hostgroup); +}; + +#endif // PGSQL_VARIABLES_H + diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h new file mode 100644 index 0000000000..691e55f708 --- /dev/null +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -0,0 +1,312 @@ +#ifndef ProxySQL_Admin_Tables_Definitions +#define ProxySQL_Admin_Tables_Definitions +// mysql_servers in v1.1.0 +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_1_0 "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" + +// mysql_servers in v1.2.0e +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_0e "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" + +// mysql_servers in v1.2.2 +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_2 "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" + +// mysql_servers in v1.4.4 +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_4_4 "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" + +// mysql_servers in v2.0.0 +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0a "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT CHECK (gtid_port <> port) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" + +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0b "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT CHECK (gtid_port <> port) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" + +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0c "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK (gtid_port <> port AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" + +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_11 "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK ((gtid_port <> port OR gtid_port=0) AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" + +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_11 + +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS_V2_6_0 "CREATE TABLE mysql_servers_ssl_params (hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , username VARCHAR NOT NULL DEFAULT '' , ssl_ca VARCHAR NOT NULL DEFAULT '' , ssl_cert VARCHAR NOT NULL DEFAULT '' , ssl_key VARCHAR NOT NULL DEFAULT '' , ssl_capath VARCHAR NOT NULL DEFAULT '' , ssl_crl VARCHAR NOT NULL DEFAULT '' , ssl_crlpath VARCHAR NOT NULL DEFAULT '' , ssl_cipher VARCHAR NOT NULL DEFAULT '' , tls_version VARCHAR NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port, username) )" + +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS_V2_6_0 + +#define ADMIN_SQLITE_TABLE_MYSQL_USERS_V1_3_0 "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 0 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" +#define ADMIN_SQLITE_TABLE_MYSQL_USERS_V1_4_0 "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" +#define ADMIN_SQLITE_TABLE_MYSQL_USERS_V2_0_0 "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" +#define ADMIN_SQLITE_TABLE_MYSQL_USERS_V2_1_0 "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" +#define ADMIN_SQLITE_TABLE_MYSQL_USERS ADMIN_SQLITE_TABLE_MYSQL_USERS_V2_1_0 + +#define ADMIN_SQLITE_RUNTIME_MYSQL_USERS "CREATE TABLE runtime_mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '', comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" + +#define ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING_V2_0_0 "CREATE TABLE mysql_ldap_mapping (priority INTEGER CHECK (priority >= 1 AND priority <= 1000000) PRIMARY KEY , frontend_entity VARCHAR NOT NULL , backend_entity VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (frontend_entity))" +#define ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING_V2_0_0 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_LDAP_MAPPING "CREATE TABLE runtime_mysql_ldap_mapping (priority INTEGER PRIMARY KEY NOT NULL , frontend_entity VARCHAR NOT NULL , backend_entity VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (frontend_entity))" + +#define ADMIN_SQLITE_RUNTIME_CHECKSUMS_VALUES "CREATE TABLE runtime_checksums_values (name VARCHAR NOT NULL , version INT NOT NULL , epoch INT NOT NULL , checksum VARCHAR NOT NULL , PRIMARY KEY (name))" + +// mysql_query_rules in v1.1.0 +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_1_0 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , delay INT UNSIGNED , error_msg VARCHAR , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" + +// mysql_query_rules in v1.2.0a +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0a "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" + +// mysql_query_rules in v1.2.0g +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0g "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" + +// mysql_query_rules in v1.2.2 +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_2 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +// mysql_query_rules in v1.3.1 +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_3_1 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +//mysql_query_rules in v1.4.0 + next_query_flagIN +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0a "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0b "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_1 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0a "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0b "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0c "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0d "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0e "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_1_0 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535) , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0) , replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_1_0 +//#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0b + + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING "CREATE TABLE mysql_query_rules_fast_routing (username VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , destination_hostgroup INT CHECK (destination_hostgroup >= 0) NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, schemaname, flagIN) )" + +#define ADMIN_SQLITE_TABLE_GLOBAL_SETTINGS "CREATE TABLE global_settings (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" + +#define ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES "CREATE TABLE global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" + +#define ADMIN_SQLITE_RUNTIME_GLOBAL_VARIABLES "CREATE TABLE runtime_global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" + +//#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR , UNIQUE (reader_hostgroup))" + +// mysql_replication_hostgroups in v1.0 +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_0 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , UNIQUE (reader_hostgroup))" + +// mysql_replication_hostgroups in v1.2.2 +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_2_2 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR , UNIQUE (reader_hostgroup))" + +// mysql_replication_hostgroups in v1.4.5 +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_4_5 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" + +// mysql_replication_hostgroups in v2.0.0 +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V2_0_0 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" + +// mysql_replication_hostgroups in v2.0.8 +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V2_0_8 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only','read_only|innodb_read_only','read_only&innodb_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" + +// mysql_replication_hostgroups current +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V2_0_8 + +#define ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS "CREATE TABLE mysql_collations (Id INTEGER NOT NULL PRIMARY KEY , Collation VARCHAR NOT NULL , Charset VARCHAR NOT NULL , `Default` VARCHAR NOT NULL)" + +#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES_V2_0_15 "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES_v2_1_0 "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , timeout_ms INTEGER CHECK (timeout_ms>=100 AND timeout_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES ADMIN_SQLITE_TABLE_RESTAPI_ROUTES_v2_1_0 + +#define ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES "CREATE TABLE runtime_restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , timeout_ms INTEGER CHECK (timeout_ms>=100 AND timeout_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_0 "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" + +#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2a "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY(id))" + +#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2b "CREATE TABLE scheduler (id INTEGER NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY(id))" + +#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2c "CREATE TABLE scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" + + +#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS_v209 "CREATE TABLE mysql_firewall_whitelist_users (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , mode VARCHAR CHECK (mode IN ('OFF','DETECTING','PROTECTING')) NOT NULL DEFAULT ('OFF') , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address) )" + +#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS_v209 + +#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES_v209 "CREATE TABLE mysql_firewall_whitelist_rules (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , digest VARCHAR NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address, schemaname, flagIN, digest) )" + +#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES_v209 + +#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS_v209 "CREATE TABLE mysql_firewall_whitelist_sqli_fingerprints (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , fingerprint VARCHAR NOT NULL , PRIMARY KEY (fingerprint) )" + +#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS_v209 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_USERS "CREATE TABLE runtime_mysql_firewall_whitelist_users (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , mode VARCHAR CHECK (mode IN ('OFF','DETECTING','PROTECTING')) NOT NULL DEFAULT ('OFF') , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address) )" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_RULES "CREATE TABLE runtime_mysql_firewall_whitelist_rules (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , digest VARCHAR NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address, schemaname, flagIN, digest) )" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS "CREATE TABLE runtime_mysql_firewall_whitelist_sqli_fingerprints (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , fingerprint VARCHAR NOT NULL , PRIMARY KEY (fingerprint) )" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK ((gtid_port <> port OR gtid_port=0) AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS "CREATE TABLE runtime_mysql_servers_ssl_params (hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , username VARCHAR NOT NULL DEFAULT '' , ssl_ca VARCHAR NOT NULL DEFAULT '' , ssl_cert VARCHAR NOT NULL DEFAULT '' , ssl_key VARCHAR NOT NULL DEFAULT '' , ssl_capath VARCHAR NOT NULL DEFAULT '' , ssl_crl VARCHAR NOT NULL DEFAULT '' , ssl_crlpath VARCHAR NOT NULL DEFAULT '' , ssl_cipher VARCHAR NOT NULL DEFAULT '' , tls_version VARCHAR NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port, username) )" + + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only','read_only|innodb_read_only','read_only&innodb_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES "CREATE TABLE runtime_mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES_FAST_ROUTING "CREATE TABLE runtime_mysql_query_rules_fast_routing (username VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , destination_hostgroup INT CHECK (destination_hostgroup >= 0) NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, schemaname, flagIN) )" + +#define ADMIN_SQLITE_TABLE_RUNTIME_SCHEDULER "CREATE TABLE runtime_scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" + +#define STATS_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE stats_mysql_query_rules (rule_id INTEGER PRIMARY KEY , hits INT NOT NULL)" +#define STATS_SQLITE_TABLE_MYSQL_USERS "CREATE TABLE stats_mysql_users (username VARCHAR PRIMARY KEY , frontend_connections INT NOT NULL , frontend_max_connections INT NOT NULL)" +#define STATS_SQLITE_TABLE_MYSQL_COMMANDS_COUNTERS "CREATE TABLE stats_mysql_commands_counters (Command VARCHAR NOT NULL PRIMARY KEY , Total_Time_us INT NOT NULL , Total_cnt INT NOT NULL , cnt_100us INT NOT NULL , cnt_500us INT NOT NULL , cnt_1ms INT NOT NULL , cnt_5ms INT NOT NULL , cnt_10ms INT NOT NULL , cnt_50ms INT NOT NULL , cnt_100ms INT NOT NULL , cnt_500ms INT NOT NULL , cnt_1s INT NOT NULL , cnt_5s INT NOT NULL , cnt_10s INT NOT NULL , cnt_INFs)" +#define STATS_SQLITE_TABLE_MYSQL_PROCESSLIST "CREATE TABLE stats_mysql_processlist (ThreadID INT NOT NULL , SessionID INTEGER PRIMARY KEY , user VARCHAR , db VARCHAR , cli_host VARCHAR , cli_port INT , hostgroup INT , l_srv_host VARCHAR , l_srv_port INT , srv_host VARCHAR , srv_port INT , command VARCHAR , time_ms INT NOT NULL , info VARCHAR , status_flags INT , extended_info VARCHAR)" +#define STATS_SQLITE_TABLE_MYSQL_CONNECTION_POOL "CREATE TABLE stats_mysql_connection_pool (hostgroup INT , srv_host VARCHAR , srv_port INT , status VARCHAR , ConnUsed INT , ConnFree INT , ConnOK INT , ConnERR INT , MaxConnUsed INT , Queries INT , Queries_GTID_sync INT , Bytes_data_sent INT , Bytes_data_recv INT , Latency_us INT)" + +#define STATS_SQLITE_TABLE_MYSQL_CONNECTION_POOL_RESET "CREATE TABLE stats_mysql_connection_pool_reset (hostgroup INT , srv_host VARCHAR , srv_port INT , status VARCHAR , ConnUsed INT , ConnFree INT , ConnOK INT , ConnERR INT , MaxConnUsed INT , Queries INT , Queries_GTID_sync INT , Bytes_data_sent INT , Bytes_data_recv INT , Latency_us INT)" + +#define STATS_SQLITE_TABLE_MYSQL_FREE_CONNECTIONS "CREATE TABLE stats_mysql_free_connections (fd INT NOT NULL , hostgroup INT NOT NULL , srv_host VARCHAR NOT NULL , srv_port INT NOT NULL , user VARCHAR NOT NULL , schema VARCHAR , init_connect VARCHAR , time_zone VARCHAR , sql_mode VARCHAR , autocommit VARCHAR , idle_ms INT , statistics VARCHAR , mysql_info VARCHAR)" + +#define STATS_SQLITE_TABLE_MYSQL_QUERY_DIGEST "CREATE TABLE stats_mysql_query_digest (hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, schemaname, username, client_address, digest))" + +#define STATS_SQLITE_TABLE_MYSQL_QUERY_DIGEST_RESET "CREATE TABLE stats_mysql_query_digest_reset (hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, schemaname, username, client_address, digest))" + +#define STATS_SQLITE_TABLE_MYSQL_GLOBAL "CREATE TABLE stats_mysql_global (Variable_Name VARCHAR NOT NULL PRIMARY KEY , Variable_Value VARCHAR NOT NULL)" + +#define STATS_SQLITE_TABLE_MEMORY_METRICS "CREATE TABLE stats_memory_metrics (Variable_Name VARCHAR NOT NULL PRIMARY KEY , Variable_Value VARCHAR NOT NULL)" + +#define STATS_SQLITE_TABLE_MYSQL_GTID_EXECUTED "CREATE TABLE stats_mysql_gtid_executed (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_executed VARCHAR , events INT NOT NULL)" + +#define STATS_SQLITE_TABLE_MYSQL_ERRORS "CREATE TABLE stats_mysql_errors (hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , errno INT NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , last_error VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup, hostname, port, username, schemaname, errno) )" +#define STATS_SQLITE_TABLE_MYSQL_ERRORS_RESET "CREATE TABLE stats_mysql_errors_reset (hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , errno INT NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , last_error VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup, hostname, port, username, schemaname, errno) )" + +#define STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE "CREATE TABLE stats_mysql_client_host_cache (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)" +#define STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE_RESET "CREATE TABLE stats_mysql_client_host_cache_reset (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)" + +#ifdef DEBUG +#define ADMIN_SQLITE_TABLE_DEBUG_LEVELS "CREATE TABLE debug_levels (module VARCHAR NOT NULL PRIMARY KEY , verbosity INT NOT NULL DEFAULT 0)" +#define ADMIN_SQLITE_TABLE_DEBUG_FILTERS "CREATE TABLE debug_filters (filename VARCHAR NOT NULL , line INT NOT NULL , funct VARCHAR NOT NULL , PRIMARY KEY (filename, line, funct) )" +#endif /* DEBUG */ + +#define ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS_V1_4 "CREATE TABLE mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS_V2_0_0 "CREATE TABLE mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS_V2_0_0 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS_V2_0_0a "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS_V2_0_0b "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS_V2_0_0b + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE runtime_mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define ADMIN_SQLITE_TABLE_COREDUMP_FILTERS "CREATE TABLE coredump_filters (filename VARCHAR NOT NULL , line INT NOT NULL , PRIMARY KEY (filename, line) )" + +#define ADMIN_SQLITE_RUNTIME_COREDUMP_FILTERS "CREATE TABLE runtime_coredump_filters (filename VARCHAR NOT NULL , line INT NOT NULL , PRIMARY KEY (filename, line) )" + +// AWS Aurora + +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_8 "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , comment VARCHAR , UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_9 "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , comment VARCHAR , UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_9 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , comment VARCHAR , UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_6_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" + +//#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_6_0 + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE runtime_mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_6_0 + +// Cluster solution + +#define ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS "CREATE TABLE proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" + +#define ADMIN_SQLITE_TABLE_RUNTIME_PROXYSQL_SERVERS "CREATE TABLE runtime_proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" + +#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CLIENTS_STATUS "CREATE TABLE stats_proxysql_servers_clients_status (uuid VARCHAR NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , admin_mysql_ifaces VARCHAR NOT NULL , last_seen_at INT NOT NULL , PRIMARY KEY (uuid, hostname, port) )" + +#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS "CREATE TABLE stats_proxysql_servers_status (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , master VARCHAR NOT NULL , global_version INT NOT NULL , check_age_us INT NOT NULL , ping_time_us INT NOT NULL, checks_OK INT NOT NULL , checks_ERR INT NOT NULL , PRIMARY KEY (hostname, port) )" + +#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_METRICS "CREATE TABLE stats_proxysql_servers_metrics (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , response_time_ms INT NOT NULL , Uptime_s INT NOT NULL , last_check_ms INT NOT NULL , Queries INT NOT NULL , Client_Connections_connected INT NOT NULL , Client_Connections_created INT NOT NULL , PRIMARY KEY (hostname, port) )" + +#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CHECKSUMS "CREATE TABLE stats_proxysql_servers_checksums (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , name VARCHAR NOT NULL , version INT NOT NULL , epoch INT NOT NULL , checksum VARCHAR NOT NULL , changed_at INT NOT NULL , updated_at INT NOT NULL , diff_check INT NOT NULL , PRIMARY KEY (hostname, port, name) )" + +#define STATS_SQLITE_TABLE_PROXYSQL_MESSAGE_METRICS "CREATE TABLE stats_proxysql_message_metrics (message_id VARCHAR NOT NULL , filename VARCHAR NOT NULL , line INT CHECK (line >= 0) NOT NULL DEFAULT 0 , func VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , PRIMARY KEY (filename, line, func) )" + +#define STATS_SQLITE_TABLE_PROXYSQL_MESSAGE_METRICS_RESET "CREATE TABLE stats_proxysql_message_metrics_reset (message_id VARCHAR NOT NULL , filename VARCHAR NOT NULL , line INT CHECK (line >= 0) NOT NULL DEFAULT 0 , func VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , PRIMARY KEY (filename, line, func) )" + +#ifdef PROXYSQLCLICKHOUSE +// ClickHouse Tables + +#define ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS_141 "CREATE TABLE clickhouse_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username))" + +#define ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS_141 + +#define ADMIN_SQLITE_TABLE_RUNTIME_CLICKHOUSE_USERS "CREATE TABLE runtime_clickhouse_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username))" +#endif /* PROXYSQLCLICKHOUSE */ + + +#define ADMIN_SQLITE_TABLE_STATS_MYSQL_PREPARED_STATEMENTS_INFO "CREATE TABLE stats_mysql_prepared_statements_info (global_stmt_id INT NOT NULL , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , digest VARCHAR NOT NULL , ref_count_client INT NOT NULL , ref_count_server INT NOT NULL , num_columns INT NOT NULL, num_params INT NOT NULL, query VARCHAR NOT NULL)" + + +// PgSQL Admin tables +#define ADMIN_SQLITE_TABLE_PGSQL_SERVERS "CREATE TABLE pgsql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 5432 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define ADMIN_SQLITE_TABLE_PGSQL_USERS "CREATE TABLE pgsql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" +#define ADMIN_SQLITE_TABLE_PGSQL_LDAP_MAPPING "CREATE TABLE pgsql_ldap_mapping (priority INTEGER CHECK (priority >= 1 AND priority <= 1000000) PRIMARY KEY , frontend_entity VARCHAR NOT NULL , backend_entity VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (frontend_entity))" +#define ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES "CREATE TABLE pgsql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , database VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535) , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0) , replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR)" +#define ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_USERS "CREATE TABLE pgsql_firewall_whitelist_users (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , mode VARCHAR CHECK (mode IN ('OFF','DETECTING','PROTECTING')) NOT NULL DEFAULT ('OFF') , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address) )" +#define ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_RULES "CREATE TABLE pgsql_firewall_whitelist_rules (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , database VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , digest VARCHAR NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address, database, flagIN, digest) )" +#define ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS "CREATE TABLE pgsql_firewall_whitelist_sqli_fingerprints (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , fingerprint VARCHAR NOT NULL , PRIMARY KEY (fingerprint) )" +#define ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES_FAST_ROUTING "CREATE TABLE pgsql_query_rules_fast_routing (username VARCHAR NOT NULL , database VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , destination_hostgroup INT CHECK (destination_hostgroup >= 0) NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, database, flagIN) )" +#define ADMIN_SQLITE_TABLE_PGSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE pgsql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" +#define ADMIN_SQLITE_TABLE_PGSQL_REPLICATION_HOSTGROUPS "CREATE TABLE pgsql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only','read_only|innodb_read_only','read_only&innodb_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" + +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_SERVERS "CREATE TABLE runtime_pgsql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 5432 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_USERS "CREATE TABLE runtime_pgsql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '', comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_LDAP_MAPPING "CREATE TABLE runtime_pgsql_ldap_mapping (priority INTEGER PRIMARY KEY NOT NULL , frontend_entity VARCHAR NOT NULL , backend_entity VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (frontend_entity))" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_QUERY_RULES "CREATE TABLE runtime_pgsql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , database VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR)" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_FIREWALL_WHITELIST_USERS "CREATE TABLE runtime_pgsql_firewall_whitelist_users (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , mode VARCHAR CHECK (mode IN ('OFF','DETECTING','PROTECTING')) NOT NULL DEFAULT ('OFF') , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address) )" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_FIREWALL_WHITELIST_RULES "CREATE TABLE runtime_pgsql_firewall_whitelist_rules (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , database VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , digest VARCHAR NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address, database, flagIN, digest) )" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS "CREATE TABLE runtime_pgsql_firewall_whitelist_sqli_fingerprints (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , fingerprint VARCHAR NOT NULL , PRIMARY KEY (fingerprint) )" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_QUERY_RULES_FAST_ROUTING "CREATE TABLE runtime_pgsql_query_rules_fast_routing (username VARCHAR NOT NULL , database VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , destination_hostgroup INT CHECK (destination_hostgroup >= 0) NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, database, flagIN) )" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE runtime_pgsql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" +#define ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_pgsql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only','read_only|innodb_read_only','read_only&innodb_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" + +/* +#define STATS_SQLITE_TABLE_PGSQL_QUERY_RULES "CREATE TABLE stats_pgsql_query_rules (rule_id INTEGER PRIMARY KEY , hits INT NOT NULL)" +#define STATS_SQLITE_TABLE_PGSQL_COMMANDS_COUNTERS "CREATE TABLE stats_pgsql_commands_counters (Command VARCHAR NOT NULL PRIMARY KEY , Total_Time_us INT NOT NULL , Total_cnt INT NOT NULL , cnt_100us INT NOT NULL , cnt_500us INT NOT NULL , cnt_1ms INT NOT NULL , cnt_5ms INT NOT NULL , cnt_10ms INT NOT NULL , cnt_50ms INT NOT NULL , cnt_100ms INT NOT NULL , cnt_500ms INT NOT NULL , cnt_1s INT NOT NULL , cnt_5s INT NOT NULL , cnt_10s INT NOT NULL , cnt_INFs)" +#define STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST "CREATE TABLE stats_pgsql_query_digest (hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, schemaname, username, client_address, digest))" +#define STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST_RESET "CREATE TABLE stats_pgsql_query_digest_reset (hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, schemaname, username, client_address, digest))" +*/ + +#define STATS_SQLITE_TABLE_PGSQL_GLOBAL "CREATE TABLE stats_pgsql_global (Variable_Name VARCHAR NOT NULL PRIMARY KEY , Variable_Value VARCHAR NOT NULL)" +#define STATS_SQLITE_TABLE_PGSQL_CONNECTION_POOL "CREATE TABLE stats_pgsql_connection_pool (hostgroup INT , srv_host VARCHAR , srv_port INT , status VARCHAR , ConnUsed INT , ConnFree INT , ConnOK INT , ConnERR INT , MaxConnUsed INT , Queries INT , Bytes_data_sent INT , Bytes_data_recv INT , Latency_us INT)" +#define STATS_SQLITE_TABLE_PGSQL_CONNECTION_POOL_RESET "CREATE TABLE stats_pgsql_connection_pool_reset (hostgroup INT , srv_host VARCHAR , srv_port INT , status VARCHAR , ConnUsed INT , ConnFree INT , ConnOK INT , ConnERR INT , MaxConnUsed INT , Queries INT , Bytes_data_sent INT , Bytes_data_recv INT , Latency_us INT)" +#define STATS_SQLITE_TABLE_PGSQL_FREE_CONNECTIONS "CREATE TABLE stats_pgsql_free_connections (fd INT NOT NULL , hostgroup INT NOT NULL , srv_host VARCHAR NOT NULL , srv_port INT NOT NULL , user VARCHAR NOT NULL , database VARCHAR , init_connect VARCHAR , time_zone VARCHAR , sql_mode VARCHAR , idle_ms INT , statistics VARCHAR , pgsql_info VARCHAR)" +#define STATS_SQLITE_TABLE_PGSQL_USERS "CREATE TABLE stats_pgsql_users (username VARCHAR PRIMARY KEY , frontend_connections INT NOT NULL , frontend_max_connections INT NOT NULL)" +#define STATS_SQLITE_TABLE_PGSQL_PROCESSLIST "CREATE TABLE stats_pgsql_processlist (ThreadID INT NOT NULL , SessionID INTEGER PRIMARY KEY , user VARCHAR , database VARCHAR , cli_host VARCHAR , cli_port INT , hostgroup INT , l_srv_host VARCHAR , l_srv_port INT , srv_host VARCHAR , srv_port INT , command VARCHAR , time_ms INT NOT NULL , info VARCHAR , status_flags INT , extended_info VARCHAR)" +#define STATS_SQLITE_TABLE_PGSQL_ERRORS "CREATE TABLE stats_pgsql_errors (hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , database VARCHAR NOT NULL , sqlstate VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , last_error VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup, hostname, port, username, database, sqlstate) )" +#define STATS_SQLITE_TABLE_PGSQL_ERRORS_RESET "CREATE TABLE stats_pgsql_errors_reset (hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , database VARCHAR NOT NULL , sqlstate VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , last_error VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup, hostname, port, username, database, sqlstate) )" +#define STATS_SQLITE_TABLE_PGSQL_CLIENT_HOST_CACHE "CREATE TABLE stats_pgsql_client_host_cache (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)" +#define STATS_SQLITE_TABLE_PGSQL_CLIENT_HOST_CACHE_RESET "CREATE TABLE stats_pgsql_client_host_cache_reset (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)" + +//#define STATS_SQLITE_TABLE_MEMORY_METRICS "CREATE TABLE stats_memory_metrics (Variable_Name VARCHAR NOT NULL PRIMARY KEY , Variable_Value VARCHAR NOT NULL)" + + +#endif // ProxySQL_Admin_Tables_Definitions diff --git a/include/ProxySQL_Cluster.hpp b/include/ProxySQL_Cluster.hpp index 0f37559f5e..05621f543e 100644 --- a/include/ProxySQL_Cluster.hpp +++ b/include/ProxySQL_Cluster.hpp @@ -104,7 +104,6 @@ class ProxySQL_Node_Address { ~ProxySQL_Node_Address(); const char* get_host_address() const; void resolve_hostname(); - void remove_dns_record(); private: char* ip_addr; }; diff --git a/include/ProxySQL_Poll.h b/include/ProxySQL_Poll.h index 5cfa3d1a48..78520ac17e 100644 --- a/include/ProxySQL_Poll.h +++ b/include/ProxySQL_Poll.h @@ -22,6 +22,7 @@ class iface_info { } }; +template class ProxySQL_Poll { private: void shrink(); @@ -31,7 +32,7 @@ class ProxySQL_Poll { unsigned int len; unsigned int size; struct pollfd *fds; - MySQL_Data_Stream **myds; + T **myds; unsigned long long *last_recv; unsigned long long *last_sent; std::atomic bootstrapping_listeners; @@ -43,7 +44,7 @@ class ProxySQL_Poll { ProxySQL_Poll(); ~ProxySQL_Poll(); - void add(uint32_t _events, int _fd, MySQL_Data_Stream *_myds, unsigned long long sent_time); + void add(uint32_t _events, int _fd, T *_myds, unsigned long long sent_time); void remove_index_fast(unsigned int i); int find_index(int fd); }; diff --git a/include/ProxySQL_Statistics.hpp b/include/ProxySQL_Statistics.hpp index 7e356b8196..945564c8c6 100644 --- a/include/ProxySQL_Statistics.hpp +++ b/include/ProxySQL_Statistics.hpp @@ -77,7 +77,18 @@ #define STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE_DAY "CREATE TABLE mysql_query_cache_day (timestamp INT NOT NULL, count_GET INT NOT NULL, count_GET_OK INT NOT NULL, count_SET INT NOT NULL, bytes_IN INT NOT NULL, bytes_OUT INT NOT NULL, Entries_Purged INT NOT NULL, Entries_In_Cache INT NOT NULL, Memory_Bytes INT NOT NULL, PRIMARY KEY (timestamp))" + +#define STATSDB_SQLITE_TABLE_HISTORY_PGSQL_STATUS_VARIABLES_V2_4_0 "CREATE TABLE history_pgsql_status_variables (timestamp INT NOT NULL , variable_id INT NOT NULL , variable_value VARCHAR NOT NULL , PRIMARY KEY (timestamp, variable_id))" + +#define STATSDB_SQLITE_TABLE_HISTORY_PGSQL_STATUS_VARIABLES STATSDB_SQLITE_TABLE_HISTORY_PGSQL_STATUS_VARIABLES_V2_4_0 + +#define STATSDB_SQLITE_TABLE_HISTORY_PGSQL_STATUS_VARIABLES_LOOKUP_V2_4_0 "CREATE TABLE history_pgsql_status_variables_lookup (variable_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, variable_name VARCHAR NOT NULL, UNIQUE (variable_name))" + +#define STATSDB_SQLITE_TABLE_HISTORY_PGSQL_STATUS_VARIABLES_LOOKUP STATSDB_SQLITE_TABLE_HISTORY_PGSQL_STATUS_VARIABLES_LOOKUP_V2_4_0 + + #define STATSDB_SQLITE_TABLE_HISTORY_MYSQL_QUERY_DIGEST "CREATE TABLE history_mysql_query_digest (dump_time INT , hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL)" +#define STATSDB_SQLITE_TABLE_HISTORY_PGSQL_QUERY_DIGEST "CREATE TABLE history_pgsql_query_digest (dump_time INT , hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL)" class ProxySQL_Statistics { SQLite3DB *statsdb_mem; // internal statistics DB diff --git a/include/QP_rule_text.h b/include/QP_rule_text.h index 9cbd7e7368..d3d4eda4c7 100644 --- a/include/QP_rule_text.h +++ b/include/QP_rule_text.h @@ -4,19 +4,22 @@ #define QP_RE_MOD_CASELESS 1 #define QP_RE_MOD_GLOBAL 2 +struct _Query_Processor_rule_t; +typedef struct _Query_Processor_rule_t QP_rule_t; + class QP_rule_text_hitsonly { public: - char **pta; + char** pta; QP_rule_text_hitsonly(QP_rule_t *QPr); ~QP_rule_text_hitsonly(); }; class QP_rule_text { public: - char **pta; + char** pta; int num_fields; - QP_rule_text(QP_rule_t *QPr); - ~QP_rule_text(); + QP_rule_text(); + /*virtual*/ ~QP_rule_text(); }; #endif // CLASS_QR_RULE_H diff --git a/include/c_tokenizer.h b/include/c_tokenizer.h index 4444ca2e6a..cd298b4675 100644 --- a/include/c_tokenizer.h +++ b/include/c_tokenizer.h @@ -23,6 +23,20 @@ tokenizer_t; enum { TOKENIZER_EMPTIES_OK, TOKENIZER_NO_EMPTIES }; +/** + * @brief Struct for holding all the configuration options used for query digests generation. + */ +typedef struct _options { + bool lowercase; + bool replace_null; + bool replace_number; + bool keep_comment; + int grouping_limit; + int groups_grouping_limit; + int max_query_length; +} options; + + #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ @@ -34,8 +48,9 @@ char * mysql_query_digest_first_stage(const char* const q, int q_len, char** con char * mysql_query_digest_second_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf); char * mysql_query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf); char * mysql_query_digest_and_first_comment_one_it(char *s , int len , char **first_comment, char *buf); -char * mysql_query_strip_comments(char *s , int len); void c_split_2(const char *in, const char *del, char **out1, char **out2); +char * query_strip_comments(char* s, int len, bool lowercase); +char * query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf, const options* opts); #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/include/cpp.h b/include/cpp.h index cb4c1dd8cd..73a9b078cb 100644 --- a/include/cpp.h +++ b/include/cpp.h @@ -1,7 +1,13 @@ +#ifndef PROXYSQL_CPP_H +#define PROXYSQL_CPP_H #include "gen_utils.h" +#include "PgSQL_Thread.h" #include "MySQL_Thread.h" +#include "Base_Session.h" #include "MySQL_Session.h" +#include "PgSQL_Session.h" #include "mysql_backend.h" +#include "PgSQL_Backend.h" #include "ProxySQL_Poll.h" //#include "MySQL_Data_Stream.h" #include "query_cache.hpp" @@ -17,13 +23,16 @@ #endif /* PROXYSQLCLICKHOUSE */ #include "fileutils.hpp" #include "configfile.hpp" -//#include "query_processor.h" -#include "proxysql_admin.h" + //#include "SQLite3_Server.h" #ifdef PROXYSQLCLICKHOUSE #include "ClickHouse_Server.h" #endif /* PROXYSQLCLICKHOUSE */ #include "MySQL_HostGroups_Manager.h" +#include "PgSQL_HostGroups_Manager.h" +#include "PgSQL_Connection.h" +#include "proxysql_admin.h" + //#include "MySQL_Logger.hpp" //#include "MySQL_PreparedStatement.h" //#include "ProxySQL_Cluster.hpp" // cluster @@ -35,3 +44,4 @@ #include #include #include +#endif // PROXYSQL_CPP_H diff --git a/include/mysql_connection.h b/include/mysql_connection.h index f88e41eb35..7eca01825c 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -4,8 +4,13 @@ #include "proxysql.h" #include "cpp.h" -#include "../deps/json/json.hpp" -using json = nlohmann::json; +//#include "../deps/json/json.hpp" +//using json = nlohmann::json; + +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON //#define STATUS_MYSQL_CONNECTION_TRANSACTION 0x00000001 // DEPRECATED #define STATUS_MYSQL_CONNECTION_COMPRESSION 0x00000002 @@ -26,8 +31,8 @@ class MySQLServers_SslParams; class Variable { public: char *value = (char*)""; - void fill_server_internal_session(json &j, int idx); - void fill_client_internal_session(json &j, int idx); + void fill_server_internal_session(nlohmann::json &j, int idx); + void fill_client_internal_session(nlohmann::json &j, int idx); }; enum charset_action { @@ -44,6 +49,7 @@ class MySQL_Connection_userinfo { uint64_t hash; char *username; char *password; + PASSWORD_TYPE::E passtype; char *schemaname; char *sha1_pass; char *fe_username; @@ -264,7 +270,7 @@ class MySQL_Connection { unsigned long get_mysql_thread_id() { return mysql ? mysql->thread_id : 0; } static void set_ssl_params(MYSQL *mysql, MySQLServers_SslParams *ssl_params); - void get_mysql_info_json(json&); - void get_backend_conn_info_json(json&); + void get_mysql_info_json(nlohmann::json&); + void get_backend_conn_info_json(nlohmann::json&); }; #endif /* __CLASS_MYSQL_CONNECTION_H */ diff --git a/include/proxysql.h b/include/proxysql.h index 8b6d1d4d50..0af0ca3962 100644 --- a/include/proxysql.h +++ b/include/proxysql.h @@ -52,7 +52,6 @@ #include "mysql.h" #include "mariadb_com.h" - #include "proxysql_mem.h" #include "proxysql_structs.h" @@ -70,16 +69,7 @@ #endif // __APPLE__ and __MACH__ #endif // NOJEM -#ifdef DEBUG -//#define VALGRIND_ENABLE_ERROR_REPORTING -//#define VALGRIND_DISABLE_ERROR_REPORTING -#include "valgrind.h" -#else -#define VALGRIND_ENABLE_ERROR_REPORTING -#define VALGRIND_DISABLE_ERROR_REPORTING -#endif /* DEBUG */ - -#include "sqlite3.h" +//#include "sqlite3.h" #include "c_tokenizer.h" diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 81f7aae6ea..e42cd406cd 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -5,7 +5,7 @@ #include "prometheus/counter.h" #include "prometheus/gauge.h" -#include "query_processor.h" +//#include "query_processor.h" #include "proxy_defines.h" #include "proxysql.h" #include "cpp.h" @@ -21,6 +21,11 @@ typedef struct { uint32_t hash; uint32_t key; } t_symstruct; class ProxySQL_Config; class ProxySQL_Restapi; +enum SERVER_TYPE { + SERVER_TYPE_MYSQL, + SERVER_TYPE_PGSQL +}; + class Scheduler_Row { public: unsigned int id; @@ -93,6 +98,7 @@ struct p_admin_gauge { fds_in_use, version_info, mysql_listener_paused, + pgsql_listener_paused, __size }; }; @@ -186,6 +192,52 @@ struct peer_mysql_servers_v2_t { peer_mysql_servers_v2_t(SQLite3_result*, const mysql_servers_v2_checksum_t&); }; + +struct incoming_pgsql_servers_t { + SQLite3_result* incoming_pgsql_servers_v2 = NULL; + SQLite3_result* runtime_pgsql_servers = NULL; + SQLite3_result* incoming_replication_hostgroups = NULL; + SQLite3_result* incoming_hostgroup_attributes = NULL; + + incoming_pgsql_servers_t(); + incoming_pgsql_servers_t(SQLite3_result*, SQLite3_result*, SQLite3_result*, SQLite3_result*); +}; + +// Separate structs for runtime pgsql server and pgsql server v2 to avoid human error +struct runtime_pgsql_servers_checksum_t { + std::string value; + time_t epoch; + + runtime_pgsql_servers_checksum_t(); + runtime_pgsql_servers_checksum_t(const std::string& value, time_t epoch); +}; + +struct pgsql_servers_v2_checksum_t { + std::string value; + time_t epoch; + + pgsql_servers_v2_checksum_t(); + pgsql_servers_v2_checksum_t(const std::string& value, time_t epoch); +}; +// + +struct peer_runtime_pgsql_servers_t { + SQLite3_result* resultset{ nullptr }; + runtime_pgsql_servers_checksum_t checksum{}; + + peer_runtime_pgsql_servers_t(); + peer_runtime_pgsql_servers_t(SQLite3_result*, const runtime_pgsql_servers_checksum_t&); +}; + +struct peer_pgsql_servers_v2_t { + SQLite3_result* resultset{ nullptr }; + pgsql_servers_v2_checksum_t checksum{}; + + peer_pgsql_servers_v2_t(); + peer_pgsql_servers_v2_t(SQLite3_result*, const pgsql_servers_v2_checksum_t&); +}; + + class ProxySQL_Admin { private: volatile int main_shutdown; @@ -214,6 +266,12 @@ class ProxySQL_Admin { rwlock_t mysql_servers_rwlock; #endif +#ifdef PA_PTHREAD_MUTEX + pthread_mutex_t pgsql_servers_lock; +#else + rwlock_t pgsql_servers_rwlock; +#endif + prometheus::SerialExposer serial_exposer; std::mutex proxysql_servers_mutex; @@ -226,6 +284,7 @@ class ProxySQL_Admin { char *stats_credentials; int refresh_interval; char *mysql_ifaces; + char *pgsql_ifaces; char *telnet_admin_ifaces; char *telnet_stats_ifaces; bool admin_read_only; @@ -258,6 +317,7 @@ class ProxySQL_Admin { int stats_system_cpu; int stats_system_memory; int mysql_show_processlist_extended; + int pgsql_show_processlist_extended; bool restapi_enabled; bool restapi_enabled_old; int restapi_port; @@ -331,8 +391,11 @@ class ProxySQL_Admin { * param 'resultset' is supplied, it will match it's value, otherwise it will be a locally created * 'SQLite3_result*' that should be freed. */ + template SQLite3_result* __add_active_users(enum cred_username_type usertype, char *user=NULL, SQLite3_result* resultset = nullptr); + template void __delete_inactive_users(enum cred_username_type usertype); + template void add_admin_users(); void __refresh_users(std::unique_ptr&& all_users = nullptr, const std::string& checksum = "", const time_t epoch = 0); void __add_active_users_ldap(); @@ -363,10 +426,14 @@ class ProxySQL_Admin { void disk_upgrade_rest_api_routes(); #ifdef DEBUG + template void add_credentials(char *type, char *credentials, int hostgroup_id); + template void delete_credentials(char *type, char *credentials); #else + template void add_credentials(char *credentials, int hostgroup_id); + template void delete_credentials(char *credentials); #endif /* DEBUG */ @@ -379,6 +446,14 @@ class ProxySQL_Admin { void flush_clickhouse_variables___database_to_runtime(SQLite3DB *db, bool replace); #endif /* PROXYSQLCLICKHOUSE */ + // PostgreSQL + void __refresh_pgsql_users(std::unique_ptr&& pgsql_users_resultset = nullptr, const std::string& checksum = "", const time_t epoch = 0); + //void __add_active_pgsql_users(char* user = NULL); + //void __delete_inactive_pgsql_users(); + void flush_pgsql_variables___runtime_to_database(SQLite3DB* db, bool replace, bool del, bool onlyifempty, bool runtime = false, bool use_lock = true); + void flush_pgsql_variables___database_to_runtime(SQLite3DB* db, bool replace, const std::string& checksum = "", const time_t epoch = 0); + // + void flush_sqliteserver_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime=false); void flush_sqliteserver_variables___database_to_runtime(SQLite3DB *db, bool replace); @@ -405,8 +480,9 @@ class ProxySQL_Admin { bool checksum_admin_variables; bool checksum_ldap_variables; } checksum_variables; + template void public_add_active_users(enum cred_username_type usertype, char *user=NULL) { - __add_active_users(usertype, user); + __add_active_users(usertype, user); } // @brief True if all ProxySQL modules have been already started. End of 'phase3'. bool all_modules_started; @@ -480,8 +556,11 @@ class ProxySQL_Admin { void save_mysql_servers_runtime_to_database(bool _runtime); void admin_shutdown(); bool is_command(std::string); - void send_MySQL_OK(MySQL_Protocol *myprot, char *msg, int rows=0); - void send_MySQL_ERR(MySQL_Protocol *myprot, char *msg, uint32_t code=1045); + + template + void send_ok_msg_to_client(S* sess, const char *msg, int rows, const char* query); + template + void send_error_msg_to_client(S* sess, const char *msg, uint16_t mysql_err_code=1045); #ifdef DEBUG // these two following functions used to just call and return one function each // this approach was replaced when we introduced debug filters @@ -531,12 +610,14 @@ class ProxySQL_Admin { char* load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_query_rules_resultset=NULL, SQLite3_result* SQLite3_query_rules_fast_routing_resultset=NULL, const std::string& checksum = "", const time_t epoch = 0); void save_mysql_query_rules_from_runtime(bool); void save_mysql_query_rules_fast_routing_from_runtime(bool); - char * load_mysql_firewall_to_runtime(); + char* load_mysql_firewall_to_runtime(); void save_mysql_firewall_from_runtime(bool); void save_mysql_firewall_whitelist_users_from_runtime(bool, SQLite3_result *); void save_mysql_firewall_whitelist_rules_from_runtime(bool, SQLite3_result *); void save_mysql_firewall_whitelist_sqli_fingerprints_from_runtime(bool, SQLite3_result *); + char* load_pgsql_firewall_to_runtime(); + void load_scheduler_to_runtime(); void save_scheduler_runtime_to_database(bool); @@ -569,6 +650,18 @@ class ProxySQL_Admin { void stats___mysql_global(); void stats___mysql_users(); + void stats___pgsql_global(); + void stats___pgsql_users(); + void stats___pgsql_free_connections(); + void stats___pgsql_connection_pool(bool _reset); + void stats___pgsql_processlist(); + void stats___pgsql_errors(bool reset); + void stats___pgsql_client_host_cache(bool reset); + int stats___save_pgsql_query_digest_to_sqlite( + const bool reset, const bool copy, const SQLite3_result* resultset, + const umap_query_digest* digest_umap, const umap_query_digest_text* digest_text_umap + ); + void stats___proxysql_servers_checksums(); void stats___proxysql_servers_metrics(); void stats___proxysql_message_metrics(bool reset); @@ -591,6 +684,9 @@ class ProxySQL_Admin { void mysql_servers_wrlock(); void mysql_servers_wrunlock(); + void pgsql_servers_wrlock(); + void pgsql_servers_wrunlock(); + char *get_variable(char *name); // wrapper to call a private function @@ -628,10 +724,47 @@ class ProxySQL_Admin { void save_clickhouse_users_runtime_to_database(bool _runtime); #endif /* PROXYSQLCLICKHOUSE */ + //PostgreSQL + void init_pgsql_servers(); + void init_pgsql_query_rules(); + void init_pgsql_firewall(); + + void init_pgsql_variables(); + void load_pgsql_variables_to_runtime(const std::string& checksum = "", const time_t epoch = 0) { flush_pgsql_variables___database_to_runtime(admindb, true, checksum, epoch); } + void save_pgsql_variables_from_runtime() { flush_pgsql_variables___runtime_to_database(admindb, true, true, false); } + void init_pgsql_users(std::unique_ptr&& pgsql_users_resultset = nullptr, const std::string& checksum = "", const time_t epoch = 0); + void flush_pgsql_users__from_memory_to_disk(); + void flush_pgsql_users__from_disk_to_memory(); + + void save_pgsql_users_runtime_to_database(bool _runtime); + + void load_pgsql_servers_to_runtime(const incoming_pgsql_servers_t& incoming_pgsql_servers = {}, const runtime_pgsql_servers_checksum_t& peer_runtime_pgsql_server = {}, + const pgsql_servers_v2_checksum_t& peer_pgsql_server_v2 = {}); + + char* load_pgsql_query_rules_to_runtime(SQLite3_result* SQLite3_query_rules_resultset = NULL, + SQLite3_result* SQLite3_query_rules_fast_routing_resultset = NULL, const std::string& checksum = "", const time_t epoch = 0); + + void save_pgsql_servers_runtime_to_database(bool _runtime); + void save_pgsql_firewall_from_runtime(bool); + void save_pgsql_query_rules_from_runtime(bool); + void save_pgsql_query_rules_fast_routing_from_runtime(bool); + void save_pgsql_firewall_whitelist_users_from_runtime(bool, SQLite3_result*); + void save_pgsql_firewall_whitelist_rules_from_runtime(bool, SQLite3_result*); + void save_pgsql_firewall_whitelist_sqli_fingerprints_from_runtime(bool, SQLite3_result*); + + void save_pgsql_ldap_mapping_runtime_to_database(bool); + // + void vacuum_stats(bool); + + template int FlushDigestTableToDisk(SQLite3DB *); bool ProxySQL_Test___Load_MySQL_Whitelist(int *, int *, int, int); + void map_test_mysql_firewall_whitelist_rules_cleanup(); + + template + void ProxySQL_Test_Handler(ProxySQL_Admin *SPA, S* sess, char *query_no_space, bool& run_query); #ifdef TEST_AURORA @@ -665,7 +798,8 @@ class ProxySQL_Admin { unsigned long long ProxySQL_Test___MySQL_HostGroups_Manager_Balancing_HG5211(); bool ProxySQL_Test___CA_Certificate_Load_And_Verify(uint64_t* duration, int cnt, const char* cacert, const char* capath); #endif - friend void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt); + template + friend void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt); // FLUSH LOGS void flush_logs(); diff --git a/include/proxysql_config.h b/include/proxysql_config.h index 1a98cce359..cf0d31b3de 100644 --- a/include/proxysql_config.h +++ b/include/proxysql_config.h @@ -16,6 +16,9 @@ class ProxySQL_Config { int Read_MySQL_Users_from_configfile(); int Read_MySQL_Query_Rules_from_configfile(); int Read_MySQL_Servers_from_configfile(); + int Read_PgSQL_Servers_from_configfile(); + int Read_PgSQL_Users_from_configfile(); + int Read_PgSQL_Query_Rules_from_configfile(); int Read_Scheduler_from_configfile(); int Read_Restapi_from_configfile(); int Read_ProxySQL_Servers_from_configfile(); @@ -25,6 +28,9 @@ class ProxySQL_Config { int Write_MySQL_Users_to_configfile(std::string& data); int Write_MySQL_Query_Rules_to_configfile(std::string& data); int Write_MySQL_Servers_to_configfile(std::string& data); + int Write_PgSQL_Servers_to_configfile(std::string& data); + int Write_PgSQL_Users_to_configfile(std::string& data); + int Write_PgSQL_Query_Rules_to_configfile(std::string& data); int Write_Scheduler_to_configfile(std::string& data); int Write_Restapi_to_configfile(std::string& data); int Write_ProxySQL_Servers_to_configfile(std::string& data); diff --git a/include/proxysql_debug.h b/include/proxysql_debug.h index bf9fa013c7..c6416497ea 100644 --- a/include/proxysql_debug.h +++ b/include/proxysql_debug.h @@ -12,6 +12,38 @@ extern int gdbg; #ifndef __PROXYSQL_DEBUG_H #define __PROXYSQL_DEBUG_H +#include +#include +#include + +#include "proxysql_macros.h" + +#if ENABLE_TIMER // this is defined in proxysql_macros.h +class TimerCount { + public: + std::chrono::duration Timer = std::chrono::seconds(0); + unsigned int Count = 0; +}; + +class Timer { + public: + Timer(TimerCount& tc) : totalTime(tc.Timer) { + start = std::chrono::high_resolution_clock::now(); + tc.Count++; + } + + ~Timer() { + auto end = std::chrono::high_resolution_clock::now(); + std::chrono::duration elapsed = end - start; + totalTime += elapsed; + } + private: + //std::atomic>& totalTime; // If using atomic , use this instead + std::chrono::duration& totalTime; + std::chrono::time_point start; +}; +#endif // ENABLE_TIMER + #ifdef DEBUG #define PROXY_TRACE() { proxy_debug(PROXY_DEBUG_GENERIC,10,"TRACE\n"); } //#define PROXY_TRACE2() { proxy_info("TRACE\n"); } diff --git a/include/proxysql_glovars.hpp b/include/proxysql_glovars.hpp index b2b2066fc3..68a51f25f7 100644 --- a/include/proxysql_glovars.hpp +++ b/include/proxysql_glovars.hpp @@ -1,7 +1,7 @@ #ifndef __CLASS_PROXYSQL_GLOVARS_H #define __CLASS_PROXYSQL_GLOVARS_H -#define CLUSTER_SYNC_INTERFACES_ADMIN "('admin-mysql_ifaces','admin-restapi_port','admin-telnet_admin_ifaces','admin-telnet_stats_ifaces','admin-web_port')" +#define CLUSTER_SYNC_INTERFACES_ADMIN "('admin-mysql_ifaces','admin-restapi_port','admin-telnet_admin_ifaces','admin-telnet_stats_ifaces','admin-web_port','admin-pgsql_ifaces')" #define CLUSTER_SYNC_INTERFACES_MYSQL "('mysql-interfaces')" #include @@ -144,6 +144,7 @@ class ProxySQL_GlobalVariables { }; struct { unsigned long stack_memory_mysql_threads; + unsigned long stack_memory_pgsql_threads; unsigned long stack_memory_admin_threads; unsigned long stack_memory_cluster_threads; } statuses; @@ -158,6 +159,11 @@ class ProxySQL_GlobalVariables { ProxySQL_Checksum_Value ldap_variables; ProxySQL_Checksum_Value proxysql_servers; ProxySQL_Checksum_Value mysql_servers_v2; + ProxySQL_Checksum_Value pgsql_query_rules; + ProxySQL_Checksum_Value pgsql_servers; + ProxySQL_Checksum_Value pgsql_users; + ProxySQL_Checksum_Value pgsql_variables; + ProxySQL_Checksum_Value pgsql_servers_v2; uint64_t global_checksum; unsigned long long updates_cnt; unsigned long long dumped_at; diff --git a/include/proxysql_macros.h b/include/proxysql_macros.h index d7090b1c61..f83cca7360 100644 --- a/include/proxysql_macros.h +++ b/include/proxysql_macros.h @@ -1,4 +1,5 @@ - +#ifndef PROXYSQL_MACROS_H +#define PROXYSQL_MACROS_H #define strdup_null(__c) ( __c ? strdup(__c) : __c ) #define char_malloc (char *)malloc #define free_null(__c) { if(__c) { free(__c); __c=NULL; } } @@ -65,3 +66,8 @@ # define unlikely(x) !!(x) #endif #endif /* PROXYSQL_LIKELY */ + +#ifndef ENABLE_TIMER +#define ENABLE_TIMER false +#endif // ENABLE_TIMER +#endif // PROXYSQL_MACROS_H diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index 0762e50cab..268f5d27e8 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -10,6 +10,10 @@ #ifndef PROXYSQL_ENUMS #define PROXYSQL_ENUMS +#define MIN_POLL_LEN 8 +#define MIN_POLL_DELETE_RATIO 8 +#define MY_EPOLL_THREAD_MAXEVENTS 128 + enum MySerStatus { MYSQL_SERVER_STATUS_ONLINE, MYSQL_SERVER_STATUS_SHUNNED, @@ -43,7 +47,7 @@ enum cred_username_type { USERNAME_BACKEND, USERNAME_FRONTEND, USERNAME_NONE }; #define PROXYSQL_USE_RESULT -enum MDB_ASYNC_ST { // MariaDB Async State Machine +enum ASYNC_ST { // MariaDB Async State Machine ASYNC_CONNECT_START, ASYNC_CONNECT_CONT, ASYNC_CONNECT_END, @@ -114,10 +118,20 @@ enum MDB_ASYNC_ST { // MariaDB Async State Machine ASYNC_CLOSE_START, ASYNC_CLOSE_CONT, ASYNC_CLOSE_END, + ASYNC_RESET_SESSION_START, + ASYNC_RESET_SESSION_CONT, + ASYNC_RESET_SESSION_END, + ASYNC_RESET_SESSION_SUCCESSFUL, + ASYNC_RESET_SESSION_FAILED, + ASYNC_RESET_SESSION_TIMEOUT, ASYNC_IDLE }; +using MDB_ASYNC_ST = ASYNC_ST; +using PG_ASYNC_ST = ASYNC_ST; + + // list of possible debugging modules enum debug_module { PROXY_DEBUG_GENERIC, @@ -244,6 +258,7 @@ enum session_status { CHANGING_USER_CLIENT, CHANGING_USER_SERVER, RESETTING_CONNECTION, + RESETTING_CONNECTION_V2, SETTING_INIT_CONNECT, SETTING_LDAP_USER_VARIABLE, SETTING_ISOLATION_LEVEL, @@ -445,6 +460,161 @@ enum MYSQL_COM_QUERY_command { MYSQL_COM_QUERY___NONE // Special marker. }; +enum PGSQL_QUERY_command { + PGSQL_QUERY_SELECT, + PGSQL_QUERY_INSERT, + PGSQL_QUERY_UPDATE, + PGSQL_QUERY_DELETE, + PGSQL_QUERY_MERGE, + PGSQL_QUERY_CREATE_TABLE, + PGSQL_QUERY_ALTER_TABLE, + PGSQL_QUERY_DROP_TABLE, + PGSQL_QUERY_TRUNCATE, + PGSQL_QUERY_COPY, + PGSQL_QUERY_CREATE_INDEX, + PGSQL_QUERY_DROP_INDEX, + PGSQL_QUERY_ALTER_INDEX, + PGSQL_QUERY_CREATE_VIEW, + PGSQL_QUERY_DROP_VIEW, + PGSQL_QUERY_ALTER_VIEW, + PGSQL_QUERY_CREATE_MATERIALIZED_VIEW, + PGSQL_QUERY_ALTER_MATERIALIZED_VIEW, + PGSQL_QUERY_REFRESH_MATERIALIZED_VIEW, + PGSQL_QUERY_DROP_MATERIALIZED_VIEW, + PGSQL_QUERY_CREATE_SEQUENCE, + PGSQL_QUERY_ALTER_SEQUENCE, + PGSQL_QUERY_DROP_SEQUENCE, + PGSQL_QUERY_CREATE_SCHEMA, + PGSQL_QUERY_DROP_SCHEMA, + PGSQL_QUERY_ALTER_SCHEMA, + PGSQL_QUERY_CREATE_FUNCTION, + PGSQL_QUERY_ALTER_FUNCTION, + PGSQL_QUERY_DROP_FUNCTION, + PGSQL_QUERY_CREATE_PROCEDURE, + PGSQL_QUERY_ALTER_PROCEDURE, + PGSQL_QUERY_CALL, + PGSQL_QUERY_DROP_PROCEDURE, + PGSQL_QUERY_CREATE_AGGREGATE, + PGSQL_QUERY_ALTER_AGGREGATE, + PGSQL_QUERY_DROP_AGGREGATE, + PGSQL_QUERY_CREATE_OPERATOR, + PGSQL_QUERY_ALTER_OPERATOR, + PGSQL_QUERY_DROP_OPERATOR, + PGSQL_QUERY_CREATE_TYPE, + PGSQL_QUERY_ALTER_TYPE, + PGSQL_QUERY_DROP_TYPE, + PGSQL_QUERY_CREATE_DOMAIN, + PGSQL_QUERY_ALTER_DOMAIN, + PGSQL_QUERY_DROP_DOMAIN, + PGSQL_QUERY_CREATE_TRIGGER, + PGSQL_QUERY_ALTER_TRIGGER, + PGSQL_QUERY_DROP_TRIGGER, + PGSQL_QUERY_CREATE_RULE, + PGSQL_QUERY_ALTER_RULE, + PGSQL_QUERY_DROP_RULE, + PGSQL_QUERY_CREATE_EXTENSION, + PGSQL_QUERY_ALTER_EXTENSION, + PGSQL_QUERY_DROP_EXTENSION, + PGSQL_QUERY_CREATE_POLICY, + PGSQL_QUERY_ALTER_POLICY, + PGSQL_QUERY_DROP_POLICY, + PGSQL_QUERY_CREATE_ROLE, + PGSQL_QUERY_ALTER_ROLE, + PGSQL_QUERY_DROP_ROLE, + PGSQL_QUERY_CREATE_USER, + PGSQL_QUERY_ALTER_USER, + PGSQL_QUERY_DROP_USER, + PGSQL_QUERY_GRANT, + PGSQL_QUERY_REVOKE, + PGSQL_QUERY_COMMENT, + PGSQL_QUERY_NOTIFY, + PGSQL_QUERY_LISTEN, + PGSQL_QUERY_UNLISTEN, + PGSQL_QUERY_LOCK, + PGSQL_QUERY_CHECKPOINT, + PGSQL_QUERY_REINDEX, + PGSQL_QUERY_VACUUM, + PGSQL_QUERY_ANALYZE, + PGSQL_QUERY_EXPLAIN, + PGSQL_QUERY_EXECUTE, + PGSQL_QUERY_PREPARE, + PGSQL_QUERY_DEALLOCATE, + PGSQL_QUERY_FETCH, + PGSQL_QUERY_MOVE, + PGSQL_QUERY_SAVEPOINT, + PGSQL_QUERY_ROLLBACK_TO_SAVEPOINT, + PGSQL_QUERY_RELEASE_SAVEPOINT, + PGSQL_QUERY_BEGIN, + PGSQL_QUERY_COMMIT, + PGSQL_QUERY_ROLLBACK, + PGSQL_QUERY_DECLARE_CURSOR, + PGSQL_QUERY_CLOSE_CURSOR, + PGSQL_QUERY_DISCARD, + PGSQL_QUERY_SHOW, + PGSQL_QUERY_SET, + PGSQL_QUERY_RESET, + PGSQL_QUERY_ALTER_DATABASE, + PGSQL_QUERY_CREATE_DATABASE, + PGSQL_QUERY_DROP_DATABASE, + PGSQL_QUERY_CREATE_COLLATION, + PGSQL_QUERY_ALTER_COLLATION, + PGSQL_QUERY_DROP_COLLATION, + PGSQL_QUERY_CREATE_TEXT_SEARCH_CONFIGURATION, + PGSQL_QUERY_ALTER_TEXT_SEARCH_CONFIGURATION, + PGSQL_QUERY_DROP_TEXT_SEARCH_CONFIGURATION, + PGSQL_QUERY_CREATE_TEXT_SEARCH_DICTIONARY, + PGSQL_QUERY_ALTER_TEXT_SEARCH_DICTIONARY, + PGSQL_QUERY_DROP_TEXT_SEARCH_DICTIONARY, + PGSQL_QUERY_CREATE_TEXT_SEARCH_TEMPLATE, + PGSQL_QUERY_ALTER_TEXT_SEARCH_TEMPLATE, + PGSQL_QUERY_DROP_TEXT_SEARCH_TEMPLATE, + PGSQL_QUERY_CREATE_TEXT_SEARCH_PARSER, + PGSQL_QUERY_ALTER_TEXT_SEARCH_PARSER, + PGSQL_QUERY_DROP_TEXT_SEARCH_PARSER, + PGSQL_QUERY_CREATE_FOREIGN_TABLE, + PGSQL_QUERY_ALTER_FOREIGN_TABLE, + PGSQL_QUERY_DROP_FOREIGN_TABLE, + PGSQL_QUERY_IMPORT_FOREIGN_SCHEMA, + PGSQL_QUERY_CREATE_SERVER, + PGSQL_QUERY_ALTER_SERVER, + PGSQL_QUERY_DROP_SERVER, + PGSQL_QUERY_CREATE_USER_MAPPING, + PGSQL_QUERY_ALTER_USER_MAPPING, + PGSQL_QUERY_DROP_USER_MAPPING, + PGSQL_QUERY_CREATE_PUBLICATION, + PGSQL_QUERY_ALTER_PUBLICATION, + PGSQL_QUERY_DROP_PUBLICATION, + PGSQL_QUERY_CREATE_SUBSCRIPTION, + PGSQL_QUERY_ALTER_SUBSCRIPTION, + PGSQL_QUERY_DROP_SUBSCRIPTION, + PGSQL_QUERY_CREATE_ACCESS_METHOD, + PGSQL_QUERY_ALTER_ACCESS_METHOD, + PGSQL_QUERY_DROP_ACCESS_METHOD, + PGSQL_QUERY_CREATE_EVENT_TRIGGER, + PGSQL_QUERY_ALTER_EVENT_TRIGGER, + PGSQL_QUERY_DROP_EVENT_TRIGGER, + PGSQL_QUERY_CREATE_TRANSFORM, + PGSQL_QUERY_ALTER_TRANSFORM, + PGSQL_QUERY_DROP_TRANSFORM, + PGSQL_QUERY_CREATE_CAST, + PGSQL_QUERY_ALTER_CAST, + PGSQL_QUERY_DROP_CAST, + PGSQL_QUERY_CREATE_OPERATOR_CLASS, + PGSQL_QUERY_ALTER_OPERATOR_CLASS, + PGSQL_QUERY_DROP_OPERATOR_CLASS, + PGSQL_QUERY_CREATE_OPERATOR_FAMILY, + PGSQL_QUERY_ALTER_OPERATOR_FAMILY, + PGSQL_QUERY_DROP_OPERATOR_FAMILY, + PGSQL_QUERY_CREATE_TABLESPACE, + PGSQL_QUERY_ALTER_TABLESPACE, + PGSQL_QUERY_DROP_TABLESPACE, + PGSQL_QUERY_CLUSTER, + PGSQL_QUERY_UNKNOWN, + PGSQL_QUERY__UNINITIALIZED, + PGSQL_QUERY___NONE // Special marker. +}; + + enum handle_unknown_charset { HANDLE_UNKNOWN_CHARSET__DISCONNECT_CLIENT, HANDLE_UNKNOWN_CHARSET__REPLACE_WITH_DEFAULT_VERBOSE, @@ -479,6 +649,17 @@ enum PROXYSQL_MYSQL_ERR { ER_PROXYSQL_SRV_NULL_REPLICATION_LAG = 9019, }; +enum proxysql_session_type { + PROXYSQL_SESSION_MYSQL, + PROXYSQL_SESSION_ADMIN, + PROXYSQL_SESSION_STATS, + PROXYSQL_SESSION_SQLITE, + PROXYSQL_SESSION_CLICKHOUSE, + PROXYSQL_SESSION_MYSQL_EMU, + PROXYSQL_SESSION_PGSQL, + PROXYSQL_SESSION_NONE +}; + #endif /* PROXYSQL_ENUMS */ @@ -500,6 +681,7 @@ typedef unsigned spinlock; typedef struct _rwlock_t rwlock_t; typedef struct _PtrSize_t PtrSize_t; typedef struct _proxysql_mysql_thread_t proxysql_mysql_thread_t; +typedef struct _proxysql_pgsql_thread_t proxysql_pgsql_thread_t; typedef struct { char * table_name; char * table_def; } table_def_t; typedef struct __SQP_query_parser_t SQP_par_t; #endif /* PROXYSQL_TYPEDEFS */ @@ -508,19 +690,25 @@ typedef struct __SQP_query_parser_t SQP_par_t; #ifndef PROXYSQL_CLASSES #define PROXYSQL_CLASSES class MySQL_Data_Stream; +class PgSQL_Data_Stream; class MySQL_Connection_userinfo; class MySQL_Session; +class PgSQL_Session; class MySQL_Backend; +class PgSQL_Backend; class MySQL_Monitor; +class PgSQL_Thread; class MySQL_Thread; class MySQL_Threads_Handler; class SQLite3DB; class SimpleKV; class AdvancedKV; +template class ProxySQL_Poll; class Query_Cache; class MySQL_Authentication; class MySQL_Connection; +class PgSQL_Connection; class MySQL_Protocol; class PtrArray; class PtrSizeArray; @@ -531,13 +719,15 @@ class SQLite3_result; class stmt_execute_metadata_t; class MySQL_STMTs_meta; class MySQL_HostGroups_Manager; +class PgSQL_HostGroups_Manager; class ProxySQL_HTTP_Server; class MySQL_STMTs_local_v14; class MySQL_STMT_Global_info; class StmtLongDataHandler; class ProxySQL_Cluster; class MySQL_ResultSet; -class Query_Processor_Output; +class MySQL_Query_Processor_Output; +class PgSQL_Query_Processor_Output; class MySrvC; class Web_Interface_plugin; class ProxySQL_Node_Address; @@ -638,6 +828,10 @@ struct _proxysql_mysql_thread_t { pthread_t thread_id; }; +struct _proxysql_pgsql_thread_t { + PgSQL_Thread* worker; + pthread_t thread_id; +}; /* Every communication between client and proxysql, and between proxysql and mysql server is * performed within a mysql_data_stream_t @@ -743,6 +937,14 @@ struct _mysql_session_t { int net_failure; }; +// Enum for primary/secondary password +struct PASSWORD_TYPE { + enum E { + PRIMARY = 0, + ADDITIONAL = 1 + }; +}; + #endif /* PROXYSQL_STRUCTS */ #ifndef EXTERN @@ -778,6 +980,107 @@ ProxySQL_GlobalVariables GloVars {}; #ifndef GLOBAL_DEFINED_HOSTGROUP #define GLOBAL_DEFINED_HOSTGROUP MySQL_HostGroups_Manager *MyHGM; +PgSQL_HostGroups_Manager* PgHGM; + +// PostgreSQL thread variables +__thread int pgsql_thread___authentication_method; +__thread int pgsql_thread___show_processlist_extended; +__thread char *pgsql_thread___server_version; +__thread char *pgsql_thread___default_client_encoding; +__thread bool pgsql_thread___have_ssl; +__thread int pgsql_thread___max_connections; +__thread bool pgsql_thread___use_tcp_keepalive; +__thread int pgsql_thread___tcp_keepalive_time; +__thread int pgsql_thread___throttle_connections_per_sec_to_hostgroup; +__thread int pgsql_thread___max_transaction_idle_time; +__thread int pgsql_thread___max_transaction_time; +__thread int pgsql_thread___threshold_query_length; +__thread int pgsql_thread___threshold_resultset_size; +__thread int pgsql_thread___poll_timeout; +__thread int pgsql_thread___poll_timeout_on_failure; +__thread int pgsql_thread___wait_timeout; +__thread int pgsql_thread___client_host_cache_size; +__thread int pgsql_thread___client_host_error_counts; +__thread int pgsql_thread___connect_retries_on_failure; +__thread int pgsql_thread___connect_retries_delay; +__thread bool pgsql_thread___multiplexing; +__thread int pgsql_thread___connection_delay_multiplex_ms; +__thread int pgsql_thread___connection_max_age_ms; +__thread int pgsql_thread___connect_timeout_client; +__thread int pgsql_thread___connect_timeout_server; +__thread int pgsql_thread___connect_timeout_server_max; +__thread bool pgsql_thread___connection_warming; +__thread bool pgsql_thread___log_unhealthy_connections; +__thread int pgsql_thread___throttle_max_bytes_per_second_to_client; +__thread int pgsql_thread___throttle_ratio_server_to_client; +__thread int pgsql_thread___shun_on_failures; +__thread int pgsql_thread___shun_recovery_time_sec; +__thread int pgsql_thread___hostgroup_manager_verbose; +__thread int pgsql_thread___default_max_latency_ms; +__thread int pgsql_thread___unshun_algorithm; +__thread int pgsql_thread___free_connections_pct; +__thread bool pgsql_thread___kill_backend_connection_when_disconnect; +__thread int pgsql_thread___max_allowed_packet; + +/* variables used for SSL , from proxy to server (p2s) */ +__thread char* pgsql_thread___ssl_p2s_ca; +__thread char* pgsql_thread___ssl_p2s_capath; +__thread char* pgsql_thread___ssl_p2s_cert; +__thread char* pgsql_thread___ssl_p2s_key; +__thread char* pgsql_thread___ssl_p2s_cipher; +__thread char* pgsql_thread___ssl_p2s_crl; +__thread char* pgsql_thread___ssl_p2s_crlpath; + +//__thread char* pgsql_thread___default_schema; + +__thread int pgsql_thread___set_query_lock_on_hostgroup; +__thread bool pgsql_thread___verbose_query_error; +__thread char* pgsql_thread___keep_multiplexing_variables; +__thread int pgsql_thread___session_idle_ms; +__thread int pgsql_thread___long_query_time; +__thread int pgsql_thread___set_parser_algorithm; +__thread bool pgsql_thread___parse_failure_logs_digest; +__thread int pgsql_thread___auto_increment_delay_multiplex; +__thread int pgsql_thread___auto_increment_delay_multiplex_timeout_ms; +__thread int pgsql_thread___default_query_delay; +__thread int pgsql_thread___default_query_timeout; +__thread int pgsql_thread___query_retries_on_failure; +__thread int pgsql_thread___ping_interval_server_msec; +__thread int pgsql_thread___ping_timeout_server; +__thread int pgsql_thread___mirror_max_concurrency; +__thread int pgsql_thread___mirror_max_queue_length; +__thread char* pgsql_thread___init_connect; +__thread bool pgsql_thread___sessions_sort; +__thread bool pgsql_thread___servers_stats; +__thread bool pgsql_thread___default_reconnect; +__thread bool pgsql_thread___automatic_detect_sqli; + +__thread bool pgsql_thread___commands_stats; +__thread bool pgsql_thread___query_digests; +__thread bool pgsql_thread___query_digests_lowercase; +__thread bool pgsql_thread___query_digests_no_digits; +__thread bool pgsql_thread___query_digests_replace_null; +__thread bool pgsql_thread___query_digests_normalize_digest_text; +__thread bool pgsql_thread___query_digests_track_hostname; +__thread bool pgsql_thread___query_digests_keep_comment; +__thread int pgsql_thread___query_digests_max_digest_length; +__thread int pgsql_thread___query_digests_max_query_length; +__thread int pgsql_thread___query_digests_grouping_limit; +__thread int pgsql_thread___query_digests_groups_grouping_limit; + +__thread bool pgsql_thread___enable_load_data_local_infile; +__thread char* pgsql_thread___auditlog_filename; +__thread int pgsql_thread___auditlog_filesize; +__thread char* pgsql_thread___eventslog_filename; +__thread int pgsql_thread___eventslog_filesize; +__thread int pgsql_thread___eventslog_default_log; +__thread int pgsql_thread___eventslog_format; +__thread char* pgsql_thread___firewall_whitelist_errormsg; +__thread bool pgsql_thread___firewall_whitelist_enabled; +__thread int pgsql_thread___query_processor_iterations; +__thread int pgsql_thread___query_processor_regex; +//--------------------------- + __thread char *mysql_thread___default_schema; __thread char *mysql_thread___server_version; __thread char *mysql_thread___keep_multiplexing_variables; @@ -951,6 +1254,105 @@ __thread unsigned int g_seed; #else extern ProxySQL_GlobalVariables GloVars; extern MySQL_HostGroups_Manager *MyHGM; +extern PgSQL_HostGroups_Manager *PgHGM; + +//PostgreSQL Thread Variables +extern __thread int pgsql_thread___authentication_method; +extern __thread int pgsql_thread___show_processlist_extended; +extern __thread char *pgsql_thread___server_version; +extern __thread char* pgsql_thread___default_client_encoding; +extern __thread bool pgsql_thread___have_ssl; +extern __thread int pgsql_thread___max_connections; +extern __thread bool pgsql_thread___use_tcp_keepalive; +extern __thread int pgsql_thread___tcp_keepalive_time; +extern __thread int pgsql_thread___throttle_connections_per_sec_to_hostgroup; +extern __thread int pgsql_thread___max_transaction_idle_time; +extern __thread int pgsql_thread___max_transaction_time; +extern __thread int pgsql_thread___threshold_query_length; +extern __thread int pgsql_thread___threshold_resultset_size; +extern __thread int pgsql_thread___poll_timeout; +extern __thread int pgsql_thread___poll_timeout_on_failure; +extern __thread int pgsql_thread___wait_timeout; +extern __thread int pgsql_thread___client_host_cache_size; +extern __thread int pgsql_thread___client_host_error_counts; +extern __thread int pgsql_thread___connect_retries_on_failure; +extern __thread int pgsql_thread___connect_retries_delay; +extern __thread bool pgsql_thread___multiplexing; +extern __thread int pgsql_thread___connection_delay_multiplex_ms; +extern __thread int pgsql_thread___connection_max_age_ms; +extern __thread int pgsql_thread___connect_timeout_client; +extern __thread int pgsql_thread___connect_timeout_server; +extern __thread int pgsql_thread___connect_timeout_server_max; +extern __thread bool pgsql_thread___connection_warming; +extern __thread bool pgsql_thread___log_unhealthy_connections; +extern __thread int pgsql_thread___throttle_max_bytes_per_second_to_client; +extern __thread int pgsql_thread___throttle_ratio_server_to_client; +extern __thread int pgsql_thread___shun_on_failures; +extern __thread int pgsql_thread___shun_recovery_time_sec; +extern __thread int pgsql_thread___hostgroup_manager_verbose; +extern __thread int pgsql_thread___default_max_latency_ms; +extern __thread int pgsql_thread___unshun_algorithm; +extern __thread int pgsql_thread___free_connections_pct; +extern __thread bool pgsql_thread___kill_backend_connection_when_disconnect; +extern __thread int pgsql_thread___max_allowed_packet; + +extern __thread char* pgsql_thread___ssl_p2s_ca; +extern __thread char* pgsql_thread___ssl_p2s_capath; +extern __thread char* pgsql_thread___ssl_p2s_cert; +extern __thread char* pgsql_thread___ssl_p2s_key; +extern __thread char* pgsql_thread___ssl_p2s_cipher; +extern __thread char* pgsql_thread___ssl_p2s_crl; +extern __thread char* pgsql_thread___ssl_p2s_crlpath; + +//extern __thread char* pgsql_thread___default_schema; +extern __thread int pgsql_thread___set_query_lock_on_hostgroup; +extern __thread bool pgsql_thread___verbose_query_error; +extern __thread char* pgsql_thread___keep_multiplexing_variables; +extern __thread int pgsql_thread___session_idle_ms; +extern __thread int pgsql_thread___long_query_time; +extern __thread int pgsql_thread___set_parser_algorithm; +extern __thread bool pgsql_thread___parse_failure_logs_digest; +extern __thread int pgsql_thread___auto_increment_delay_multiplex; +extern __thread int pgsql_thread___auto_increment_delay_multiplex_timeout_ms; +extern __thread int pgsql_thread___default_query_delay; +extern __thread int pgsql_thread___default_query_timeout; +extern __thread int pgsql_thread___query_retries_on_failure; +extern __thread int pgsql_thread___ping_interval_server_msec; +extern __thread int pgsql_thread___ping_timeout_server; +extern __thread int pgsql_thread___mirror_max_concurrency; +extern __thread int pgsql_thread___mirror_max_queue_length; +extern __thread char* pgsql_thread___init_connect; +extern __thread bool pgsql_thread___sessions_sort; +extern __thread bool pgsql_thread___servers_stats; +extern __thread bool pgsql_thread___default_reconnect; +extern __thread bool pgsql_thread___automatic_detect_sqli; + +extern __thread bool pgsql_thread___commands_stats; +extern __thread bool pgsql_thread___query_digests; +extern __thread bool pgsql_thread___query_digests_lowercase; +extern __thread bool pgsql_thread___query_digests_no_digits; +extern __thread bool pgsql_thread___query_digests_replace_null; +extern __thread bool pgsql_thread___query_digests_normalize_digest_text; +extern __thread bool pgsql_thread___query_digests_track_hostname; +extern __thread bool pgsql_thread___query_digests_keep_comment; +extern __thread int pgsql_thread___query_digests_max_digest_length; +extern __thread int pgsql_thread___query_digests_max_query_length; +extern __thread int pgsql_thread___query_digests_grouping_limit; +extern __thread int pgsql_thread___query_digests_groups_grouping_limit; + +extern __thread bool pgsql_thread___enable_load_data_local_infile; +extern __thread char* pgsql_thread___auditlog_filename; +extern __thread int pgsql_thread___auditlog_filesize; +extern __thread char* pgsql_thread___eventslog_filename; +extern __thread int pgsql_thread___eventslog_filesize; +extern __thread int pgsql_thread___eventslog_default_log; +extern __thread int pgsql_thread___eventslog_format; +extern __thread char* pgsql_thread___firewall_whitelist_errormsg; +extern __thread bool pgsql_thread___firewall_whitelist_enabled; +extern __thread int pgsql_thread___query_processor_iterations; +extern __thread int pgsql_thread___query_processor_regex; +//--------------------------- + extern __thread char *mysql_thread___default_schema; extern __thread char *mysql_thread___server_version; extern __thread char *mysql_thread___keep_multiplexing_variables; diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index 187a7c0f1e..a15031359d 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -14,6 +14,7 @@ #include #include "sqlite3db.h" +#include "../deps/json/json.hpp" #ifndef ProxySQL_Checksum_Value_LENGTH #define ProxySQL_Checksum_Value_LENGTH 20 @@ -259,6 +260,53 @@ void close_all_non_term_fd(std::vector excludeFDs); */ std::pair get_dollar_quote_error(const char* version); +/** + * @brief Extracts a nested JSON element from the supplied path. + * @param j The JSON from which to extract the element. + * @param p The path to trasverse to find the element. + * @return Pointer to the element if found, 'nullptr' otherwise. + */ +const nlohmann::json* get_nested_elem(const nlohmann::json& j, const std::vector& p); + +/** + * @brief Retrieves the value of a nested JSON element given a path of keys. + * + * @tparam T The type of the value to retrieve from the JSON element. + * + * @param j The JSON object to search through. + * @param p A vector of strings representing the path to the nested element. + * @param def_val The default value to return if the nested element is not found, + * is null, or cannot be converted to type `T`. + * + * @return The value of the nested JSON element as type `T` if it exists and is + * of the correct type, otherwise returns `def_val`. + */ +template +T get_nested_elem_val(const nlohmann::json& j, const std::vector& p, const T def_val) { + const nlohmann::json* next_step = get_nested_elem(j, p); + + try { + if (next_step != nullptr && !next_step->is_null()) { + return next_step->get(); + } else { + return def_val; + } + } catch (std::exception&) { + return def_val; + } +} + +/** + * @brief Helper type for freeing memory allocated by 'malloc' managed by smart pointers. + */ +struct free_deleter { + void operator()(void* x) { free(x); } + void operator()(const void* x) { free(const_cast(x)); } +}; + +template +using mf_unique_ptr = std::unique_ptr; + static inline void set_thread_name(const char name[16]) { #if defined(__linux__) || defined(__FreeBSD__) int rc; diff --git a/include/query_processor.h b/include/query_processor.h index 9d62330291..4b95e115d4 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -2,7 +2,7 @@ #define __CLASS_QUERY_PROCESSOR_H #include "proxysql.h" #include "cpp.h" - +#include #include // Optimization introduced in 2.0.6 @@ -10,8 +10,12 @@ #define DIGEST_STATS_FAST_MINSIZE 100000 #define DIGEST_STATS_FAST_THREADS 4 -#include "../deps/json/json.hpp" +//#include "../deps/json/json.hpp" +#ifndef PROXYJSON +#define PROXYJSON +#include "../deps/json/json_fwd.hpp" +#endif // PROXYJSON #include "khash.h" KHASH_MAP_INIT_STR(khStrInt, int) @@ -23,6 +27,19 @@ KHASH_MAP_INIT_STR(khStrInt, int) #define WUS_DETECTING 2 // allow the query but log it #define WUS_PROTECTING 3 // block the query +// Utilizing SFINAE (Substitution Failure Is Not An Error) to check if a class contains a specific method +#define DEFINE_HAS_METHOD_STRUCT(METHODNAME) \ + template \ + class has_##METHODNAME \ + { \ + private: \ + typedef char YesType[1]; \ + typedef char NoType[2]; \ + template static YesType& test(decltype(&C::METHODNAME)); \ + template static NoType& test(...); \ + public: \ + enum { value = sizeof(test(0)) == sizeof(YesType) }; \ + }; typedef struct _query_digest_stats_pointers_t { char *pta[14]; @@ -38,7 +55,6 @@ typedef struct _query_digest_stats_pointers_t { char rows_sent[24]; } query_digest_stats_pointers_t; - class QP_query_digest_stats { public: uint64_t digest; @@ -58,7 +74,8 @@ class QP_query_digest_stats { unsigned long long rows_affected; unsigned long long rows_sent; int hid; - QP_query_digest_stats(char *u, char *s, uint64_t d, char *dt, int h, char *ca); + QP_query_digest_stats(const char* _user, const char* _schema, uint64_t _digest, const char* _digest_text, + int _hid, const char* _client_addr, int query_digests_max_digest_length); void add_time( unsigned long long t, unsigned long long n, unsigned long long ra, unsigned long long rs, unsigned long long cnt = 1 @@ -68,7 +85,7 @@ class QP_query_digest_stats { char **get_row(umap_query_digest_text *digest_text_umap, query_digest_stats_pointers_t *qdsp); }; -struct _Query_Processor_rule_t { +typedef struct _Query_Processor_rule_t { int rule_id; bool active; char *username; @@ -100,21 +117,18 @@ struct _Query_Processor_rule_t { char *OK_msg; int sticky_conn; int multiplex; - int gtid_from_hostgroup; int log; bool apply; char* attributes; - char *comment; // #643 + char *comment; // #643 void *regex_engine1; void *regex_engine2; uint64_t hits; struct _Query_Processor_rule_t *parent; // pointer to parent, to speed up parent update - std::vector * flagOUT_ids; - std::vector * flagOUT_weights; + std::vector* flagOUT_ids; + std::vector* flagOUT_weights; int flagOUT_weights_total; -}; - -typedef struct _Query_Processor_rule_t QP_rule_t; +} QP_rule_t; class Query_Processor_Output { public: @@ -135,13 +149,12 @@ class Query_Processor_Output { char *OK_msg; int sticky_conn; int multiplex; - int gtid_from_hostgroup; long long max_lag_ms; int log; int firewall_whitelist_mode; char *attributes; char *comment; // #643 - char *min_gtid; + bool create_new_conn; std::string *new_query; void * operator new(size_t size) { @@ -151,10 +164,10 @@ class Query_Processor_Output { l_free(sizeof(Query_Processor_Output),ptr); } Query_Processor_Output() { - init(); + //init(); } ~Query_Processor_Output() { - destroy(); + //destroy(); } void init() { ptr=NULL; @@ -172,7 +185,6 @@ class Query_Processor_Output { delay=-1; sticky_conn=-1; multiplex=-1; - gtid_from_hostgroup=-1; max_lag_ms=-1; log=-1; new_query=NULL; @@ -180,7 +192,6 @@ class Query_Processor_Output { OK_msg=NULL; attributes=NULL; comment=NULL; // #643 - min_gtid=NULL; firewall_whitelist_mode = WUS_NOT_FOUND; create_new_conn=0; } @@ -193,10 +204,6 @@ class Query_Processor_Output { free(OK_msg); OK_msg=NULL; } - if (min_gtid) { - free(min_gtid); - min_gtid = NULL; - } if (attributes) { free(attributes); } @@ -204,70 +211,7 @@ class Query_Processor_Output { free(comment); } } - void get_info_json(nlohmann::json& j) { - j["create_new_connection"] = create_new_conn; - j["reconnect"] = reconnect; - j["sticky_conn"] = sticky_conn; - j["cache_timeout"] = cache_timeout; - j["cache_ttl"] = cache_ttl; - j["delay"] = delay; - j["destination_hostgroup"] = destination_hostgroup; - j["firewall_whitelist_mode"] = firewall_whitelist_mode; - j["multiplex"] = multiplex; - j["timeout"] = timeout; - j["retries"] = retries; - j["max_lag_ms"] = max_lag_ms; - } -}; - -static char *commands_counters_desc[MYSQL_COM_QUERY___NONE]; - -class Command_Counter { - private: - int cmd_idx; - int _add_idx(unsigned long long t) { - if (t<=100) return 0; - if (t<=500) return 1; - if (t<=1000) return 2; - if (t<=5000) return 3; - if (t<=10000) return 4; - if (t<=50000) return 5; - if (t<=100000) return 6; - if (t<=500000) return 7; - if (t<=1000000) return 8; - if (t<=5000000) return 9; - if (t<=10000000) return 10; - return 11; - } - public: - unsigned long long total_time; - unsigned long long counters[13]; - Command_Counter(int a) { - total_time=0; - cmd_idx=a; - total_time=0; - for (int i=0; i<13; i++) { - counters[i]=0; - } - } - unsigned long long add_time(unsigned long long t) { - total_time+=t; - counters[0]++; - int i=_add_idx(t); - counters[i+1]++; - return total_time; - } - char **get_row() { - char **pta=(char **)malloc(sizeof(char *)*15); - pta[0]=commands_counters_desc[cmd_idx]; - itostr(pta[1],total_time); - for (int i=0;i<13;i++) itostr(pta[i+2], counters[i]); - return pta; - } - void free_row(char **pta) { - for (int i=1;i<15;i++) free(pta[i]); - free(pta); - } + void get_info_json(nlohmann::json& j); }; /** @@ -297,101 +241,73 @@ struct rules_mem_sts_t { khash_t(khStrInt)* rules_fast_routing; }; -class Query_Processor { - private: - char rand_del[16]; - umap_query_digest digest_umap; - umap_query_digest_text digest_text_umap; - pthread_rwlock_t digest_rwlock; - enum MYSQL_COM_QUERY_command __query_parser_command_type(SQP_par_t *qp); - protected: - pthread_rwlock_t rwlock; - std::vector rules; - khash_t(khStrInt) * rules_fast_routing; - char * rules_fast_routing___keys_values; - unsigned long long rules_fast_routing___keys_values___size; - unsigned long long rules_fast_routing___number; - Command_Counter * commands_counters[MYSQL_COM_QUERY___NONE]; +class MySQL_Query_Processor; +class PgSQL_Query_Processor; +class MySQL_Connection_userinfo; +class PgSQL_Connection_userinfo; +class MySQL_Session; +class PgSQL_Session; +class MySQL_Query_Processor_Output; +class PgSQL_Query_Processor_Output; +struct _MySQL_Query_processor_Rule_t; +struct PgSQL_Query_Processor_Rule_t; +typedef struct _MySQL_Query_processor_Rule_t MySQL_Query_Processor_Rule_t; - // firewall - pthread_mutex_t global_mysql_firewall_whitelist_mutex; - std::unordered_mapglobal_mysql_firewall_whitelist_users; - std::unordered_map global_mysql_firewall_whitelist_rules; - std::vector global_mysql_firewall_whitelist_sqli_fingerprints; - SQLite3_result * global_mysql_firewall_whitelist_users_runtime; - SQLite3_result * global_mysql_firewall_whitelist_rules_runtime; - SQLite3_result * global_mysql_firewall_whitelist_sqli_fingerprints_runtime; - unsigned long long global_mysql_firewall_whitelist_users_map___size; - unsigned long long global_mysql_firewall_whitelist_users_result___size; - unsigned long long global_mysql_firewall_whitelist_rules_map___size; - unsigned long long global_mysql_firewall_whitelist_rules_result___size; - volatile unsigned int version; - unsigned long long rules_mem_used; - unsigned long long new_req_conns_count; - public: - Query_Processor(); +/** + * @brief Query Processor class. + * @details This class is responsible for managing the query rules and processing the incoming queries. + */ +template +class Query_Processor { + static_assert(std::is_same_v || std::is_same_v, + "Invalid QP_DERIVED Query Processor type"); + using TypeSession = typename std::conditional,MySQL_Session,PgSQL_Session>::type; + using TypeConnInfo = typename std::conditional,MySQL_Connection_userinfo,PgSQL_Connection_userinfo>::type; + using TypeQPOutput = typename std::conditional,MySQL_Query_Processor_Output,PgSQL_Query_Processor_Output>::type; + using TypeQueryRule = typename std::conditional,MySQL_Query_Processor_Rule_t,PgSQL_Query_Processor_Rule_t>::type; +public: + Query_Processor(int _query_rules_fast_routing_algorithm); ~Query_Processor(); - void print_version(); - rules_mem_sts_t reset_all(bool lock=true); - void wrlock(); // explicit write lock, to be used in multi-insert - void wrunlock(); // explicit write unlock - bool insert(QP_rule_t *qr, bool lock=true); // insert a new rule. Uses a generic void pointer to a structure that may vary depending from the Query Processor - QP_rule_t * new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *client_addr, char *proxy_addr, int proxy_port, char *digest, char *match_digest, char *match_pattern, bool negate_match_pattern, char *re_modifiers, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int cache_empty_result, int cache_timeout, int reconnect, int timeout, int retries, int delay, int next_query_flagIN, int mirror_hostgroup, int mirror_flagOUT, char *error_msg, char *OK_msg, int sticky_conn, int multiplex, int gtid_from_hostgroup, int log, bool apply, char* attributes, char *comment); // to use a generic query rule struct, this is generated by this function and returned as generic void pointer - void delete_query_rule(QP_rule_t *qr); // destructor - Query_Processor_Output * process_mysql_query(MySQL_Session *sess, void *ptr, unsigned int size, Query_Info *qi); - void delete_QP_out(Query_Processor_Output *o); - void sort(bool lock=true); - - void init_thread(); - void end_thread(); - void commit(); // this applies all the changes in memory - SQLite3_result * get_current_query_rules(); - SQLite3_result * get_stats_query_rules(); - - void update_query_processor_stats(); - - void query_parser_init(SQP_par_t *qp, char *query, int query_length, int flags); - enum MYSQL_COM_QUERY_command query_parser_command_type(SQP_par_t *qp); - bool query_parser_first_comment(Query_Processor_Output *qpo, char *fc); - void query_parser_free(SQP_par_t *qp); - char * get_digest_text(SQP_par_t *qp); - uint64_t get_digest(SQP_par_t *qp); - bool is_valid_gtid(char *gtid, size_t gtid_len); - - void update_query_digest(SQP_par_t *qp, int hid, MySQL_Connection_userinfo *ui, unsigned long long t, unsigned long long n, MySQL_STMT_Global_info *_stmt_info, MySQL_Session *sess); - - unsigned long long query_parser_update_counters(MySQL_Session *sess, enum MYSQL_COM_QUERY_command c, SQP_par_t *qp, unsigned long long t); - - SQLite3_result * get_stats_commands_counters(); - SQLite3_result * get_query_digests(); - SQLite3_result * get_query_digests_reset(); - std::pair get_query_digests_v2(const bool use_resultset = true); - std::pair get_query_digests_reset_v2( - const bool copy, const bool use_resultset = true - ); - void get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt); - unsigned long long purge_query_digests(bool async_purge, bool parallel, char **msg); - unsigned long long purge_query_digests_async(char **msg); - unsigned long long purge_query_digests_sync(bool parallel); + void print_version(); + rules_mem_sts_t reset_all(bool lock = true); + void delete_QP_out(Query_Processor_Output* o); + void query_parser_init(SQP_par_t* qp, const char* query, int query_length, int flags); + void query_parser_free(SQP_par_t* qp); + char* get_digest_text(SQP_par_t* qp); + uint64_t get_digest(SQP_par_t* qp); + void update_query_digest(uint64_t digest_total, uint64_t digest, char* digest_text, int hid, + TypeConnInfo* ui, unsigned long long t, unsigned long long n, const char* client_addr, + unsigned long long rows_affected, unsigned long long rows_sent); + std::pair get_query_digests_v2(const bool use_resultset = true); + std::pair get_query_digests_reset_v2(const bool copy, const bool use_resultset = true); + void get_query_digests_reset(umap_query_digest* uqd, umap_query_digest_text* uqdt); + unsigned long long purge_query_digests(bool async_purge, bool parallel, char** msg); + + void save_query_rules(SQLite3_result* resultset); + + void wrlock(); // explicit write lock, to be used in multi-insert + void rdlock(); // explicit read lock + void wrunlock(); // explicit unlock + void commit(); // this applies all the changes in memory unsigned long long get_query_digests_total_size(); unsigned long long get_rules_mem_used(); unsigned long long get_new_req_conns_count(); - SQLite3_result * query_rules_resultset; // here we save a copy of resultset for query rules - void save_query_rules(SQLite3_result *resultset); - SQLite3_result * get_current_query_rules_inner(); - - // fast routing - SQLite3_result * fast_routing_resultset; // here we save a copy of resultset for query rules fast routing - uint32_t query_rules_fast_routing_algorithm = 1; + SQLite3_result* get_current_query_rules_inner(); + SQLite3_result* get_stats_query_rules(); + SQLite3_result* get_query_digests(); + SQLite3_result* get_query_digests_reset(); + + /** * @brief Creates a hashmap for 'rules_fast_routing' from the provided resultset. * @param resultset A resulset from which to create a hashmap. * @return A hashmap encapsulated into the 'fast_routing_hashmap_t' type. */ fast_routing_hashmap_t create_fast_routing_hashmap(SQLite3_result* resultset); + /** * @brief Swaps the current 'rules_fast_routing' hashmap, updating all the required related info. * @details This function assumes caller has taken write access over '' @@ -399,6 +315,88 @@ class Query_Processor { * @return Old 'fast_routing_resultset' that has been replaced. Required to be freed by caller. */ SQLite3_result* load_fast_routing(const fast_routing_hashmap_t& fast_routing_hashmap); + + SQLite3_result* get_current_query_rules_fast_routing(); + SQLite3_result* get_current_query_rules_fast_routing_inner(); + int get_current_query_rules_fast_routing_count(); + + bool insert(QP_rule_t* qr, bool lock = true); // insert a new rule. Uses a generic void pointer to a structure that may vary depending from the Query Processor + void delete_query_rule(QP_rule_t* qr); // destructor + void sort(bool lock = true); + + int testing___find_HG_in_mysql_query_rules_fast_routing(char* username, char* schemaname, int flagIN); + int testing___find_HG_in_mysql_query_rules_fast_routing_dual(khash_t(khStrInt)* _rules_fast_routing, char* username, char* schemaname, int flagIN, bool lock); + + // firewall + void load_firewall(SQLite3_result* u, SQLite3_result* r, SQLite3_result* sf); + void load_firewall_users(SQLite3_result*); + void load_firewall_rules(SQLite3_result*); + void load_firewall_sqli_fingerprints(SQLite3_result*); + + unsigned long long get_firewall_memory_users_table(); + unsigned long long get_firewall_memory_users_config(); + unsigned long long get_firewall_memory_rules_table(); + unsigned long long get_firewall_memory_rules_config(); + void get_current_firewall_whitelist(SQLite3_result** u, SQLite3_result** r, SQLite3_result** sf); + int find_firewall_whitelist_user(char* username, char* client); + bool find_firewall_whitelist_rule(char* username, char* client_address, char* schemaname, int flagIN, uint64_t digest); + + SQLite3_result* get_firewall_whitelist_users(); + SQLite3_result* get_firewall_whitelist_rules(); + bool whitelisted_sqli_fingerprint(char*); + + uint32_t query_rules_fast_routing_algorithm = 1; + +protected: + volatile unsigned int version; + std::vector rules; + + Query_Processor_Output* process_query(TypeSession* sess, bool stmt_exec, const char* query, unsigned int len, + Query_Processor_Output*, SQP_par_t* qp); + void init_thread(); + void end_thread(); + void update_query_processor_stats(); + void query_parser_update_counters(TypeSession* sess, uint64_t digest_total, uint64_t digest, char* digest_text, unsigned long long t); + bool query_parser_first_comment(Query_Processor_Output* qpo, char* fc); + +private: + char rand_del[16]; + umap_query_digest digest_umap; + umap_query_digest_text digest_text_umap; + pthread_rwlock_t digest_rwlock; + pthread_rwlock_t rwlock; + khash_t(khStrInt)* rules_fast_routing; + char* rules_fast_routing___keys_values; + unsigned long long rules_fast_routing___keys_values___size; + unsigned long long rules_fast_routing___number; + + // firewall + pthread_mutex_t global_firewall_whitelist_mutex; + std::unordered_map global_firewall_whitelist_users; + std::unordered_map global_firewall_whitelist_rules; + std::vector global_firewall_whitelist_sqli_fingerprints; + SQLite3_result* global_firewall_whitelist_users_runtime; + SQLite3_result* global_firewall_whitelist_rules_runtime; + SQLite3_result* global_firewall_whitelist_sqli_fingerprints_runtime; + + unsigned long long global_firewall_whitelist_users_map___size; + unsigned long long global_firewall_whitelist_users_result___size; + unsigned long long global_firewall_whitelist_rules_map___size; + unsigned long long global_firewall_whitelist_rules_result___size; + + unsigned long long rules_mem_used; + unsigned long long new_req_conns_count; + + SQLite3_result* query_rules_resultset; // here we save a copy of resultset for query rules + // fast routing + SQLite3_result* fast_routing_resultset; // here we save a copy of resultset for query rules fast routing + + DEFINE_HAS_METHOD_STRUCT(query_parser_first_comment_extended); + DEFINE_HAS_METHOD_STRUCT(process_query_extended); + + unsigned long long purge_query_digests_async(char** msg); + unsigned long long purge_query_digests_sync(bool parallel); + /** * @brief Searches for a matching rule in the supplied map, returning the destination hostgroup. * @details This functions takes a pointer to the hashmap pointer. This is because it performs a @@ -415,31 +413,8 @@ class Query_Processor { int search_rules_fast_routing_dest_hg( khash_t(khStrInt)** __rules_fast_routing, const char* u, const char* s, int flagIN, bool lock ); - SQLite3_result * get_current_query_rules_fast_routing(); - SQLite3_result * get_current_query_rules_fast_routing_inner(); - int get_current_query_rules_fast_routing_count(); - int testing___find_HG_in_mysql_query_rules_fast_routing(char *username, char *schemaname, int flagIN); - int testing___find_HG_in_mysql_query_rules_fast_routing_dual(khash_t(khStrInt)* _rules_fast_routing, char *username, char *schemaname, int flagIN, bool lock); - - // firewall - void load_mysql_firewall(SQLite3_result *u, SQLite3_result *r, SQLite3_result *sf); - void load_mysql_firewall_users(SQLite3_result *); - void load_mysql_firewall_rules(SQLite3_result *); - void load_mysql_firewall_sqli_fingerprints(SQLite3_result *); - unsigned long long get_mysql_firewall_memory_users_table(); - unsigned long long get_mysql_firewall_memory_users_config(); - unsigned long long get_mysql_firewall_memory_rules_table(); - unsigned long long get_mysql_firewall_memory_rules_config(); - void get_current_mysql_firewall_whitelist(SQLite3_result **u, SQLite3_result **r, SQLite3_result **sf); - int find_firewall_whitelist_user(char *username, char *client); - bool find_firewall_whitelist_rule(char *username, char *client_address, char *schemaname, int flagIN, uint64_t digest); - SQLite3_result * get_mysql_firewall_whitelist_users(); - SQLite3_result * get_mysql_firewall_whitelist_rules(); - SQLite3_result * get_mysql_firewall_whitelist_sqli_fingerprints(); - bool whitelisted_sqli_fingerprint(char *); + friend Web_Interface_plugin; }; -typedef Query_Processor * create_Query_Processor_t(); - #endif /* __CLASS_QUERY_PROCESSOR_H */ diff --git a/lib/Admin_Bootstrap.cpp b/lib/Admin_Bootstrap.cpp new file mode 100644 index 0000000000..f5ed48433b --- /dev/null +++ b/lib/Admin_Bootstrap.cpp @@ -0,0 +1,1051 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include // std::cout +#include // std::stringstream +#include +#include // std::sort +#include +#include // std::vector +#include +#include "prometheus/exposer.h" +#include "prometheus/counter.h" +#include "openssl/ssl.h" +#include "openssl/err.h" + +#include "Base_Thread.h" + +#include "MySQL_HostGroups_Manager.h" +#include "PgSQL_HostGroups_Manager.h" +#include "mysql.h" +#include "proxysql_admin.h" +#include "re2/re2.h" +#include "re2/regexp.h" +#include "proxysql.h" +#include "proxysql_config.h" +#include "proxysql_restapi.h" +#include "proxysql_utils.h" +#include "prometheus_helpers.h" +#include "cpp.h" + +#include "MySQL_Data_Stream.h" +#include "PgSQL_Data_Stream.h" +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" +#include "ProxySQL_HTTP_Server.hpp" // HTTP server +#include "MySQL_Authentication.hpp" +#include "PgSQL_Authentication.h" +#include "MySQL_LDAP_Authentication.hpp" +#include "MySQL_PreparedStatement.h" +#include "ProxySQL_Cluster.hpp" +#include "ProxySQL_Statistics.hpp" +#include "MySQL_Logger.hpp" +#include "PgSQL_Logger.hpp" +#include "SQLite3_Server.h" +#include "Web_Interface.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif + +#include +#include + +#include "platform.h" +#include "microhttpd.h" + +#if (defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__) || defined(__mips__)) && defined(__linux) +// currently only support x86-32, x86-64, ARM, and MIPS on Linux +#include "coredumper/coredumper.h" +#endif + +#include + +#include "PgSQL_Protocol.h" +//#include "usual/time.h" + +using std::string; +using std::unique_ptr; + +#ifdef WITHGCOV +extern "C" void __gcov_dump(); +extern "C" void __gcov_reset(); +#endif + + +#ifdef DEBUG +//#define BENCHMARK_FASTROUTING_LOAD +#endif // DEBUG + +//#define MYSQL_THREAD_IMPLEMENTATION + +#define SELECT_VERSION_COMMENT "select @@version_comment limit 1" +#define SELECT_VERSION_COMMENT_LEN 32 +#define SELECT_DB_USER "select DATABASE(), USER() limit 1" +#define SELECT_DB_USER_LEN 33 +#define SELECT_CHARSET_VARIOUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" +#define SELECT_CHARSET_VARIOUS_LEN 115 + +#define READ_ONLY_OFF "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0e\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x03\x4f\x46\x46\x05\x00\x00\x06\xfe\x00\x00\x02\x00" +#define READ_ONLY_ON "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0d\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x02\x4f\x4e\x05\x00\x00\x06\xfe\x00\x00\x02\x00" + +#define READ_ONLY_0 "\x01\x00\x00\x01\x01\x28\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x12\x40\x40\x67\x6c\x6f\x62\x61\x6c\x2e\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x00\x0c\x3f\x00\x01\x00\x00\x00\x08\x80\x00\x00\x00\x00\x05\x00\x00\x03\xfe\x00\x00\x02\x00\x02\x00\x00\x04\x01\x30\x05\x00\x00\x05\xfe\x00\x00\x02\x00" + +#define READ_ONLY_1 "\x01\x00\x00\x01\x01\x28\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x12\x40\x40\x67\x6c\x6f\x62\x61\x6c\x2e\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x00\x0c\x3f\x00\x01\x00\x00\x00\x08\x80\x00\x00\x00\x00\x05\x00\x00\x03\xfe\x00\x00\x02\x00\x02\x00\x00\x04\x01\x31\x05\x00\x00\x05\xfe\x00\x00\x02\x00" + +extern struct MHD_Daemon *Admin_HTTP_Server; + +extern ProxySQL_Statistics *GloProxyStats; + +int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg); + +extern char *ssl_key_fp; +extern char *ssl_cert_fp; +extern char *ssl_ca_fp; + +// ProxySQL_Admin shared variables +extern int admin___web_verbosity; +extern char * proxysql_version; + +#include "proxysql_find_charset.h" + +template ::value, bool>::type = true> +T j_get_srv_default_int_val( +const json& j, uint32_t hid, const string& key, const function& val_check); + +struct cpu_timer +{ + cpu_timer() { + begin = monotonic_time(); + } + ~cpu_timer() + { + unsigned long long end = monotonic_time(); +#ifdef DEBUG + std::cerr << double( end - begin ) / 1000000 << " secs.\n" ; +#endif + begin=end-begin; // make the compiler happy + }; + unsigned long long begin; +}; + +extern int admin_load_main_; +extern bool admin_nostart_; + +extern Query_Cache *GloQC; +extern MySQL_Authentication *GloMyAuth; +extern PgSQL_Authentication *GloPgAuth; +extern MySQL_LDAP_Authentication *GloMyLdapAuth; +extern ProxySQL_Admin *GloAdmin; +extern MySQL_Query_Processor* GloMyQPro; +extern PgSQL_Query_Processor* GloPgQPro; +extern MySQL_Threads_Handler *GloMTH; +extern MySQL_Logger *GloMyLogger; +extern PgSQL_Logger* GloPgSQL_Logger; +extern MySQL_STMT_Manager_v14 *GloMyStmt; +extern MySQL_Monitor *GloMyMon; +extern PgSQL_Threads_Handler* GloPTH; + +extern void (*flush_logs_function)(); + +extern Web_Interface *GloWebInterface; + +extern ProxySQL_Cluster *GloProxyCluster; +#ifdef PROXYSQLCLICKHOUSE +extern ClickHouse_Authentication *GloClickHouseAuth; +extern ClickHouse_Server *GloClickHouseServer; +#endif /* PROXYSQLCLICKHOUSE */ + +extern SQLite3_Server *GloSQLite3Server; + +extern char * binary_sha1; + +extern int ProxySQL_create_or_load_TLS(bool bootstrap, std::string& msg); + +#include "ProxySQL_Admin_Tables_Definitions.h" + +extern void * (*child_func[3]) (void *arg); + +bootstrap_info_t::~bootstrap_info_t() { + if (servers != nullptr) { + mysql_free_result(servers); + } + if (users != nullptr) { + mysql_free_result(users); + } +} + +#include "Admin_ifaces.h" +extern admin_main_loop_listeners S_amll; + +static void flush_logs_handler() { + GloAdmin->flush_logs(); +} + +extern void * admin_main_loop(void *arg); + +struct boot_srv_info_t { + string member_id; + string member_host; + uint32_t member_port; + string member_state; + string member_role; + string member_version; +}; + +struct BOOT_SRV_INFO_T { + enum { + MEMBER_ID, + MEMBER_HOST, + MEMBER_PORT, + MEMBER_STATE, + MEMBER_ROLE, + MEMBER_VERSION + }; +}; + +struct boot_user_info_t { + string user; + string ssl_type; + string auth_string; + string auth_plugin; + bool password_expired; +}; + +struct BOOT_USER_INFO_T { + enum { + USER, + SSL_TYPE, + AUTH_STRING, + AUTH_PLUGIN, + PASSWORD_EXPIRED + }; +}; + +struct srv_defs_t { + int64_t weight; + int64_t max_conns; + int32_t use_ssl; +}; + +using boot_srv_cnf_t = pair; + +vector extract_boot_servers_info(MYSQL_RES* servers) { + vector servers_info {}; + + while (MYSQL_ROW row = mysql_fetch_row(servers)) { + servers_info.push_back({ + string { row[BOOT_SRV_INFO_T::MEMBER_ID] }, + string { row[BOOT_SRV_INFO_T::MEMBER_HOST] }, + static_cast(stoi(row[BOOT_SRV_INFO_T::MEMBER_PORT])), + string { row[BOOT_SRV_INFO_T::MEMBER_STATE] }, + string { row[BOOT_SRV_INFO_T::MEMBER_ROLE] }, + string { row[BOOT_SRV_INFO_T::MEMBER_VERSION] }, + }); + } + + return servers_info; +} + +string build_boot_servers_insert(const vector& srvs_info_defs) { + const string t_srvs_insert { + "INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,max_connections,use_ssl) VALUES " + }; + string t_srvs_values {}; + + for (const auto& info_defs : srvs_info_defs) { + const boot_srv_info_t& srv_info = info_defs.first; + const srv_defs_t& srv_defs = info_defs.second; + + const char t_values[] { "(%d, \"%s\", %d, \"%s\", %ld, %ld, %d)" }; + string srv_values = cstr_format( + t_values, + srv_info.member_role == "PRIMARY" ? 0 : 1, // HOSTGROUP_ID + srv_info.member_host.c_str(), // HOSTNAME + srv_info.member_port, // PORT + srv_info.member_state.c_str(), // STATUS + srv_defs.weight, // Weight + srv_defs.max_conns, // Max Connections + srv_defs.use_ssl // UseSSL + ).str; + + if (&info_defs != &srvs_info_defs.back()) { + srv_values += ","; + } + + t_srvs_values += srv_values; + } + + const string servers_insert { t_srvs_insert + t_srvs_values }; + + return servers_insert; +} + +string build_boot_users_insert(MYSQL_RES* users) { + vector users_info {}; + + while (MYSQL_ROW row = mysql_fetch_row(users)) { + users_info.push_back({ + string { row[BOOT_USER_INFO_T::USER] }, + string { row[BOOT_USER_INFO_T::SSL_TYPE] }, + string { row[BOOT_USER_INFO_T::AUTH_STRING] }, + string { row[BOOT_USER_INFO_T::AUTH_PLUGIN] }, + static_cast(atoi(row[BOOT_USER_INFO_T::PASSWORD_EXPIRED])) + }); + } + + // MySQL Users + const string t_users_insert { + "INSERT INTO mysql_users (username,password,active,use_ssl) VALUES " + }; + string t_users_values {}; + + for (const boot_user_info_t& user : users_info) { + uint32_t use_ssl = user.ssl_type.empty() ? 0 : 1; + const char t_values[] { "(\"%s\", \"%s\", %d, %d)" }; + + string srv_values = cstr_format( + t_values, + user.user.c_str(), // USERNAME + user.auth_string.c_str(), // HOSTNAME + 1, // ACTIVE: Always ON + use_ssl // USE_SSL: Dependent on backend user + ).str; + + if (&user != &users_info.back()) { + srv_values += ","; + } + + t_users_values += srv_values; + } + + const string users_insert { t_users_insert + t_users_values }; + + return users_insert; +} + +map get_cur_hg_attrs(SQLite3DB* admindb) { + map res {}; + + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + admindb->execute_statement( + "SELECT hostgroup_id,servers_defaults FROM mysql_hostgroup_attributes", + &error, &cols, &affected_rows, &resultset + ); + + for (SQLite3_row* row : resultset->rows) { + const int32_t hid = atoi(row->fields[0]); + srv_defs_t srv_defs {}; + srv_defs.weight = 1; + srv_defs.max_conns = 512; + srv_defs.use_ssl = 1; + + nlohmann::json j_srv_defs = nlohmann::json::parse(row->fields[1]); + + const auto weight_check = [] (int64_t weight) -> bool { return weight >= 0; }; + srv_defs.weight = j_get_srv_default_int_val(j_srv_defs, hid, "weight", weight_check); + + const auto max_conns_check = [] (int64_t max_conns) -> bool { return max_conns >= 0; }; + srv_defs.max_conns = j_get_srv_default_int_val(j_srv_defs, hid, "max_connections", max_conns_check); + + const auto use_ssl_check = [] (int32_t use_ssl) -> bool { return use_ssl == 0 || use_ssl == 1; }; + srv_defs.use_ssl = j_get_srv_default_int_val(j_srv_defs, hid, "use_ssl", use_ssl_check); + + res.insert({ hid , srv_defs }); + } + + delete resultset; + + return res; +} + +vector build_srvs_info_with_defs( + const vector& srvs_info, + const map& hgid_defs, + const srv_defs_t global_defs +) { + vector res {}; + + for (const boot_srv_info_t& srv_info : srvs_info) { + if (srv_info.member_role == "PRIMARY") { + const auto hg_it = hgid_defs.find(0); + + if (hg_it != hgid_defs.end()) { + res.push_back({ srv_info, hg_it->second }); + } else { + res.push_back({ srv_info, global_defs }); + } + } else { + const auto hg_it = hgid_defs.find(1); + + if (hg_it != hgid_defs.end()) { + res.push_back({ srv_info, hg_it->second }); + } else { + res.push_back({ srv_info, global_defs }); + } + } + } + + return res; +} + +/** + * @brief Helper function used to check if tables are already filled with data. + * @details Handles the boilerplate operations of executing 'SELECT COUNT(*)' alike queries. + * @param admindb An already initialized instance of a SQLite3DB object to 'mem_admindb'. + * @param query The query to be executed, it's required to be 'SELECT COUNT(*)' alike. + * @return The resulting int of the 'COUNT(*)' in case of success, '-1' otherwise. In case of error, error + * cause are logged, and `assert` is called. + */ +int check_if_user_config(SQLite3DB* admindb, const char* query) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error( + "Aborting due to failed query over SQLite3 - db: '%s', query: '%s', err: %s", admindb->get_url(), query, error + ); + assert(0); + } + + int count = -1; + + if (resultset != nullptr && !resultset->rows.empty() && resultset->rows[0]->cnt >= 0) { + const char* s_count = resultset->rows[0]->fields[0]; + char* p_end = nullptr; + + count = std::strtol(s_count, &p_end, 10); + + if (p_end == s_count || errno == ERANGE) { + proxy_error( + "Aborting due to invalid query output, expected single INT (E.g. 'COUNT(*)') - query: '%s'", query + ); + count = -1; + } + } + + if (count == -1) { + assert(0); + } + + delete resultset; + return count; +}; + +/** + * @brief Definition of an auxiliary table used to store bootstrap variables. + * @details Table is used only to store in configdb bootstrap variables that are required to persist between + * executions. + */ +#define ADMIN_SQLITE_TABLE_BOOTSTRAP_VARIABLES "CREATE TABLE IF NOT EXISTS bootstrap_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" + + +extern void *child_mysql(void *arg); +extern void *child_telnet(void *arg); +extern void *child_postgres(void *arg); + + +bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { + cpu_timer cpt; + + if (flush_logs_function == NULL) { + flush_logs_function = flush_logs_handler; + } + + Admin_HTTP_Server = NULL; + AdminRestApiServer = NULL; + AdminHTTPServer = NULL; + +/* + AdminRestApiServer = new ProxySQL_RESTAPI_Server(); + AdminRestApiServer->print_version(); +*/ + + child_func[0]=child_mysql; + child_func[1]=child_telnet; + child_func[2]=child_postgres; + main_shutdown=0; + main_poll_nfds=0; + main_poll_fds=NULL; + main_callback_func=NULL; + + { + int rc=pipe(pipefd); + if (rc) { + perror("Call to pipe() failed"); + exit(EXIT_FAILURE); + } + } + + main_callback_func=(int *)malloc(sizeof(int)*MAX_ADMIN_LISTENERS); + main_poll_fds=(struct pollfd *)malloc(sizeof(struct pollfd)*MAX_ADMIN_LISTENERS); + main_poll_nfds=0; + + pthread_attr_t attr; + pthread_attr_init(&attr); + //pthread_attr_setstacksize (&attr, mystacksize); + + admindb=new SQLite3DB(); + admindb->open((char *)"file:mem_admindb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + admindb->execute("PRAGMA cache_size = -50000"); + //sqlite3_enable_load_extension(admindb->get_db(),1); + //sqlite3_auto_extension( (void(*)(void))sqlite3_json_init); + statsdb=new SQLite3DB(); + statsdb->open((char *)"file:mem_statsdb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + + // check if file exists , see #617 + bool admindb_file_exists=Proxy_file_exists(GloVars.admindb); + + configdb=new SQLite3DB(); + configdb->open((char *)GloVars.admindb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + // Fully synchronous is not required. See to #1055 + // https://sqlite.org/pragma.html#pragma_synchronous + configdb->execute("PRAGMA synchronous=0"); + + monitordb = new SQLite3DB(); + monitordb->open((char *)"file:mem_monitordb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + + statsdb_disk = new SQLite3DB(); + statsdb_disk->open((char *)GloVars.statsdb_disk, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); +// char *dbname = (char *)malloc(strlen(GloVars.statsdb_disk)+50); +// sprintf(dbname,"%s?mode=memory&cache=shared",GloVars.statsdb_disk); +// statsdb_disk->open(dbname, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_FULLMUTEX); +// free(dbname); + + statsdb_disk->execute("PRAGMA synchronous=0"); +// GloProxyStats->statsdb_disk = configdb; + GloProxyStats->init(); + + tables_defs_admin=new std::vector; + tables_defs_stats=new std::vector; + tables_defs_config=new std::vector; + + insert_into_tables_defs(tables_defs_admin,"mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_servers", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS); + insert_into_tables_defs(tables_defs_admin,"mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_users", ADMIN_SQLITE_RUNTIME_MYSQL_USERS); + insert_into_tables_defs(tables_defs_admin,"runtime_checksums_values", ADMIN_SQLITE_RUNTIME_CHECKSUMS_VALUES); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); + insert_into_tables_defs(tables_defs_admin,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS); + insert_into_tables_defs(tables_defs_admin,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); + insert_into_tables_defs(tables_defs_admin,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_query_rules", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES_FAST_ROUTING); + insert_into_tables_defs(tables_defs_admin,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); + insert_into_tables_defs(tables_defs_admin,"runtime_global_variables", ADMIN_SQLITE_RUNTIME_GLOBAL_VARIABLES); + insert_into_tables_defs(tables_defs_admin,"mysql_collations", ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS); + insert_into_tables_defs(tables_defs_admin,"scheduler", ADMIN_SQLITE_TABLE_SCHEDULER); + insert_into_tables_defs(tables_defs_admin,"runtime_scheduler", ADMIN_SQLITE_TABLE_RUNTIME_SCHEDULER); + insert_into_tables_defs(tables_defs_admin,"mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_USERS); + insert_into_tables_defs(tables_defs_admin,"mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_RULES); + insert_into_tables_defs(tables_defs_admin,"mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + insert_into_tables_defs(tables_defs_admin,"restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES); + insert_into_tables_defs(tables_defs_admin,"runtime_restapi_routes", ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES); + insert_into_tables_defs(tables_defs_admin,"coredump_filters", ADMIN_SQLITE_TABLE_COREDUMP_FILTERS); + insert_into_tables_defs(tables_defs_admin,"runtime_coredump_filters", ADMIN_SQLITE_RUNTIME_COREDUMP_FILTERS); +#ifdef DEBUG + insert_into_tables_defs(tables_defs_admin,"debug_levels", ADMIN_SQLITE_TABLE_DEBUG_LEVELS); + insert_into_tables_defs(tables_defs_admin,"debug_filters", ADMIN_SQLITE_TABLE_DEBUG_FILTERS); +#endif /* DEBUG */ +#ifdef PROXYSQLCLICKHOUSE + // ClickHouse + if (GloVars.global.clickhouse_server) { + insert_into_tables_defs(tables_defs_admin,"clickhouse_users", ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS); + insert_into_tables_defs(tables_defs_admin,"runtime_clickhouse_users", ADMIN_SQLITE_TABLE_RUNTIME_CLICKHOUSE_USERS); + } +#endif /* PROXYSQLCLICKHOUSE */ + + // PgSQL + insert_into_tables_defs(tables_defs_admin, "pgsql_servers", ADMIN_SQLITE_TABLE_PGSQL_SERVERS); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_servers", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_SERVERS); + insert_into_tables_defs(tables_defs_admin, "pgsql_users", ADMIN_SQLITE_TABLE_PGSQL_USERS); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_users", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_USERS); + insert_into_tables_defs(tables_defs_admin, "pgsql_ldap_mapping", ADMIN_SQLITE_TABLE_PGSQL_LDAP_MAPPING); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_ldap_mapping", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_LDAP_MAPPING); + insert_into_tables_defs(tables_defs_admin, "pgsql_query_rules", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_query_rules", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_QUERY_RULES); + insert_into_tables_defs(tables_defs_admin, "pgsql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES_FAST_ROUTING); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_QUERY_RULES_FAST_ROUTING); + insert_into_tables_defs(tables_defs_admin, "pgsql_hostgroup_attributes", ADMIN_SQLITE_TABLE_PGSQL_HOSTGROUP_ATTRIBUTES); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_hostgroup_attributes", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_HOSTGROUP_ATTRIBUTES); + insert_into_tables_defs(tables_defs_admin, "pgsql_replication_hostgroups", ADMIN_SQLITE_TABLE_PGSQL_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_replication_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_REPLICATION_HOSTGROUPS); + + insert_into_tables_defs(tables_defs_admin, "pgsql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_USERS); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_FIREWALL_WHITELIST_USERS); + insert_into_tables_defs(tables_defs_admin, "pgsql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_RULES); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_FIREWALL_WHITELIST_RULES); + insert_into_tables_defs(tables_defs_admin, "pgsql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + insert_into_tables_defs(tables_defs_admin, "runtime_pgsql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_RUNTIME_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + + insert_into_tables_defs(tables_defs_config, "pgsql_servers", ADMIN_SQLITE_TABLE_PGSQL_SERVERS); + insert_into_tables_defs(tables_defs_config, "pgsql_users", ADMIN_SQLITE_TABLE_PGSQL_USERS); + insert_into_tables_defs(tables_defs_config, "pgsql_ldap_mapping", ADMIN_SQLITE_TABLE_PGSQL_LDAP_MAPPING); + insert_into_tables_defs(tables_defs_config, "pgsql_query_rules", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES); + insert_into_tables_defs(tables_defs_config, "pgsql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_PGSQL_QUERY_RULES_FAST_ROUTING); + insert_into_tables_defs(tables_defs_config, "pgsql_hostgroup_attributes", ADMIN_SQLITE_TABLE_PGSQL_HOSTGROUP_ATTRIBUTES); + insert_into_tables_defs(tables_defs_config, "pgsql_replication_hostgroups", ADMIN_SQLITE_TABLE_PGSQL_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config, "pgsql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_USERS); + insert_into_tables_defs(tables_defs_config, "pgsql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_RULES); + insert_into_tables_defs(tables_defs_config, "pgsql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_PGSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + // + + insert_into_tables_defs(tables_defs_config,"mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS); + insert_into_tables_defs(tables_defs_config,"mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS); + insert_into_tables_defs(tables_defs_config,"mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); + insert_into_tables_defs(tables_defs_config,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); + insert_into_tables_defs(tables_defs_config,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); + insert_into_tables_defs(tables_defs_config,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); + insert_into_tables_defs(tables_defs_config,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); + insert_into_tables_defs(tables_defs_config,"global_settings", ADMIN_SQLITE_TABLE_GLOBAL_SETTINGS); + // the table is not required to be present on disk. Removing it due to #1055 + insert_into_tables_defs(tables_defs_config,"mysql_collations", ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS); + insert_into_tables_defs(tables_defs_config,"scheduler", ADMIN_SQLITE_TABLE_SCHEDULER); + insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS); + insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES); + insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + insert_into_tables_defs(tables_defs_config, "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES); +#ifdef DEBUG + insert_into_tables_defs(tables_defs_config,"debug_levels", ADMIN_SQLITE_TABLE_DEBUG_LEVELS); + insert_into_tables_defs(tables_defs_config,"debug_filters", ADMIN_SQLITE_TABLE_DEBUG_FILTERS); +#endif /* DEBUG */ +#ifdef PROXYSQLCLICKHOUSE + // ClickHouse + if (GloVars.global.clickhouse_server) { + insert_into_tables_defs(tables_defs_config,"clickhouse_users", ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS); + } +#endif /* PROXYSQLCLICKHOUSE */ + + insert_into_tables_defs(tables_defs_stats,"stats_mysql_query_rules", STATS_SQLITE_TABLE_MYSQL_QUERY_RULES); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_commands_counters", STATS_SQLITE_TABLE_MYSQL_COMMANDS_COUNTERS); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_processlist", STATS_SQLITE_TABLE_MYSQL_PROCESSLIST); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_connection_pool", STATS_SQLITE_TABLE_MYSQL_CONNECTION_POOL); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_connection_pool_reset", STATS_SQLITE_TABLE_MYSQL_CONNECTION_POOL_RESET); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_free_connections", STATS_SQLITE_TABLE_MYSQL_FREE_CONNECTIONS); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_query_digest", STATS_SQLITE_TABLE_MYSQL_QUERY_DIGEST); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_query_digest_reset", STATS_SQLITE_TABLE_MYSQL_QUERY_DIGEST_RESET); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_errors", STATS_SQLITE_TABLE_MYSQL_ERRORS); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_errors_reset", STATS_SQLITE_TABLE_MYSQL_ERRORS_RESET); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_global", STATS_SQLITE_TABLE_MYSQL_GLOBAL); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_gtid_executed", STATS_SQLITE_TABLE_MYSQL_GTID_EXECUTED); + insert_into_tables_defs(tables_defs_stats,"stats_memory_metrics", STATS_SQLITE_TABLE_MEMORY_METRICS); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_users", STATS_SQLITE_TABLE_MYSQL_USERS); + insert_into_tables_defs(tables_defs_stats,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); // workaround for issue #708 + insert_into_tables_defs(tables_defs_stats,"stats_mysql_prepared_statements_info", ADMIN_SQLITE_TABLE_STATS_MYSQL_PREPARED_STATEMENTS_INFO); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_client_host_cache", STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_client_host_cache_reset", STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE_RESET); + + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_global", STATS_SQLITE_TABLE_PGSQL_GLOBAL); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_connection_pool", STATS_SQLITE_TABLE_PGSQL_CONNECTION_POOL); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_connection_pool_reset", STATS_SQLITE_TABLE_PGSQL_CONNECTION_POOL_RESET); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_free_connections", STATS_SQLITE_TABLE_PGSQL_FREE_CONNECTIONS); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_users", STATS_SQLITE_TABLE_PGSQL_USERS); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_processlist", STATS_SQLITE_TABLE_PGSQL_PROCESSLIST); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_errors", STATS_SQLITE_TABLE_PGSQL_ERRORS); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_errors_reset", STATS_SQLITE_TABLE_PGSQL_ERRORS_RESET); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_client_host_cache", STATS_SQLITE_TABLE_PGSQL_CLIENT_HOST_CACHE); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_client_host_cache_reset", STATS_SQLITE_TABLE_PGSQL_CLIENT_HOST_CACHE_RESET); + + // ProxySQL Cluster + insert_into_tables_defs(tables_defs_admin,"proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS); + insert_into_tables_defs(tables_defs_config,"proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS); + insert_into_tables_defs(tables_defs_admin,"runtime_proxysql_servers", ADMIN_SQLITE_TABLE_RUNTIME_PROXYSQL_SERVERS); + insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_checksums", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CHECKSUMS); + insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_metrics", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_METRICS); + insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_status", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS); + insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_clients_status", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CLIENTS_STATUS); + insert_into_tables_defs(tables_defs_stats,"stats_proxysql_message_metrics", STATS_SQLITE_TABLE_PROXYSQL_MESSAGE_METRICS); + insert_into_tables_defs(tables_defs_stats,"stats_proxysql_message_metrics_reset", STATS_SQLITE_TABLE_PROXYSQL_MESSAGE_METRICS_RESET); + + // init ldap here + init_ldap(); + + // upgrade mysql_servers if needed (upgrade from previous version) + disk_upgrade_mysql_servers(); + + // upgrade mysql_users if needed (upgrade from previous version) + disk_upgrade_mysql_users(); + + // upgrade mysql_query_rules if needed (upgrade from previous version) + disk_upgrade_mysql_query_rules(); + + // upgrade scheduler if needed (upgrade from previous version) + disk_upgrade_scheduler(); + + // upgrade restapi_routes if needed (upgrade from previous version) + disk_upgrade_rest_api_routes(); + + check_and_build_standard_tables(admindb, tables_defs_admin); + check_and_build_standard_tables(configdb, tables_defs_config); + check_and_build_standard_tables(statsdb, tables_defs_stats); + + __attach_db(admindb, configdb, (char *)"disk"); + __attach_db(admindb, statsdb, (char *)"stats"); + __attach_db(admindb, monitordb, (char *)"monitor"); + __attach_db(statsdb, monitordb, (char *)"monitor"); + __attach_db(admindb, statsdb_disk, (char *)"stats_history"); + __attach_db(statsdb, statsdb_disk, (char *)"stats_history"); + + dump_mysql_collations(); + +#ifdef DEBUG + admindb->execute("ATTACH DATABASE 'file:mem_mydb?mode=memory&cache=shared' AS myhgm"); + admindb->execute("ATTACH DATABASE 'file:mem_monitor_internal_db?mode=memory&cache=shared' AS 'monitor_internal'"); + { + string debugdb_disk_path = string(GloVars.datadir) + "/" + "proxysql_debug.db"; + debugdb_disk = new SQLite3DB(); + debugdb_disk->open((char *)debugdb_disk_path.c_str(), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + debugdb_disk->execute("CREATE TABLE IF NOT EXISTS debug_log (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , time INT NOT NULL , lapse INT NOT NULL , thread INT NOT NULL , file VARCHAR NOT NULL , line INT NOT NULL , funct VARCHAR NOT NULL , modnum INT NOT NULL , modname VARCHAR NOT NULL , verbosity INT NOT NULL , message VARCHAR , note VARCHAR , backtrace VARCHAR)"); +/* + // DO NOT CREATE INDEX. + // We can create index on a running instance or an archived DB if needed + debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_time ON debug_log (time)"); + debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_thread ON debug_log (thread)"); + debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_file ON debug_log (file)"); + debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_file_line ON debug_log (file,line)"); + debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_funct ON debug_log (funct)"); + debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_modnum ON debug_log (modnum)"); +*/ + debugdb_disk->execute("PRAGMA synchronous=0"); + debugdb_disk->execute("PRAGMA journal_mode=OFF"); +/* + // DO NOT ATTACH DATABASE + // it seems sqlite starts randomly failing. For example these 2 TAP tests: + // - admin_show_fields_from-t + // - admin_show_table_status-t + string cmd = "ATTACH DATABASE '" + debugdb_disk_path + "' AS debugdb_disk"; + admindb->execute(cmd.c_str()); +*/ + proxysql_set_admin_debugdb_disk(debugdb_disk); + } +#endif /* DEBUG */ + +#ifdef DEBUG + flush_debug_levels_runtime_to_database(configdb, false); + flush_debug_levels_runtime_to_database(admindb, true); +#endif /* DEBUG */ + + // Set default values for the module variables in the target 'dbs' + flush_mysql_variables___runtime_to_database(configdb, false, false, false); + flush_mysql_variables___runtime_to_database(admindb, false, true, false); + + flush_admin_variables___runtime_to_database(configdb, false, false, false); + flush_admin_variables___runtime_to_database(admindb, false, true, false); + + flush_pgsql_variables___runtime_to_database(configdb, false, false, false); + flush_pgsql_variables___runtime_to_database(admindb, false, true, false); + + load_or_update_global_settings(configdb); + + // Insert or update the configuration from 'disk' + __insert_or_replace_maintable_select_disktable(); + + // removing this line of code. It seems redundant + //flush_admin_variables___database_to_runtime(admindb,true); + + // workaround for issue #708 + statsdb->execute("INSERT OR IGNORE INTO global_variables VALUES('mysql-max_allowed_packet',4194304)"); + + +#ifdef DEBUG + if (GloVars.global.gdbg==false && GloVars.__cmd_proxysql_gdbg) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Enabling GloVars.global.gdbg because GloVars.__cmd_proxysql_gdbg==%d\n", GloVars.__cmd_proxysql_gdbg); + GloVars.global.gdbg=true; + } + load_debug_to_runtime(); +#endif /* DEBUG */ + + if (GloVars.__cmd_proxysql_reload || GloVars.__cmd_proxysql_initial || admindb_file_exists==false) { // see #617 + if (GloVars.configfile_open) { + proxysql_config().Read_MySQL_Servers_from_configfile(); + proxysql_config().Read_MySQL_Users_from_configfile(); + proxysql_config().Read_MySQL_Query_Rules_from_configfile(); + proxysql_config().Read_Global_Variables_from_configfile("admin"); + proxysql_config().Read_Global_Variables_from_configfile("mysql"); + + proxysql_config().Read_PgSQL_Servers_from_configfile(); + proxysql_config().Read_PgSQL_Users_from_configfile(); + proxysql_config().Read_PgSQL_Query_Rules_from_configfile(); + proxysql_config().Read_Global_Variables_from_configfile("pgsql"); + + proxysql_config().Read_Scheduler_from_configfile(); + proxysql_config().Read_Restapi_from_configfile(); + proxysql_config().Read_ProxySQL_Servers_from_configfile(); + __insert_or_replace_disktable_select_maintable(); + } + } + + /** + * @brief Inserts a default 'mysql_group_replication_hostgroup'. + * @details Uses the following defaults: + * - writer_hostgroup: 0 + * - reader_hostgroup: 1 + * - backup_writer_hostgroup: 2 + * - offline_hostgroup: 3 + * - max_writers: 9 + * - writer_is_also_reader: 0 -> Keep hostgroups separated + * - max_transactions_behind: 0 + * + * The number of writers in 'multi_primary_mode' wont be restricted, user should tune this value to + * convenience. By default 'max_writers' is set to 9, as is the current member limitation for Group + * Replication. + */ + const char insert_def_gr_hgs[] { + "INSERT INTO mysql_group_replication_hostgroups (" + "writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers," + "writer_is_also_reader" + ") VALUES (0,2,1,3,1,9,0)" + }; + vector servers_info {}; + + if (GloVars.global.gr_bootstrap_mode) { + // Check if user config is present for 'mysql_group_replication_hostgroups' + bool user_gr_hg_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_group_replication_hostgroups"); + if (user_gr_hg_cnf == false) { + admindb->execute(insert_def_gr_hgs); + } else { + proxy_info("Bootstrap config, found previous user 'mysql_group_replication_hostgroups' config, reusing...\n"); + } + + // Stores current user config for 'mysql_hostgroup_attributes::servers_defaults' + map hgid_defs {}; + // Check if user config is present for 'mysql_hostgroup_attributes' + bool user_gr_hg_attrs_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_hostgroup_attributes"); + int32_t have_ssl = 1; + + // SSL explicitly disabled by user for backend connections + if (GloVars.global.gr_bootstrap_ssl_mode) { + if (strcasecmp(GloVars.global.gr_bootstrap_ssl_mode, "DISABLED") == 0) { + have_ssl = 0; + } + } + + const int64_t DEF_GR_SRV_WEIGHT = 1; + const int64_t DEF_GR_SRV_MAX_CONNS = 512; + const int32_t DEF_GR_SRV_USE_SSL = have_ssl; + + // Update 'mysql_hostgroup_attributes' with sensible defaults for the new discovered instances + if (user_gr_hg_attrs_cnf == false) { + const nlohmann::json j_def_attrs { + { "weight", DEF_GR_SRV_WEIGHT }, + { "max_connections", DEF_GR_SRV_MAX_CONNS }, + { "use_ssl", DEF_GR_SRV_USE_SSL } + }; + const string str_def_attrs { j_def_attrs.dump() }; + const string insert_def_hg_attrs { + "INSERT INTO mysql_hostgroup_attributes (hostgroup_id, servers_defaults) VALUES" + " (0,'"+ str_def_attrs + "'), (1,'" + str_def_attrs + "')" + }; + admindb->execute(insert_def_hg_attrs.c_str()); + } else { + proxy_info("Bootstrap config, found previous user 'mysql_hostgroup_attributes' config, reusing...\n"); + hgid_defs = get_cur_hg_attrs(admindb); + } + + // Define the 'global defaults'. Either pure defaults, or user specified (argument). These values are + // supersede if previous user config is found for 'mysql_hostgroup_attributes::servers_defaults'. + srv_defs_t global_srvs_defs {}; + global_srvs_defs.weight = DEF_GR_SRV_WEIGHT; + global_srvs_defs.max_conns = DEF_GR_SRV_MAX_CONNS; + global_srvs_defs.use_ssl = DEF_GR_SRV_USE_SSL; + + servers_info = extract_boot_servers_info(bootstrap_info.servers); + auto full_srvs_info = build_srvs_info_with_defs(servers_info, hgid_defs, global_srvs_defs); + const string servers_insert { build_boot_servers_insert(full_srvs_info) }; + + admindb->execute("DELETE FROM mysql_servers"); + admindb->execute(servers_insert.c_str()); + + const string users_insert { build_boot_users_insert(bootstrap_info.users) }; + admindb->execute("DELETE FROM mysql_users"); + admindb->execute(users_insert.c_str()); + + // Make the configuration persistent + flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); + flush_mysql_users__from_memory_to_disk(); + } + + // Admin variables 'bootstrap' modifications + if (GloVars.global.gr_bootstrap_mode) { + // TODO-NOTE: This MUST go away; 'admin-hash_passwords' will be deprecated + admindb->execute("UPDATE global_variables SET variable_value='false' WHERE variable_name='admin-hash_passwords'"); + } + flush_admin_variables___database_to_runtime(admindb,true); + + if (GloVars.global.gr_bootstrap_mode) { + flush_admin_variables___runtime_to_database(configdb, false, true, false); + } + + // MySQL variables / MySQL Query Rules 'bootstrap' modifications + if (GloVars.global.gr_bootstrap_mode && !servers_info.empty()) { + const uint64_t base_port { + GloVars.global.gr_bootstrap_conf_base_port == 0 ? 6446 : + GloVars.global.gr_bootstrap_conf_base_port + }; + const string bind_addr { + GloVars.global.gr_bootstrap_conf_bind_address == nullptr ? "0.0.0.0" : + string { GloVars.global.gr_bootstrap_conf_bind_address } + }; + const string s_rw_port { std::to_string(base_port) }; + const string s_ro_port { std::to_string(base_port + 1) }; + const string rw_addr { bind_addr + ":" + s_rw_port }; + const string ro_addr { bind_addr + ":" + s_ro_port }; + const string mysql_interfaces { rw_addr + ";" + ro_addr }; + + // Look for the default collation + const MARIADB_CHARSET_INFO* charset_info = proxysql_find_charset_nr(bootstrap_info.server_language); + const char* server_charset = charset_info == nullptr ? "" : charset_info->csname; + const char* server_collation = charset_info == nullptr ? "" : charset_info->name; + + // Holds user specified values, defaults, and implications of variables over others + const map bootstrap_mysql_vars { + { "mysql-server_version", bootstrap_info.server_version.c_str() }, + { "mysql-default_charset", server_charset }, + { "mysql-default_collation_connection", server_collation }, + { "mysql-interfaces", mysql_interfaces.c_str() }, + { "mysql-monitor_username", bootstrap_info.mon_user.c_str() }, + { "mysql-monitor_password", bootstrap_info.mon_pass.c_str() }, + { "mysql-have_ssl", "true" }, + { "mysql-ssl_p2s_ca", GloVars.global.gr_bootstrap_ssl_ca }, + { "mysql-ssl_p2s_capath", GloVars.global.gr_bootstrap_ssl_capath }, + { "mysql-ssl_p2s_cert", GloVars.global.gr_bootstrap_ssl_cert }, + { "mysql-ssl_p2s_cipher", GloVars.global.gr_bootstrap_ssl_cipher }, + { "mysql-ssl_p2s_crl", GloVars.global.gr_bootstrap_ssl_crl }, + { "mysql-ssl_p2s_crlpath", GloVars.global.gr_bootstrap_ssl_crlpath }, + { "mysql-ssl_p2s_key", GloVars.global.gr_bootstrap_ssl_key } + }; + + for (const pair& p_var_val : bootstrap_mysql_vars) { + if (p_var_val.second != nullptr) { + const string& name { p_var_val.first }; + const string& value { p_var_val.second }; + const string update_mysql_var { + "UPDATE global_variables SET variable_value='" + value + "' WHERE variable_name='" + name + "'" + }; + + admindb->execute(update_mysql_var.c_str()); + } + } + + // MySQL Query Rules - Port based RW split + { + // TODO: This should be able to contain in the future Unix socket based rules + const string insert_rw_split_rules { + "INSERT INTO mysql_query_rules (rule_id,active,proxy_port,destination_hostgroup,apply) VALUES " + " (0,1," + s_rw_port + ",0,1), (1,1," + s_ro_port + ",1,1)" + }; + + // Preserve previous user config targeting hostgroups 0/1 + bool user_qr_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_query_rules"); + if (user_qr_cnf == false) { + admindb->execute(insert_rw_split_rules.c_str()); + } else { + proxy_info("Bootstrap config, found previous user 'mysql_query_rules' config, reusing...\n"); + } + + flush_GENERIC__from_to("mysql_query_rules", "memory_to_disk"); + } + + // Store the 'bootstrap_variables' + if (bootstrap_info.rand_gen_user) { + configdb->execute(ADMIN_SQLITE_TABLE_BOOTSTRAP_VARIABLES); + + const string insert_bootstrap_user { + "INSERT INTO bootstrap_variables (variable_name,variable_value) VALUES" + " ('bootstrap_username','" + string { bootstrap_info.mon_user } + "')" + }; + const string insert_bootstrap_pass { + "INSERT INTO bootstrap_variables (variable_name,variable_value) VALUES" + " ('bootstrap_password','" + string { bootstrap_info.mon_pass } + "')" + }; + + configdb->execute("DELETE FROM bootstrap_variables WHERE variable_name='bootstrap_username'"); + configdb->execute(insert_bootstrap_user.c_str()); + configdb->execute("DELETE FROM bootstrap_variables WHERE variable_name='bootstrap_password'"); + configdb->execute(insert_bootstrap_pass.c_str()); + } + } + flush_mysql_variables___database_to_runtime(admindb,true); + if (GloVars.global.gr_bootstrap_mode) { + flush_mysql_variables___runtime_to_database(configdb, false, true, false); + } + flush_pgsql_variables___database_to_runtime(admindb, true); +#ifdef PROXYSQLCLICKHOUSE + flush_clickhouse_variables___database_to_runtime(admindb,true); +#endif /* PROXYSQLCLICKHOUSE */ + flush_sqliteserver_variables___database_to_runtime(admindb,true); + + if (GloVars.__cmd_proxysql_admin_socket) { + set_variable((char *)"mysql_ifaces",GloVars.__cmd_proxysql_admin_socket); + } + + S_amll.update_ifaces(variables.mysql_ifaces, &S_amll.ifaces_mysql); + S_amll.update_ifaces(variables.pgsql_ifaces, &S_amll.ifaces_pgsql); + S_amll.update_ifaces(variables.telnet_admin_ifaces, &S_amll.ifaces_telnet_admin); + S_amll.update_ifaces(variables.telnet_stats_ifaces, &S_amll.ifaces_telnet_stats); + + + +// pthread_t admin_thr; + struct _main_args *arg=(struct _main_args *)malloc(sizeof(struct _main_args)); + arg->nfds=main_poll_nfds; + arg->fds=main_poll_fds; + arg->shutdown=&main_shutdown; + arg->callback_func=main_callback_func; + if (pthread_create(&admin_thr, &attr, admin_main_loop, (void *)arg) !=0 ) { + perror("Thread creation"); + exit(EXIT_FAILURE); + } + do { usleep(50); } while (__sync_fetch_and_sub(&admin_load_main_,0)==0); + admin_load_main_=0; + + // Register the global prometheus registry in the 'serial_exposer' + if (registered_prometheus_collectable == false) { + this->serial_exposer.RegisterCollectable(GloVars.prometheus_registry); + registered_prometheus_collectable = true; + } + +#ifdef DEBUG + std::cerr << "Admin initialized in "; +#endif +return true; +}; diff --git a/lib/Admin_FlushVariables.cpp b/lib/Admin_FlushVariables.cpp new file mode 100644 index 0000000000..e6c9f81456 --- /dev/null +++ b/lib/Admin_FlushVariables.cpp @@ -0,0 +1,1175 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include // std::cout +#include // std::stringstream +#include +#include // std::sort +#include +#include // std::vector +#include +#include "prometheus/exposer.h" +#include "prometheus/counter.h" +#include "openssl/ssl.h" +#include "openssl/err.h" + +#include "Base_Thread.h" + +#include "MySQL_HostGroups_Manager.h" +#include "PgSQL_HostGroups_Manager.h" +#include "mysql.h" +#include "proxysql_admin.h" +#include "re2/re2.h" +#include "re2/regexp.h" +#include "proxysql.h" +#include "proxysql_config.h" +#include "proxysql_restapi.h" +#include "proxysql_utils.h" +#include "prometheus_helpers.h" +#include "cpp.h" + +#include "MySQL_Data_Stream.h" +#include "PgSQL_Data_Stream.h" +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" +#include "ProxySQL_HTTP_Server.hpp" // HTTP server +#include "MySQL_Authentication.hpp" +#include "PgSQL_Authentication.h" +#include "MySQL_LDAP_Authentication.hpp" +#include "MySQL_PreparedStatement.h" +#include "ProxySQL_Cluster.hpp" +#include "ProxySQL_Statistics.hpp" +#include "MySQL_Logger.hpp" +#include "PgSQL_Logger.hpp" +#include "SQLite3_Server.h" +#include "Web_Interface.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif + +#include +#include + +#include "platform.h" +#include "microhttpd.h" + +#if (defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__) || defined(__mips__)) && defined(__linux) +// currently only support x86-32, x86-64, ARM, and MIPS on Linux +#include "coredumper/coredumper.h" +#endif + +#include + +#include "PgSQL_Protocol.h" +//#include "usual/time.h" + +using std::string; +using std::unique_ptr; + +#ifdef WITHGCOV +extern "C" void __gcov_dump(); +extern "C" void __gcov_reset(); +#endif + + +#ifdef DEBUG +//#define BENCHMARK_FASTROUTING_LOAD +#endif // DEBUG + +//#define MYSQL_THREAD_IMPLEMENTATION + +#define SELECT_VERSION_COMMENT "select @@version_comment limit 1" +#define SELECT_VERSION_COMMENT_LEN 32 +#define SELECT_DB_USER "select DATABASE(), USER() limit 1" +#define SELECT_DB_USER_LEN 33 +#define SELECT_CHARSET_VARIOUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" +#define SELECT_CHARSET_VARIOUS_LEN 115 + +#define READ_ONLY_OFF "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0e\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x03\x4f\x46\x46\x05\x00\x00\x06\xfe\x00\x00\x02\x00" +#define READ_ONLY_ON "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0d\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x02\x4f\x4e\x05\x00\x00\x06\xfe\x00\x00\x02\x00" + +#define READ_ONLY_0 "\x01\x00\x00\x01\x01\x28\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x12\x40\x40\x67\x6c\x6f\x62\x61\x6c\x2e\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x00\x0c\x3f\x00\x01\x00\x00\x00\x08\x80\x00\x00\x00\x00\x05\x00\x00\x03\xfe\x00\x00\x02\x00\x02\x00\x00\x04\x01\x30\x05\x00\x00\x05\xfe\x00\x00\x02\x00" + +#define READ_ONLY_1 "\x01\x00\x00\x01\x01\x28\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x12\x40\x40\x67\x6c\x6f\x62\x61\x6c\x2e\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x00\x0c\x3f\x00\x01\x00\x00\x00\x08\x80\x00\x00\x00\x00\x05\x00\x00\x03\xfe\x00\x00\x02\x00\x02\x00\x00\x04\x01\x31\x05\x00\x00\x05\xfe\x00\x00\x02\x00" + +extern struct MHD_Daemon *Admin_HTTP_Server; + +extern ProxySQL_Statistics *GloProxyStats; + +int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg); + +extern char *ssl_key_fp; +extern char *ssl_cert_fp; +extern char *ssl_ca_fp; + +// ProxySQL_Admin shared variables +extern int admin___web_verbosity; +extern char * proxysql_version; + +#include "proxysql_find_charset.h" + +extern Query_Cache *GloQC; +extern MySQL_Authentication *GloMyAuth; +extern PgSQL_Authentication *GloPgAuth; +extern MySQL_LDAP_Authentication *GloMyLdapAuth; +extern ProxySQL_Admin *GloAdmin; +extern MySQL_Query_Processor* GloMyQPro; +extern PgSQL_Query_Processor* GloPgQPro; +extern MySQL_Threads_Handler *GloMTH; +extern MySQL_Logger *GloMyLogger; +extern PgSQL_Logger* GloPgSQL_Logger; +extern MySQL_STMT_Manager_v14 *GloMyStmt; +extern MySQL_Monitor *GloMyMon; +extern PgSQL_Threads_Handler* GloPTH; + +extern void (*flush_logs_function)(); + +extern Web_Interface *GloWebInterface; + +extern ProxySQL_Cluster *GloProxyCluster; +#ifdef PROXYSQLCLICKHOUSE +extern ClickHouse_Authentication *GloClickHouseAuth; +extern ClickHouse_Server *GloClickHouseServer; +#endif /* PROXYSQLCLICKHOUSE */ + +extern SQLite3_Server *GloSQLite3Server; + +extern char * binary_sha1; + +extern int ProxySQL_create_or_load_TLS(bool bootstrap, std::string& msg); + +bool ProxySQL_Admin::flush_GENERIC_variables__retrieve__database_to_runtime(const std::string& modname, char* &error, int& cols, int& affected_rows, SQLite3_result* &resultset) { + string q = "SELECT substr(variable_name," + to_string(modname.length()+2) + ") vn, variable_value FROM global_variables WHERE variable_name LIKE '" + modname + "-%'"; + admindb->execute_statement(q.c_str(), &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", q.c_str(), error); + free(error); + return false; + } + return true; +} + +void ProxySQL_Admin::flush_GENERIC_variables__process__database_to_runtime( + const string& modname, SQLite3DB *db, SQLite3_result* resultset, + const bool& lock, const bool& replace, + const std::unordered_set& variables_read_only, + const std::unordered_set& variables_to_delete_silently, + const std::unordered_set& variables_deprecated, + const std::unordered_set& variables_special_values, + std::function special_variable_action +) { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + bool rc = false; + if (modname == "admin") { + rc = set_variable(r->fields[0],r->fields[1], lock); + } else if (modname == "mysql") { + rc = GloMTH->set_variable(r->fields[0],r->fields[1]); + } else if (modname == "sqliteserver") { + rc = GloSQLite3Server->set_variable(r->fields[0],r->fields[1]); +#ifdef PROXYSQLCLICKHOUSE + } else if (modname == "clickhouse") { + rc = GloClickHouseServer->set_variable(r->fields[0],r->fields[1]); +#endif // PROXYSQLCLICKHOUSE + } else if (modname == "ldap") { + rc = GloMyLdapAuth->set_variable(r->fields[0],r->fields[1]); + } + const string v = string(r->fields[0]); + if (rc==false) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Impossible to set variable %s with value \"%s\"\n", r->fields[0],r->fields[1]); + if (replace) { + char *val = NULL; + if (modname == "admin") { + val = get_variable(r->fields[0]); + } else if (modname == "mysql") { + val = GloMTH->get_variable(r->fields[0]); + } else if (modname == "sqliteserver") { + val = GloSQLite3Server->get_variable(r->fields[0]); +#ifdef PROXYSQLCLICKHOUSE + } else if (modname == "clickhouse") { + val = GloClickHouseServer->get_variable(r->fields[0]); +#endif // PROXYSQLCLICKHOUSE + } else if (modname == "ldap") { + val = GloMyLdapAuth->get_variable(r->fields[0]); + } + char q[1000]; + if (val) { + if (variables_read_only.count(v) > 0) { + proxy_warning("Impossible to set read-only variable %s with value \"%s\". Resetting to current \"%s\".\n", r->fields[0],r->fields[1], val); + } else { + proxy_warning("Impossible to set variable %s with value \"%s\". Resetting to current \"%s\".\n", r->fields[0],r->fields[1], val); + } + sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"%s-%s\",\"%s\")", modname.c_str(), r->fields[0],val); + db->execute(q); + free(val); + } else { + if (variables_to_delete_silently.count(v) > 0) { + sprintf(q,"DELETE FROM disk.global_variables WHERE variable_name=\"%s-%s\"", modname.c_str(), r->fields[0]); + db->execute(q); + } else if (variables_deprecated.count(v) > 0) { + proxy_error("Global variable %s-%s is deprecated.\n", modname.c_str(), r->fields[0]); + sprintf(q,"DELETE FROM disk.global_variables WHERE variable_name=\"%s-%s\"", modname.c_str(), r->fields[0]); + db->execute(q); + } else { + proxy_warning("Impossible to set not existing variable %s with value \"%s\". Deleting. If the variable name is correct, this version doesn't support it\n", r->fields[0],r->fields[1]); + } + sprintf(q,"DELETE FROM global_variables WHERE variable_name=\"%s-%s\"", modname.c_str(), r->fields[0]); + db->execute(q); + } + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Set variable %s with value \"%s\"\n", r->fields[0],r->fields[1]); + if (variables_special_values.count(v) > 0) { + if (special_variable_action != nullptr) { + special_variable_action(v, r->fields[1], db); + } + } + } + } +} + +void ProxySQL_Admin::flush_admin_variables___database_to_runtime( + SQLite3DB *db, bool replace, const string& checksum, const time_t epoch, bool lock +) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ADMIN variables. Replace:%d\n", replace); + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (flush_GENERIC_variables__retrieve__database_to_runtime("admin", error, cols, affected_rows, resultset) == true) { + wrlock(); + flush_GENERIC_variables__process__database_to_runtime("admin", db, resultset, lock, replace, {"version"}, {"debug"}, {}, {}); + //commit(); NOT IMPLEMENTED + + // Checksums are always generated - 'admin-checksum_*' deprecated + + { + // generate checksum for cluster + pthread_mutex_lock(&GloVars.checksum_mutex); + flush_admin_variables___runtime_to_database(admindb, false, false, false, true); + flush_GENERIC_variables__checksum__database_to_runtime("admin", checksum, epoch); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + wrunlock(); + { + load_http_server(); + load_restapi_server(); + // Update the admin variable for 'web_verbosity' + admin___web_verbosity = variables.web_verbosity; + } + } + if (resultset) delete resultset; +} + +void ProxySQL_Admin::flush_pgsql_variables___runtime_to_database(SQLite3DB* db, bool replace, bool del, bool onlyifempty, bool runtime, bool use_lock) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing PgSQL variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); + if (onlyifempty) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + char* q = (char*)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'pgsql-%'"; + db->execute_statement(q, &error, &cols, &affected_rows, &resultset); + int matching_rows = 0; + if (error) { + proxy_error("Error on %s : %s\n", q, error); + return; + } + else { + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + matching_rows += atoi(r->fields[0]); + } + } + if (resultset) delete resultset; + if (matching_rows) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has PgSQL variables - skipping\n"); + return; + } + } + if (del) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting PgSQL variables from global_variables\n"); + db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'pgsql-%'"); + } + static char* a; + static char* b; + if (replace) { + a = (char*)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(?1, ?2)"; + } + else { + a = (char*)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(?1, ?2)"; + } + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement2 = NULL; + //sqlite3 *mydb3=db->get_db(); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, a, -1, &statement1, 0); + rc = db->prepare_v2(a, &statement1); + ASSERT_SQLITE_OK(rc, db); + if (runtime) { + db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'pgsql-%'"); + b = (char*)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(?1, ?2)"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, b, -1, &statement2, 0); + rc = db->prepare_v2(b, &statement2); + ASSERT_SQLITE_OK(rc, db); + } + if (use_lock) { + GloPTH->wrlock(); + db->execute("BEGIN"); + } + char** varnames = GloPTH->get_variables_list(); + for (int i = 0; varnames[i]; i++) { + char* val = GloPTH->get_variable(varnames[i]); + char* qualified_name = (char*)malloc(strlen(varnames[i]) + 12); + sprintf(qualified_name, "pgsql-%s", varnames[i]); + rc = (*proxy_sqlite3_bind_text)(statement1, 1, qualified_name, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, (val ? val : (char*)""), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, db); + if (runtime) { + rc = (*proxy_sqlite3_bind_text)(statement2, 1, qualified_name, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_bind_text)(statement2, 2, (val ? val : (char*)""), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + SAFE_SQLITE3_STEP2(statement2); + rc = (*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, db); + rc = (*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, db); + } + if (val) + free(val); + free(qualified_name); + } + if (use_lock) { + db->execute("COMMIT"); + GloPTH->wrunlock(); + } + (*proxy_sqlite3_finalize)(statement1); + if (runtime) + (*proxy_sqlite3_finalize)(statement2); + for (int i = 0; varnames[i]; i++) { + free(varnames[i]); + } + free(varnames); +} + +void ProxySQL_Admin::flush_GENERIC_variables__checksum__database_to_runtime(const string& modname, const string& checksum, const time_t epoch) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + std::string q; + q="SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE '" + modname + "-\%' "; + if (modname == "mysql") { + q += " AND variable_name NOT IN ('mysql-threads')"; + if (GloVars.cluster_sync_interfaces == false) { + q += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_MYSQL); + } + } else if (modname == "admin") { + if (GloVars.cluster_sync_interfaces == false) { + q += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_ADMIN); + } + } + q += " ORDER BY variable_name"; + admindb->execute_statement(q.c_str(), &error , &cols , &affected_rows , &resultset); + uint64_t hash1 = resultset->raw_checksum(); + uint32_t d32[2]; + char buf[20]; + memcpy(&d32, &hash1, sizeof(hash1)); + sprintf(buf,"0x%0X%0X", d32[0], d32[1]); + ProxySQL_Checksum_Value *checkvar = NULL; + if (modname == "admin") { + checkvar = &GloVars.checksums_values.admin_variables; + } else if (modname == "mysql") { + checkvar = &GloVars.checksums_values.mysql_variables; + } else if (modname == "ldap") { + checkvar = &GloVars.checksums_values.ldap_variables; + } + assert(checkvar != NULL); + checkvar->set_checksum(buf); + checkvar->version++; + time_t t = time(NULL); + if (epoch != 0 && checksum != "" && checkvar->checksum == checksum) { + checkvar->epoch = epoch; + } else { + checkvar->epoch = t; + } + GloVars.epoch_version = t; + GloVars.generate_global_checksum(); + GloVars.checksums_values.updates_cnt++; + string modnameupper = modname; + for (char &c : modnameupper) { c = std::toupper(c); } + proxy_info( + "Computed checksum for 'LOAD %s VARIABLES TO RUNTIME' was '%s', with epoch '%llu'\n", + modnameupper.c_str(), checkvar->checksum, checkvar->epoch + ); + delete resultset; +} + +void ProxySQL_Admin::flush_mysql_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum, const time_t epoch) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing MySQL variables. Replace:%d\n", replace); + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (flush_GENERIC_variables__retrieve__database_to_runtime("mysql", error, cols, affected_rows, resultset) == true) { + GloMTH->wrlock(); + char * previous_default_charset = GloMTH->get_variable_string((char *)"default_charset"); + char * previous_default_collation_connection = GloMTH->get_variable_string((char *)"default_collation_connection"); + assert(previous_default_charset); + assert(previous_default_collation_connection); + flush_GENERIC_variables__process__database_to_runtime("mysql", db, resultset, false, replace, {}, {"session_debug"}, {"forward_autocommit"}, + {"default_collation_connection", "default_charset", "show_processlist_extended"}, + [](const std::string& varname, const char *varvalue, SQLite3DB* db) { + if (varname == "default_collation_connection" || varname == "default_charset") { + char *val=GloMTH->get_variable((char *)varname.c_str()); + if (val) { + if (strcmp(val,varvalue)) { + char q[1000]; + proxy_warning("Variable %s with value \"%s\" is being replaced with value \"%s\".\n", varname.c_str(), varvalue, val); + sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-%s\",\"%s\")", varname.c_str() ,val); + db->execute(q); + } + free(val); + } + } else if (varname == "show_processlist_extended") { + GloAdmin->variables.mysql_show_processlist_extended = atoi(varvalue); + } + } + ); + char q[1000]; + char * default_charset = GloMTH->get_variable_string((char *)"default_charset"); + char * default_collation_connection = GloMTH->get_variable_string((char *)"default_collation_connection"); + assert(default_charset); + assert(default_collation_connection); + MARIADB_CHARSET_INFO * ci = NULL; + ci = proxysql_find_charset_name(default_charset); + if (ci == NULL) { + // invalid charset + proxy_error("Found an incorrect value for mysql-default_charset: %s\n", default_charset); + // let's try to get a charset from collation connection + ci = proxysql_find_charset_collate(default_collation_connection); + if (ci == NULL) { + proxy_error("Found an incorrect value for mysql-default_collation_connection: %s\n", default_collation_connection); + const char *p = mysql_tracked_variables[SQL_CHARACTER_SET].default_value; + ci = proxysql_find_charset_name(p); + assert(ci); + proxy_info("Resetting mysql-default_charset to hardcoded default value: %s\n", ci->csname); + sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_charset\",\"%s\")", ci->csname); + db->execute(q); + GloMTH->set_variable((char *)"default_charset",ci->csname); + proxy_info("Resetting mysql-default_collation_connection to hardcoded default value: %s\n", ci->name); + sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_collation_connection\",\"%s\")", ci->name); + db->execute(q); + GloMTH->set_variable((char *)"default_collation_connection",ci->name); + } else { + proxy_info("Changing mysql-default_charset to %s using configured mysql-default_collation_connection %s\n", ci->csname, ci->name); + sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_charset\",\"%s\")", ci->csname); + db->execute(q); + GloMTH->set_variable((char *)"default_charset",ci->csname); + } + } else { + MARIADB_CHARSET_INFO * cic = NULL; + cic = proxysql_find_charset_collate(default_collation_connection); + if (cic == NULL) { + proxy_error("Found an incorrect value for mysql-default_collation_connection: %s\n", default_collation_connection); + proxy_info("Changing mysql-default_collation_connection to %s using configured mysql-default_charset: %s\n", ci->name, ci->csname); + sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_collation_connection\",\"%s\")", ci->name); + db->execute(q); + GloMTH->set_variable((char *)"default_collation_connection",ci->name); + } else { + if (strcmp(cic->csname,ci->csname)==0) { + // mysql-default_collation_connection and mysql-default_charset are compatible + } else { + proxy_error("Found incompatible values for mysql-default_charset (%s) and mysql-default_collation_connection (%s)\n", default_charset, default_collation_connection); + bool use_collation = true; + if (strcmp(default_charset, previous_default_charset)) { // charset changed + if (strcmp(default_collation_connection, previous_default_collation_connection)==0) { // collation didn't change + // the user has changed the charset but not the collation + // we use charset as source of truth + use_collation = false; + } + } + if (use_collation) { + proxy_info("Changing mysql-default_charset to %s using configured mysql-default_collation_connection %s\n", cic->csname, cic->name); + sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_charset\",\"%s\")", cic->csname); + db->execute(q); + GloMTH->set_variable((char *)"default_charset",cic->csname); + } else { + proxy_info("Changing mysql-default_collation_connection to %s using configured mysql-default_charset: %s\n", ci->name, ci->csname); + sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_collation_connection\",\"%s\")", ci->name); + db->execute(q); + GloMTH->set_variable((char *)"default_collation_connection",ci->name); + } + } + } + } + free(default_charset); + free(default_collation_connection); + free(previous_default_charset); + free(previous_default_collation_connection); + GloMTH->commit(); + GloMTH->wrunlock(); + + { + // NOTE: 'GloMTH->wrunlock()' should have been called before this point to avoid possible + // deadlocks. See issue #3847. + pthread_mutex_lock(&GloVars.checksum_mutex); + // generate checksum for cluster + flush_mysql_variables___runtime_to_database(admindb, false, false, false, true, true); + flush_GENERIC_variables__checksum__database_to_runtime("mysql", checksum, epoch); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + } + if (resultset) delete resultset; +} + +void ProxySQL_Admin::flush_sqliteserver_variables___database_to_runtime(SQLite3DB *db, bool replace) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing SQLiteServer variables. Replace:%d\n", replace); + if ( + (GloVars.global.sqlite3_server == false) + || + ( GloSQLite3Server == NULL ) + ) { + return; + } + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (flush_GENERIC_variables__retrieve__database_to_runtime("sqliteserver", error, cols, affected_rows, resultset) == true) { + GloSQLite3Server->wrlock(); + flush_GENERIC_variables__process__database_to_runtime("sqliteserver", db, resultset, false, replace, {}, {"session_debug"}, {}, {}); + //GloClickHouse->commit(); + GloSQLite3Server->wrunlock(); + } + if (resultset) delete resultset; +} + +void ProxySQL_Admin::flush_sqliteserver_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ClickHouse variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); + if (GloVars.global.sqlite3_server == false) { + return; + } + if (onlyifempty) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'sqliteserver-%'"; + db->execute_statement(q, &error , &cols , &affected_rows , &resultset); + int matching_rows=0; + if (error) { + proxy_error("Error on %s : %s\n", q, error); + return; + } else { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + matching_rows+=atoi(r->fields[0]); + } + } + if (resultset) delete resultset; + if (matching_rows) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has ClickHouse variables - skipping\n"); + return; + } + } + if (del) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting ClickHouse variables from global_variables\n"); + db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'sqliteserver-%'"); + } + if (runtime) { + db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'sqliteserver-%'"); + } + char *a; + char *b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(\"sqliteserver-%s\",\"%s\")"; + if (replace) { + a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(\"sqliteserver-%s\",\"%s\")"; + } else { + a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(\"sqliteserver-%s\",\"%s\")"; + } + int l=strlen(a)+200; + GloSQLite3Server->wrlock(); + char **varnames=GloSQLite3Server->get_variables_list(); + for (int i=0; varnames[i]; i++) { + char *val=GloSQLite3Server->get_variable(varnames[i]); + l+=( varnames[i] ? strlen(varnames[i]) : 6); + l+=( val ? strlen(val) : 6); + char *query=(char *)malloc(l); + sprintf(query, a, varnames[i], val); + if (runtime) { + db->execute(query); + sprintf(query, b, varnames[i], val); + } + db->execute(query); + if (val) + free(val); + free(query); + } + GloSQLite3Server->wrunlock(); + for (int i=0; varnames[i]; i++) { + free(varnames[i]); + } + free(varnames); +} + + +#ifdef PROXYSQLCLICKHOUSE +void ProxySQL_Admin::flush_clickhouse_variables___database_to_runtime(SQLite3DB *db, bool replace) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ClickHouse variables. Replace:%d\n", replace); + if ( + (GloVars.global.clickhouse_server == false) + || + ( GloClickHouseServer == NULL ) + ) { + return; + } + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (flush_GENERIC_variables__retrieve__database_to_runtime("clickhouse", error, cols, affected_rows, resultset) == true) { + GloClickHouseServer->wrlock(); + flush_GENERIC_variables__process__database_to_runtime("clickhouse", db, resultset, false, replace, {}, {"session_debug"}, {}, {}); + //GloClickHouse->commit(); + GloClickHouseServer->wrunlock(); + } + if (resultset) delete resultset; +} + +void ProxySQL_Admin::flush_clickhouse_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ClickHouse variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); + if ( + (GloVars.global.clickhouse_server == false) + || + ( GloClickHouseServer == NULL ) + ) { + return; + } + if (onlyifempty) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'clickhouse-%'"; + db->execute_statement(q, &error , &cols , &affected_rows , &resultset); + int matching_rows=0; + if (error) { + proxy_error("Error on %s : %s\n", q, error); + return; + } else { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + matching_rows+=atoi(r->fields[0]); + } + } + if (resultset) delete resultset; + if (matching_rows) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has ClickHouse variables - skipping\n"); + return; + } + } + if (del) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting ClickHouse variables from global_variables\n"); + db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'clickhouse-%'"); + } + if (runtime) { + db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'clickhouse-%'"); + } + char *a; + char *b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(\"clickhouse-%s\",\"%s\")"; + if (replace) { + a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(\"clickhouse-%s\",\"%s\")"; + } else { + a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(\"clickhouse-%s\",\"%s\")"; + } + int l=strlen(a)+200; + GloClickHouseServer->wrlock(); + char **varnames=GloClickHouseServer->get_variables_list(); + for (int i=0; varnames[i]; i++) { + char *val=GloClickHouseServer->get_variable(varnames[i]); + l+=( varnames[i] ? strlen(varnames[i]) : 6); + l+=( val ? strlen(val) : 6); + char *query=(char *)malloc(l); + sprintf(query, a, varnames[i], val); + if (runtime) { + db->execute(query); + sprintf(query, b, varnames[i], val); + } + db->execute(query); + if (val) + free(val); + free(query); + } + GloClickHouseServer->wrunlock(); + for (int i=0; varnames[i]; i++) { + free(varnames[i]); + } + free(varnames); +} +#endif /* PROXYSQLCLICKHOUSE */ + +void ProxySQL_Admin::flush_pgsql_variables___database_to_runtime(SQLite3DB* db, bool replace, const std::string& checksum, const time_t epoch) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing PgSQL variables. Replace:%d\n", replace); + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + char* q = (char*)"SELECT substr(variable_name,7) vn, variable_value FROM global_variables WHERE variable_name LIKE 'pgsql-%'"; + admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", q, error); + return; + } + else { + GloPTH->wrlock(); + char* previous_default_charset = GloPTH->get_variable_string((char*)"default_charset"); + char* previous_default_collation_connection = GloPTH->get_variable_string((char*)"default_collation_connection"); + assert(previous_default_charset); + assert(previous_default_collation_connection); + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + const char* value = r->fields[1]; + bool rc = GloPTH->set_variable(r->fields[0], value); + if (rc == false) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Impossible to set variable %s with value \"%s\"\n", r->fields[0], value); + if (replace) { + char* val = GloPTH->get_variable(r->fields[0]); + char q[1000]; + if (val) { + if (strcmp(val, value)) { + proxy_warning("Impossible to set variable %s with value \"%s\". Resetting to current \"%s\".\n", r->fields[0], value, val); + sprintf(q, "INSERT OR REPLACE INTO global_variables VALUES(\"pgsql-%s\",\"%s\")", r->fields[0], val); + db->execute(q); + } + free(val); + } + else { + if (strcmp(r->fields[0], (char*)"session_debug") == 0) { + sprintf(q, "DELETE FROM disk.global_variables WHERE variable_name=\"pgsql-%s\"", r->fields[0]); + db->execute(q); + } + else { + if (strcmp(r->fields[0], (char*)"forward_autocommit") == 0) { + if (strcasecmp(value, "true") == 0 || strcasecmp(value, "1") == 0) { + proxy_error("Global variable pgsql-forward_autocommit is deprecated. See issue #3253\n"); + } + sprintf(q, "DELETE FROM disk.global_variables WHERE variable_name=\"pgsql-%s\"", r->fields[0]); + db->execute(q); + } + else { + proxy_warning("Impossible to set not existing variable %s with value \"%s\". Deleting. If the variable name is correct, this version doesn't support it\n", r->fields[0], r->fields[1]); + } + } + sprintf(q, "DELETE FROM global_variables WHERE variable_name=\"pgsql-%s\"", r->fields[0]); + db->execute(q); + } + } + } + else { + if ( + (strcmp(r->fields[0], "default_collation_connection") == 0) + || (strcmp(r->fields[0], "default_charset") == 0) + ) { + char* val = GloPTH->get_variable(r->fields[0]); + char q[1000]; + if (val) { + if (strcmp(val, value)) { + proxy_warning("Variable %s with value \"%s\" is being replaced with value \"%s\".\n", r->fields[0], value, val); + sprintf(q, "INSERT OR REPLACE INTO global_variables VALUES(\"pgsql-%s\",\"%s\")", r->fields[0], val); + db->execute(q); + } + free(val); + } + } + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Set variable %s with value \"%s\"\n", r->fields[0], value); + if (strcmp(r->fields[0], (char*)"show_processlist_extended") == 0) { + variables.pgsql_show_processlist_extended = atoi(value); + } + } + // } + } + + char q[1000]; + char* default_charset = GloPTH->get_variable_string((char*)"default_charset"); + char* default_collation_connection = GloPTH->get_variable_string((char*)"default_collation_connection"); + assert(default_charset); + assert(default_collation_connection); + MARIADB_CHARSET_INFO* ci = NULL; + ci = proxysql_find_charset_name(default_charset); + if (ci == NULL) { + // invalid charset + proxy_error("Found an incorrect value for pgsql-default_charset: %s\n", default_charset); + // let's try to get a charset from collation connection + ci = proxysql_find_charset_collate(default_collation_connection); + if (ci == NULL) { + proxy_error("Found an incorrect value for pgsql-default_collation_connection: %s\n", default_collation_connection); + const char* p = mysql_tracked_variables[SQL_CHARACTER_SET].default_value; + ci = proxysql_find_charset_name(p); + assert(ci); + proxy_info("Resetting pgsql-default_charset to hardcoded default value: %s\n", ci->csname); + sprintf(q, "INSERT OR REPLACE INTO global_variables VALUES(\"pgsql-default_charset\",\"%s\")", ci->csname); + db->execute(q); + GloPTH->set_variable((char*)"default_charset", ci->csname); + proxy_info("Resetting pgsql-default_collation_connection to hardcoded default value: %s\n", ci->name); + sprintf(q, "INSERT OR REPLACE INTO global_variables VALUES(\"pgsql-default_collation_connection\",\"%s\")", ci->name); + db->execute(q); + GloPTH->set_variable((char*)"default_collation_connection", ci->name); + } + else { + proxy_info("Changing pgsql-default_charset to %s using configured pgsql-default_collation_connection %s\n", ci->csname, ci->name); + sprintf(q, "INSERT OR REPLACE INTO global_variables VALUES(\"pgsql-default_charset\",\"%s\")", ci->csname); + db->execute(q); + GloPTH->set_variable((char*)"default_charset", ci->csname); + } + } + else { + MARIADB_CHARSET_INFO* cic = NULL; + cic = proxysql_find_charset_collate(default_collation_connection); + if (cic == NULL) { + proxy_error("Found an incorrect value for pgsql-default_collation_connection: %s\n", default_collation_connection); + proxy_info("Changing pgsql-default_collation_connection to %s using configured pgsql-default_charset: %s\n", ci->name, ci->csname); + sprintf(q, "INSERT OR REPLACE INTO global_variables VALUES(\"pgsql-default_collation_connection\",\"%s\")", ci->name); + db->execute(q); + GloPTH->set_variable((char*)"default_collation_connection", ci->name); + } + else { + if (strcmp(cic->csname, ci->csname) == 0) { + // pgsql-default_collation_connection and pgsql-default_charset are compatible + } + else { + proxy_error("Found incompatible values for pgsql-default_charset (%s) and pgsql-default_collation_connection (%s)\n", default_charset, default_collation_connection); + bool use_collation = true; + if (strcmp(default_charset, previous_default_charset)) { // charset changed + if (strcmp(default_collation_connection, previous_default_collation_connection) == 0) { // collation didn't change + // the user has changed the charset but not the collation + // we use charset as source of truth + use_collation = false; + } + } + if (use_collation) { + proxy_info("Changing pgsql-default_charset to %s using configured pgsql-default_collation_connection %s\n", cic->csname, cic->name); + sprintf(q, "INSERT OR REPLACE INTO global_variables VALUES(\"pgsql-default_charset\",\"%s\")", cic->csname); + db->execute(q); + GloPTH->set_variable((char*)"default_charset", cic->csname); + } + else { + proxy_info("Changing pgsql-default_collation_connection to %s using configured pgsql-default_charset: %s\n", ci->name, ci->csname); + sprintf(q, "INSERT OR REPLACE INTO global_variables VALUES(\"pgsql-default_collation_connection\",\"%s\")", ci->name); + db->execute(q); + GloPTH->set_variable((char*)"default_collation_connection", ci->name); + } + } + } + } + free(default_charset); + free(default_collation_connection); + free(previous_default_charset); + free(previous_default_collation_connection); + GloPTH->commit(); + GloPTH->wrunlock(); + + /* Checksums are always generated - 'admin-checksum_*' deprecated + { + // NOTE: 'GloPTH->wrunlock()' should have been called before this point to avoid possible + // deadlocks. See issue #3847. + pthread_mutex_lock(&GloVars.checksum_mutex); + // generate checksum for cluster + flush_mysql_variables___runtime_to_database(admindb, false, false, false, true, true); + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + std::string q; + q = "SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'mysql-\%' AND variable_name NOT IN ('mysql-threads')"; + if (GloVars.cluster_sync_interfaces == false) { + q += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_MYSQL); + } + q += " ORDER BY variable_name"; + admindb->execute_statement(q.c_str(), &error, &cols, &affected_rows, &resultset); + uint64_t hash1 = resultset->raw_checksum(); + uint32_t d32[2]; + char buf[20]; + memcpy(&d32, &hash1, sizeof(hash1)); + sprintf(buf, "0x%0X%0X", d32[0], d32[1]); + GloVars.checksums_values.mysql_variables.set_checksum(buf); + GloVars.checksums_values.mysql_variables.version++; + time_t t = time(NULL); + if (epoch != 0 && checksum != "" && GloVars.checksums_values.mysql_variables.checksum == checksum) { + GloVars.checksums_values.mysql_variables.epoch = epoch; + } + else { + GloVars.checksums_values.mysql_variables.epoch = t; + } + GloVars.epoch_version = t; + GloVars.generate_global_checksum(); + GloVars.checksums_values.updates_cnt++; + pthread_mutex_unlock(&GloVars.checksum_mutex); + delete resultset; + } + proxy_info( + "Computed checksum for 'LOAD MYSQL VARIABLES TO RUNTIME' was '%s', with epoch '%llu'\n", + GloVars.checksums_values.mysql_variables.checksum, GloVars.checksums_values.mysql_variables.epoch + ); + */ + } + if (resultset) delete resultset; +} + +void ProxySQL_Admin::flush_mysql_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime, bool use_lock) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing MySQL variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); + if (onlyifempty) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'mysql-%'"; + db->execute_statement(q, &error , &cols , &affected_rows , &resultset); + int matching_rows=0; + if (error) { + proxy_error("Error on %s : %s\n", q, error); + return; + } else { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + matching_rows+=atoi(r->fields[0]); + } + } + if (resultset) delete resultset; + if (matching_rows) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has MySQL variables - skipping\n"); + return; + } + } + if (del) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting MySQL variables from global_variables\n"); + db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'mysql-%'"); + } + static char *a; + static char *b; + if (replace) { + a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(?1, ?2)"; + } else { + a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(?1, ?2)"; + } + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement2=NULL; + + rc=db->prepare_v2(a, &statement1); + ASSERT_SQLITE_OK(rc, db); + if (runtime) { + db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'mysql-%'"); + b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(?1, ?2)"; + + rc=db->prepare_v2(b, &statement2); + ASSERT_SQLITE_OK(rc, db); + } + if (use_lock) { + GloMTH->wrlock(); + db->execute("BEGIN"); + } + char **varnames=GloMTH->get_variables_list(); + for (int i=0; varnames[i]; i++) { + char *val=GloMTH->get_variable(varnames[i]); + char *qualified_name=(char *)malloc(strlen(varnames[i])+7); + sprintf(qualified_name, "mysql-%s", varnames[i]); + rc=(*proxy_sqlite3_bind_text)(statement1, 1, qualified_name, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, (val ? val : (char *)""), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, db); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, db); + if (runtime) { + rc=(*proxy_sqlite3_bind_text)(statement2, 1, qualified_name, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + rc=(*proxy_sqlite3_bind_text)(statement2, 2, (val ? val : (char *)""), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); + SAFE_SQLITE3_STEP2(statement2); + rc=(*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, db); + rc=(*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, db); + } + if (val) + free(val); + free(qualified_name); + } + if (use_lock) { + db->execute("COMMIT"); + GloMTH->wrunlock(); + } + (*proxy_sqlite3_finalize)(statement1); + if (runtime) + (*proxy_sqlite3_finalize)(statement2); + for (int i=0; varnames[i]; i++) { + free(varnames[i]); + } + free(varnames); +} + +void ProxySQL_Admin::flush_ldap_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum, const time_t epoch) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing LDAP variables. Replace:%d\n", replace); + if (GloMyLdapAuth == NULL) { + return; + } + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (flush_GENERIC_variables__retrieve__database_to_runtime("ldap", error, cols, affected_rows, resultset) == true) { + GloMyLdapAuth->wrlock(); + flush_GENERIC_variables__process__database_to_runtime("admin", db, resultset, false, replace, {}, {}, {}, {}); + GloMyLdapAuth->wrunlock(); + + // Checksums are always generated - 'admin-checksum_*' deprecated + { + pthread_mutex_lock(&GloVars.checksum_mutex); + // generate checksum for cluster + flush_ldap_variables___runtime_to_database(admindb, false, false, false, true); + flush_GENERIC_variables__checksum__database_to_runtime("ldap", checksum, epoch); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + } + if (resultset) delete resultset; +} + +void ProxySQL_Admin::flush_ldap_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing LDAP variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); + if (GloMyLdapAuth == NULL) { + return; + } + if (onlyifempty) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'ldap-%'"; + db->execute_statement(q, &error , &cols , &affected_rows , &resultset); + int matching_rows=0; + if (error) { + proxy_error("Error on %s : %s\n", q, error); + return; + } else { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + matching_rows+=atoi(r->fields[0]); + } + } + if (resultset) delete resultset; + if (matching_rows) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has LDAP variables - skipping\n"); + return; + } + } + if (del) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting LDAP variables from global_variables\n"); + db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'ldap-%'"); + } + if (runtime) { + db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'ldap-%'"); + } + char *a; + char *b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(\"ldap-%s\",\"%s\")"; + if (replace) { + a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(\"ldap-%s\",\"%s\")"; + } else { + a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(\"ldap-%s\",\"%s\")"; + } + int l=strlen(a)+200; + GloMyLdapAuth->wrlock(); + char **varnames=GloMyLdapAuth->get_variables_list(); + for (int i=0; varnames[i]; i++) { + char *val=GloMyLdapAuth->get_variable(varnames[i]); + l+=( varnames[i] ? strlen(varnames[i]) : 6); + l+=( val ? strlen(val) : 6); + char *query=(char *)malloc(l); + sprintf(query, a, varnames[i], val); + if (runtime) { + db->execute(query); + sprintf(query, b, varnames[i], val); + } + db->execute(query); + if (val) + free(val); + free(query); + } + GloMyLdapAuth->wrunlock(); + for (int i=0; varnames[i]; i++) { + free(varnames[i]); + } + free(varnames); +} + +void ProxySQL_Admin::flush_admin_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ADMIN variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); + if (onlyifempty) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'admin-%'"; + db->execute_statement(q, &error , &cols , &affected_rows , &resultset); + int matching_rows=0; + if (error) { + proxy_error("Error on %s : %s\n", q, error); + return; + } else { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + matching_rows+=atoi(r->fields[0]); + } + } + if (resultset) delete resultset; + if (matching_rows) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has ADMIN variables - skipping\n"); + return; + } + } + if (del) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting ADMIN variables from global_variables\n"); + db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'admin-%'"); + } + if (runtime) { + db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'admin-%'"); + } + char *a; + char *b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(\"admin-%s\",\"%s\")"; + if (replace) { + a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(\"admin-%s\",\"%s\")"; + } else { + a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(\"admin-%s\",\"%s\")"; + } + int l=strlen(a)+200; + + char **varnames=get_variables_list(); + for (int i=0; varnames[i]; i++) { + char *val=get_variable(varnames[i]); + l+=( varnames[i] ? strlen(varnames[i]) : 6); + l+=( val ? strlen(val) : 6); + char *query=(char *)malloc(l); + sprintf(query, a, varnames[i], val); + db->execute(query); + if (runtime) { + sprintf(query, b, varnames[i], val); + db->execute(query); + } + if (val) + free(val); + free(query); + } + for (int i=0; varnames[i]; i++) { + free(varnames[i]); + } + free(varnames); + +} diff --git a/lib/Admin_Handler.cpp b/lib/Admin_Handler.cpp new file mode 100644 index 0000000000..7d11946a0b --- /dev/null +++ b/lib/Admin_Handler.cpp @@ -0,0 +1,3721 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include // std::cout +#include // std::stringstream +#include +#include // std::sort +#include +#include // std::vector +#include +#include "prometheus/exposer.h" +#include "prometheus/counter.h" +#include "openssl/ssl.h" +#include "openssl/err.h" + +#include "Base_Thread.h" + +#include "MySQL_HostGroups_Manager.h" +#include "PgSQL_HostGroups_Manager.h" +#include "mysql.h" +#include "proxysql_admin.h" +#include "re2/re2.h" +#include "re2/regexp.h" +#include "proxysql.h" +#include "proxysql_config.h" +#include "proxysql_restapi.h" +#include "proxysql_utils.h" +#include "prometheus_helpers.h" +#include "cpp.h" + +#include "MySQL_Data_Stream.h" +#include "PgSQL_Data_Stream.h" +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" +#include "ProxySQL_HTTP_Server.hpp" // HTTP server +#include "MySQL_Authentication.hpp" +#include "PgSQL_Authentication.h" +#include "MySQL_LDAP_Authentication.hpp" +#include "MySQL_PreparedStatement.h" +#include "ProxySQL_Cluster.hpp" +#include "ProxySQL_Statistics.hpp" +#include "MySQL_Logger.hpp" +#include "PgSQL_Logger.hpp" +#include "SQLite3_Server.h" +#include "Web_Interface.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif + +#include +#include + +#include "platform.h" +#include "microhttpd.h" + +#if (defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__) || defined(__mips__)) && defined(__linux) +// currently only support x86-32, x86-64, ARM, and MIPS on Linux +#include "coredumper/coredumper.h" +#endif + +#include + +#include "PgSQL_Protocol.h" +//#include "usual/time.h" + +using std::string; +using std::unique_ptr; + +#ifdef WITHGCOV +extern "C" void __gcov_dump(); +extern "C" void __gcov_reset(); +#endif + + +#ifdef DEBUG +//#define BENCHMARK_FASTROUTING_LOAD +#endif // DEBUG + +//#define MYSQL_THREAD_IMPLEMENTATION + +#define SELECT_VERSION_COMMENT "select @@version_comment limit 1" +#define SELECT_VERSION_COMMENT_LEN 32 +#define SELECT_DB_USER "select DATABASE(), USER() limit 1" +#define SELECT_DB_USER_LEN 33 +#define SELECT_CHARSET_VARIOUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" +#define SELECT_CHARSET_VARIOUS_LEN 115 + +#define READ_ONLY_OFF "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0e\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x03\x4f\x46\x46\x05\x00\x00\x06\xfe\x00\x00\x02\x00" +#define READ_ONLY_ON "\x01\x00\x00\x01\x02\x23\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x0d\x56\x61\x72\x69\x61\x62\x6c\x65\x5f\x6e\x61\x6d\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x1b\x00\x00\x03\x03\x64\x65\x66\x00\x00\x00\x05\x56\x61\x6c\x75\x65\x00\x0c\x21\x00\x0f\x00\x00\x00\xfd\x01\x00\x1f\x00\x00\x05\x00\x00\x04\xfe\x00\x00\x02\x00\x0d\x00\x00\x05\x09\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x02\x4f\x4e\x05\x00\x00\x06\xfe\x00\x00\x02\x00" + +#define READ_ONLY_0 "\x01\x00\x00\x01\x01\x28\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x12\x40\x40\x67\x6c\x6f\x62\x61\x6c\x2e\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x00\x0c\x3f\x00\x01\x00\x00\x00\x08\x80\x00\x00\x00\x00\x05\x00\x00\x03\xfe\x00\x00\x02\x00\x02\x00\x00\x04\x01\x30\x05\x00\x00\x05\xfe\x00\x00\x02\x00" + +#define READ_ONLY_1 "\x01\x00\x00\x01\x01\x28\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x12\x40\x40\x67\x6c\x6f\x62\x61\x6c\x2e\x72\x65\x61\x64\x5f\x6f\x6e\x6c\x79\x00\x0c\x3f\x00\x01\x00\x00\x00\x08\x80\x00\x00\x00\x00\x05\x00\x00\x03\xfe\x00\x00\x02\x00\x02\x00\x00\x04\x01\x31\x05\x00\x00\x05\xfe\x00\x00\x02\x00" + +extern struct MHD_Daemon *Admin_HTTP_Server; + +extern ProxySQL_Statistics *GloProxyStats; + +int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg); + +extern char *ssl_key_fp; +extern char *ssl_cert_fp; +extern char *ssl_ca_fp; + +// ProxySQL_Admin shared variables +extern int admin___web_verbosity; +extern char * proxysql_version; + +#include "proxysql_find_charset.h" + +extern int admin_load_main_; +extern bool admin_nostart_; + +extern int __admin_refresh_interval; + +extern bool admin_proxysql_mysql_paused; +extern bool admin_proxysql_pgsql_paused; +extern int admin_old_wait_timeout; + + +extern Query_Cache *GloQC; +extern MySQL_Authentication *GloMyAuth; +extern PgSQL_Authentication *GloPgAuth; +extern MySQL_LDAP_Authentication *GloMyLdapAuth; +extern ProxySQL_Admin *GloAdmin; +extern MySQL_Query_Processor* GloMyQPro; +extern PgSQL_Query_Processor* GloPgQPro; +extern MySQL_Threads_Handler *GloMTH; +extern MySQL_Logger *GloMyLogger; +extern PgSQL_Logger* GloPgSQL_Logger; +extern MySQL_STMT_Manager_v14 *GloMyStmt; +extern MySQL_Monitor *GloMyMon; +extern PgSQL_Threads_Handler* GloPTH; + +extern void (*flush_logs_function)(); + +extern Web_Interface *GloWebInterface; + +extern ProxySQL_Cluster *GloProxyCluster; +#ifdef PROXYSQLCLICKHOUSE +extern ClickHouse_Authentication *GloClickHouseAuth; +extern ClickHouse_Server *GloClickHouseServer; +#endif /* PROXYSQLCLICKHOUSE */ + +extern SQLite3_Server *GloSQLite3Server; + +extern char * binary_sha1; + +extern int ProxySQL_create_or_load_TLS(bool bootstrap, std::string& msg); + + +#define PANIC(msg) { perror(msg); exit(EXIT_FAILURE); } + +extern pthread_mutex_t users_mutex; + +extern ProxySQL_Admin *SPA; + +const std::vector LOAD_ADMIN_VARIABLES_TO_MEMORY = { + "LOAD ADMIN VARIABLES TO MEMORY" , + "LOAD ADMIN VARIABLES TO MEM" , + "LOAD ADMIN VARIABLES FROM DISK" }; + +const std::vector SAVE_ADMIN_VARIABLES_FROM_MEMORY = { + "SAVE ADMIN VARIABLES FROM MEMORY" , + "SAVE ADMIN VARIABLES FROM MEM" , + "SAVE ADMIN VARIABLES TO DISK" }; + +const std::vector LOAD_ADMIN_VARIABLES_FROM_MEMORY = { + "LOAD ADMIN VARIABLES FROM MEMORY" , + "LOAD ADMIN VARIABLES FROM MEM" , + "LOAD ADMIN VARIABLES TO RUNTIME" , + "LOAD ADMIN VARIABLES TO RUN" }; + +const std::vector SAVE_ADMIN_VARIABLES_TO_MEMORY = { + "SAVE ADMIN VARIABLES TO MEMORY" , + "SAVE ADMIN VARIABLES TO MEM" , + "SAVE ADMIN VARIABLES FROM RUNTIME" , + "SAVE ADMIN VARIABLES FROM RUN" }; + +const std::vector LOAD_MYSQL_SERVERS_FROM_MEMORY = { + "LOAD MYSQL SERVERS FROM MEMORY" , + "LOAD MYSQL SERVERS FROM MEM" , + "LOAD MYSQL SERVERS TO RUNTIME" , + "LOAD MYSQL SERVERS TO RUN" }; + +const std::vector SAVE_MYSQL_SERVERS_TO_MEMORY = { + "SAVE MYSQL SERVERS TO MEMORY" , + "SAVE MYSQL SERVERS TO MEM" , + "SAVE MYSQL SERVERS FROM RUNTIME" , + "SAVE MYSQL SERVERS FROM RUN" }; + +const std::vector LOAD_MYSQL_USERS_FROM_MEMORY = { + "LOAD MYSQL USERS FROM MEMORY" , + "LOAD MYSQL USERS FROM MEM" , + "LOAD MYSQL USERS TO RUNTIME" , + "LOAD MYSQL USERS TO RUN" }; + +const std::vector SAVE_MYSQL_USERS_TO_MEMORY = { + "SAVE MYSQL USERS TO MEMORY" , + "SAVE MYSQL USERS TO MEM" , + "SAVE MYSQL USERS FROM RUNTIME" , + "SAVE MYSQL USERS FROM RUN" }; + +const std::vector LOAD_MYSQL_VARIABLES_FROM_MEMORY = { + "LOAD MYSQL VARIABLES FROM MEMORY" , + "LOAD MYSQL VARIABLES FROM MEM" , + "LOAD MYSQL VARIABLES TO RUNTIME" , + "LOAD MYSQL VARIABLES TO RUN" }; + +const std::vector SAVE_MYSQL_VARIABLES_TO_MEMORY = { + "SAVE MYSQL VARIABLES TO MEMORY" , + "SAVE MYSQL VARIABLES TO MEM" , + "SAVE MYSQL VARIABLES FROM RUNTIME" , + "SAVE MYSQL VARIABLES FROM RUN" }; + +// PgSQL +const std::vector LOAD_PGSQL_SERVERS_FROM_MEMORY = { + "LOAD PGSQL SERVERS FROM MEMORY" , + "LOAD PGSQL SERVERS FROM MEM" , + "LOAD PGSQL SERVERS TO RUNTIME" , + "LOAD PGSQL SERVERS TO RUN" }; + +const std::vector SAVE_PGSQL_SERVERS_TO_MEMORY = { + "SAVE PGSQL SERVERS TO MEMORY" , + "SAVE PGSQL SERVERS TO MEM" , + "SAVE PGSQL SERVERS FROM RUNTIME" , + "SAVE PGSQL SERVERS FROM RUN" }; + +const std::vector LOAD_PGSQL_USERS_FROM_MEMORY = { + "LOAD PGSQL USERS FROM MEMORY" , + "LOAD PGSQL USERS FROM MEM" , + "LOAD PGSQL USERS TO RUNTIME" , + "LOAD PGSQL USERS TO RUN" }; + +const std::vector SAVE_PGSQL_USERS_TO_MEMORY = { + "SAVE PGSQL USERS TO MEMORY" , + "SAVE PGSQL USERS TO MEM" , + "SAVE PGSQL USERS FROM RUNTIME" , + "SAVE PGSQL USERS FROM RUN" }; + +const std::vector LOAD_PGSQL_VARIABLES_FROM_MEMORY = { + "LOAD PGSQL VARIABLES FROM MEMORY" , + "LOAD PGSQL VARIABLES FROM MEM" , + "LOAD PGSQL VARIABLES TO RUNTIME" , + "LOAD PGSQL VARIABLES TO RUN" }; + +const std::vector SAVE_PGSQL_VARIABLES_TO_MEMORY = { + "SAVE PGSQL VARIABLES TO MEMORY" , + "SAVE PGSQL VARIABLES TO MEM" , + "SAVE PGSQL VARIABLES FROM RUNTIME" , + "SAVE PGSQL VARIABLES FROM RUN" }; +// +const std::vector LOAD_COREDUMP_FROM_MEMORY = { + "LOAD COREDUMP FROM MEMORY" , + "LOAD COREDUMP FROM MEM" , + "LOAD COREDUMP TO RUNTIME" , + "LOAD COREDUMP TO RUN" }; + +extern unordered_map, vector>> load_save_disk_commands; + +bool is_admin_command_or_alias(const std::vector& cmds, char *query_no_space, int query_no_space_length) { + for (std::vector::const_iterator it=cmds.begin(); it!=cmds.end(); ++it) { + if ((unsigned int)query_no_space_length==it->length() && !strncasecmp(it->c_str(), query_no_space, query_no_space_length)) { + proxy_info("Received %s command\n", query_no_space); + return true; + } + } + return false; +} + + +template +bool FlushCommandWrapper(S* sess, const std::vector& cmds, char *query_no_space, int query_no_space_length, const string& name, const string& direction) { + if ( is_admin_command_or_alias(cmds, query_no_space, query_no_space_length) ) { + ProxySQL_Admin *SPA = GloAdmin; + SPA->flush_GENERIC__from_to(name, direction); +#ifdef DEBUG + string msg = "Loaded " + name + " "; + if (direction == "memory_to_disk") + msg += "from MEMORY to DISK"; + else if (direction == "disk_to_memory") + msg += "from DISK to MEMORY"; + else + assert(0); + msg += "\n"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s", msg.c_str()); +#endif // DEBUG + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return true; + } + return false; +} + +template +bool FlushCommandWrapper(S* sess, const string& modname, char *query_no_space, int query_no_space_length) { + assert(load_save_disk_commands.find(modname) != load_save_disk_commands.end()); + tuple, vector>& t = load_save_disk_commands[modname]; + if (FlushCommandWrapper(sess, get<1>(t), query_no_space, query_no_space_length, modname, "disk_to_memory") == true) + return true; + if (FlushCommandWrapper(sess, get<2>(t), query_no_space, query_no_space_length, modname, "memory_to_disk") == true) + return true; + return false; +} + +template +bool admin_handler_command_kill_connection(char *query_no_space, unsigned int query_no_space_length, S* sess, ProxySQL_Admin *pa) { + uint32_t id=atoi(query_no_space+16); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Trying to kill session %u\n", id); + bool rc=GloMTH->kill_session(id); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (rc) { + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + char buf[1024]; + sprintf(buf,"Unknown thread id: %u", id); + SPA->send_error_msg_to_client(sess, buf); + } + return false; +} + +template +bool admin_handler_command_proxysql(char *query_no_space, unsigned int query_no_space_length, S* sess, ProxySQL_Admin *pa) { + +#if (defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__) || defined(__mips__)) && defined(__linux) + // currently only support x86-32, x86-64, ARM, and MIPS on Linux + if (!(strncasecmp("PROXYSQL COREDUMP", query_no_space, strlen("PROXYSQL COREDUMP")))) { + string filename = "core"; + if (query_no_space_length > strlen("PROXYSQL COREDUMP")) { + if (query_no_space[strlen("PROXYSQL COREDUMP")] == ' ') { + filename = string(query_no_space+strlen("PROXYSQL COREDUMP ")); + } else { + filename = ""; + } + } + if (filename == "") { + proxy_error("Received incorrect PROXYSQL COREDUMP command: %s\n", query_no_space); + } else { + proxy_info("Received PROXYSQL COREDUMP command: %s\n", query_no_space); + // generates a core dump + WriteCoreDump(filename.c_str()); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + string msg = "Coredump: " + filename; + SPA->send_ok_msg_to_client(sess, (char *)msg.c_str(), 0, query_no_space); + return false; + } + } + if (!(strncasecmp("PROXYSQL COMPRESSEDCOREDUMP", query_no_space, strlen("PROXYSQL COMPRESSEDCOREDUMP")))) { + string filename = "core"; + if (query_no_space_length > strlen("PROXYSQL COMPRESSEDCOREDUMP")) { + if (query_no_space[strlen("PROXYSQL COMPRESSEDCOREDUMP")] == ' ') { + filename = string(query_no_space+strlen("PROXYSQL COMPRESSEDCOREDUMP ")); + } else { + filename = ""; + } + } + if (filename == "") { + proxy_error("Received incorrect PROXYSQL COMPRESSEDCOREDUMP command: %s\n", query_no_space); + } else { + proxy_info("Received PROXYSQL COMPRESSEDCOREDUMP command: %s\n", query_no_space); + // generates a compressed core dump + WriteCompressedCoreDump(filename.c_str(), SIZE_MAX, COREDUMPER_COMPRESSED, NULL); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + string msg = "Coredump: " + filename; + SPA->send_ok_msg_to_client(sess, (char *)msg.c_str(), 0, query_no_space); + return false; + } + } +#endif + + if (!(strncasecmp("PROXYSQL CLUSTER_NODE_UUID ", query_no_space, strlen("PROXYSQL CLUSTER_NODE_UUID ")))) { + int l = strlen("PROXYSQL CLUSTER_NODE_UUID "); + if (sess->client_myds->addr.port == 0) { + proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID not from TCP socket. Exiting client\n"); + SPA->send_error_msg_to_client(sess, (char *)"Received PROXYSQL CLUSTER_NODE_UUID not from TCP socket"); + sess->client_myds->shut_soft(); + return false; + } + if (query_no_space_length >= (unsigned int)l+36+2) { + uuid_t uu; + char *A_uuid = NULL; + char *B_interface = NULL; + c_split_2(query_no_space+l, " ", &A_uuid, &B_interface); // we split the value + if (uuid_parse(A_uuid, uu)==0 && B_interface && strlen(B_interface)) { + proxy_info("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d : %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); + if (sess->proxysql_node_address==NULL) { + sess->proxysql_node_address = new ProxySQL_Node_Address(sess->client_myds->addr.addr, sess->client_myds->addr.port); + sess->proxysql_node_address->uuid = strdup(A_uuid); + if (sess->proxysql_node_address->admin_mysql_ifaces) { + free(sess->proxysql_node_address->admin_mysql_ifaces); + } + sess->proxysql_node_address->admin_mysql_ifaces = strdup(B_interface); + proxy_info("Created new link with Cluster node %s:%d : %s at interface %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid, B_interface); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + free(A_uuid); + free(B_interface); + return false; + } else { + if (strcmp(A_uuid, sess->proxysql_node_address->uuid)) { + proxy_error("Cluster node %s:%d is sending a new UUID : %s . Former UUID : %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid, sess->proxysql_node_address->uuid); + SPA->send_error_msg_to_client(sess, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with a new UUID not matching the previous one"); + sess->client_myds->shut_soft(); + free(A_uuid); + free(B_interface); + return false; + } else { + proxy_info("Cluster node %s:%d is sending again its UUID : %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + free(A_uuid); + free(B_interface); + return false; + } + } + free(A_uuid); + free(B_interface); + return false; + } else { + proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d with invalid format: %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); + SPA->send_error_msg_to_client(sess, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with invalid format"); + sess->client_myds->shut_soft(); + return false; + } + } else { + proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d with invalid format: %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); + SPA->send_error_msg_to_client(sess, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with invalid format"); + sess->client_myds->shut_soft(); + return false; + } + } + if (query_no_space_length==strlen("PROXYSQL READONLY") && !strncasecmp("PROXYSQL READONLY",query_no_space, query_no_space_length)) { + // this command enables admin_read_only , so the admin module is in read_only mode + proxy_info("Received PROXYSQL READONLY command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->set_read_only(true); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + if (query_no_space_length==strlen("PROXYSQL READWRITE") && !strncasecmp("PROXYSQL READWRITE",query_no_space, query_no_space_length)) { + // this command disables admin_read_only , so the admin module won't be in read_only mode + proxy_info("Received PROXYSQL WRITE command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->set_read_only(false); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + if (query_no_space_length==strlen("PROXYSQL START") && !strncasecmp("PROXYSQL START",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL START command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + bool rc=false; + if (admin_nostart_) { + rc=__sync_bool_compare_and_swap(&GloVars.global.nostart,1,0); + } + if (rc) { + // Set the status variable 'threads_initialized' to 0 because it's initialized back + // in main 'init_phase3'. After GloMTH have been initialized again. + __sync_bool_compare_and_swap(&GloMTH->status_variables.threads_initialized, 1, 0); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Starting ProxySQL following PROXYSQL START command\n"); + while(__sync_fetch_and_add(&GloMTH->status_variables.threads_initialized, 0) == 1) { + usleep(1000); + } + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + proxy_warning("ProxySQL was already started when received PROXYSQL START command\n"); + SPA->send_error_msg_to_client(sess, (char *)"ProxySQL already started"); + } + return false; + } + + if (query_no_space_length==strlen("PROXYSQL RESTART") && !strncasecmp("PROXYSQL RESTART",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL RESTART command\n"); + // This function was introduced into 'prometheus::Registry' for being + // able to do a complete reset of all the 'prometheus counters'. It + // shall only be used during ProxySQL shutdown phases. + GloVars.prometheus_registry->ResetCounters(); + __sync_bool_compare_and_swap(&glovars.shutdown,0,1); + glovars.reload=1; + return false; + } + + if (query_no_space_length==strlen("PROXYSQL STOP") && !strncasecmp("PROXYSQL STOP",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL STOP command\n"); + // to speed up this process we first change wait_timeout to 0 + // MySQL_thread will call poll() with a maximum timeout of 100ms + admin_old_wait_timeout=GloMTH->get_variable_int((char *)"wait_timeout"); + GloMTH->set_variable((char *)"wait_timeout",(char *)"0"); + GloMTH->commit(); + GloMTH->signal_all_threads(0); + GloMTH->stop_listeners(); + char buf[32]; + sprintf(buf,"%d",admin_old_wait_timeout); + GloMTH->set_variable((char *)"wait_timeout",buf); + GloMTH->commit(); + glovars.reload=2; + // This function was introduced into 'prometheus::Registry' for being + // able to do a complete reset of all the 'prometheus counters'. It + // shall only be used during ProxySQL shutdown phases. + GloVars.prometheus_registry->ResetCounters(); + __sync_bool_compare_and_swap(&glovars.shutdown,0,1); + // After setting the shutdown flag, we should wake all threads and wait for + // the shutdown phase to complete. + GloMTH->signal_all_threads(0); + while (__sync_fetch_and_add(&glovars.shutdown,0)==1) { + usleep(1000); + } + // After shutdown phase is completed, we must to send a 'OK' to the + // mysql client, otherwise, since this session might not be drop due + // to the waiting condition, the client wont disconnect and will + // keep forever waiting for acknowledgement. + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if (query_no_space_length==strlen("PROXYSQL PAUSE") && !strncasecmp("PROXYSQL PAUSE",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL PAUSE command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (admin_nostart_) { + if (__sync_fetch_and_add((uint8_t *)(&GloVars.global.nostart),0)) { + SPA->send_error_msg_to_client(sess, (char *)"ProxySQL MySQL module not running, impossible to pause"); + return false; + } + } + if (admin_proxysql_mysql_paused==false) { + // to speed up this process we first change poll_timeout to 10 + // MySQL_thread will call poll() with a maximum timeout of 10ms + admin_old_wait_timeout=GloMTH->get_variable_int((char *)"poll_timeout"); + GloMTH->set_variable((char *)"poll_timeout",(char *)"10"); + GloMTH->commit(); + GloMTH->signal_all_threads(0); + GloMTH->stop_listeners(); + admin_proxysql_mysql_paused=true; + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + // we now rollback poll_timeout + char buf[32]; + sprintf(buf,"%d",admin_old_wait_timeout); + GloMTH->set_variable((char *)"poll_timeout",buf); + GloMTH->commit(); + } else { + SPA->send_error_msg_to_client(sess, (char *)"ProxySQL MySQL module is already paused, impossible to pause"); + } + + if (admin_proxysql_pgsql_paused == false) { + // to speed up this process we first change poll_timeout to 10 + // PgSQL_thread will call poll() with a maximum timeout of 10ms + admin_old_wait_timeout = GloPTH->get_variable_int((char*)"poll_timeout"); + GloPTH->set_variable((char*)"poll_timeout", (char*)"10"); + GloPTH->commit(); + GloPTH->signal_all_threads(0); + GloPTH->stop_listeners(); + admin_proxysql_pgsql_paused = true; + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + // we now rollback poll_timeout + char buf[32]; + sprintf(buf, "%d", admin_old_wait_timeout); + GloPTH->set_variable((char*)"poll_timeout", buf); + GloPTH->commit(); + } + else { + SPA->send_error_msg_to_client(sess, (char*)"ProxySQL PgSQL module is already paused, impossible to pause"); + } + return false; + } + + if (query_no_space_length==strlen("PROXYSQL RESUME") && !strncasecmp("PROXYSQL RESUME",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL RESUME command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (admin_nostart_) { + if (__sync_fetch_and_add((uint8_t *)(&GloVars.global.nostart),0)) { + SPA->send_error_msg_to_client(sess, (char *)"ProxySQL MySQL module not running, impossible to resume"); + return false; + } + } + if (admin_proxysql_mysql_paused==true) { + // to speed up this process we first change poll_timeout to 10 + // MySQL_thread will call poll() with a maximum timeout of 10ms + admin_old_wait_timeout=GloMTH->get_variable_int((char *)"poll_timeout"); + GloMTH->set_variable((char *)"poll_timeout",(char *)"10"); + GloMTH->commit(); + GloMTH->signal_all_threads(0); + GloMTH->start_listeners(); + //char buf[32]; + //sprintf(buf,"%d",old_wait_timeout); + //GloMTH->set_variable((char *)"poll_timeout",buf); + //GloMTH->commit(); + admin_proxysql_mysql_paused=false; + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + // we now rollback poll_timeout + char buf[32]; + sprintf(buf,"%d",admin_old_wait_timeout); + GloMTH->set_variable((char *)"poll_timeout",buf); + GloMTH->commit(); + } else { + SPA->send_error_msg_to_client(sess, (char *)"ProxySQL MySQL module is not paused, impossible to resume"); + } + + if (admin_proxysql_pgsql_paused == true) { + // to speed up this process we first change poll_timeout to 10 + // MySQL_thread will call poll() with a maximum timeout of 10ms + admin_old_wait_timeout = GloPTH->get_variable_int((char*)"poll_timeout"); + GloPTH->set_variable((char*)"poll_timeout", (char*)"10"); + GloPTH->commit(); + GloPTH->signal_all_threads(0); + GloPTH->start_listeners(); + //char buf[32]; + //sprintf(buf,"%d",old_wait_timeout); + //GloPTH->set_variable((char *)"poll_timeout",buf); + //GloPTH->commit(); + admin_proxysql_pgsql_paused = false; + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + // we now rollback poll_timeout + char buf[32]; + sprintf(buf, "%d", admin_old_wait_timeout); + GloPTH->set_variable((char*)"poll_timeout", buf); + GloPTH->commit(); + } + else { + SPA->send_error_msg_to_client(sess, (char*)"ProxySQL MySQL module is not paused, impossible to resume"); + } + return false; + } + + if (query_no_space_length==strlen("PROXYSQL SHUTDOWN SLOW") && !strncasecmp("PROXYSQL SHUTDOWN SLOW",query_no_space, query_no_space_length)) { + glovars.proxy_restart_on_error=false; + glovars.reload=0; + proxy_info("Received PROXYSQL SHUTDOWN SLOW command\n"); + __sync_bool_compare_and_swap(&glovars.shutdown,0,1); + return false; + } + + if (query_no_space_length==strlen("PROXYSQL FLUSH LOGS") && !strncasecmp("PROXYSQL FLUSH LOGS",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL FLUSH LOGS command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->flush_logs(); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if (query_no_space_length==strlen("PROXYSQL FLUSH QUERY CACHE") && !strncasecmp("PROXYSQL FLUSH QUERY CACHE",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL FLUSH QUERY CACHE command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (GloQC) { + GloQC->flush(); + } + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if (!strcasecmp("PROXYSQL FLUSH MYSQL CLIENT HOSTS", query_no_space)) { + proxy_info("Received PROXYSQL FLUSH MYSQL CLIENT HOSTS command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (GloMTH) { + GloMTH->flush_client_host_cache(); + } + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("PROXYSQL FLUSH CONFIGDB") && !strncasecmp("PROXYSQL FLUSH CONFIGDB",query_no_space, query_no_space_length)) // see #923 + ) { + proxy_info("Received %s command\n", query_no_space); + proxy_warning("A misconfigured configdb will cause undefined behaviors\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->flush_configdb(); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if (strcasecmp("PROXYSQL RELOAD TLS",query_no_space) == 0) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + std::string s; + int rc = ProxySQL_create_or_load_TLS(false, s); + if (rc == 0) { + SPA->send_ok_msg_to_client(sess, s.length() ? (char*)s.c_str() : NULL, 0, query_no_space); + } else { + SPA->send_error_msg_to_client(sess, s.length() ? (char *)s.c_str() : (char *)"RELOAD TLS failed"); + } + return false; + } + +#ifndef NOJEM + if (query_no_space_length==strlen("PROXYSQL MEMPROFILE START") && !strncasecmp("PROXYSQL MEMPROFILE START",query_no_space, query_no_space_length)) { + bool en=true; + mallctl("prof.active", NULL, NULL, &en, sizeof(bool)); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + if (query_no_space_length==strlen("PROXYSQL MEMPROFILE STOP") && !strncasecmp("PROXYSQL MEMPROFILE STOP",query_no_space, query_no_space_length)) { + bool en=false; + mallctl("prof.active", NULL, NULL, &en, sizeof(bool)); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } +#endif + +#ifdef WITHGCOV + if (query_no_space_length==strlen("PROXYSQL GCOV DUMP") && !strncasecmp("PROXYSQL GCOV DUMP",query_no_space, query_no_space_length)) { + proxy_info("Received %s command\n", query_no_space); + __gcov_dump(); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + if (query_no_space_length==strlen("PROXYSQL GCOV RESET") && !strncasecmp("PROXYSQL GCOV RESET",query_no_space, query_no_space_length)) { + proxy_info("Received %s command\n", query_no_space); + __gcov_reset(); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } +#endif + + if (query_no_space_length==strlen("PROXYSQL KILL") && !strncasecmp("PROXYSQL KILL",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL KILL command\n"); + exit(EXIT_SUCCESS); + } + + if (query_no_space_length==strlen("PROXYSQL SHUTDOWN") && !strncasecmp("PROXYSQL SHUTDOWN",query_no_space, query_no_space_length)) { + // in 2.1 , PROXYSQL SHUTDOWN behaves like PROXYSQL KILL : quick exit + // the former PROXYQL SHUTDOWN is now replaced with PROXYSQL SHUTDOWN SLOW + proxy_info("Received PROXYSQL SHUTDOWN command\n"); + exit(EXIT_SUCCESS); + } + + return true; +} + +// Returns true if the given name is either a know mysql or admin global variable. +bool is_valid_global_variable(const char *var_name) { + if (strlen(var_name) > 6 && !strncmp(var_name, "mysql-", 6) && GloMTH->has_variable(var_name + 6)) { + return true; + } else if (strlen(var_name) > 6 && !strncmp(var_name, "pgsql-", 6) && GloPTH->has_variable(var_name + 6)) { + return true; + } else if (strlen(var_name) > 6 && !strncmp(var_name, "admin-", 6) && SPA->has_variable(var_name + 6)) { + return true; + } else if (strlen(var_name) > 5 && !strncmp(var_name, "ldap-", 5) && GloMyLdapAuth && GloMyLdapAuth->has_variable(var_name + 5)) { + return true; + } else if (strlen(var_name) > 13 && !strncmp(var_name, "sqliteserver-", 13) && GloSQLite3Server && GloSQLite3Server->has_variable(var_name + 13)) { + return true; +#ifdef PROXYSQLCLICKHOUSE + } else if (strlen(var_name) > 11 && !strncmp(var_name, "clickhouse-", 11) && GloClickHouseServer && GloClickHouseServer->has_variable(var_name + 11)) { + return true; +#endif /* PROXYSQLCLICKHOUSE */ + } else { + return false; + } +} + + +// This method translates a 'SET variable=value' command into an equivalent UPDATE. It doesn't yes support setting +// multiple variables at once. +// +// It modifies the original query. +template +bool admin_handler_command_set(char *query_no_space, unsigned int query_no_space_length, S* sess, ProxySQL_Admin *pa, char **q, unsigned int *ql) { + if (!strstr(query_no_space,(char *)"password")) { // issue #599 + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received command %s\n", query_no_space); + if (strncasecmp(query_no_space,(char *)"set autocommit",strlen((char *)"set autocommit"))) { + if (strncasecmp(query_no_space,(char *)"SET @@session.autocommit",strlen((char *)"SET @@session.autocommit"))) { + proxy_info("Received command %s\n", query_no_space); + } + } + } + // Get a pointer to the beginnig of var=value entry and split to get var name and value + char *set_entry = query_no_space + strlen("SET "); + char *untrimmed_var_name=NULL; + char *var_value=NULL; + c_split_2(set_entry, "=", &untrimmed_var_name, &var_value); + + // Trim spaces from var name to allow writing like 'var = value' + char *var_name = trim_spaces_in_place(untrimmed_var_name); + + if (strstr(var_name,(char *)"password") || strcmp(var_name,(char *)"mysql-default_authentication_plugin")==0) { + proxy_info("Received SET command for %s\n", var_name); + } + + bool run_query = false; + // Check if the command tries to set a non-existing variable. + if (strcmp(var_name,"mysql-init_connect")==0) { + char *err_msg_fmt = (char *) "ERROR: Global variable '%s' is not configurable using SET command. You must run UPDATE global_variables"; + size_t buff_len = strlen(err_msg_fmt) + strlen(var_name) + 1; + char *buff = (char *) malloc(buff_len); + snprintf(buff, buff_len, err_msg_fmt, var_name); + SPA->send_error_msg_to_client(sess, buff); + free(buff); + run_query = false; + } else { + if (!is_valid_global_variable(var_name)) { + char *err_msg_fmt = (char *) "ERROR: Unknown global variable: '%s'."; + size_t buff_len = strlen(err_msg_fmt) + strlen(var_name) + 1; + char *buff = (char *) malloc(buff_len); + snprintf(buff, buff_len, err_msg_fmt, var_name); + SPA->send_ok_msg_to_client(sess, buff, 0, query_no_space); + free(buff); + run_query = false; + } else { + const char *update_format = (char *)"UPDATE global_variables SET variable_value=%s WHERE variable_name='%s'"; + // Computed length is more than needed since it also counts the format modifiers (%s). + size_t query_len = strlen(update_format) + strlen(var_name) + strlen(var_value) + 1; + char *query = (char *)l_alloc(query_len); + snprintf(query, query_len, update_format, var_value, var_name); + + run_query = true; + l_free(*ql,*q); + *q = query; + *ql = strlen(*q) + 1; + } + } + free(untrimmed_var_name); + free(var_value); + return run_query; +} + +/* Note: + * This function can modify the original query + */ +template +bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query_no_space_length, S* sess, ProxySQL_Admin *pa, char **q, unsigned int *ql) { + proxy_debug(PROXY_DEBUG_ADMIN, 5, "Received command %s\n", query_no_space); + +#ifdef DEBUG + if ((query_no_space_length>11) && ( (!strncasecmp("SAVE DEBUG ", query_no_space, 11)) || (!strncasecmp("LOAD DEBUG ", query_no_space, 11))) ) { + if ( + (query_no_space_length==strlen("LOAD DEBUG TO MEMORY") && !strncasecmp("LOAD DEBUG TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD DEBUG TO MEM") && !strncasecmp("LOAD DEBUG TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD DEBUG FROM DISK") && !strncasecmp("LOAD DEBUG FROM DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + // we are now copying the data from memory to disk + // tables involved are: + // * debug_levels + // * debug_filters + // We only delete from filters and not from levels because the + // levels are hardcoded and fixed in number, while filters can + // be arbitrary + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->admindb->execute("DELETE FROM main.debug_filters"); + SPA->admindb->execute("INSERT OR REPLACE INTO main.debug_levels SELECT * FROM disk.debug_levels"); + SPA->admindb->execute("INSERT INTO main.debug_filters SELECT * FROM disk.debug_filters"); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded debug levels/filters to MEMORY\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("SAVE DEBUG FROM MEMORY") && !strncasecmp("SAVE DEBUG FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE DEBUG FROM MEM") && !strncasecmp("SAVE DEBUG FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE DEBUG TO DISK") && !strncasecmp("SAVE DEBUG TO DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + // we are now copying the data from disk to memory + // tables involved are: + // * debug_levels + // * debug_filters + // We only delete from filters and not from levels because the + // levels are hardcoded and fixed in number, while filters can + // be arbitrary + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->admindb->execute("DELETE FROM disk.debug_filters"); + SPA->admindb->execute("INSERT OR REPLACE INTO disk.debug_levels SELECT * FROM main.debug_levels"); + SPA->admindb->execute("INSERT INTO disk.debug_filters SELECT * FROM main.debug_filters"); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved debug levels/filters to DISK\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD DEBUG FROM MEMORY") && !strncasecmp("LOAD DEBUG FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD DEBUG FROM MEM") && !strncasecmp("LOAD DEBUG FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD DEBUG TO RUNTIME") && !strncasecmp("LOAD DEBUG TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD DEBUG TO RUN") && !strncasecmp("LOAD DEBUG TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rc=SPA->load_debug_to_runtime(); + if (rc) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded debug levels/filters to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 1, "Error while loading debug levels/filters to RUNTIME\n"); + SPA->send_error_msg_to_client(sess, (char *)"Error while loading debug levels/filters to RUNTIME"); + } + return false; + } + + if ( + (query_no_space_length==strlen("SAVE DEBUG TO MEMORY") && !strncasecmp("SAVE DEBUG TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE DEBUG TO MEM") && !strncasecmp("SAVE DEBUG TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE DEBUG FROM RUNTIME") && !strncasecmp("SAVE DEBUG FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE DEBUG FROM RUN") && !strncasecmp("SAVE DEBUG FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_debug_from_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved debug levels/filters from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + } +#endif /* DEBUG */ + + if ((query_no_space_length>13) && ( (!strncasecmp("SAVE RESTAPI ", query_no_space, 13)) || (!strncasecmp("LOAD RESTAPI ", query_no_space, 13))) ) { + + if (FlushCommandWrapper(sess, "restapi", query_no_space, query_no_space_length) == true) + return false; + + if ( + (query_no_space_length==strlen("LOAD RESTAPI FROM MEMORY") && !strncasecmp("LOAD RESTAPI FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI FROM MEM") && !strncasecmp("LOAD RESTAPI FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI TO RUNTIME") && !strncasecmp("LOAD RESTAPI TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI TO RUN") && !strncasecmp("LOAD RESTAPI TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->proxysql_restapi().load_restapi_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded restapito RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD RESTAPI FROM CONFIG") && !strncasecmp("LOAD RESTAPI FROM CONFIG",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + rows=SPA->proxysql_config().Read_Restapi_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded restapi from CONFIG\n"); + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + if ( + (query_no_space_length==strlen("SAVE RESTAPI TO MEMORY") && !strncasecmp("SAVE RESTAPI TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI TO MEM") && !strncasecmp("SAVE RESTAPI TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI FROM RUNTIME") && !strncasecmp("SAVE RESTAPI FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI FROM RUN") && !strncasecmp("SAVE RESTAPI FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_scheduler_runtime_to_database(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved scheduler from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + + if ((query_no_space_length>15) && ( (!strncasecmp("SAVE SCHEDULER ", query_no_space, 15)) || (!strncasecmp("LOAD SCHEDULER ", query_no_space, 15))) ) { + + if (FlushCommandWrapper(sess, "scheduler", query_no_space, query_no_space_length) == true) + return false; + + if ( + (query_no_space_length==strlen("LOAD SCHEDULER FROM MEMORY") && !strncasecmp("LOAD SCHEDULER FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD SCHEDULER FROM MEM") && !strncasecmp("LOAD SCHEDULER FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD SCHEDULER TO RUNTIME") && !strncasecmp("LOAD SCHEDULER TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD SCHEDULER TO RUN") && !strncasecmp("LOAD SCHEDULER TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->load_scheduler_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded scheduler to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD SCHEDULER FROM CONFIG") && !strncasecmp("LOAD SCHEDULER FROM CONFIG",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + rows=SPA->proxysql_config().Read_Scheduler_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded scheduler from CONFIG\n"); + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + if ( + (query_no_space_length==strlen("SAVE SCHEDULER TO MEMORY") && !strncasecmp("SAVE SCHEDULER TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE SCHEDULER TO MEM") && !strncasecmp("SAVE SCHEDULER TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE SCHEDULER FROM RUNTIME") && !strncasecmp("SAVE SCHEDULER FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE SCHEDULER FROM RUN") && !strncasecmp("SAVE SCHEDULER FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_scheduler_runtime_to_database(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved scheduler from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + } + if ((query_no_space_length>16) && (!strncasecmp("LOAD MYSQL USER ", query_no_space, 16)) ) { + if (query_no_space_length>27) { + if (!strncasecmp(" TO RUNTIME", query_no_space+query_no_space_length-11, 11)) { + char *name=(char *)malloc(query_no_space_length-27+1); + strncpy(name,query_no_space+16,query_no_space_length-27); + name[query_no_space_length-27]=0; + int i=0; + int s=strlen(name); + bool legitname=true; + for (i=0; i= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + ( (c == '-') || (c == '+') || (c == '_')) + ) { + v=true; + } + if (v==false) { + legitname=false; + } + } + if (legitname) { + proxy_info("Loading user %s\n", name); + pthread_mutex_lock(&users_mutex); + + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') { + SPA->public_add_active_users(USERNAME_BACKEND, name); + SPA->public_add_active_users(USERNAME_FRONTEND, name); + } else { + SPA->public_add_active_users(USERNAME_BACKEND, name); + SPA->public_add_active_users(USERNAME_FRONTEND, name); + } + + pthread_mutex_unlock(&users_mutex); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + proxy_info("Tried to load invalid user %s\n", name); + char *s=(char *)"Invalid name %s"; + char *m=(char *)malloc(strlen(s)+strlen(name)+1); + sprintf(m,s,name); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + free(name); + return false; + } + } + } +#ifdef PROXYSQLCLICKHOUSE + if ( ( GloVars.global.clickhouse_server == true ) && (query_no_space_length>22) && ( (!strncasecmp("SAVE CLICKHOUSE USERS ", query_no_space, 22)) || (!strncasecmp("LOAD CLICKHOUSE USERS ", query_no_space, 22))) ) { + if ( + (query_no_space_length==strlen("LOAD CLICKHOUSE USERS TO MEMORY") && !strncasecmp("LOAD CLICKHOUSE USERS TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE USERS TO MEM") && !strncasecmp("LOAD CLICKHOUSE USERS TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE USERS FROM DISK") && !strncasecmp("LOAD CLICKHOUSE USERS FROM DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->flush_clickhouse_users__from_disk_to_memory(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading clickhouse users to MEMORY\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("SAVE CLICKHOUSE USERS FROM MEMORY") && !strncasecmp("SAVE CLICKHOUSE USERS FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE USERS FROM MEM") && !strncasecmp("SAVE CLICKHOUSE USERS FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE USERS TO DISK") && !strncasecmp("SAVE CLICKHOUSE USERS TO DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->flush_clickhouse_users__from_memory_to_disk(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving clickhouse users to DISK\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD CLICKHOUSE USERS FROM MEMORY") && !strncasecmp("LOAD CLICKHOUSE USERS FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE USERS FROM MEM") && !strncasecmp("LOAD CLICKHOUSE USERS FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE USERS TO RUNTIME") && !strncasecmp("LOAD CLICKHOUSE USERS TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE USERS TO RUN") && !strncasecmp("LOAD CLICKHOUSE USERS TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->init_clickhouse_users(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded clickhouse users to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("SAVE CLICKHOUSE USERS TO MEMORY") && !strncasecmp("SAVE CLICKHOUSE USERS TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE USERS TO MEM") && !strncasecmp("SAVE CLICKHOUSE USERS TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE USERS FROM RUNTIME") && !strncasecmp("SAVE CLICKHOUSE USERS FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE USERS FROM RUN") && !strncasecmp("SAVE CLICKHOUSE USERS FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_clickhouse_users_runtime_to_database(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved clickhouse users from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + } +#endif /* PROXYSQLCLICKHOUSE */ + + if ((query_no_space_length>17) && ( (!strcasecmp("SAVE MYSQL DIGEST TO DISK", query_no_space) ) )) { + proxy_info("Received %s command\n", query_no_space); + unsigned long long curtime1=monotonic_time(); + int r1 = SPA->FlushDigestTableToDisk(SPA->statsdb_disk); + unsigned long long curtime2=monotonic_time(); + curtime1 = curtime1/1000; + curtime2 = curtime2/1000; + proxy_info("Saved stats_mysql_query_digest to disk: %llums to write %u entries\n", curtime2-curtime1, r1); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + return false; + } + + if ((query_no_space_length > 17) && ((!strcasecmp("SAVE PGSQL DIGEST TO DISK", query_no_space)))) { + proxy_info("Received %s command\n", query_no_space); + unsigned long long curtime1 = monotonic_time(); + int r1 = SPA->FlushDigestTableToDisk(SPA->statsdb_disk); + unsigned long long curtime2 = monotonic_time(); + curtime1 = curtime1 / 1000; + curtime2 = curtime2 / 1000; + proxy_info("Saved stats_mysql_query_digest to disk: %llums to write %u entries\n", curtime2 - curtime1, r1); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + return false; + } + + if ((query_no_space_length>17) && + ((!strncasecmp("SAVE MYSQL USERS ", query_no_space, 17)) || (!strncasecmp("LOAD MYSQL USERS ", query_no_space, 17)) || + ((!strncasecmp("SAVE PGSQL USERS ", query_no_space, 17)) || (!strncasecmp("LOAD PGSQL USERS ", query_no_space, 17))))) { + + const bool is_pgsql = (query_no_space[5] == 'P' || query_no_space[5] == 'p') ? true : false; + const std::string modname = is_pgsql ? "pgsql_users" : "mysql_users"; + + tuple, vector>& t = load_save_disk_commands[modname]; + if ( is_admin_command_or_alias(get<1>(t), query_no_space, query_no_space_length) ) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + + if (is_pgsql) + SPA->flush_pgsql_users__from_disk_to_memory(); + else + SPA->flush_mysql_users__from_disk_to_memory(); + + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading %s to MEMORY\n", modname.c_str()); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + if ( is_admin_command_or_alias(get<2>(t), query_no_space, query_no_space_length) ) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + + if (is_pgsql) + SPA->flush_pgsql_users__from_memory_to_disk(); + else + SPA->flush_mysql_users__from_memory_to_disk(); + + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving %s to DISK\n", modname.c_str()); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if (is_pgsql) { + if (is_admin_command_or_alias(LOAD_PGSQL_USERS_FROM_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->init_pgsql_users(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql users to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } else { + if (is_admin_command_or_alias(LOAD_MYSQL_USERS_FROM_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->init_users(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql users to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + + if ( + (query_no_space_length==strlen("LOAD MYSQL USERS FROM CONFIG") && (!strncasecmp("LOAD MYSQL USERS FROM CONFIG",query_no_space, query_no_space_length) || + !strncasecmp("LOAD PGSQL USERS FROM CONFIG", query_no_space, query_no_space_length))) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') { + rows=SPA->proxysql_config().Read_PgSQL_Users_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql users from CONFIG\n"); + } else { + rows=SPA->proxysql_config().Read_MySQL_Users_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql users from CONFIG\n"); + } + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + if (is_pgsql) { + if (is_admin_command_or_alias(SAVE_PGSQL_USERS_TO_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->save_pgsql_users_runtime_to_database(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved pgsql users from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } else { + if (is_admin_command_or_alias(SAVE_MYSQL_USERS_TO_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->save_mysql_users_runtime_to_database(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql users from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + } + if ((query_no_space_length>28) && ( (!strncasecmp("SAVE SQLITESERVER VARIABLES ", query_no_space, 28)) || (!strncasecmp("LOAD SQLITESERVER VARIABLES ", query_no_space, 28))) ) { + + if ( + (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES TO MEMORY") && !strncasecmp("LOAD SQLITESERVER VARIABLES TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES TO MEM") && !strncasecmp("LOAD SQLITESERVER VARIABLES TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES FROM DISK") && !strncasecmp("LOAD SQLITESERVER VARIABLES FROM DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + l_free(*ql,*q); + *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'sqliteserver-%'"); + *ql=strlen(*q)+1; + return true; + } + + if ( + (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES FROM MEMORY") && !strncasecmp("SAVE SQLITESERVER VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES FROM MEM") && !strncasecmp("SAVE SQLITESERVER VARIABLES FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES TO DISK") && !strncasecmp("SAVE SQLITESERVER VARIABLES TO DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + l_free(*ql,*q); + *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'sqliteserver-%'"); + *ql=strlen(*q)+1; + return true; + } + + if ( + (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES FROM MEMORY") && !strncasecmp("LOAD SQLITESERVER VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES FROM MEM") && !strncasecmp("LOAD SQLITESERVER VARIABLES FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES TO RUNTIME") && !strncasecmp("LOAD SQLITESERVER VARIABLES TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES TO RUN") && !strncasecmp("LOAD SQLITESERVER VARIABLES TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->load_sqliteserver_variables_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded SQLiteServer variables to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES TO MEMORY") && !strncasecmp("SAVE SQLITESERVER VARIABLES TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES TO MEM") && !strncasecmp("SAVE SQLITESERVER VARIABLES TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES FROM RUNTIME") && !strncasecmp("SAVE SQLITESERVER VARIABLES FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES FROM RUN") && !strncasecmp("SAVE SQLITESERVER VARIABLES FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_sqliteserver_variables_from_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved SQLiteServer variables from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } +#ifdef PROXYSQLCLICKHOUSE + if ((query_no_space_length>26) && ( (!strncasecmp("SAVE CLICKHOUSE VARIABLES ", query_no_space, 26)) || (!strncasecmp("LOAD CLICKHOUSE VARIABLES ", query_no_space, 26))) ) { + + if ( + (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES TO MEMORY") && !strncasecmp("LOAD CLICKHOUSE VARIABLES TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES TO MEM") && !strncasecmp("LOAD CLICKHOUSE VARIABLES TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES FROM DISK") && !strncasecmp("LOAD CLICKHOUSE VARIABLES FROM DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + l_free(*ql,*q); + *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'clickhouse-%'"); + *ql=strlen(*q)+1; + return true; + } + + if ( + (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES FROM MEMORY") && !strncasecmp("SAVE CLICKHOUSE VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES FROM MEM") && !strncasecmp("SAVE CLICKHOUSE VARIABLES FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES TO DISK") && !strncasecmp("SAVE CLICKHOUSE VARIABLES TO DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + l_free(*ql,*q); + *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'clickhouse-%'"); + *ql=strlen(*q)+1; + return true; + } + + if ( + (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES FROM MEMORY") && !strncasecmp("LOAD CLICKHOUSE VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES FROM MEM") && !strncasecmp("LOAD CLICKHOUSE VARIABLES FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES TO RUNTIME") && !strncasecmp("LOAD CLICKHOUSE VARIABLES TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES TO RUN") && !strncasecmp("LOAD CLICKHOUSE VARIABLES TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->load_clickhouse_variables_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded clickhouse variables to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES TO MEMORY") && !strncasecmp("SAVE CLICKHOUSE VARIABLES TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES TO MEM") && !strncasecmp("SAVE CLICKHOUSE VARIABLES TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES FROM RUNTIME") && !strncasecmp("SAVE CLICKHOUSE VARIABLES FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES FROM RUN") && !strncasecmp("SAVE CLICKHOUSE VARIABLES FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_clickhouse_variables_from_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved clickhouse variables from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } +#endif /* PROXYSQLCLICKHOUSE */ + + if (GloMyLdapAuth) { + if ((query_no_space_length>20) && ( (!strncasecmp("SAVE LDAP VARIABLES ", query_no_space, 20)) || (!strncasecmp("LOAD LDAP VARIABLES ", query_no_space, 20))) ) { + + if ( + (query_no_space_length==strlen("LOAD LDAP VARIABLES TO MEMORY") && !strncasecmp("LOAD LDAP VARIABLES TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD LDAP VARIABLES TO MEM") && !strncasecmp("LOAD LDAP VARIABLES TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD LDAP VARIABLES FROM DISK") && !strncasecmp("LOAD LDAP VARIABLES FROM DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + l_free(*ql,*q); + *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'ldap-%'"); + *ql=strlen(*q)+1; + return true; + } + + if ( + (query_no_space_length==strlen("SAVE LDAP VARIABLES FROM MEMORY") && !strncasecmp("SAVE LDAP VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE LDAP VARIABLES FROM MEM") && !strncasecmp("SAVE LDAP VARIABLES FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE LDAP VARIABLES TO DISK") && !strncasecmp("SAVE LDAP VARIABLES TO DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + l_free(*ql,*q); + *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'ldap-%'"); + *ql=strlen(*q)+1; + return true; + } + + if ( + (query_no_space_length==strlen("LOAD LDAP VARIABLES FROM MEMORY") && !strncasecmp("LOAD LDAP VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD LDAP VARIABLES FROM MEM") && !strncasecmp("LOAD LDAP VARIABLES FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD LDAP VARIABLES TO RUNTIME") && !strncasecmp("LOAD LDAP VARIABLES TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD LDAP VARIABLES TO RUN") && !strncasecmp("LOAD LDAP VARIABLES TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->load_ldap_variables_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded ldap variables to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("SAVE LDAP VARIABLES TO MEMORY") && !strncasecmp("SAVE LDAP VARIABLES TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE LDAP VARIABLES TO MEM") && !strncasecmp("SAVE LDAP VARIABLES TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE LDAP VARIABLES FROM RUNTIME") && !strncasecmp("SAVE LDAP VARIABLES FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE LDAP VARIABLES FROM RUN") && !strncasecmp("SAVE LDAP VARIABLES FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_ldap_variables_from_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved ldap variables from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + } + + if ((query_no_space_length > 21) && ((!strncasecmp("SAVE MYSQL VARIABLES ", query_no_space, 21)) || (!strncasecmp("LOAD MYSQL VARIABLES ", query_no_space, 21)) || + (!strncasecmp("SAVE PGSQL VARIABLES ", query_no_space, 21)) || (!strncasecmp("LOAD PGSQL VARIABLES ", query_no_space, 21)))) { + + const bool is_pgsql = (query_no_space[5] == 'P' || query_no_space[5] == 'p') ? true : false; + const std::string modname = is_pgsql ? "pgsql_variables" : "mysql_variables"; + + tuple, vector>& t = load_save_disk_commands[modname]; + if (is_admin_command_or_alias(get<1>(t), query_no_space, query_no_space_length)) { + l_free(*ql, *q); + if (is_pgsql) { + *q = l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'pgsql-%'"); + } + else { + *q = l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'mysql-%'"); + } + *ql = strlen(*q) + 1; + return true; + } + + if (is_admin_command_or_alias(get<2>(t), query_no_space, query_no_space_length)) { + l_free(*ql, *q); + if (is_pgsql) { + *q = l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'pgsql-%'"); + } + else { + *q = l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'mysql-%'"); + } + *ql = strlen(*q) + 1; + return true; + } + + if (is_pgsql) { + if (is_admin_command_or_alias(LOAD_PGSQL_VARIABLES_FROM_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->load_pgsql_variables_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql variables to RUNTIME\n"); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql variables to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } else { + if (is_admin_command_or_alias(LOAD_MYSQL_VARIABLES_FROM_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->load_mysql_variables_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql variables to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + + if ( + (query_no_space_length==strlen("LOAD MYSQL VARIABLES FROM CONFIG") && (!strncasecmp("LOAD MYSQL VARIABLES FROM CONFIG",query_no_space, query_no_space_length) || + !strncasecmp("LOAD PGSQL VARIABLES FROM CONFIG", query_no_space, query_no_space_length))) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + int rows=0; + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') { + rows=SPA->proxysql_config().Read_Global_Variables_from_configfile("pgsql"); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql global variables from CONFIG\n"); + } else { + rows = SPA->proxysql_config().Read_Global_Variables_from_configfile("mysql"); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql global variables from CONFIG\n"); + } + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + if (is_pgsql) { + if (is_admin_command_or_alias(SAVE_PGSQL_VARIABLES_TO_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->save_pgsql_variables_from_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved pgsql variables from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } else { + if (is_admin_command_or_alias(SAVE_MYSQL_VARIABLES_TO_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->save_mysql_variables_from_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql variables from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + } + + if ((query_no_space_length > 14) && (!strncasecmp("LOAD COREDUMP ", query_no_space, 14))) { + + if ( is_admin_command_or_alias(LOAD_COREDUMP_FROM_MEMORY, query_no_space, query_no_space_length) ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + bool rc = SPA->load_coredump_to_runtime(); + if (rc) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded coredump filters to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 1, "Error while loading coredump filters to RUNTIME\n"); + SPA->send_error_msg_to_client(sess, (char*)"Error while loading coredump filters to RUNTIME"); + } + return false; + } + } + + if ((query_no_space_length>19) && ( (!strncasecmp("SAVE MYSQL SERVERS ", query_no_space, 19)) || (!strncasecmp("LOAD MYSQL SERVERS ", query_no_space, 19)) || + (!strncasecmp("SAVE PGSQL SERVERS ", query_no_space, 19)) || (!strncasecmp("LOAD PGSQL SERVERS ", query_no_space, 19)))) { + + const bool is_pgsql = (query_no_space[5] == 'P' || query_no_space[5] == 'p') ? true : false; + const std::string modname = is_pgsql ? "pgsql_servers" : "mysql_servers"; + + if (FlushCommandWrapper(sess, modname, query_no_space, query_no_space_length) == true) + return false; + + if (is_pgsql) { + if (is_admin_command_or_alias(LOAD_PGSQL_SERVERS_FROM_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->pgsql_servers_wrlock(); + SPA->load_pgsql_servers_to_runtime(); + SPA->pgsql_servers_wrunlock(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql servers to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } else { + if (is_admin_command_or_alias(LOAD_MYSQL_SERVERS_FROM_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->mysql_servers_wrlock(); + SPA->load_mysql_servers_to_runtime(); + SPA->mysql_servers_wrunlock(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql servers to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + + if ( + (query_no_space_length==strlen("LOAD MYSQL SERVERS FROM CONFIG") && (!strncasecmp("LOAD MYSQL SERVERS FROM CONFIG",query_no_space, query_no_space_length) || + !strncasecmp("LOAD PGSQL SERVERS FROM CONFIG", query_no_space, query_no_space_length) ))) { + + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + if (is_pgsql) { + rows=SPA->proxysql_config().Read_PgSQL_Servers_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql servers from CONFIG\n"); + } else { + rows=SPA->proxysql_config().Read_MySQL_Servers_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql servers from CONFIG\n"); + } + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + if (is_pgsql) { + if (is_admin_command_or_alias(SAVE_PGSQL_SERVERS_TO_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->pgsql_servers_wrlock(); + SPA->save_pgsql_servers_runtime_to_database(false); + SPA->pgsql_servers_wrunlock(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved pgsql servers from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } else { + if (is_admin_command_or_alias(SAVE_MYSQL_SERVERS_TO_MEMORY, query_no_space, query_no_space_length)) { + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + SPA->mysql_servers_wrlock(); + SPA->save_mysql_servers_runtime_to_database(false); + SPA->mysql_servers_wrunlock(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql servers from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + } + + if ((query_no_space_length>22) && ( (!strncasecmp("SAVE PROXYSQL SERVERS ", query_no_space, 22)) || (!strncasecmp("LOAD PROXYSQL SERVERS ", query_no_space, 22))) ) { + + if (FlushCommandWrapper(sess, "proxysql_servers", query_no_space, query_no_space_length) == true) + return false; +/* + string modname = "proxysql_servers"; + tuple, vector>& t = load_save_disk_commands[modname]; + if (FlushCommandWrapper(sess, get<1>(t), query_no_space, query_no_space_length, modname, "disk_to_memory") == true) + return false; + + if (FlushCommandWrapper(sess, get<2>(t), query_no_space, query_no_space_length, modname, "memory_to_disk") == true) + return false; +*/ + if ( + (query_no_space_length==strlen("LOAD PROXYSQL SERVERS FROM MEMORY") && !strncasecmp("LOAD PROXYSQL SERVERS FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD PROXYSQL SERVERS FROM MEM") && !strncasecmp("LOAD PROXYSQL SERVERS FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD PROXYSQL SERVERS TO RUNTIME") && !strncasecmp("LOAD PROXYSQL SERVERS TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD PROXYSQL SERVERS TO RUN") && !strncasecmp("LOAD PROXYSQL SERVERS TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + //SPA->mysql_servers_wrlock(); + // before calling load_proxysql_servers_to_runtime() we release + // sql_query_global_mutex to prevent a possible deadlock due to + // a race condition + // load_proxysql_servers_to_runtime() calls ProxySQL_Cluster::load_servers_list() + // that then calls ProxySQL_Cluster_Nodes::load_servers_list(), holding a mutex + pthread_mutex_unlock(&SPA->sql_query_global_mutex); + SPA->load_proxysql_servers_to_runtime(true); + // we re-acquired the mutex because it will be released by the calling function + pthread_mutex_lock(&SPA->sql_query_global_mutex); + //SPA->mysql_servers_wrunlock(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded ProxySQL servers to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + if ( + (query_no_space_length==strlen("SAVE PROXYSQL SERVERS TO MEMORY") && !strncasecmp("SAVE PROXYSQL SERVERS TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE PROXYSQL SERVERS TO MEM") && !strncasecmp("SAVE PROXYSQL SERVERS TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE PROXYSQL SERVERS FROM RUNTIME") && !strncasecmp("SAVE PROXYSQL SERVERS FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE PROXYSQL SERVERS FROM RUN") && !strncasecmp("SAVE PROXYSQL SERVERS FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + //SPA->mysql_servers_wrlock(); + // before save_proxysql_servers_runtime_to_database() we release + // sql_query_global_mutex to prevent a possible deadlock due to + // a race condition + // save_proxysql_servers_runtime_to_database() calls ProxySQL_Cluster::dump_table_proxysql_servers() + // that then holds a mutex + pthread_mutex_unlock(&SPA->sql_query_global_mutex); + SPA->save_proxysql_servers_runtime_to_database(false); + // we re-acquired the mutex because it will be released by the calling function + pthread_mutex_lock(&SPA->sql_query_global_mutex); + //SPA->mysql_servers_wrunlock(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved ProxySQL servers from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD PROXYSQL SERVERS FROM CONFIG") && !strncasecmp("LOAD PROXYSQL SERVERS FROM CONFIG",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + rows=SPA->proxysql_config().Read_ProxySQL_Servers_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded ProxySQL servers from CONFIG\n"); + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + } + + if ((query_no_space_length>20) && (( (!strncasecmp("SAVE MYSQL FIREWALL ", query_no_space, 20)) || (!strncasecmp("LOAD MYSQL FIREWALL ", query_no_space, 20))) || + (!strncasecmp("SAVE PGSQL FIREWALL ", query_no_space, 20) || (!strncasecmp("LOAD PGSQL FIREWALL ", query_no_space, 20)))) ) { + + const std::string modname = (query_no_space[5] == 'P' || query_no_space[5] == 'p') ? "pgsql_firewall" : "mysql_firewall"; + + if (FlushCommandWrapper(sess, modname, query_no_space, query_no_space_length) == true) + return false; + + if ( + (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM CONFIG") && (!strncasecmp("LOAD MYSQL FIREWALL FROM CONFIG",query_no_space, query_no_space_length) || + !strncasecmp("LOAD PGSQL FIREWALL FROM CONFIG", query_no_space, query_no_space_length))) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + // FIXME: not implemented yet + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') { + // rows=SPA->proxysql_config().Read_PgSQL_Firewall_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql firewall from CONFIG\n"); + } else { + // rows=SPA->proxysql_config().Read_MySQL_Firewall_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql firewall from CONFIG\n"); + } + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + if ( + (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM MEMORY") && !strncasecmp("LOAD MYSQL FIREWALL FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM MEM") && !strncasecmp("LOAD MYSQL FIREWALL FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD MYSQL FIREWALL TO RUNTIME") && !strncasecmp("LOAD MYSQL FIREWALL TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD MYSQL FIREWALL TO RUN") && !strncasecmp("LOAD MYSQL FIREWALL TO RUN",query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("LOAD PGSQL FIREWALL FROM MEMORY") && !strncasecmp("LOAD PGSQL FIREWALL FROM MEMORY", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("LOAD PGSQL FIREWALL FROM MEM") && !strncasecmp("LOAD PGSQL FIREWALL FROM MEM", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("LOAD PGSQL FIREWALL TO RUNTIME") && !strncasecmp("LOAD PGSQL FIREWALL TO RUNTIME", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("LOAD PGSQL FIREWALL TO RUN") && !strncasecmp("LOAD PGSQL FIREWALL TO RUN", query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + const char* err = (query_no_space[5] == 'P' || query_no_space[5] == 'p') ? SPA->load_pgsql_firewall_to_runtime() : SPA->load_mysql_firewall_to_runtime(); + if (err==NULL) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql firewall to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + SPA->send_error_msg_to_client(sess, err); + } + return false; + } + + if ( + (query_no_space_length==strlen("SAVE MYSQL FIREWALL TO MEMORY") && !strncasecmp("SAVE MYSQL FIREWALL TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE MYSQL FIREWALL TO MEM") && !strncasecmp("SAVE MYSQL FIREWALL TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE MYSQL FIREWALL FROM RUNTIME") && !strncasecmp("SAVE MYSQL FIREWALL FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE MYSQL FIREWALL FROM RUN") && !strncasecmp("SAVE MYSQL FIREWALL FROM RUN",query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE PGSQL FIREWALL TO MEMORY") && !strncasecmp("SAVE PGSQL FIREWALL TO MEMORY", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE PGSQL FIREWALL TO MEM") && !strncasecmp("SAVE PGSQL FIREWALL TO MEM", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE PGSQL FIREWALL FROM RUNTIME") && !strncasecmp("SAVE PGSQL FIREWALL FROM RUNTIME", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE PGSQL FIREWALL FROM RUN") && !strncasecmp("SAVE PGSQL FIREWALL FROM RUN", query_no_space, query_no_space_length)) + + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') { + SPA->save_pgsql_firewall_from_runtime(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved pgsql firewall from RUNTIME\n"); + } else { + SPA->save_mysql_firewall_from_runtime(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql firewall from RUNTIME\n"); + } + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + + if ((query_no_space_length>23) && ( (!strncasecmp("SAVE MYSQL QUERY RULES ", query_no_space, 23)) || (!strncasecmp("LOAD MYSQL QUERY RULES ", query_no_space, 23)) || + (!strncasecmp("SAVE PGSQL QUERY RULES ", query_no_space, 23)) || (!strncasecmp("LOAD PGSQL QUERY RULES ", query_no_space, 23)) + ) ) { + + const std::string modname = (query_no_space[5] == 'P' || query_no_space[5] == 'p') ? "pgsql_query_rules" : "mysql_query_rules"; + if (FlushCommandWrapper(sess, modname, query_no_space, query_no_space_length) == true) + return false; + + if ( + (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM CONFIG") && (!strncasecmp("LOAD MYSQL QUERY RULES FROM CONFIG",query_no_space, query_no_space_length) || + !strncasecmp("LOAD PGSQL QUERY RULES FROM CONFIG", query_no_space, query_no_space_length))) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') { + rows = SPA->proxysql_config().Read_PgSQL_Query_Rules_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql query rules from CONFIG\n"); + } else { + rows = SPA->proxysql_config().Read_MySQL_Query_Rules_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql query rules from CONFIG\n"); + } + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + if ( + (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM MEMORY") && !strncasecmp("LOAD MYSQL QUERY RULES FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM MEM") && !strncasecmp("LOAD MYSQL QUERY RULES FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD MYSQL QUERY RULES TO RUNTIME") && !strncasecmp("LOAD MYSQL QUERY RULES TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD MYSQL QUERY RULES TO RUN") && !strncasecmp("LOAD MYSQL QUERY RULES TO RUN",query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("LOAD PGSQL QUERY RULES FROM MEMORY") && !strncasecmp("LOAD PGSQL QUERY RULES FROM MEMORY", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("LOAD PGSQL QUERY RULES FROM MEM") && !strncasecmp("LOAD PGSQL QUERY RULES FROM MEM", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("LOAD PGSQL QUERY RULES TO RUNTIME") && !strncasecmp("LOAD PGSQL QUERY RULES TO RUNTIME", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("LOAD PGSQL QUERY RULES TO RUN") && !strncasecmp("LOAD PGSQL QUERY RULES TO RUN", query_no_space, query_no_space_length)) + + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + char* err = NULL; + + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') + err = SPA->load_pgsql_query_rules_to_runtime(); + else + err = SPA->load_mysql_query_rules_to_runtime(); + + if (err==NULL) { + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded pgsql query rules to RUNTIME\n"); + else + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql query rules to RUNTIME\n"); + + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + SPA->send_error_msg_to_client(sess, err); + } + return false; + } + + if ( + (query_no_space_length == strlen("SAVE MYSQL QUERY RULES TO MEMORY") && !strncasecmp("SAVE MYSQL QUERY RULES TO MEMORY", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE MYSQL QUERY RULES TO MEM") && !strncasecmp("SAVE MYSQL QUERY RULES TO MEM", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE MYSQL QUERY RULES FROM RUNTIME") && !strncasecmp("SAVE MYSQL QUERY RULES FROM RUNTIME", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE MYSQL QUERY RULES FROM RUN") && !strncasecmp("SAVE MYSQL QUERY RULES FROM RUN", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE PGSQL QUERY RULES TO MEMORY") && !strncasecmp("SAVE PGSQL QUERY RULES TO MEMORY", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE PGSQL QUERY RULES TO MEM") && !strncasecmp("SAVE PGSQL QUERY RULES TO MEM", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE PGSQL QUERY RULES FROM RUNTIME") && !strncasecmp("SAVE PGSQL QUERY RULES FROM RUNTIME", query_no_space, query_no_space_length)) + || + (query_no_space_length == strlen("SAVE PGSQL QUERY RULES FROM RUN") && !strncasecmp("SAVE PGSQL QUERY RULES FROM RUN", query_no_space, query_no_space_length)) + + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; + if (query_no_space[5] == 'P' || query_no_space[5] == 'p') { + SPA->save_pgsql_query_rules_from_runtime(false); + SPA->save_pgsql_query_rules_fast_routing_from_runtime(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved pgsql query rules from RUNTIME\n"); + } else { + SPA->save_mysql_query_rules_from_runtime(false); + SPA->save_mysql_query_rules_fast_routing_from_runtime(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql query rules from RUNTIME\n"); + } + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + } + + if ((query_no_space_length>21) && ( (!strncasecmp("SAVE ADMIN VARIABLES ", query_no_space, 21)) || (!strncasecmp("LOAD ADMIN VARIABLES ", query_no_space, 21))) ) { + + if ( is_admin_command_or_alias(LOAD_ADMIN_VARIABLES_TO_MEMORY, query_no_space, query_no_space_length) ) { + l_free(*ql,*q); + *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'admin-%'"); + *ql=strlen(*q)+1; + return true; + } + + if ( is_admin_command_or_alias(SAVE_ADMIN_VARIABLES_FROM_MEMORY, query_no_space, query_no_space_length) ) { + l_free(*ql,*q); + *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'admin-%'"); + *ql=strlen(*q)+1; + return true; + } + + if ( is_admin_command_or_alias(LOAD_ADMIN_VARIABLES_FROM_MEMORY, query_no_space, query_no_space_length) ) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->load_admin_variables_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded admin variables to RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD ADMIN VARIABLES FROM CONFIG") && !strncasecmp("LOAD ADMIN VARIABLES FROM CONFIG",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + int rows=0; + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + rows=SPA->proxysql_config().Read_Global_Variables_from_configfile("admin"); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded admin variables from CONFIG\n"); + SPA->send_ok_msg_to_client(sess, NULL, rows, query_no_space); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_error_msg_to_client(sess, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_error_msg_to_client(sess, (char *)"Config file unknown"); + } + return false; + } + + if ( is_admin_command_or_alias(SAVE_ADMIN_VARIABLES_TO_MEMORY, query_no_space, query_no_space_length) ) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_admin_variables_from_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved admin variables from RUNTIME\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + return false; + } + + } + + if (!strncasecmp("SAVE CONFIG TO FILE", query_no_space, strlen("SAVE CONFIG TO FILE"))) { + std::string fileName = query_no_space + strlen("SAVE CONFIG TO FILE"); + + fileName.erase(0, fileName.find_first_not_of("\t\n\v\f\r ")); + fileName.erase(fileName.find_last_not_of("\t\n\v\f\r ") + 1); + if (fileName.size() == 0) { + proxy_error("ProxySQL Admin Error: empty file name\n"); + SPA->send_error_msg_to_client(sess, (char *)"ProxySQL Admin Error: empty file name"); + return false; + } + std::string data; + data.reserve(100000); + data += config_header; + int rc = pa->proxysql_config().Write_Global_Variables_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Users_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Users_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Query_Rules_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Servers_to_configfile(data); + rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); + rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); + if (rc) { + std::stringstream ss; + proxy_error("ProxySQL Admin Error: Cannot extract configuration\n"); + SPA->send_error_msg_to_client(sess, (char *)"ProxySQL Admin Error: Cannot extract configuration"); + return false; + } else { + std::ofstream out; + out.open(fileName); + if (out.is_open()) { + out << data; + out.close(); + if (!out) { + std::stringstream ss; + ss << "ProxySQL Admin Error: Error writing file " << fileName; + proxy_error("%s\n", ss.str().c_str()); + SPA->send_error_msg_to_client(sess, (char*)ss.str().c_str()); + return false; + } else { + std::stringstream ss; + ss << "File " << fileName << " is saved."; + SPA->send_ok_msg_to_client(sess, (char*)ss.str().c_str(), 0, query_no_space); + return false; + } + } else { + std::stringstream ss; + ss << "ProxySQL Admin Error: Cannot open file " << fileName; + proxy_error("%s\n", ss.str().c_str()); + SPA->send_error_msg_to_client(sess, (char*)ss.str().c_str()); + return false; + } + } + } + + return true; +} + +/** + * @brief Helper function that converts the current timezone + * expressed in seconds into a string of the format: + * - '[-]HH:MM:00'. + * Following the same pattern as the possible values returned by the SQL query + * 'SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())' in a MySQL server. + * @return A string holding the specified representation of the + * supplied timezone. + */ +std::string timediff_timezone_offset() { + std::string time_zone_offset {}; + char result[8]; + time_t rawtime; + struct tm *info; + int offset; + + time(&rawtime); + info = localtime(&rawtime); + strftime(result, 8, "%z", info); + offset = (result[0] == '+') ? 1 : 0; + time_zone_offset = ((std::string)(result)).substr(offset, 3-offset) + ":" + ((std::string)(result)).substr(3, 2) + ":00"; + + return time_zone_offset; +} + + + +template +void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt) { + + ProxySQL_Admin *pa=(ProxySQL_Admin *)_pa; + bool needs_vacuum = false; + char *error=NULL; + int cols; + int affected_rows = 0; + bool run_query=true; + SQLite3_result *resultset=NULL; + char *strA=NULL; + char *strB=NULL; + int strAl, strBl; + char *query=NULL; + unsigned int query_length = 0; + + if constexpr (std::is_same_v) { + query_length = pkt->size - sizeof(mysql_hdr); + query = (char*)l_alloc(query_length); + memcpy(query, (char*)pkt->ptr + sizeof(mysql_hdr) + 1, query_length - 1); + } else if constexpr (std::is_same_v) { + assert(sess->client_myds); + + pgsql_hdr hdr{}; + if (sess->client_myds->myprot.get_header((unsigned char*)pkt->ptr, pkt->size, &hdr) == false) { + //error + proxy_warning("Malformed packet\n"); + SPA->send_error_msg_to_client(sess, "Malformed packet"); + return; + } + + switch (hdr.type) { + case PG_PKT_STARTUP_V2: + case PG_PKT_STARTUP: + case PG_PKT_CANCEL: + case PG_PKT_SSLREQ: + case PG_PKT_GSSENCREQ: + //error + return; + } + + query_length = hdr.data.size; + query = (char*)l_alloc(query_length); + memcpy(query, (char*)hdr.data.ptr, query_length - 1); + } else { + assert(0); + } + + query[query_length-1]=0; + + char *query_no_space=(char *)l_alloc(query_length); + memcpy(query_no_space,query,query_length); + + unsigned int query_no_space_length=remove_spaces(query_no_space); + //fprintf(stderr,"%s----\n",query_no_space); + + if (query_no_space_length) { + // fix bug #925 + while (query_no_space[query_no_space_length-1]==';' || query_no_space[query_no_space_length-1]==' ') { + query_no_space_length--; + query_no_space[query_no_space_length]=0; + } + } + + // add global mutex, see bug #1188 + pthread_mutex_lock(&pa->sql_query_global_mutex); + + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + if (!strncasecmp("LOGENTRY ", query_no_space, strlen("LOGENTRY "))) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received command LOGENTRY: %s\n", query_no_space + strlen("LOGENTRY ")); + proxy_info("Received command LOGENTRY: %s\n", query_no_space + strlen("LOGENTRY ")); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + run_query=false; + goto __run_query; + } + } + + // handle special queries from Cluster + // for bug #1188 , ProxySQL Admin needs to know the exact query + + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + string tn = ""; + if (!strncasecmp(CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS))) { + tn = "cluster_mysql_servers"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS))) { + tn = "mysql_replication_hostgroups"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS))) { + tn = "mysql_group_replication_hostgroups"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_GALERA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GALERA))) { + tn = "mysql_galera_hostgroups"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_AURORA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_AURORA))) { + tn = "mysql_aws_aurora_hostgroups"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES, query_no_space, strlen(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES))) { + tn = "mysql_hostgroup_attributes"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS))) { + tn = "mysql_servers_ssl_params"; + } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_V2, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_V2))) { + tn = "mysql_servers_v2"; + } + if (tn != "") { + GloAdmin->mysql_servers_wrlock(); + resultset = MyHGM->get_current_mysql_table(tn); + GloAdmin->mysql_servers_wrunlock(); + + if (resultset == nullptr) { + // 'mysql_servers_v2' is a virtual table that represents the latest 'main.mysql_servers' + // records promoted by the user. This section shouldn't be reached, since the initial resulset + // for this table ('MySQL_HostGroups_Manager::incoming_mysql_servers') is generated during + // initialization, and it's only updated in subsequent user config promotions. In case we + // reach here, an empty resultset should be replied, as it would mean that no user + // config has ever been promoted to runtime, and thus, this virtual table should remain empty. + if (tn == "mysql_servers_v2") { + const string query_empty_resultset { + string { MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS } + " LIMIT 0" + }; + + char *error=NULL; + int cols=0; + int affected_rows=0; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + GloAdmin->mysql_servers_wrlock(); + GloAdmin->admindb->execute_statement(query_empty_resultset.c_str(), &error, &cols, &affected_rows, &resultset); + GloAdmin->mysql_servers_wrunlock(); + } else { + resultset = MyHGM->dump_table_mysql(tn); + } + + if (resultset) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query=false; + goto __run_query; + } + } else { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + run_query=false; + goto __run_query; + } + + } + } + + if (!strncasecmp(CLUSTER_QUERY_MYSQL_USERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_USERS))) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { + pthread_mutex_lock(&users_mutex); + resultset = GloMyAuth->get_current_mysql_users(); + pthread_mutex_unlock(&users_mutex); + if (resultset != nullptr) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + run_query=false; + goto __run_query; + } + } + } + + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + if (!strncasecmp(CLUSTER_QUERY_MYSQL_QUERY_RULES, query_no_space, strlen(CLUSTER_QUERY_MYSQL_QUERY_RULES))) { + GloMyQPro->wrlock(); + resultset = GloMyQPro->get_current_query_rules_inner(); + if (resultset == NULL) { + GloMyQPro->wrunlock(); // unlock first + resultset = GloMyQPro->get_current_query_rules(); + if (resultset) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query=false; + goto __run_query; + } + } else { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + //delete resultset; // DO NOT DELETE . This is the inner resultset of Query_Processor + GloMyQPro->wrunlock(); + run_query=false; + goto __run_query; + } + } + if (!strncasecmp(CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING, query_no_space, strlen(CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING))) { + GloMyQPro->wrlock(); + resultset = GloMyQPro->get_current_query_rules_fast_routing_inner(); + if (resultset == NULL) { + GloMyQPro->wrunlock(); // unlock first + resultset = GloMyQPro->get_current_query_rules_fast_routing(); + if (resultset) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query=false; + goto __run_query; + } + } else { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + //delete resultset; // DO NOT DELETE . This is the inner resultset of Query_Processor + GloMyQPro->wrunlock(); + run_query=false; + goto __run_query; + } + } + } + + // if the client simply executes: + // SELECT COUNT(*) FROM runtime_mysql_query_rules_fast_routing + // we just return the count + if (strcmp("SELECT COUNT(*) FROM runtime_mysql_query_rules_fast_routing", query_no_space)==0) { + int cnt = GloMyQPro->get_current_query_rules_fast_routing_count(); + l_free(query_length,query); + char buf[256]; + sprintf(buf,"SELECT %d AS 'COUNT(*)'", cnt); + query=l_strdup(buf); + query_length=strlen(query)+1; + goto __run_query; + } + + if (!strncasecmp("TRUNCATE ", query_no_space, strlen("TRUNCATE "))) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + if (strstr(query_no_space,"stats_mysql_query_digest")) { + bool truncate_digest_table = false; + static char * truncate_digest_table_queries[] = { + (char *)"TRUNCATE TABLE stats.stats_mysql_query_digest", + (char *)"TRUNCATE TABLE stats.stats_mysql_query_digest_reset", + (char *)"TRUNCATE TABLE stats_mysql_query_digest", + (char *)"TRUNCATE TABLE stats_mysql_query_digest_reset", + (char *)"TRUNCATE stats.stats_mysql_query_digest", + (char *)"TRUNCATE stats.stats_mysql_query_digest_reset", + (char *)"TRUNCATE stats_mysql_query_digest", + (char *)"TRUNCATE stats_mysql_query_digest_reset" + }; + size_t l=sizeof(truncate_digest_table_queries)/sizeof(char *); + unsigned int i; + for (i=0;iadmindb->execute("DELETE FROM stats.stats_mysql_query_digest"); + SPA->admindb->execute("DELETE FROM stats.stats_mysql_query_digest_reset"); + SPA->vacuum_stats(true); + // purge the digest map, asynchronously, in single thread + char *msg = NULL; + int r1 = ProxySQL_Test___PurgeDigestTable(true, false, &msg); + SPA->send_ok_msg_to_client(sess, msg, r1, query_no_space); + free(msg); + run_query=false; + goto __run_query; + } + } + } + } +#ifdef DEBUG + /** + * @brief Handles the 'PROXYSQL_SIMULATOR' command. Performing the operation specified in the payload + * format. + * @details The 'PROXYSQL_SIMULATOR' command is specified the following format. Allowing to perform a + * certain internal state changing operation. Payload spec: + * ``` + * PROXYSQL_SIMULATOR ${operation} ${hg} ${address}:${port} ${operation_params} + * ``` + * + * Supported operations include: + * - mysql_error: Find the server specified by 'hostname:port' in the specified hostgroup and calls + * 'MySrvC::connect_error()' with the provider 'error_code'. + * + * Payload example: + * ``` + * PROXYSQL_SIMULATOR mysql_error 1 127.0.0.1 3306 1234 + * ``` + */ + if (!strncasecmp("PROXYSQL_SIMULATOR ", query_no_space, strlen("PROXYSQL_SIMULATOR "))) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + proxy_warning("Received PROXYSQL_SIMULATOR command: %s\n", query_no_space); + + re2::RE2::Options opts = re2::RE2::Options(RE2::Quiet); + re2::RE2 pattern("\\s*(\\w+) (\\d+) (\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+) (\\d+)\\s*\\;*", opts); + re2::StringPiece input(query_no_space + strlen("PROXYSQL_SIMULATOR")); + + std::string command, s_hg, srv_addr, s_port, s_errcode {}; + bool c_res = re2::RE2::Consume(&input, pattern, &command, &s_hg, &srv_addr, &s_port, &s_errcode); + + long i_hg = 0; + long i_port = 0; + long i_errcode = 0; + + if (c_res == true) { + char* endptr = nullptr; + i_hg = std::strtol(s_hg.c_str(), &endptr, 10); + if (errno == ERANGE || errno == EINVAL) i_hg = LONG_MIN; + i_port = std::strtol(s_port.c_str(), &endptr, 10); + if (errno == ERANGE || errno == EINVAL) i_port = LONG_MIN; + i_errcode = std::strtol(s_errcode.c_str(), &endptr, 10); + if (errno == ERANGE || errno == EINVAL) i_errcode = LONG_MIN; + } + + if (c_res == true && i_hg != LONG_MIN && i_port != LONG_MIN && i_errcode != LONG_MIN) { + if constexpr (std::is_same_v) { + MyHGM->wrlock(); + + MySrvC* mysrvc = MyHGM->find_server_in_hg(i_hg, srv_addr, i_port); + if (mysrvc != nullptr) { + int backup_shun_on_failures; + backup_shun_on_failures = mysql_thread___shun_on_failures; + mysql_thread___shun_on_failures = 1; + // Set the error twice to surpass 'mysql_thread___shun_on_failures' value. + mysrvc->connect_error(i_errcode, false); + mysrvc->connect_error(i_errcode, false); + mysql_thread___shun_on_failures = backup_shun_on_failures; + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + std::string t_err_msg{ "Supplied server '%s:%d' wasn't found in hg '%d'" }; + std::string err_msg{}; + string_format(t_err_msg, err_msg, srv_addr.c_str(), i_port, i_hg); + + proxy_info("%s\n", err_msg.c_str()); + SPA->send_error_msg_to_client(sess, const_cast(err_msg.c_str())); + } + MyHGM->wrunlock(); + } else if constexpr (std::is_same_v) { + PgHGM->wrlock(); + + PgSQL_SrvC* pgsrvc = PgHGM->find_server_in_hg(i_hg, srv_addr, i_port); + + if (pgsrvc != nullptr) { + int backup_shun_on_failures; + backup_shun_on_failures = pgsql_thread___shun_on_failures; + pgsql_thread___shun_on_failures = 1; + // Set the error twice to surpass 'pgsql_thread___shun_on_failures' value. + pgsrvc->connect_error(i_errcode, false); + pgsrvc->connect_error(i_errcode, false); + pgsql_thread___shun_on_failures = backup_shun_on_failures; + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + std::string t_err_msg{ "Supplied server '%s:%d' wasn't found in hg '%d'" }; + std::string err_msg{}; + string_format(t_err_msg, err_msg, srv_addr.c_str(), i_port, i_hg); + + proxy_info("%s\n", err_msg.c_str()); + SPA->send_error_msg_to_client(sess, const_cast(err_msg.c_str())); + } + PgHGM->wrunlock(); + } else { + assert(0); + } + } else { + SPA->send_error_msg_to_client(sess, (char*)"Invalid arguments supplied with query 'PROXYSQL_SIMULATOR'"); + } + + run_query=false; + goto __run_query; + } + } +#endif // DEBUG + if (!strncasecmp("PROXYSQLTEST ", query_no_space, strlen("PROXYSQLTEST "))) { + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->ProxySQL_Test_Handler(SPA, sess, query_no_space, run_query); + goto __run_query; + } + } + { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + needs_vacuum = SPA->GenericRefreshStatistics(query_no_space,query_no_space_length, ( sess->session_type == PROXYSQL_SESSION_ADMIN ? true : false ) ); + } + + + if (!strncasecmp("SHOW GLOBAL VARIABLES LIKE 'read_only'", query_no_space, strlen("SHOW GLOBAL VARIABLES LIKE 'read_only'"))) { + l_free(query_length,query); + char *q=(char *)"SELECT 'read_only' Variable_name, '%s' Value FROM global_variables WHERE Variable_name='admin-read_only'"; + query_length=strlen(q)+5; + query=(char *)l_alloc(query_length); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + bool ro=SPA->get_read_only(); + //sprintf(query,q,( ro ? "ON" : "OFF")); + PtrSize_t pkt_2; + if (ro) { + pkt_2.size=110; + pkt_2.ptr=l_alloc(pkt_2.size); + memcpy(pkt_2.ptr,READ_ONLY_ON,pkt_2.size); + } else { + pkt_2.size=111; + pkt_2.ptr=l_alloc(pkt_2.size); + memcpy(pkt_2.ptr,READ_ONLY_OFF,pkt_2.size); + } + sess->status=WAITING_CLIENT_DATA; + sess->client_myds->DSS=STATE_SLEEP; + sess->client_myds->PSarrayOUT->add(pkt_2.ptr,pkt_2.size); + run_query=false; + goto __run_query; + } + + if (!strncasecmp("SELECT @@global.read_only", query_no_space, strlen("SELECT @@global.read_only"))) { + l_free(query_length,query); + char *q=(char *)"SELECT 'read_only' Variable_name, '%s' Value FROM global_variables WHERE Variable_name='admin-read_only'"; + query_length=strlen(q)+5; + query=(char *)l_alloc(query_length); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + bool ro=SPA->get_read_only(); + //sprintf(query,q,( ro ? "ON" : "OFF")); + PtrSize_t pkt_2; + if (ro) { + pkt_2.size=73; + pkt_2.ptr=l_alloc(pkt_2.size); + memcpy(pkt_2.ptr,READ_ONLY_1,pkt_2.size); + } else { + pkt_2.size=73; + pkt_2.ptr=l_alloc(pkt_2.size); + memcpy(pkt_2.ptr,READ_ONLY_0,pkt_2.size); + } + sess->status=WAITING_CLIENT_DATA; + sess->client_myds->DSS=STATE_SLEEP; + sess->client_myds->PSarrayOUT->add(pkt_2.ptr,pkt_2.size); + run_query=false; + goto __run_query; + } + + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + if ((query_no_space_length>13) && (!strncasecmp("PULL VERSION ", query_no_space, 13))) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received PULL command\n"); + if ((query_no_space_length>27) && (!strncasecmp("PULL VERSION MYSQL SERVERS ", query_no_space, 27))) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received PULL VERSION MYSQL SERVERS command\n"); + unsigned int wait_mysql_servers_version = 0; + unsigned int wait_timeout = 0; + int rc = sscanf(query_no_space+27,"%u %u",&wait_mysql_servers_version, &wait_timeout); + if (rc < 2) { + SPA->send_error_msg_to_client(sess, (char *)"Invalid argument"); + run_query=false; + goto __run_query; + } else { + MyHGM->wait_servers_table_version(wait_mysql_servers_version, wait_timeout); + l_free(query_length,query); + unsigned int curver = MyHGM->get_servers_table_version(); + char buf[256]; + sprintf(buf,"SELECT %u AS 'version'", curver); + query=l_strdup(buf); + query_length=strlen(query)+1; + //SPA->send_ok_msg_to_client(sess, , NULL); + //run_query=false; + goto __run_query; + } + } + } + + + if ((query_no_space_length == strlen("SELECT GLOBAL_CHECKSUM()")) && (!strncasecmp("SELECT GLOBAL_CHECKSUM()", query_no_space, strlen("SELECT GLOBAL_CHECKSUM()")))) { + char buf[32]; + pthread_mutex_lock(&GloVars.checksum_mutex); + sprintf(buf,"%lu",GloVars.checksums_values.global_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + uint16_t setStatus = 0; + auto *myds=sess->client_myds; + auto *myprot=&sess->client_myds->myprot; + myds->DSS=STATE_QUERY_SENT_DS; + int sid=1; + myprot->generate_pkt_column_count(true,NULL,NULL,sid,1); sid++; + myprot->generate_pkt_field(true,NULL,NULL,sid,(char *)"",(char *)"",(char *)"",(char *)"CHECKSUM",(char *)"",63,31,MYSQL_TYPE_LONGLONG,161,0,false,0,NULL); sid++; + myds->DSS=STATE_COLUMN_DEFINITION; + myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + char **p=(char **)malloc(sizeof(char*)*1); + unsigned long *l=(unsigned long *)malloc(sizeof(unsigned long *)*1); + l[0]=strlen(buf);; + p[0]=buf; + myprot->generate_pkt_row(true,NULL,NULL,sid,1,l,p); sid++; + myds->DSS=STATE_ROW; + myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + myds->DSS=STATE_SLEEP; + run_query=false; + free(l); + free(p); + goto __run_query; + } + + + if ((query_no_space_length>8) && (!strncasecmp("PROXYSQL ", query_no_space, 8))) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received PROXYSQL command\n"); + //pthread_mutex_lock(&admin_mutex); + run_query=admin_handler_command_proxysql(query_no_space, query_no_space_length, sess, pa); + //pthread_mutex_unlock(&admin_mutex); + goto __run_query; + } + if ((query_no_space_length>5) && ( (!strncasecmp("SAVE ", query_no_space, 5)) || (!strncasecmp("LOAD ", query_no_space, 5))) ) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received LOAD or SAVE command\n"); + run_query=admin_handler_command_load_or_save(query_no_space, query_no_space_length, sess, pa, &query, &query_length); + goto __run_query; + } + if ((query_no_space_length>16) && ( (!strncasecmp("KILL CONNECTION ", query_no_space, 16)) || (!strncasecmp("KILL CONNECTION ", query_no_space, 16))) ) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received KILL CONNECTION command\n"); + run_query=admin_handler_command_kill_connection(query_no_space, query_no_space_length, sess, pa); + goto __run_query; + } + + + // queries generated by mysqldump + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if ( + !strncmp("/*!40014 SET ", query_no_space, 13) || + !strncmp("/*!40101 SET ", query_no_space, 13) || + !strncmp("/*!40103 SET ", query_no_space, 13) || + !strncmp("/*!40111 SET ", query_no_space, 13) || + !strncmp("/*!80000 SET ", query_no_space, 13) || + !strncmp("/*!50503 SET ", query_no_space, 13) || + !strncmp("/*!50717 SET ", query_no_space, 13) || + !strncmp("/*!50717 SELECT ", query_no_space, strlen("/*!50717 SELECT ")) || + !strncmp("/*!50717 PREPARE ", query_no_space, strlen("/*!50717 PREPARE ")) || + !strncmp("/*!50717 EXECUTE ", query_no_space, strlen("/*!50717 EXECUTE ")) || + !strncmp("/*!50717 DEALLOCATE ", query_no_space, strlen("/*!50717 DEALLOCATE ")) || + !strncmp("/*!50112 SET ", query_no_space, strlen("/*!50112 SET ")) || + !strncmp("/*!50112 PREPARE ", query_no_space, strlen("/*!50112 PREPARE ")) || + !strncmp("/*!50112 EXECUTE ", query_no_space, strlen("/*!50112 EXECUTE ")) || + !strncmp("/*!50112 DEALLOCATE ", query_no_space, strlen("/*!50112 DEALLOCATE ")) || + !strncmp("/*!40000 ALTER TABLE", query_no_space, strlen("/*!40000 ALTER TABLE")) + || + !strncmp("/*!40100 SET @@SQL_MODE='' */", query_no_space, strlen("/*!40100 SET @@SQL_MODE='' */")) + || + !strncmp("/*!40103 SET TIME_ZONE=", query_no_space, strlen("/*!40103 SET TIME_ZONE=")) + || + !strncmp("LOCK TABLES", query_no_space, strlen("LOCK TABLES")) + || + !strncmp("UNLOCK TABLES", query_no_space, strlen("UNLOCK TABLES")) + || + !strncmp("SET SQL_QUOTE_SHOW_CREATE=1", query_no_space, strlen("SET SQL_QUOTE_SHOW_CREATE=1")) + || + !strncmp("SET SESSION character_set_results", query_no_space, strlen("SET SESSION character_set_results")) + || + !strncasecmp("USE ", query_no_space, strlen("USE ")) // this applies to all clients, not only mysqldump + ) { + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + run_query=false; + goto __run_query; + } + + if (!strncmp("SHOW STATUS LIKE 'binlog_snapshot_gtid_executed'", query_no_space, strlen("SHOW STATUS LIKE 'binlog_snapshot_gtid_executed'"))) { + l_free(query_length, query); + query = l_strdup("SELECT variable_name AS Variable_name, Variable_value AS Value FROM global_variables WHERE 1=0"); + query_length = strlen(query)+1; + goto __run_query; + } + if (!strncmp("SELECT COLUMN_NAME, JSON_EXTRACT(HISTOGRAM, '$.\"number-of-buckets-specified\"') FROM information_schema.COLUMN_STATISTICS", query_no_space, strlen("SELECT COLUMN_NAME, JSON_EXTRACT(HISTOGRAM, '$.\"number-of-buckets-specified\"') FROM information_schema.COLUMN_STATISTICS"))) { + l_free(query_length, query); + query = l_strdup("SELECT variable_name AS COLUMN_NAME, Variable_value AS 'JSON_EXTRACT(HISTOGRAM, ''$.\"number-of-buckets-specified\"'')' FROM global_variables WHERE 1=0"); + query_length = strlen(query)+1; + goto __run_query; + } + if (!strncmp("SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'performance_schema' AND table_name = 'session_variables'", query_no_space, strlen("SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'performance_schema' AND table_name = 'session_variables'"))) { + l_free(query_length,query); + query=l_strdup("SELECT 0 as 'COUNT(*)'"); + query_length=strlen(query)+1; + goto __run_query; + } + if (!strncmp("SHOW VARIABLES LIKE 'gtid\\_mode'", query_no_space, strlen("SHOW VARIABLES LIKE 'gtid\\_mode'"))) { + l_free(query_length,query); + query=l_strdup("SELECT variable_name Variable_name, Variable_value Value FROM global_variables WHERE Variable_name='gtid_mode'"); + query_length=strlen(query)+1; + goto __run_query; + } + if (!strncmp("select @@collation_database", query_no_space, strlen("select @@collation_database"))) { + l_free(query_length,query); + query=l_strdup("SELECT Collation '@@collation_database' FROM mysql_collations WHERE Collation='utf8_general_ci' LIMIT 1"); + query_length=strlen(query)+1; + goto __run_query; + } + if (!strncmp("SHOW VARIABLES LIKE 'ndbinfo\\_version'", query_no_space, strlen("SHOW VARIABLES LIKE 'ndbinfo\\_version'"))) { + l_free(query_length,query); + query=l_strdup("SELECT variable_name Variable_name, Variable_value Value FROM global_variables WHERE Variable_name='ndbinfo_version'"); + query_length=strlen(query)+1; + goto __run_query; + } + if (!strncasecmp("show table status like '", query_no_space, strlen("show table status like '"))) { + char *strA=query_no_space+24; + int strAl=strlen(strA); + if (strAl<2) { // error + goto __run_query; + } + char *err=NULL; + SQLite3_result *resultset=SPA->generate_show_table_status(strA, &err); + sess->SQLite3_to_MySQL(resultset, err, 0, &sess->client_myds->myprot); + if (resultset) delete resultset; + if (err) free(err); + run_query=false; + goto __run_query; + } + if (!strncasecmp("show fields from ", query_no_space, strlen("show fields from "))) { + char *strA=query_no_space+17; + int strAl=strlen(strA); + if (strAl==0) { // error + goto __run_query; + } + if (strA[0]=='`') { + strA++; + strAl--; + } + if (strAl<2) { // error + goto __run_query; + } + char *err=NULL; + SQLite3_result *resultset=SPA->generate_show_fields_from(strA, &err); + sess->SQLite3_to_MySQL(resultset, err, 0, &sess->client_myds->myprot); + if (resultset) delete resultset; + if (err) free(err); + run_query=false; + goto __run_query; + } + } + + // FIXME: this should be removed, it is just a POC for issue #253 . What is important is the call to GloMTH->signal_all_threads(); + if (!strncasecmp("SIGNAL MYSQL THREADS", query_no_space, strlen("SIGNAL MYSQL THREADS"))) { + GloMTH->signal_all_threads(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_admin_variables_from_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Sent signal to all mysql threads\n"); + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + run_query=false; + goto __run_query; + } + + // fix bug #442 + if (!strncmp("SET SQL_SAFE_UPDATES=1", query_no_space, strlen("SET SQL_SAFE_UPDATES=1"))) { + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + run_query=false; + goto __run_query; + } + + // fix bug #1047 + if ( + (!strncasecmp("BEGIN", query_no_space, strlen("BEGIN"))) + || + (!strncasecmp("START TRANSACTION", query_no_space, strlen("START TRANSACTION"))) + || + (!strncasecmp("COMMIT", query_no_space, strlen("COMMIT"))) + || + (!strncasecmp("ROLLBACK", query_no_space, strlen("ROLLBACK"))) + || + (!strncasecmp("SET character_set_results", query_no_space, strlen("SET character_set_results"))) + || + (!strncasecmp("SET SQL_AUTO_IS_NULL", query_no_space, strlen("SET SQL_AUTO_IS_NULL"))) + || + (!strncasecmp("SET NAMES", query_no_space, strlen("SET NAMES"))) + || + (!strncasecmp("SET AUTOCOMMIT", query_no_space, strlen("SET AUTOCOMMIT"))) + ) { + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + run_query=false; + goto __run_query; + } + + // MySQL client check command for dollars quote support, starting at version '8.1.0'. See #4300. + if (!strncasecmp("SELECT $$", query_no_space, strlen("SELECT $$"))) { + pair err_info { get_dollar_quote_error(mysql_thread___server_version) }; + SPA->send_error_msg_to_client(sess, const_cast(err_info.second), err_info.first); + run_query=false; + goto __run_query; + } + + if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { + if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT '(ProxySQL Admin Module)'"); + query_length=strlen(query)+1; + goto __run_query; + } + } + + if (!strncasecmp("select concat(@@version, ' ', @@version_comment)", query_no_space, strlen("select concat(@@version, ' ', @@version_comment)"))) { + l_free(query_length,query); + char *q = const_cast("SELECT '%s Admin Module'"); + query_length = strlen(q) + strlen(PROXYSQL_VERSION) + 1; + query = static_cast(l_alloc(query_length)); + sprintf(query, q, PROXYSQL_VERSION); + goto __run_query; + } + + // add support for SELECT current_user() and SELECT user() + // see https://github.com/sysown/proxysql/issues/1105#issuecomment-990940585 + if ( + (strcasecmp("SELECT current_user()", query_no_space) == 0) + || + (strcasecmp("SELECT user()", query_no_space) == 0) + ) { + bool current = false; + if (strcasestr(query_no_space, "current") != NULL) + current = true; + l_free(query_length,query); + std::string s = "SELECT '"; + s += sess->client_myds->myconn->userinfo->username ; + if (strlen(sess->client_myds->addr.addr) > 0) { + s += "@"; + s += sess->client_myds->addr.addr; + } + s += "' AS '"; + if (current == true) { + s+= "current_"; + } + s += "user()'"; + query=l_strdup(s.c_str()); + query_length=strlen(query)+1; + goto __run_query; + } + + if (!strncasecmp("select @@sql_mode", query_no_space, strlen("select @@sql_mode"))) { + l_free(query_length,query); + char *q = const_cast("SELECT \"\" as \"@@sql_mode\""); + query_length = strlen(q) + strlen(PROXYSQL_VERSION) + 1; + query = static_cast(l_alloc(query_length)); + sprintf(query, q, PROXYSQL_VERSION); + goto __run_query; + } + + // trivial implementation for 'connection_id()' to support 'mycli'. See #3247 + if (!strncasecmp("select connection_id()", query_no_space, strlen("select connection_id()"))) { + l_free(query_length,query); + // 'connection_id()' is always forced to be '0' + query=l_strdup("SELECT 0 AS 'CONNECTION_ID()'"); + query_length=strlen(query)+1; + goto __run_query; + } + + // implementation for 'SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())' in order to support'csharp' connector. See #2543 + if (!strncasecmp("SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())", query_no_space, strlen("SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())"))) { + l_free(query_length,query); + char *query1=(char*)"SELECT '%s' as 'TIMEDIFF(NOW(), UTC_TIMESTAMP()'"; + + // compute the timezone diff + std::string timezone_offset_str = timediff_timezone_offset(); + char *query2=(char *)malloc(strlen(query1) + strlen(timezone_offset_str.c_str()) + 1); + + // format the query + sprintf(query2, query1, timezone_offset_str.c_str()); + + // copy the resulting query + query=l_strdup(query2); + query_length=strlen(query2) + 1; + + // free the buffer used to format + free(query2); + goto __run_query; + } + + // implementation for '"select @@max_allowed_packet, @@character_set_client, @@character_set_connection, @@license, @@sql_mode, @@lower_case_table_names"' + // in order to support 'csharp' connector. See #2543 + if ( + !strncasecmp( + "select @@max_allowed_packet, @@character_set_client, @@character_set_connection, @@license, @@sql_mode, @@lower_case_table_names", + query_no_space, + strlen("select @@max_allowed_packet, @@character_set_client, @@character_set_connection, @@license, @@sql_mode, @@lower_case_table_names") + ) + ) { + l_free(query_length,query); + char *query1= + const_cast( + "select '67108864' as '@@max_allowed_packet', 'utf8' as '@@character_set_client', 'utf8' as '@@character_set_connection', '' as '@@license', '' as '@@sql_mode', '' as '@@lower_case_table_names'" + ); + query=l_strdup(query1); + query_length=strlen(query1)+1; + goto __run_query; + } + + if (query_no_space_length==SELECT_DB_USER_LEN) { + if (!strncasecmp(SELECT_DB_USER, query_no_space, query_no_space_length)) { + l_free(query_length,query); + char *query1=(char *)"SELECT \"admin\" AS 'DATABASE()', \"%s\" AS 'USER()'"; + char *query2=(char *)malloc(strlen(query1)+strlen(sess->client_myds->myconn->userinfo->username)+10); + sprintf(query2,query1,sess->client_myds->myconn->userinfo->username); + query=l_strdup(query2); + query_length=strlen(query2)+1; + free(query2); + goto __run_query; + } + } + + if (query_no_space_length==SELECT_CHARSET_VARIOUS_LEN) { + if (!strncasecmp(SELECT_CHARSET_VARIOUS, query_no_space, query_no_space_length)) { + l_free(query_length,query); + char *query1=(char *)"select 'utf8' as '@@character_set_client', 'utf8' as '@@character_set_connection', 'utf8' as '@@character_set_server', 'utf8' as '@@character_set_database' limit 1"; + query=l_strdup(query1); + query_length=strlen(query1)+1; + goto __run_query; + } + } + + if (!strncasecmp("SELECT @@version", query_no_space, strlen("SELECT @@version"))) { + l_free(query_length,query); + char *q=(char *)"SELECT '%s' AS '@@version'"; + if (GloMyLdapAuth == nullptr) { + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); + } else { + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen("-Enterprise"); + } + query=(char *)l_alloc(query_length); + if (GloMyLdapAuth == nullptr) { + sprintf(query, q, PROXYSQL_VERSION); + } else { + sprintf(query, q, PROXYSQL_VERSION"-Enterprise"); + } + goto __run_query; + } + + if (!strncasecmp("SELECT version()", query_no_space, strlen("SELECT version()"))) { + l_free(query_length,query); + char *q=(char *)"SELECT '%s' AS 'version()'"; + if (GloMyLdapAuth == nullptr) { + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); + } else { + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen("-Enterprise"); + } + query=(char *)l_alloc(query_length); + if (GloMyLdapAuth == nullptr) { + sprintf(query, q, PROXYSQL_VERSION); + } else { + sprintf(query, q, PROXYSQL_VERSION"-Enterprise"); + } + goto __run_query; + } + + if (!strncasecmp("SHOW VARIABLES WHERE Variable_name in", query_no_space, strlen("SHOW VARIABLES WHERE Variable_name in"))) { + // Allow MariaDB ConnectorJ to connect to Admin #743 + if (!strncasecmp("SHOW VARIABLES WHERE Variable_name in ('max_allowed_packet','system_time_zone','time_zone','sql_mode')", query_no_space, strlen("SHOW VARIABLES WHERE Variable_name in ('max_allowed_packet','system_time_zone','time_zone','sql_mode')"))) { + l_free(query_length,query); + char *q=(char *)"SELECT 'max_allowed_packet' Variable_name,'4194304' Value UNION ALL SELECT 'sql_mode', 'STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION' UNION ALL SELECT 'system_time_zone', 'UTC' UNION ALL SELECT 'time_zone','SYSTEM'"; + query_length=strlen(q)+20; + query=(char *)l_alloc(query_length); + sprintf(query,q,PROXYSQL_VERSION); + goto __run_query; + } + // Allow MariaDB ConnectorJ 2.4.1 to connect to Admin #2009 + if (!strncasecmp("SHOW VARIABLES WHERE Variable_name in ('max_allowed_packet','system_time_zone','time_zone','auto_increment_increment')", query_no_space, strlen("SHOW VARIABLES WHERE Variable_name in ('max_allowed_packet','system_time_zone','time_zone','auto_increment_increment')"))) { + l_free(query_length,query); + char *q=(char *)"SELECT 'max_allowed_packet' Variable_name,'4194304' Value UNION ALL SELECT 'auto_increment_increment', '1' UNION ALL SELECT 'system_time_zone', 'UTC' UNION ALL SELECT 'time_zone','SYSTEM'"; + query_length=strlen(q)+20; + query=(char *)l_alloc(query_length); + sprintf(query,q,PROXYSQL_VERSION); + goto __run_query; + } + } + + { + bool rc; + rc=RE2::PartialMatch(query_no_space,*(RE2 *)(pa->match_regexes.re[0])); + if (rc) { + string *new_query=new std::string(query_no_space); + RE2::Replace(new_query,(char *)"^(\\w+)\\s+@@(\\w+)\\s*",(char *)"SELECT variable_value AS '@@max_allowed_packet' FROM global_variables WHERE variable_name='mysql-max_allowed_packet'"); + free(query); + query_length=new_query->length()+1; + query=(char *)malloc(query_length); + memcpy(query,new_query->c_str(),query_length-1); + query[query_length-1]='\0'; + delete new_query; + goto __run_query; + } + } + { + bool rc; + rc=RE2::PartialMatch(query_no_space,*(RE2 *)(pa->match_regexes.re[1])); + if (rc) { + string *new_query=new std::string(query_no_space); + RE2::Replace(new_query,(char *)"^(\\w+) *@@([0-9A-Za-z_-]+) *",(char *)"SELECT variable_value AS '@@\\2' FROM global_variables WHERE variable_name='\\2' COLLATE NOCASE UNION ALL SELECT variable_value AS '@@\\2' FROM stats.stats_mysql_global WHERE variable_name='\\2' COLLATE NOCASE"); + free(query); + query_length=new_query->length()+1; + query=(char *)malloc(query_length); + memcpy(query,new_query->c_str(),query_length-1); + query[query_length-1]='\0'; + GloAdmin->stats___mysql_global(); + delete new_query; + goto __run_query; + } + } + { + bool rc; + rc = RE2::PartialMatch(query_no_space, *(RE2*)(pa->match_regexes.re[1])); + if (rc) { + string* new_query = new std::string(query_no_space); + RE2::Replace(new_query, (char*)"^(\\w+) *@@([0-9A-Za-z_-]+) *", (char*)"SELECT variable_value AS '@@\\2' FROM global_variables WHERE variable_name='\\2' COLLATE NOCASE UNION ALL SELECT variable_value AS '@@\\2' FROM stats.stats_pgsql_global WHERE variable_name='\\2' COLLATE NOCASE"); + free(query); + query_length = new_query->length() + 1; + query = (char*)malloc(query_length); + memcpy(query, new_query->c_str(), query_length - 1); + query[query_length - 1] = '\0'; + GloAdmin->stats___pgsql_global(); + delete new_query; + goto __run_query; + } + } + { + bool rc; + rc=RE2::PartialMatch(query_no_space,*(RE2 *)(pa->match_regexes.re[2])); + if (rc) { + string *new_query=new std::string(query_no_space); + RE2::Replace(new_query,(char *)"([Ss][Hh][Oo][Ww]\\s+[Vv][Aa][Rr][Ii][Aa][Bb][Ll][Ee][Ss]\\s+[Ww][Hh][Ee][Rr][Ee])",(char *)"SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables WHERE"); + free(query); + query_length=new_query->length()+1; + query=(char *)malloc(query_length); + memcpy(query,new_query->c_str(),query_length-1); + query[query_length-1]='\0'; + delete new_query; + goto __run_query; + } + } + { + bool rc; + rc=RE2::PartialMatch(query_no_space,*(RE2 *)(pa->match_regexes.re[3])); + if (rc) { + string *new_query=new std::string(query_no_space); + RE2::Replace(new_query,(char *)"([Ss][Hh][Oo][Ww]\\s+[Vv][Aa][Rr][Ii][Aa][Bb][Ll][Ee][Ss]\\s+[Ll][Ii][Kk][Ee])",(char *)"SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables WHERE variable_name LIKE"); + free(query); + query_length=new_query->length()+1; + query=(char *)malloc(query_length); + memcpy(query,new_query->c_str(),query_length-1); + query[query_length-1]='\0'; + delete new_query; + goto __run_query; + } + } + + if (!strncasecmp("SET ", query_no_space, 4)) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received SET\n"); + run_query = admin_handler_command_set(query_no_space, query_no_space_length, sess, pa, &query, &query_length); + goto __run_query; + } + + if(!strncasecmp("CHECKSUM ", query_no_space, 9)){ + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received CHECKSUM command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SQLite3_result *resultset=NULL; + char *tablename=NULL; + char *error=NULL; + int affected_rows=0; + int cols=0; + if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL SERVERS") && !strncasecmp("CHECKSUM DISK MYSQL SERVERS", query_no_space, strlen(query_no_space))){ + char *q=(char *)"SELECT * FROM mysql_servers ORDER BY hostgroup_id, hostname, port"; + tablename=(char *)"MYSQL SERVERS"; + SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL USERS") && !strncasecmp("CHECKSUM DISK MYSQL USERS", query_no_space, strlen(query_no_space))){ + char *q=(char *)"SELECT * FROM mysql_users ORDER BY username"; + tablename=(char *)"MYSQL USERS"; + SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL QUERY RULES") && !strncasecmp("CHECKSUM DISK MYSQL QUERY RULES", query_no_space, strlen(query_no_space))){ + char *q=(char *)"SELECT * FROM mysql_query_rules ORDER BY rule_id"; + tablename=(char *)"MYSQL QUERY RULES"; + SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL VARIABLES") && !strncasecmp("CHECKSUM DISK MYSQL VARIABLES", query_no_space, strlen(query_no_space))){ + char *q=(char *)"SELECT * FROM global_variables WHERE variable_name LIKE 'mysql-%' ORDER BY variable_name"; + tablename=(char *)"MYSQL VARIABLES"; + SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM DISK MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))){ + char *q=(char *)"SELECT * FROM mysql_replication_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL REPLICATION HOSTGROUPS"; + SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL SERVERS") && !strncasecmp("CHECKSUM MEMORY MYSQL SERVERS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL SERVERS") && !strncasecmp("CHECKSUM MEM MYSQL SERVERS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL SERVERS") && !strncasecmp("CHECKSUM MYSQL SERVERS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_servers ORDER BY hostgroup_id, hostname, port"; + tablename=(char *)"MYSQL SERVERS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL USERS") && !strncasecmp("CHECKSUM MEMORY MYSQL USERS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL USERS") && !strncasecmp("CHECKSUM MEM MYSQL USERS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL USERS") && !strncasecmp("CHECKSUM MYSQL USERS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_users ORDER BY username"; + tablename=(char *)"MYSQL USERS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL QUERY RULES") && !strncasecmp("CHECKSUM MEMORY MYSQL QUERY RULES", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL QUERY RULES") && !strncasecmp("CHECKSUM MEM MYSQL QUERY RULES", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL QUERY RULES") && !strncasecmp("CHECKSUM MYSQL QUERY RULES", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_query_rules ORDER BY rule_id"; + tablename=(char *)"MYSQL QUERY RULES"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL VARIABLES") && !strncasecmp("CHECKSUM MEMORY MYSQL VARIABLES", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL VARIABLES") && !strncasecmp("CHECKSUM MEM MYSQL VARIABLES", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL VARIABLES") && !strncasecmp("CHECKSUM MYSQL VARIABLES", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM global_variables WHERE variable_name LIKE 'mysql-%' ORDER BY variable_name"; + tablename=(char *)"MYSQL VARIABLES"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_replication_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL REPLICATION HOSTGROUPS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL GROUP REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL GROUP REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL GROUP REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL GROUP REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL GROUP REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL GROUP REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_group_replication_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL GROUP REPLICATION HOSTGROUPS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL GALERA HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL GALERA HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL GALERA HOSTGROUPS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_galera_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL GALERA HOSTGROUPS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL AURORA HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL AURORA HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL AURORA HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL AURORA HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL AURORA HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL AURORA HOSTGROUPS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL AURORA HOSTGROUPS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; + tablename=(char *)"MYSQL HOSTGROUP ATTRIBUTES"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL SERVERS SSL PARAMS") && !strncasecmp("CHECKSUM MEMORY MYSQL SERVERS SSL PARAMS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL SERVERS SSL PARAMS") && !strncasecmp("CHECKSUM MEM MYSQL SERVERS SSL PARAMS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL SERVERS SSL PARAMS") && !strncasecmp("CHECKSUM MYSQL SERVERS SSL PARAMS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_servers_ssl_params ORDER BY hostname, port, username"; + tablename=(char *)"MYSQL HOSTGROUP ATTRIBUTES"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + + if (error) { + proxy_error("Error: %s\n", error); + char buf[1024]; + sprintf(buf,"%s", error); + SPA->send_error_msg_to_client(sess, buf); + run_query=false; + } else if (resultset) { + l_free(query_length,query); + char *q=(char *)"SELECT '%s' AS 'table', '%s' AS 'checksum'"; + char *checksum=(char *)resultset->checksum(); + query=(char *)malloc(strlen(q)+strlen(tablename)+strlen(checksum)+1); + sprintf(query,q,tablename,checksum); + query_length = strlen(query); + free(checksum); + delete resultset; + } + goto __run_query; + } + + if (!strncasecmp("SELECT CONFIG INTO OUTFILE", query_no_space, strlen("SELECT CONFIG INTO OUTFILE"))) { + std::string fileName = query_no_space + strlen("SELECT CONFIG INTO OUTFILE"); + fileName.erase(0, fileName.find_first_not_of("\t\n\v\f\r ")); + fileName.erase(fileName.find_last_not_of("\t\n\v\f\r ") + 1); + if (fileName.size() == 0) { + std::stringstream ss; + ss << "ProxySQL Admin Error: empty file name"; + sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); + } + std::string data; + data.reserve(100000); + data += config_header; + int rc = pa->proxysql_config().Write_Global_Variables_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Users_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Users_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Query_Rules_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Servers_to_configfile(data); + rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); + rc = pa->proxysql_config().Write_Restapi_to_configfile(data); + rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); + if (rc) { + std::stringstream ss; + ss << "ProxySQL Admin Error: Cannot extract configuration"; + sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); + } else { + std::ofstream out; + out.open(fileName); + if (out.is_open()) { + out << data; + out.close(); + if (!out) { + std::stringstream ss; + ss << "ProxySQL Admin Error: Error writing file " << fileName; + sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); + } else { + std::stringstream ss; + ss << "File " << fileName << " is saved."; + SPA->send_ok_msg_to_client(sess, (char*)ss.str().c_str(), data.size(), query_no_space); + } + } else { + std::stringstream ss; + ss << "ProxySQL Admin Error: Cannot open file " << fileName; + sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); + } + } + run_query = false; + goto __run_query; + } + + if (query_no_space_length==strlen("SELECT CONFIG FILE") && !strncasecmp("SELECT CONFIG FILE", query_no_space, query_no_space_length)) { + std::string data; + data.reserve(100000); + data += config_header; + int rc = pa->proxysql_config().Write_Global_Variables_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Users_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); + rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Users_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Query_Rules_to_configfile(data); + rc = pa->proxysql_config().Write_PgSQL_Servers_to_configfile(data); + rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); + rc = pa->proxysql_config().Write_Restapi_to_configfile(data); + rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); + if (rc) { + std::stringstream ss; + ss << "ProxySQL Admin Error: Cannot write proxysql.cnf"; + sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); + } else { + char *pta[1]; + pta[0]=NULL; + pta[0]=(char*)data.c_str(); + SQLite3_result* resultset = new SQLite3_result(1); + resultset->add_column_definition(SQLITE_TEXT,"Data"); + resultset->add_row(pta); + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + } + run_query = false; + goto __run_query; + } + + if (strncasecmp("SHOW ", query_no_space, 5)) { + goto __end_show_commands; // in the next block there are only SHOW commands + } + + if (!strncasecmp("SHOW PROMETHEUS METRICS", query_no_space, strlen("SHOW PROMETHEUS METRICS"))) { + char* pta[1]; + pta[0] = NULL; + SQLite3_result* resultset = new SQLite3_result(1); + resultset->add_column_definition(SQLITE_TEXT,"Data"); + + if (__sync_fetch_and_add(&GloMTH->status_variables.threads_initialized, 0) == 1) { + auto result = pa->serial_exposer({}); + pta[0] = (char*)result.second.c_str(); + resultset->add_row(pta); + } else { + resultset->add_row(pta); + } + + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + delete resultset; + run_query = false; + + goto __run_query; + } + + if (!strncasecmp("SHOW GLOBAL VARIABLES LIKE 'version'", query_no_space, strlen("SHOW GLOBAL VARIABLES LIKE 'version'"))) { + l_free(query_length,query); + char *q=(char *)"SELECT 'version' Variable_name, '%s' Value FROM global_variables WHERE Variable_name='admin-version'"; + query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); + query=(char *)l_alloc(query_length); + sprintf(query,q,PROXYSQL_VERSION); + goto __run_query; + } + + + if (query_no_space_length==strlen("SHOW TABLES") && !strncasecmp("SHOW TABLES",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT name AS tables FROM sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (query_no_space_length==strlen("SHOW CHARSET") && !strncasecmp("SHOW CHARSET",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT Charset, Collation AS 'Default collation' FROM mysql_collations WHERE `Default`='Yes'"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (query_no_space_length==strlen("SHOW COLLATION") && !strncasecmp("SHOW COLLATION",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT * FROM mysql_collations"); + query_length=strlen(query)+1; + goto __run_query; + } + + if ((query_no_space_length>15) && (!strncasecmp("SHOW TABLES IN ", query_no_space, 15))) { + strA=query_no_space+15; + strAl=strlen(strA); + strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"; + strBl=strlen(strB); + int l=strBl+strAl-2; + char *b=(char *)l_alloc(l+1); + snprintf(b,l+1,strB,strA); + b[l]=0; + l_free(query_length,query); + query=b; + query_length=l+1; + goto __run_query; + } + + if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES FROM ", query_no_space, 17))) { + strA=query_no_space+17; + strAl=strlen(strA); + strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"; + strBl=strlen(strB); + int l=strBl+strAl-2; + char *b=(char *)l_alloc(l+1); + snprintf(b,l+1,strB,strA); + b[l]=0; + l_free(query_length,query); + query=b; + query_length=l+1; + goto __run_query; + } + + if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES LIKE ", query_no_space, 17))) { + strA=query_no_space+17; + strAl=strlen(strA); + strB=(char *)"SELECT name AS tables FROM sqlite_master WHERE type='table' AND name LIKE '%s'"; + strBl=strlen(strB); + char *tn=NULL; // tablename + tn=(char *)malloc(strAl+1); + unsigned int i=0, j=0; + while (i<(unsigned int)strAl) { + if (strA[i]!='\\' && strA[i]!='`' && strA[i]!='\'') { + tn[j]=strA[i]; + j++; + } + i++; + } + tn[j]=0; + int l=strBl+strlen(tn)-2; + char *b=(char *)l_alloc(l+1); + snprintf(b,l+1,strB,tn); + b[l]=0; + free(tn); + l_free(query_length,query); + query=b; + query_length=l+1; + goto __run_query; + } + + if (query_no_space_length==strlen("SHOW MYSQL USERS") && !strncasecmp("SHOW MYSQL USERS",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT * FROM mysql_users ORDER BY username, active DESC, username ASC"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (query_no_space_length==strlen("SHOW MYSQL SERVERS") && !strncasecmp("SHOW MYSQL SERVERS",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT * FROM mysql_servers ORDER BY hostgroup_id, hostname, port"); + query_length=strlen(query)+1; + goto __run_query; + } + + if ( + (query_no_space_length==strlen("SHOW GLOBAL VARIABLES") && !strncasecmp("SHOW GLOBAL VARIABLES",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SHOW ALL VARIABLES") && !strncasecmp("SHOW ALL VARIABLES",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SHOW VARIABLES") && !strncasecmp("SHOW VARIABLES",query_no_space, query_no_space_length)) + ) { + l_free(query_length,query); + query=l_strdup("SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables ORDER BY variable_name"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (GloMyLdapAuth) { + if (query_no_space_length==strlen("SHOW LDAP VARIABLES") && !strncasecmp("SHOW LDAP VARIABLES",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables WHERE variable_name LIKE 'ldap-\%' ORDER BY variable_name"); + query_length=strlen(query)+1; + goto __run_query; + } + } + + if (query_no_space_length==strlen("SHOW ADMIN VARIABLES") && !strncasecmp("SHOW ADMIN VARIABLES",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables WHERE variable_name LIKE 'admin-\%' ORDER BY variable_name"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (query_no_space_length==strlen("SHOW MYSQL VARIABLES") && !strncasecmp("SHOW MYSQL VARIABLES",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables WHERE variable_name LIKE 'mysql-\%' ORDER BY variable_name"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (query_no_space_length==strlen("SHOW MYSQL STATUS") && !strncasecmp("SHOW MYSQL STATUS",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT Variable_Name AS Variable_name, Variable_Value AS Value FROM stats_mysql_global ORDER BY variable_name"); + query_length=strlen(query)+1; + GloAdmin->stats___mysql_global(); + goto __run_query; + } + + if (query_no_space_length == strlen("SHOW PGSQL VARIABLES") && !strncasecmp("SHOW PGSQL VARIABLES", query_no_space, query_no_space_length)) { + l_free(query_length, query); + query = l_strdup("SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables WHERE variable_name LIKE 'pgsql-\%' ORDER BY variable_name"); + query_length = strlen(query) + 1; + goto __run_query; + } + + if (query_no_space_length == strlen("SHOW PGSQL STATUS") && !strncasecmp("SHOW PGSQL STATUS", query_no_space, query_no_space_length)) { + l_free(query_length, query); + query = l_strdup("SELECT Variable_Name AS Variable_name, Variable_Value AS Value FROM stats_pgsql_global ORDER BY variable_name"); + query_length = strlen(query) + 1; + GloAdmin->stats___pgsql_global(); + goto __run_query; + } + + strA=(char *)"SHOW CREATE TABLE "; + strB=(char *)"SELECT name AS 'table' , REPLACE(REPLACE(sql,' , ', X'2C0A20202020'),'CREATE TABLE %s (','CREATE TABLE %s ('||X'0A20202020') AS 'Create Table' FROM %s.sqlite_master WHERE type='table' AND name='%s'"; + strAl=strlen(strA); + if (strncasecmp("SHOW CREATE TABLE ", query_no_space, strAl)==0) { + strBl=strlen(strB); + char *dbh=NULL; + char *tbh=NULL; + c_split_2(query_no_space+strAl,".",&dbh,&tbh); + + if (strlen(tbh)==0) { + free(tbh); + tbh=dbh; + dbh=strdup("main"); + } + if (strlen(tbh)>=3 && tbh[0]=='`' && tbh[strlen(tbh)-1]=='`') { // tablename is quoted + char *tbh_tmp=(char *)malloc(strlen(tbh)-1); + strncpy(tbh_tmp,tbh+1,strlen(tbh)-2); + tbh_tmp[strlen(tbh)-2]=0; + free(tbh); + tbh=tbh_tmp; + } + int l=strBl+strlen(tbh)*3+strlen(dbh)-8; + char *buff=(char *)l_alloc(l+1); + snprintf(buff,l+1,strB,tbh,tbh,dbh,tbh); + buff[l]=0; + free(tbh); + free(dbh); + l_free(query_length,query); + query=buff; + query_length=l+1; + goto __run_query; + } + + if ( + (query_no_space_length==strlen("SHOW DATABASES") && !strncasecmp("SHOW DATABASES",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SHOW SCHEMAS") && !strncasecmp("SHOW SCHEMAS",query_no_space, query_no_space_length)) + ) { + l_free(query_length,query); + query=l_strdup("PRAGMA DATABASE_LIST"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (query_no_space_length==strlen("SHOW FULL PROCESSLIST") && !strncasecmp("SHOW FULL PROCESSLIST",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT * FROM stats_mysql_processlist"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (query_no_space_length==strlen("SHOW PROCESSLIST") && !strncasecmp("SHOW PROCESSLIST",query_no_space, query_no_space_length)) { + l_free(query_length,query); + query=l_strdup("SELECT SessionID, user, db, hostgroup, command, time_ms, SUBSTR(info,0,100) info FROM stats_mysql_processlist"); + query_length=strlen(query)+1; + goto __run_query; + } + + if (query_no_space_length == strlen("SHOW FULL PGSQL PROCESSLIST") && !strncasecmp("SHOW FULL PGSQL PROCESSLIST", query_no_space, query_no_space_length)) { + l_free(query_length, query); + query = l_strdup("SELECT * FROM stats_pgsql_processlist"); + query_length = strlen(query) + 1; + goto __run_query; + } + + if (query_no_space_length == strlen("SHOW PGSQL PROCESSLIST") && !strncasecmp("SHOW PGSQL PROCESSLIST", query_no_space, query_no_space_length)) { + l_free(query_length, query); + query = l_strdup("SELECT SessionID, user, database, hostgroup, command, time_ms, SUBSTR(info,0,100) info FROM stats_pgsql_processlist"); + query_length = strlen(query) + 1; + goto __run_query; + } + +__end_show_commands: + + if (query_no_space_length==strlen("SELECT DATABASE()") && !strncasecmp("SELECT DATABASE()",query_no_space, query_no_space_length)) { + l_free(query_length,query); + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + query=l_strdup("SELECT \"admin\" AS 'DATABASE()'"); + } else { + query=l_strdup("SELECT \"stats\" AS 'DATABASE()'"); + } + query_length=strlen(query)+1; + goto __run_query; + } + + // see issue #1022 + if (query_no_space_length==strlen("SELECT DATABASE() AS name") && !strncasecmp("SELECT DATABASE() AS name",query_no_space, query_no_space_length)) { + l_free(query_length,query); + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + query=l_strdup("SELECT \"admin\" AS 'name'"); + } else { + query=l_strdup("SELECT \"stats\" AS 'name'"); + } + query_length=strlen(query)+1; + goto __run_query; + } + + if (sess->session_type == PROXYSQL_SESSION_STATS) { // no admin + if ( + (strncasecmp("PRAGMA",query_no_space,6)==0) + || + (strncasecmp("ATTACH",query_no_space,6)==0) + ) { + proxy_error("[WARNING]: Commands executed from stats interface in Admin Module: \"%s\"\n", query_no_space); + SPA->send_error_msg_to_client(sess, (char *)"Command not allowed"); + run_query=false; + } + } + +__run_query: + if (sess->proxysql_node_address && (__sync_fetch_and_add(&glovars.shutdown,0)==0)) { + if (sess->client_myds->active) { + const string uuid { sess->proxysql_node_address->uuid }; + const string hostname { sess->proxysql_node_address->hostname }; + const string port { std::to_string(sess->proxysql_node_address->port) }; + const string mysql_ifaces { sess->proxysql_node_address->admin_mysql_ifaces }; + + time_t now = time(NULL); + string q = "INSERT OR REPLACE INTO stats_proxysql_servers_clients_status (uuid, hostname, port, admin_mysql_ifaces, last_seen_at) VALUES (\""; + q += uuid; + q += "\",\""; + q += hostname; + q += "\","; + q += port; + q += ",\""; + q += mysql_ifaces; + q += "\","; + q += std::to_string(now) + ")"; + SPA->statsdb->execute(q.c_str()); + + std::map m_labels { { "uuid", uuid }, { "hostname", hostname }, { "port", port } }; + const string m_id { uuid + ":" + hostname + ":" + port }; + + p_update_map_gauge( + SPA->metrics.p_proxysql_servers_clients_status_map, + SPA->metrics.p_dyn_gauge_array[p_admin_dyn_gauge::proxysql_servers_clients_status_last_seen_at], + m_id, m_labels, now + ); + } + } + if (run_query) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats + if (SPA->get_read_only()) { // disable writes if the admin interface is in read_only mode + SPA->admindb->execute("PRAGMA query_only = ON"); + SPA->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + SPA->admindb->execute("PRAGMA query_only = OFF"); + } else { + SPA->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + } + if (needs_vacuum) { + SPA->vacuum_stats(true); + } + } else { + SPA->statsdb->execute("PRAGMA query_only = ON"); + SPA->statsdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + SPA->statsdb->execute("PRAGMA query_only = OFF"); + if (needs_vacuum) { + SPA->vacuum_stats(false); + } + } + if (error == NULL) { + + if constexpr (std::is_same_v) { + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + } else if constexpr (std::is_same_v) { + SQLite3_to_Postgres(sess->client_myds->PSarrayOUT, resultset, error, affected_rows, query); + } else { + assert(0); + } + } else { + char *a = (char *)"ProxySQL Admin Error: "; + char *new_msg = (char *)malloc(strlen(error)+strlen(a)+1); + sprintf(new_msg, "%s%s", a, error); + + if constexpr (std::is_same_v) { + sess->SQLite3_to_MySQL(resultset, new_msg, affected_rows, &sess->client_myds->myprot); + } else if constexpr (std::is_same_v) { + SQLite3_to_Postgres(sess->client_myds->PSarrayOUT, resultset, new_msg, affected_rows, query); + } else { + assert(0); + } + + free(new_msg); + free(error); + } + delete resultset; + } + if (run_query == true) { + pthread_mutex_unlock(&pa->sql_query_global_mutex); + } else { + // The admin module may have already been freed in case of "PROXYSQL STOP" + if (strcasecmp("PROXYSQL STOP",query_no_space)) + pthread_mutex_unlock(&pa->sql_query_global_mutex); + } + l_free(pkt->size-sizeof(mysql_hdr),query_no_space); // it is always freed here + l_free(query_length,query); +} + +// Explicitly instantiate the required template class and member functions +template void admin_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *pkt); +template void admin_session_handler(PgSQL_Session* sess, void *_pa, PtrSize_t *pkt); + diff --git a/lib/BaseHGC.cpp b/lib/BaseHGC.cpp new file mode 100644 index 0000000000..54b0d4ca5b --- /dev/null +++ b/lib/BaseHGC.cpp @@ -0,0 +1,131 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include "Base_HostGroups_Manager.h" + + +template BaseHGC::BaseHGC(int); +template BaseHGC::~BaseHGC(); +template void BaseHGC::log_num_online_server_count_error(); +template void BaseHGC::reset_attributes(); +template void BaseHGC::refresh_online_server_count(); +template BaseHGC::BaseHGC(int); +template BaseHGC::~BaseHGC(); +template void BaseHGC::log_num_online_server_count_error(); +template void BaseHGC::reset_attributes(); +template void BaseHGC::refresh_online_server_count(); + +template +using TypeSrvC = typename std::conditional< + std::is_same_v, MySrvC, PgSQL_SrvC +>::type; + +template +using TypeSess = typename std::conditional< + std::is_same_v, MySQL_Session, PgSQL_Session +>::type; + + +#include "MySQL_HostGroups_Manager.h" + + +#ifdef TEST_AURORA +if constexpr (std::is_same_v) { +static unsigned long long array_mysrvc_total = 0; +static unsigned long long array_mysrvc_cands = 0; +} +#endif // TEST_AURORA + +extern MySQL_Threads_Handler *GloMTH; + + +template +BaseHGC::BaseHGC(int _hid) { + hid=_hid; + if constexpr (std::is_same_v) { + mysrvs=new MySrvList(static_cast(this)); + } else if constexpr (std::is_same_v) { + mysrvs=new PgSQL_SrvList(static_cast(this)); + } else { + assert(0); + } + current_time_now = 0; + new_connections_now = 0; + attributes.initialized = false; + reset_attributes(); + // Uninitialized server defaults. Should later be initialized via 'mysql_hostgroup_attributes'. + servers_defaults.weight = -1; + servers_defaults.max_connections = -1; + servers_defaults.use_ssl = -1; + num_online_servers.store(0, std::memory_order_relaxed);; + last_log_time_num_online_servers = 0; +} + + +template +void BaseHGC::reset_attributes() { + if (attributes.initialized == false) { + attributes.init_connect = NULL; + attributes.comment = NULL; + attributes.ignore_session_variables_text = NULL; + } + attributes.initialized = true; + attributes.configured = false; + attributes.max_num_online_servers = 1000000; + attributes.throttle_connections_per_sec = 1000000; + attributes.autocommit = -1; + attributes.free_connections_pct = 10; + attributes.handle_warnings = -1; + attributes.monitor_slave_lag_when_null = -1; + attributes.multiplex = true; + attributes.connection_warming = false; + free(attributes.init_connect); + attributes.init_connect = NULL; + free(attributes.comment); + attributes.comment = NULL; + free(attributes.ignore_session_variables_text); + attributes.ignore_session_variables_text = NULL; + if (attributes.ignore_session_variables_json) { + delete attributes.ignore_session_variables_json; + attributes.ignore_session_variables_json = NULL; + } +} + +template +BaseHGC::~BaseHGC() { + reset_attributes(); // free all memory + delete mysrvs; +} + +template +void BaseHGC::refresh_online_server_count() { + if (__sync_fetch_and_add(&glovars.shutdown, 0) != 0) + return; +#ifdef DEBUG + assert(MyHGM->is_locked); +#endif + unsigned int online_servers_count = 0; + if constexpr (std::is_same_v) { // FIXME: this logic for now is enabled only for MySQL + for (unsigned int i = 0; i < mysrvs->servers->len; i++) { + TypeSrvC* mysrvc = (TypeSrvC*)mysrvs->servers->index(i); + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { + online_servers_count++; + } + } + } + num_online_servers.store(online_servers_count, std::memory_order_relaxed); +} + +template +void BaseHGC::log_num_online_server_count_error() { + const time_t curtime = time(NULL); + // if this is the first time the method is called or if more than 10 seconds have passed since the last log + if (last_log_time_num_online_servers == 0 || + ((curtime - last_log_time_num_online_servers) > 10)) { + last_log_time_num_online_servers = curtime; + proxy_error( + "Number of online servers detected in a hostgroup exceeds the configured maximum online servers. hostgroup:%u, num_online_servers:%u, max_online_servers:%u\n", + hid, num_online_servers.load(std::memory_order_relaxed), attributes.max_num_online_servers); + } +} diff --git a/lib/BaseSrvList.cpp b/lib/BaseSrvList.cpp new file mode 100644 index 0000000000..991861192f --- /dev/null +++ b/lib/BaseSrvList.cpp @@ -0,0 +1,72 @@ +#include "Base_HostGroups_Manager.h" + +template +using TypeSrvC = typename std::conditional< + std::is_same_v, MySrvC, PgSQL_SrvC +>::type; + +template BaseSrvList::BaseSrvList(MyHGC*); +template BaseSrvList::~BaseSrvList(); +template void BaseSrvList::add(MySrvC*); + +template BaseSrvList::BaseSrvList(PgSQL_HGC*); +template BaseSrvList::~BaseSrvList(); +template void BaseSrvList::add(PgSQL_SrvC*); + + +template +BaseSrvList::BaseSrvList(HGC *_myhgc) { + myhgc=_myhgc; + servers=new PtrArray(); +} + +template +void BaseSrvList::add(TypeSrvC *s) { + if (s->myhgc==NULL) { + s->myhgc=myhgc; + } + servers->add(s); + if constexpr (std::is_same_v) { + myhgc->refresh_online_server_count(); + } else if constexpr (std::is_same_v) { + //myhgc->refresh_online_server_count(); FIXME: not implemented + } else { + assert(0); + } +} + + +template +int BaseSrvList::find_idx(TypeSrvC *s) { + for (unsigned int i=0; ilen; i++) { + TypeSrvC *mysrv=(TypeSrvC *)servers->index(i); + if (mysrv==s) { + return (unsigned int)i; + } + } + return -1; +} + +template +void BaseSrvList::remove(TypeSrvC *s) { + int i=find_idx(s); + assert(i>=0); + servers->remove_index_fast((unsigned int)i); + if constexpr (std::is_same_v) { + myhgc->refresh_online_server_count(); + } else if constexpr (std::is_same_v) { + //myhgc->refresh_online_server_count(); FIXME: not implemented + } else { + assert(0); + } +} + +template +BaseSrvList::~BaseSrvList() { + myhgc=NULL; + while (servers->len) { + TypeSrvC *mysrvc=(TypeSrvC *)servers->remove_index_fast(0); + delete mysrvc; + } + delete servers; +} diff --git a/lib/Base_HostGroups_Manager.cpp b/lib/Base_HostGroups_Manager.cpp new file mode 100644 index 0000000000..4a27e78887 --- /dev/null +++ b/lib/Base_HostGroups_Manager.cpp @@ -0,0 +1,4797 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include "MySQL_HostGroups_Manager.h" +#include "proxysql.h" +#include "cpp.h" + +#include "MySQL_PreparedStatement.h" +#include "MySQL_Data_Stream.h" + +#include +#include +#include + +#include "prometheus/counter.h" +#include "prometheus/detail/builder.h" +#include "prometheus/family.h" +#include "prometheus/gauge.h" + +#include "prometheus_helpers.h" +#include "proxysql_utils.h" + +#define char_malloc (char *)malloc +#define itostr(__s, __i) { __s=char_malloc(32); sprintf(__s, "%lld", __i); } + +#include "thread.h" +#include "wqueue.h" + +#include "ev.h" + +#include +#include +#include + +using std::function; + +#include "Base_HostGroups_Manager.h" + + +template Base_HostGroups_Manager::Base_HostGroups_Manager(); +template MyHGC * Base_HostGroups_Manager::MyHGC_find(unsigned int); +template MyHGC * Base_HostGroups_Manager::MyHGC_create(unsigned int); +template MyHGC * Base_HostGroups_Manager::MyHGC_lookup(unsigned int); +template void Base_HostGroups_Manager::wrlock(); +template void Base_HostGroups_Manager::wrunlock(); + +template Base_HostGroups_Manager::Base_HostGroups_Manager(); +template PgSQL_HGC * Base_HostGroups_Manager::MyHGC_find(unsigned int); +template PgSQL_HGC * Base_HostGroups_Manager::MyHGC_create(unsigned int); +template PgSQL_HGC * Base_HostGroups_Manager::MyHGC_lookup(unsigned int); +template void Base_HostGroups_Manager::wrlock(); +template void Base_HostGroups_Manager::wrunlock(); + +template SQLite3_result * Base_HostGroups_Manager::execute_query(char*, char**); + +#if 0 +#define SAFE_SQLITE3_STEP(_stmt) do {\ + do {\ + rc=(*proxy_sqlite3_step)(_stmt);\ + if (rc!=SQLITE_DONE) {\ + assert(rc==SQLITE_LOCKED);\ + usleep(100);\ + }\ + } while (rc!=SQLITE_DONE);\ +} while (0) + +extern ProxySQL_Admin *GloAdmin; + +extern MySQL_Threads_Handler *GloMTH; + +extern MySQL_Monitor *GloMyMon; + +class MySrvConnList; +class MySrvC; +class MySrvList; +class MyHGC; + +const int MYSQL_ERRORS_STATS_FIELD_NUM = 11; + + +static int wait_for_mysql(MYSQL *mysql, int status) { + struct pollfd pfd; + int timeout, res; + + pfd.fd = mysql_get_socket(mysql); + pfd.events = + (status & MYSQL_WAIT_READ ? POLLIN : 0) | + (status & MYSQL_WAIT_WRITE ? POLLOUT : 0) | + (status & MYSQL_WAIT_EXCEPT ? POLLPRI : 0); + timeout = 1; + res = poll(&pfd, 1, timeout); + if (res == 0) + return MYSQL_WAIT_TIMEOUT | status; + else if (res < 0) + return MYSQL_WAIT_TIMEOUT; + else { + int status = 0; + if (pfd.revents & POLLIN) status |= MYSQL_WAIT_READ; + if (pfd.revents & POLLOUT) status |= MYSQL_WAIT_WRITE; + if (pfd.revents & POLLPRI) status |= MYSQL_WAIT_EXCEPT; + return status; + } +} + +/** + * @brief Helper function used to try to extract a value from the JSON field 'servers_defaults'. + * + * @param j JSON object constructed from 'servers_defaults' field. + * @param hid Hostgroup for which the 'servers_defaults' is defined in 'mysql_hostgroup_attributes'. Used for + * error logging. + * @param key The key for the value to be extracted. + * @param val_check A validation function, checks if the value is within a expected range. + * + * @return The value extracted from the supplied JSON. In case of error '-1', and error cause is logged. + */ +template ::value, bool>::type = true> +T j_get_srv_default_int_val( + const json& j, uint32_t hid, const string& key, const function& val_check +) { + if (j.find(key) != j.end()) { + const json::value_t val_type = j[key].type(); + const char* type_name = j[key].type_name(); + + if (val_type == json::value_t::number_integer || val_type == json::value_t::number_unsigned) { + T val = j[key].get(); + + if (val_check(val)) { + return val; + } else { + proxy_error( + "Invalid value %ld supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." + " Value NOT UPDATED.\n", + static_cast(val), key.c_str(), hid + ); + } + } else { + proxy_error( + "Invalid type '%s'(%hhu) supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." + " Value NOT UPDATED.\n", + type_name, static_cast(val_type), key.c_str(), hid + ); + } + } + + return static_cast(-1); +} + + +//static void * HGCU_thread_run() { +static void * HGCU_thread_run() { + PtrArray *conn_array=new PtrArray(); + while(1) { + MySQL_Connection *myconn= NULL; + myconn = (MySQL_Connection *)MyHGM->queue.remove(); + if (myconn==NULL) { + // intentionally exit immediately + delete conn_array; + return NULL; + } + conn_array->add(myconn); + while (MyHGM->queue.size()) { + myconn=(MySQL_Connection *)MyHGM->queue.remove(); + if (myconn==NULL) { + delete conn_array; + return NULL; + } + conn_array->add(myconn); + } + unsigned int l=conn_array->len; + int *errs=(int *)malloc(sizeof(int)*l); + int *statuses=(int *)malloc(sizeof(int)*l); + my_bool *ret=(my_bool *)malloc(sizeof(my_bool)*l); + int i; + for (i=0;i<(int)l;i++) { + myconn->reset(); + MyHGM->increase_reset_counter(); + myconn=(MySQL_Connection *)conn_array->index(i); + if (myconn->mysql->net.pvio && myconn->mysql->net.fd && myconn->mysql->net.buff) { + MySQL_Connection_userinfo *userinfo = myconn->userinfo; + char *auth_password = NULL; + if (userinfo->password) { + if (userinfo->password[0]=='*') { // we don't have the real password, let's pass sha1 + auth_password=userinfo->sha1_pass; + } else { + auth_password=userinfo->password; + } + } + //async_exit_status = mysql_change_user_start(&ret_bool,mysql,_ui->username, auth_password, _ui->schemaname); + // we first reset the charset to a default one. + // this to solve the problem described here: + // https://github.com/sysown/proxysql/pull/3249#issuecomment-761887970 + if (myconn->mysql->charset->nr >= 255) + mysql_options(myconn->mysql, MYSQL_SET_CHARSET_NAME, myconn->mysql->charset->csname); + statuses[i]=mysql_change_user_start(&ret[i], myconn->mysql, myconn->userinfo->username, auth_password, myconn->userinfo->schemaname); + if (myconn->mysql->net.pvio==NULL || myconn->mysql->net.fd==0 || myconn->mysql->net.buff==NULL) { + statuses[i]=0; ret[i]=1; + } + } else { + statuses[i]=0; + ret[i]=1; + } + } + for (i=0;i<(int)conn_array->len;i++) { + if (statuses[i]==0) { + myconn=(MySQL_Connection *)conn_array->remove_index_fast(i); + if (!ret[i]) { + MyHGM->push_MyConn_to_pool(myconn); + } else { + myconn->send_quit=false; + MyHGM->destroy_MyConn_from_pool(myconn); + } + statuses[i]=statuses[conn_array->len]; + ret[i]=ret[conn_array->len]; + i--; + } + } + unsigned long long now=monotonic_time(); + while (conn_array->len && ((monotonic_time() - now) < 1000000)) { + usleep(50); + for (i=0;i<(int)conn_array->len;i++) { + myconn=(MySQL_Connection *)conn_array->index(i); + if (myconn->mysql->net.pvio && myconn->mysql->net.fd && myconn->mysql->net.buff) { + statuses[i]=wait_for_mysql(myconn->mysql, statuses[i]); + if (myconn->mysql->net.pvio && myconn->mysql->net.fd && myconn->mysql->net.buff) { + if ((statuses[i] & MYSQL_WAIT_TIMEOUT) == 0) { + statuses[i]=mysql_change_user_cont(&ret[i], myconn->mysql, statuses[i]); + if (myconn->mysql->net.pvio==NULL || myconn->mysql->net.fd==0 || myconn->mysql->net.buff==NULL ) { + statuses[i]=0; ret[i]=1; + } + } + } else { + statuses[i]=0; ret[i]=1; + } + } else { + statuses[i]=0; ret[i]=1; + } + } + for (i=0;i<(int)conn_array->len;i++) { + if (statuses[i]==0) { + myconn=(MySQL_Connection *)conn_array->remove_index_fast(i); + if (!ret[i]) { + myconn->reset(); + MyHGM->push_MyConn_to_pool(myconn); + } else { + myconn->send_quit=false; + MyHGM->destroy_MyConn_from_pool(myconn); + } + statuses[i]=statuses[conn_array->len]; + ret[i]=ret[conn_array->len]; + i--; + } + } + } + while (conn_array->len) { + // we reached here, and there are still connections + myconn=(MySQL_Connection *)conn_array->remove_index_fast(0); + myconn->send_quit=false; + MyHGM->destroy_MyConn_from_pool(myconn); + } + free(statuses); + free(errs); + free(ret); + } + delete conn_array; +} + + +using metric_name = std::string; +using metric_help = std::string; +using metric_tags = std::map; + +using hg_counter_tuple = + std::tuple< + p_hg_counter::metric, + metric_name, + metric_help, + metric_tags + >; + +using hg_gauge_tuple = + std::tuple< + p_hg_gauge::metric, + metric_name, + metric_help, + metric_tags + >; + +using hg_dyn_counter_tuple = + std::tuple< + p_hg_dyn_counter::metric, + metric_name, + metric_help, + metric_tags + >; + +using hg_dyn_gauge_tuple = + std::tuple< + p_hg_dyn_gauge::metric, + metric_name, + metric_help, + metric_tags + >; + +using hg_counter_vector = std::vector; +using hg_gauge_vector = std::vector; +using hg_dyn_counter_vector = std::vector; +using hg_dyn_gauge_vector = std::vector; + +/** + * @brief Metrics map holding the metrics for the 'MySQL_HostGroups_Manager' module. + * + * @note Many metrics in this map, share a common "id name", because + * they differ only by label, because of this, HELP is shared between + * them. For better visual identification of this groups they are + * sepparated using a line separator comment. + */ +const std::tuple< + hg_counter_vector, + hg_gauge_vector, + hg_dyn_counter_vector, + hg_dyn_gauge_vector +> +hg_metrics_map = std::make_tuple( + hg_counter_vector { + std::make_tuple ( + p_hg_counter::servers_table_version, + "proxysql_servers_table_version_total", + "Number of times the \"servers_table\" have been modified.", + metric_tags {} + ), + + // ==================================================================== + std::make_tuple ( + p_hg_counter::server_connections_created, + "proxysql_server_connections_total", + "Total number of server connections (created|delayed|aborted).", + metric_tags { + { "status", "created" } + } + ), + std::make_tuple ( + p_hg_counter::server_connections_delayed, + "proxysql_server_connections_total", + "Total number of server connections (created|delayed|aborted).", + metric_tags { + { "status", "delayed" } + } + ), + std::make_tuple ( + p_hg_counter::server_connections_aborted, + "proxysql_server_connections_total", + "Total number of server connections (created|delayed|aborted).", + metric_tags { + { "status", "aborted" } + } + ), + // ==================================================================== + + // ==================================================================== + std::make_tuple ( + p_hg_counter::client_connections_created, + "proxysql_client_connections_total", + "Total number of client connections created.", + metric_tags { + { "status", "created" } + } + ), + std::make_tuple ( + p_hg_counter::client_connections_aborted, + "proxysql_client_connections_total", + "Total number of client failed connections (or closed improperly).", + metric_tags { + { "status", "aborted" } + } + ), + // ==================================================================== + + std::make_tuple ( + p_hg_counter::com_autocommit, + "proxysql_com_autocommit_total", + "Total queries autocommited.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_autocommit_filtered, + "proxysql_com_autocommit_filtered_total", + "Total queries filtered autocommit.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_rollback, + "proxysql_com_rollback_total", + "Total queries rollbacked.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_rollback_filtered, + "proxysql_com_rollback_filtered_total", + "Total queries filtered rollbacked.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_backend_change_user, + "proxysql_com_backend_change_user_total", + "Total CHANGE_USER queries backend.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_backend_init_db, + "proxysql_com_backend_init_db_total", + "Total queries backend INIT DB.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_backend_set_names, + "proxysql_com_backend_set_names_total", + "Total queries backend SET NAMES.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_frontend_init_db, + "proxysql_com_frontend_init_db_total", + "Total INIT DB queries frontend.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_frontend_set_names, + "proxysql_com_frontend_set_names_total", + "Total SET NAMES frontend queries.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_frontend_use_db, + "proxysql_com_frontend_use_db_total", + "Total USE DB queries frontend.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_commit_cnt, + "proxysql_com_commit_cnt_total", + "Total queries commit.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::com_commit_cnt_filtered, + "proxysql_com_commit_cnt_filtered_total", + "Total queries commit filtered.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::selects_for_update__autocommit0, + "proxysql_selects_for_update__autocommit0_total", + "Total queries that are SELECT for update or equivalent.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::access_denied_wrong_password, + "proxysql_access_denied_wrong_password_total", + "Total access denied \"wrong password\".", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::access_denied_max_connections, + "proxysql_access_denied_max_connections_total", + "Total access denied \"max connections\".", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::access_denied_max_user_connections, + "proxysql_access_denied_max_user_connections_total", + "Total access denied \"max user connections\".", + metric_tags {} + ), + + // ==================================================================== + std::make_tuple ( + p_hg_counter::myhgm_myconnpool_get, + "proxysql_myhgm_myconnpool_get_total", + "The number of requests made to the connection pool.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::myhgm_myconnpool_get_ok, + "proxysql_myhgm_myconnpool_get_ok_total", + "The number of successful requests to the connection pool (i.e. where a connection was available).", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::myhgm_myconnpool_get_ping, + "proxysql_myhgm_myconnpool_get_ping_total", + "The number of connections that were taken from the pool to run a ping to keep them alive.", + metric_tags {} + ), + // ==================================================================== + + std::make_tuple ( + p_hg_counter::myhgm_myconnpool_push, + "proxysql_myhgm_myconnpool_push_total", + "The number of connections returned to the connection pool.", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::myhgm_myconnpool_reset, + "proxysql_myhgm_myconnpool_reset_total", + "The number of connections that have been reset / re-initialized using \"COM_CHANGE_USER\"", + metric_tags {} + ), + std::make_tuple ( + p_hg_counter::myhgm_myconnpool_destroy, + "proxysql_myhgm_myconnpool_destroy_total", + "The number of connections considered unhealthy and therefore closed.", + metric_tags {} + ), + + // ==================================================================== + + std::make_tuple ( + p_hg_counter::auto_increment_delay_multiplex, + "proxysql_myhgm_auto_increment_multiplex_total", + "The number of times that 'auto_increment_delay_multiplex' has been triggered.", + metric_tags {} + ), + }, + // prometheus gauges + hg_gauge_vector { + std::make_tuple ( + p_hg_gauge::server_connections_connected, + "proxysql_server_connections_connected", + "Backend connections that are currently connected.", + metric_tags {} + ), + std::make_tuple ( + p_hg_gauge::client_connections_connected, + "proxysql_client_connections_connected", + "Client connections that are currently connected.", + metric_tags {} + ) + }, + // prometheus dynamic counters + hg_dyn_counter_vector { + // connection_pool + // ==================================================================== + + // ==================================================================== + std::make_tuple ( + p_hg_dyn_counter::conn_pool_bytes_data_recv, + "proxysql_connpool_data_bytes_total", + "Amount of data (sent|recv) from the backend, excluding metadata.", + metric_tags { + { "traffic_flow", "recv" } + } + ), + std::make_tuple ( + p_hg_dyn_counter::conn_pool_bytes_data_sent, + "proxysql_connpool_data_bytes_total", + "Amount of data (sent|recv) from the backend, excluding metadata.", + metric_tags { + { "traffic_flow", "sent" } + } + ), + // ==================================================================== + + // ==================================================================== + std::make_tuple ( + p_hg_dyn_counter::connection_pool_conn_err, + "proxysql_connpool_conns_total", + "How many connections have been tried to be established.", + metric_tags { + { "status", "err" } + } + ), + std::make_tuple ( + p_hg_dyn_counter::connection_pool_conn_ok, + "proxysql_connpool_conns_total", + "How many connections have been tried to be established.", + metric_tags { + { "status", "ok" } + } + ), + // ==================================================================== + + std::make_tuple ( + p_hg_dyn_counter::connection_pool_queries, + "proxysql_connpool_conns_queries_total", + "The number of queries routed towards this particular backend server.", + metric_tags {} + ), + // gtid + std::make_tuple ( + p_hg_dyn_counter::gtid_executed, + "proxysql_gtid_executed_total", + "Tracks the number of executed gtid per host and port.", + metric_tags {} + ), + // mysql_error + std::make_tuple ( + p_hg_dyn_counter::proxysql_mysql_error, + "proxysql_mysql_error_total", + "Tracks the mysql errors generated by proxysql.", + metric_tags {} + ), + std::make_tuple ( + p_hg_dyn_counter::mysql_error, + "mysql_error_total", + "Tracks the mysql errors encountered.", + metric_tags {} + ) + }, + // prometheus dynamic gauges + hg_dyn_gauge_vector { + std::make_tuple ( + p_hg_dyn_gauge::connection_pool_conn_free, + "proxysql_connpool_conns", + "How many backend connections are currently (free|used).", + metric_tags { + { "status", "free" } + } + ), + std::make_tuple ( + p_hg_dyn_gauge::connection_pool_conn_used, + "proxysql_connpool_conns", + "How many backend connections are currently (free|used).", + metric_tags { + { "status", "used" } + } + ), + std::make_tuple ( + p_hg_dyn_gauge::connection_pool_latency_us, + "proxysql_connpool_conns_latency_us", + "The currently ping time in microseconds, as reported from Monitor.", + metric_tags {} + ), + std::make_tuple ( + p_hg_dyn_gauge::connection_pool_status, + "proxysql_connpool_conns_status", + "The status of the backend server (1 - ONLINE, 2 - SHUNNED, 3 - OFFLINE_SOFT, 4 - OFFLINE_HARD).", + metric_tags {} + ) + } +); +#endif // 0 + +template +Base_HostGroups_Manager::Base_HostGroups_Manager() { + pthread_mutex_init(&readonly_mutex, NULL); + pthread_mutex_init(&lock, NULL); + admindb=NULL; // initialized only if needed + mydb=new SQLite3DB(); +} + +#if 0 +MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { + status.client_connections=0; + status.client_connections_aborted=0; + status.client_connections_created=0; + status.server_connections_connected=0; + status.server_connections_aborted=0; + status.server_connections_created=0; + status.server_connections_delayed=0; + status.servers_table_version=0; + pthread_mutex_init(&status.servers_table_version_lock, NULL); + pthread_cond_init(&status.servers_table_version_cond, NULL); + status.myconnpoll_get=0; + status.myconnpoll_get_ok=0; + status.myconnpoll_get_ping=0; + status.myconnpoll_push=0; + status.myconnpoll_destroy=0; + status.myconnpoll_reset=0; + status.autocommit_cnt=0; + status.commit_cnt=0; + status.rollback_cnt=0; + status.autocommit_cnt_filtered=0; + status.commit_cnt_filtered=0; + status.rollback_cnt_filtered=0; + status.backend_change_user=0; + status.backend_init_db=0; + status.backend_set_names=0; + status.frontend_init_db=0; + status.frontend_set_names=0; + status.frontend_use_db=0; + status.access_denied_wrong_password=0; + status.access_denied_max_connections=0; + status.access_denied_max_user_connections=0; + status.select_for_update_or_equivalent=0; + status.auto_increment_delay_multiplex=0; + pthread_mutex_init(&readonly_mutex, NULL); + pthread_mutex_init(&Group_Replication_Info_mutex, NULL); + pthread_mutex_init(&Galera_Info_mutex, NULL); + pthread_mutex_init(&AWS_Aurora_Info_mutex, NULL); + pthread_mutex_init(&lock, NULL); + admindb=NULL; // initialized only if needed + mydb=new SQLite3DB(); +#ifdef DEBUG + mydb->open((char *)"file:mem_mydb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); +#else + mydb->open((char *)"file:mem_mydb?mode=memory", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); +#endif /* DEBUG */ + mydb->execute(MYHGM_MYSQL_SERVERS); + mydb->execute(MYHGM_MYSQL_SERVERS_INCOMING); + mydb->execute(MYHGM_MYSQL_REPLICATION_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_GALERA_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES); + mydb->execute(MYHGM_MYSQL_SERVERS_SSL_PARAMS); + mydb->execute("CREATE INDEX IF NOT EXISTS idx_mysql_servers_hostname_port ON mysql_servers (hostname,port)"); + MyHostGroups=new PtrArray(); + runtime_mysql_servers=NULL; + incoming_replication_hostgroups=NULL; + incoming_group_replication_hostgroups=NULL; + incoming_galera_hostgroups=NULL; + incoming_aws_aurora_hostgroups = NULL; + incoming_hostgroup_attributes = NULL; + incoming_mysql_servers_ssl_params = NULL; + incoming_mysql_servers_v2 = NULL; + pthread_rwlock_init(>id_rwlock, NULL); + gtid_missing_nodes = false; + gtid_ev_loop=NULL; + gtid_ev_timer=NULL; + gtid_ev_async = (struct ev_async *)malloc(sizeof(struct ev_async)); + mysql_servers_to_monitor = NULL; + + { + static const char alphanum[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + rand_del[0] = '-'; + for (int i = 1; i < 6; i++) { + rand_del[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; + } + rand_del[6] = '-'; + rand_del[7] = 0; + } + pthread_mutex_init(&mysql_errors_mutex, NULL); + + // Initialize prometheus metrics + init_prometheus_counter_array(hg_metrics_map, this->status.p_counter_array); + init_prometheus_gauge_array(hg_metrics_map, this->status.p_gauge_array); + init_prometheus_dyn_counter_array(hg_metrics_map, this->status.p_dyn_counter_array); + init_prometheus_dyn_gauge_array(hg_metrics_map, this->status.p_dyn_gauge_array); + + pthread_mutex_init(&mysql_errors_mutex, NULL); +} + +void MySQL_HostGroups_Manager::init() { + //conn_reset_queue = NULL; + //conn_reset_queue = new wqueue(); + HGCU_thread = new std::thread(&HGCU_thread_run); + //pthread_create(&HGCU_thread_id, NULL, HGCU_thread_run , NULL); + + // gtid initialization; + GTID_syncer_thread = new std::thread(>ID_syncer_run); + + //pthread_create(>ID_syncer_thread_id, NULL, GTID_syncer_run , NULL); +} + +void MySQL_HostGroups_Manager::shutdown() { + queue.add(NULL); + HGCU_thread->join(); + delete HGCU_thread; + ev_async_send(gtid_ev_loop, gtid_ev_async); + GTID_syncer_thread->join(); + delete GTID_syncer_thread; +} + +MySQL_HostGroups_Manager::~MySQL_HostGroups_Manager() { + while (MyHostGroups->len) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->remove_index_fast(0); + delete myhgc; + } + delete MyHostGroups; + delete mydb; + if (admindb) { + delete admindb; + } + for (auto info : AWS_Aurora_Info_Map) + delete info.second; + free(gtid_ev_async); + if (gtid_ev_loop) + ev_loop_destroy(gtid_ev_loop); + if (gtid_ev_timer) + free(gtid_ev_timer); + pthread_mutex_destroy(&lock); +} + +#endif // 0 + +// wrlock() is only required during commit() +template +void Base_HostGroups_Manager::wrlock() { + pthread_mutex_lock(&lock); +#ifdef DEBUG + is_locked = true; +#endif +} + +#if 0 +void MySQL_HostGroups_Manager::p_update_mysql_error_counter(p_mysql_error_type err_type, unsigned int hid, char* address, uint16_t port, unsigned int code) { + p_hg_dyn_counter::metric metric = p_hg_dyn_counter::mysql_error; + if (err_type == p_mysql_error_type::proxysql) { + metric = p_hg_dyn_counter::proxysql_mysql_error; + } + + std::string s_hostgroup = std::to_string(hid); + std::string s_address = std::string(address); + std::string s_port = std::to_string(port); + // TODO: Create switch here to classify error codes + std::string s_code = std::to_string(code); + std::string metric_id = s_hostgroup + ":" + address + ":" + s_port + ":" + s_code; + std::map metric_labels { + { "hostgroup", s_hostgroup }, + { "address", address }, + { "port", s_port }, + { "code", s_code } + }; + + pthread_mutex_lock(&mysql_errors_mutex); + + p_inc_map_counter( + status.p_mysql_errors_map, + status.p_dyn_counter_array[metric], + metric_id, + metric_labels + ); + + pthread_mutex_unlock(&mysql_errors_mutex); +} +#endif // 0 + +template +void Base_HostGroups_Manager::wrunlock() { +#ifdef DEBUG + is_locked = false; +#endif + pthread_mutex_unlock(&lock); +} + +#if 0 +void MySQL_HostGroups_Manager::wait_servers_table_version(unsigned v, unsigned w) { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + //ts.tv_sec += w; + unsigned int i = 0; + int rc = 0; + pthread_mutex_lock(&status.servers_table_version_lock); + while ((rc == 0 || rc == ETIMEDOUT) && (i < w) && (__sync_fetch_and_add(&glovars.shutdown,0)==0) && (__sync_fetch_and_add(&status.servers_table_version,0) < v)) { + i++; + ts.tv_sec += 1; + rc = pthread_cond_timedwait( &status.servers_table_version_cond, &status.servers_table_version_lock, &ts); + } + pthread_mutex_unlock(&status.servers_table_version_lock); +} + +unsigned int MySQL_HostGroups_Manager::get_servers_table_version() { + return __sync_fetch_and_add(&status.servers_table_version,0); +} + +// we always assume that the calling thread has acquired a rdlock() +int MySQL_HostGroups_Manager::servers_add(SQLite3_result *resultset) { + if (resultset==NULL) { + return 0; + } + int rc; + mydb->execute("DELETE FROM mysql_servers_incoming"); + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + //sqlite3 *mydb3=mydb->get_db(); + char *query1=(char *)"INSERT INTO mysql_servers_incoming VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + std::string query32s = "INSERT INTO mysql_servers_incoming VALUES " + generate_multi_rows_query(32,12); + char *query32 = (char *)query32s.c_str(); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = mydb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, mydb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = mydb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, mydb); + MySerStatus status1=MYSQL_SERVER_STATUS_ONLINE; + int row_idx=0; + int max_bulk_row_idx=resultset->rows_count/32; + max_bulk_row_idx=max_bulk_row_idx*32; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + status1=MYSQL_SERVER_STATUS_ONLINE; + if (strcasecmp(r1->fields[4],"ONLINE")) { + if (!strcasecmp(r1->fields[4],"SHUNNED")) { + status1=MYSQL_SERVER_STATUS_SHUNNED; + } else { + if (!strcasecmp(r1->fields[4],"OFFLINE_SOFT")) { + status1=MYSQL_SERVER_STATUS_OFFLINE_SOFT; + } else { + if (!strcasecmp(r1->fields[4],"OFFLINE_HARD")) { + status1=MYSQL_SERVER_STATUS_OFFLINE_HARD; + } + } + } + } + int idx=row_idx%32; + if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+5, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+6, status1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+10, atoi(r1->fields[9])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*12)+11, atoi(r1->fields[10])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*12)+12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, mydb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, status1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoi(r1->fields[9])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoi(r1->fields[10])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + return 0; +} +#endif // 0 + +/** + * @brief Execute a SQL query and retrieve the resultset. + * + * This function executes a SQL query using the provided query string and returns the resultset obtained from the + * database operation. It also provides an optional error parameter to capture any error messages encountered during + * query execution. + * + * @param query A pointer to a null-terminated string containing the SQL query to be executed. + * @param error A pointer to a char pointer where any error message encountered during query execution will be stored. + * Pass nullptr if error handling is not required. + * @return A pointer to a SQLite3_result object representing the resultset obtained from the query execution. This + * pointer may be nullptr if the query execution fails or returns an empty result. + */ +template +SQLite3_result * Base_HostGroups_Manager::execute_query(char *query, char **error) { + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + wrlock(); + mydb->execute_statement(query, error , &cols , &affected_rows , &resultset); + wrunlock(); + return resultset; +} + +#if 0 +/** + * @brief Calculate and update the checksum for a specified table in the database. + * + * This function calculates the checksum for a specified table in the database using the provided SpookyHash object. + * The checksum is computed based on the table's contents, sorted by the specified column name. If the initialization + * flag is false, the SpookyHash object is initialized with predefined parameters. The calculated checksum is stored + * in the raw_checksum parameter. + * + * @param myhash A reference to the SpookyHash object used for calculating the checksum. + * @param init A reference to a boolean flag indicating whether the SpookyHash object has been initialized. + * @param TableName The name of the table for which the checksum is to be calculated. + * @param ColumnName The name of the column to be used for sorting the table before calculating the checksum. + * @param raw_checksum A reference to a uint64_t variable where the calculated checksum will be stored. + */ +void MySQL_HostGroups_Manager::CUCFT1( + SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum +) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + string query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + mydb->execute_statement(query.c_str(), &error , &cols , &affected_rows , &resultset); + if (resultset) { + if (resultset->rows_count) { + if (init == false) { + init = true; + myhash.Init(19,3); + } + uint64_t hash1_ = resultset->raw_checksum(); + raw_checksum = hash1_; + myhash.Update(&hash1_, sizeof(hash1_)); + proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), hash1_); + } + delete resultset; + } else { + proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), (long unsigned int)0); + } +} + +/** + * @brief Compute and update checksum values for specified tables. + * + * This function computes checksum values for specified tables by executing checksum calculation queries for each table. + * It updates the checksum values in the `table_resultset_checksum` array. + * + * @param myhash A reference to a SpookyHash object used for computing the checksums. + * @param init A reference to a boolean flag indicating whether the checksum computation has been initialized. + * @note This function resets the current checksum values for all tables except MYSQL_SERVERS and MYSQL_SERVERS_V2 + * before recomputing the checksums. + * @note The computed checksum values are stored in the `table_resultset_checksum` array. + */ +void MySQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& myhash, bool& init) { + // Always reset the current table values before recomputing + for (size_t i = 0; i < table_resultset_checksum.size(); i++) { + if (i != HGM_TABLES::MYSQL_SERVERS && i != HGM_TABLES::MYSQL_SERVERS_V2) { + table_resultset_checksum[i] = 0; + } + } + + CUCFT1(myhash,init,"mysql_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); + CUCFT1(myhash,init,"mysql_group_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GROUP_REPLICATION_HOSTGROUPS]); + CUCFT1(myhash,init,"mysql_galera_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_GALERA_HOSTGROUPS]); + CUCFT1(myhash,init,"mysql_aws_aurora_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::MYSQL_AWS_AURORA_HOSTGROUPS]); + CUCFT1(myhash,init,"mysql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::MYSQL_HOSTGROUP_ATTRIBUTES]); + CUCFT1(myhash,init,"mysql_servers_ssl_params","hostname,port,username", table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_SSL_PARAMS]); +} + +/** + * @brief This code updates the 'hostgroup_server_mapping' table with the most recent mysql_servers and mysql_replication_hostgroups + * records while utilizing checksums to prevent unnecessary updates. + * + * IMPORTANT: Make sure wrlock() is called before calling this method. + * +*/ +void MySQL_HostGroups_Manager::update_hostgroup_manager_mappings() { + + if (hgsm_mysql_servers_checksum != table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] || + hgsm_mysql_replication_hostgroups_checksum != table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]) + { + proxy_info("Rebuilding 'Hostgroup_Manager_Mapping' due to checksums change - mysql_servers { old: 0x%lX, new: 0x%lX }, mysql_replication_hostgroups { old:0x%lX, new:0x%lX }\n", + hgsm_mysql_servers_checksum, table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS], + hgsm_mysql_replication_hostgroups_checksum, table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]); + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + hostgroup_server_mapping.clear(); + + const char* query = "SELECT DISTINCT hostname, port, '1' is_writer, status, reader_hostgroup, writer_hostgroup, mem_pointer FROM mysql_replication_hostgroups JOIN mysql_servers ON hostgroup_id=writer_hostgroup WHERE status<>3 \ + UNION \ + SELECT DISTINCT hostname, port, '0' is_writer, status, reader_hostgroup, writer_hostgroup, mem_pointer FROM mysql_replication_hostgroups JOIN mysql_servers ON hostgroup_id=reader_hostgroup WHERE status<>3 \ + ORDER BY hostname, port"; + + mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + + if (resultset && resultset->rows_count) { + std::string fetched_server_id; + HostGroup_Server_Mapping* fetched_server_mapping = NULL; + + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + + const std::string& server_id = std::string(r->fields[0]) + ":::" + r->fields[1]; + + if (fetched_server_mapping == NULL || server_id != fetched_server_id) { + + auto itr = hostgroup_server_mapping.find(server_id); + + if (itr == hostgroup_server_mapping.end()) { + std::unique_ptr server_mapping(new HostGroup_Server_Mapping(this)); + fetched_server_mapping = server_mapping.get(); + hostgroup_server_mapping.insert( std::pair> { + server_id, std::move(server_mapping) + } ); + } else { + fetched_server_mapping = itr->second.get(); + } + + fetched_server_id = server_id; + } + + HostGroup_Server_Mapping::Node node; + //node.server_status = static_cast(atoi(r->fields[3])); + node.reader_hostgroup_id = atoi(r->fields[4]); + node.writer_hostgroup_id = atoi(r->fields[5]); + node.srv = reinterpret_cast(atoll(r->fields[6])); + + HostGroup_Server_Mapping::Type type = (r->fields[2] && r->fields[2][0] == '1') ? HostGroup_Server_Mapping::Type::WRITER : HostGroup_Server_Mapping::Type::READER; + fetched_server_mapping->add(type, node); + } + } + delete resultset; + + hgsm_mysql_servers_checksum = table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; + hgsm_mysql_replication_hostgroups_checksum = table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS]; + } +} + +/** + * @brief Generates a resultset holding the current Admin 'runtime_mysql_servers' as reported by Admin. + * @details Requires caller to hold the mutex 'MySQL_HostGroups_Manager::wrlock'. + * @param mydb The db in which to perform the query, typically 'MySQL_HostGroups_Manager::mydb'. + * @return An SQLite3 resultset for the query 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. + */ +unique_ptr get_admin_runtime_mysql_servers(SQLite3DB* mydb) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = nullptr; + + mydb->execute_statement(MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + + if (error) { + proxy_error("SQLite3 query generating 'runtime_mysql_servers' resultset failed with error '%s'\n", error); + assert(0); + } + + return unique_ptr(resultset); +} + +/** + * @brief Generates a resultset with holding the current 'mysql_servers_v2' table. + * @details Requires caller to hold the mutex 'ProxySQL_Admin::mysql_servers_wrlock'. + * @return A resulset holding 'mysql_servers_v2'. + */ +unique_ptr get_mysql_servers_v2() { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = nullptr; + + if (GloAdmin && GloAdmin->admindb) { + GloAdmin->admindb->execute_statement( + MYHGM_GEN_CLUSTER_ADMIN_MYSQL_SERVERS, &error, &cols, &affected_rows, &resultset + ); + } + + return unique_ptr(resultset); +} + +static void update_glovars_checksum_with_peers( + ProxySQL_Checksum_Value& module_checksum, + const string& new_checksum, + const string& peer_checksum_value, + time_t new_epoch, + time_t peer_checksum_epoch, + bool update_version +) { + module_checksum.set_checksum(const_cast(new_checksum.c_str())); + + if (update_version) + module_checksum.version++; + + bool computed_checksum_matches = + peer_checksum_value != "" && module_checksum.checksum == peer_checksum_value; + + if (peer_checksum_epoch != 0 && computed_checksum_matches) { + module_checksum.epoch = peer_checksum_epoch; + } else { + module_checksum.epoch = new_epoch; + } +} + +/** + * @brief Updates the global 'mysql_servers' module checksum. + * @details If the new computed checksum matches the supplied 'cluster_checksum', the epoch used for the + * checksum is the supplied epoch instead of current time. This way we ensure the preservation of the + * checksum and epoch fetched from the ProxySQL cluster peer node. + * + * IMPORTANT: This function also generates a new 'global_checksum'. This is because everytime + * 'runtime_mysql_servers' change, updating the global checksum is unconditional. + * @param new_checksum The new computed checksum for 'runtime_mysql_servers'. + * @param peer_checksum A checksum fetched from another ProxySQL cluster node, holds the checksum value + * and its epoch. Should be empty if no remote checksum is being considered. + * @param epoch The epoch to be preserved in case the supplied 'peer_checksum' matches the new computed + * checksum. + */ +static void update_glovars_mysql_servers_checksum( + const string& new_checksum, + const runtime_mysql_servers_checksum_t& peer_checksum = {}, + bool update_version = false +) { + time_t new_epoch = time(NULL); + + update_glovars_checksum_with_peers( + GloVars.checksums_values.mysql_servers, + new_checksum, + peer_checksum.value, + new_epoch, + peer_checksum.epoch, + update_version + ); + + GloVars.checksums_values.updates_cnt++; + GloVars.generate_global_checksum(); + GloVars.epoch_version = new_epoch; +} + +/** + * @brief Updates the global 'mysql_servers_v2' module checksum. + * @details Unlike 'update_glovars_mysql_servers_checksum' this function doesn't generate a new + * 'global_checksum'. It's caller responsibility to ensure that 'global_checksum' is updated. + * @param new_checksum The new computed checksum for 'mysql_servers_v2'. + * @param peer_checksum A checksum fetched from another ProxySQL cluster node, holds the checksum value + * and its epoch. Should be empty if no remote checksum is being considered. + * @param epoch The epoch to be preserved in case the supplied 'peer_checksum' matches the new computed + * checksum. + */ +static void update_glovars_mysql_servers_v2_checksum( + const string& new_checksum, + const mysql_servers_v2_checksum_t& peer_checksum = {}, + bool update_version = false +) { + time_t new_epoch = time(NULL); + + update_glovars_checksum_with_peers( + GloVars.checksums_values.mysql_servers_v2, + new_checksum, + peer_checksum.value, + new_epoch, + peer_checksum.epoch, + update_version + ); +} + +/** + * @brief Commit and update checksum from the MySQL servers. + * + * This function commits updates and calculates the checksum from the MySQL servers. It performs the following steps: + * 1. Deletes existing data from the 'mysql_servers' table. + * 2. Generates a new 'mysql_servers' table. + * 3. Saves the runtime MySQL servers data obtained from the provided result set or from the database if the result set is null. + * 4. Calculates the checksum of the runtime MySQL servers data and updates the checksum value in the 'table_resultset_checksum' array. + * + * @param runtime_mysql_servers A pointer to the result set containing runtime MySQL servers data. + * @return The raw checksum value calculated from the runtime MySQL servers data. + */ +uint64_t MySQL_HostGroups_Manager::commit_update_checksum_from_mysql_servers(SQLite3_result* runtime_mysql_servers) { + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + if (runtime_mysql_servers == nullptr) { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + save_runtime_mysql_servers(resultset.release()); + } else { + save_runtime_mysql_servers(runtime_mysql_servers); + } + + uint64_t raw_checksum = this->runtime_mysql_servers ? this->runtime_mysql_servers->raw_checksum() : 0; + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = raw_checksum; + + return raw_checksum; +} + +/** + * @brief Commit and update checksum from the MySQL servers V2. + * + * This function commits updates and calculates the checksum from the MySQL servers V2 data. It performs the following steps: + * 1. Saves the provided MySQL servers V2 data if not null, or retrieves and saves the data from the database. + * 2. Calculates the checksum of the MySQL servers V2 data and updates the checksum value in the 'table_resultset_checksum' array. + * + * @param mysql_servers_v2 A pointer to the result set containing MySQL servers V2 data. + * @return The raw checksum value calculated from the MySQL servers V2 data. + */ +uint64_t MySQL_HostGroups_Manager::commit_update_checksum_from_mysql_servers_v2(SQLite3_result* mysql_servers_v2) { + if (mysql_servers_v2 == nullptr) { + unique_ptr resultset { get_mysql_servers_v2() }; + save_mysql_servers_v2(resultset.release()); + } else { + save_mysql_servers_v2(mysql_servers_v2); + } + + uint64_t raw_checksum = this->incoming_mysql_servers_v2 ? this->incoming_mysql_servers_v2->raw_checksum() : 0; + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_V2] = raw_checksum; + + return raw_checksum; +} + +std::string MySQL_HostGroups_Manager::gen_global_mysql_servers_v2_checksum(uint64_t servers_v2_hash) { + bool init = false; + SpookyHash global_hash {}; + + if (servers_v2_hash != 0) { + if (init == false) { + init = true; + global_hash.Init(19, 3); + } + + global_hash.Update(&servers_v2_hash, sizeof(servers_v2_hash)); + } + + commit_update_checksums_from_tables(global_hash, init); + + uint64_t hash_1 = 0, hash_2 = 0; + if (init) { + global_hash.Final(&hash_1,&hash_2); + } + + string mysrvs_checksum { get_checksum_from_hash(hash_1) }; + return mysrvs_checksum; +} + +bool MySQL_HostGroups_Manager::commit() { + return commit({},{}); +} + +bool MySQL_HostGroups_Manager::commit( + const peer_runtime_mysql_servers_t& peer_runtime_mysql_servers, + const peer_mysql_servers_v2_t& peer_mysql_servers_v2, + bool only_commit_runtime_mysql_servers, + bool update_version +) { + // if only_commit_runtime_mysql_servers is true, mysql_servers_v2 resultset will not be entertained and will cause memory leak. + if (only_commit_runtime_mysql_servers) { + proxy_info("Generating runtime mysql servers records only.\n"); + } else { + proxy_info("Generating runtime mysql servers and mysql servers v2 records.\n"); + } + + unsigned long long curtime1=monotonic_time(); + wrlock(); + // purge table + purge_mysql_servers_table(); + // if any server has gtid_port enabled, use_gtid is set to true + // and then has_gtid_port is set too + bool use_gtid = false; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (GloMTH->variables.hostgroup_manager_verbose) { + mydb->execute_statement((char *)"SELECT * FROM mysql_servers_incoming", &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on read from mysql_servers_incoming : %s\n", error); + } else { + if (resultset) { + proxy_info("Dumping mysql_servers_incoming\n"); + resultset->dump_to_stderr(); + } + } + if (resultset) { delete resultset; resultset=NULL; } + } + char *query=NULL; + query=(char *)"SELECT mem_pointer, t1.hostgroup_id, t1.hostname, t1.port FROM mysql_servers t1 LEFT OUTER JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE t2.hostgroup_id IS NULL"; + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Dumping mysql_servers LEFT JOIN mysql_servers_incoming\n"); + resultset->dump_to_stderr(); + } + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + long long ptr=atoll(r->fields[0]); + proxy_warning("Removed server at address %lld, hostgroup %s, address %s port %s. Setting status OFFLINE HARD and immediately dropping all free connections. Used connections will be dropped when trying to use them\n", ptr, r->fields[1], r->fields[2], r->fields[3]); + MySrvC *mysrvc=(MySrvC *)ptr; + mysrvc->set_status(MYSQL_SERVER_STATUS_OFFLINE_HARD); + mysrvc->ConnectionsFree->drop_all_connections(); + char *q1=(char *)"DELETE FROM mysql_servers WHERE mem_pointer=%lld"; + char *q2=(char *)malloc(strlen(q1)+32); + sprintf(q2,q1,ptr); + mydb->execute(q2); + free(q2); + } + } + if (resultset) { delete resultset; resultset=NULL; } + + // This seems unnecessary. Removed as part of issue #829 + //mydb->execute("DELETE FROM mysql_servers"); + //generate_mysql_servers_table(); + + mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming"); + + // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) + query=(char *)"SELECT t1.*, t2.gtid_port, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms, t2.comment FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.gtid_port<>t2.gtid_port OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms or t1.comment<>t2.comment"; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Dumping mysql_servers JOIN mysql_servers_incoming\n"); + resultset->dump_to_stderr(); + } + // optimization #829 + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement2=NULL; + //sqlite3 *mydb3=mydb->get_db(); + char *query1=(char *)"UPDATE mysql_servers SET mem_pointer = ?1 WHERE hostgroup_id = ?2 AND hostname = ?3 AND port = ?4"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = mydb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, mydb); + char *query2=(char *)"UPDATE mysql_servers SET weight = ?1 , status = ?2 , compression = ?3 , max_connections = ?4 , max_replication_lag = ?5 , use_ssl = ?6 , max_latency_ms = ?7 , comment = ?8 , gtid_port = ?9 WHERE hostgroup_id = ?10 AND hostname = ?11 AND port = ?12"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query2, -1, &statement2, 0); + rc = mydb->prepare_v2(query2, &statement2); + ASSERT_SQLITE_OK(rc, mydb); + + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + long long ptr=atoll(r->fields[12]); // increase this index every time a new column is added + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d , weight=%d, status=%d, mem_pointer=%llu, hostgroup=%d, compression=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), atoi(r->fields[5]), ptr, atoi(r->fields[0]), atoi(r->fields[6])); + //fprintf(stderr,"%lld\n", ptr); + if (ptr==0) { + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), atoi(r->fields[5])); + } + MySrvC *mysrvc=new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus)atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), atoi(r->fields[10]), r->fields[11]); // add new fields here if adding more columns in mysql_servers + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), atoi(r->fields[5]), mysrvc, atoi(r->fields[0])); + add(mysrvc,atoi(r->fields[0])); + ptr=(uintptr_t)mysrvc; + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, ptr); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); + if (mysrvc->gtid_port) { + // this server has gtid_port configured, we set use_gtid + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid , mysrvc->address, mysrvc->port); + use_gtid = true; + } + } else { + bool run_update=false; + MySrvC *mysrvc=(MySrvC *)ptr; + // carefully increase the 2nd index by 1 for every new column added + if (atoi(r->fields[3])!=atoi(r->fields[13])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing gtid_port for server %u:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]) , mysrvc->gtid_port , atoi(r->fields[13])); + mysrvc->gtid_port=atoi(r->fields[13]); + } + + if (atoi(r->fields[4])!=atoi(r->fields[14])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %d:%s:%d (%s:%d) from %d (%ld) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]) , mysrvc->weight , atoi(r->fields[14])); + mysrvc->weight=atoi(r->fields[14]); + } + if (atoi(r->fields[5])!=atoi(r->fields[15])) { + bool change_server_status = true; + if (GloMTH->variables.evaluate_replication_lag_on_servers_load == 1) { + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG && // currently server is shunned due to replication lag + (MySerStatus)atoi(r->fields[15]) == MYSQL_SERVER_STATUS_ONLINE) { // new server status is online + if (mysrvc->cur_replication_lag != -2) { // Master server? Seconds_Behind_Master column is not present + const unsigned int new_max_repl_lag = atoi(r->fields[18]); + if (mysrvc->cur_replication_lag < 0 || + (new_max_repl_lag > 0 && + ((unsigned int)mysrvc->cur_replication_lag > new_max_repl_lag))) { // we check if current replication lag is greater than new max_replication_lag + change_server_status = false; + } + } + } + } + if (change_server_status == true) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing status for server %d:%s:%d (%s:%d) from %d (%d) to %d\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[5]), (int)mysrvc->get_status(), atoi(r->fields[15])); + mysrvc->set_status((MySerStatus)atoi(r->fields[15])); + } + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED) { + mysrvc->shunned_automatic=false; + } + } + if (atoi(r->fields[6])!=atoi(r->fields[16])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing compression for server %d:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[6]) , mysrvc->compression , atoi(r->fields[16])); + mysrvc->compression=atoi(r->fields[16]); + } + if (atoi(r->fields[7])!=atoi(r->fields[17])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_connections for server %d:%s:%d (%s:%d) from %d (%ld) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[7]) , mysrvc->max_connections , atoi(r->fields[17])); + mysrvc->max_connections=atoi(r->fields[17]); + } + if (atoi(r->fields[8])!=atoi(r->fields[18])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_replication_lag for server %u:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[8]) , mysrvc->max_replication_lag , atoi(r->fields[18])); + mysrvc->max_replication_lag=atoi(r->fields[18]); + if (mysrvc->max_replication_lag == 0) { // we just changed it to 0 + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { + // the server is currently shunned due to replication lag + // but we reset max_replication_lag to 0 + // therefore we immediately reset the status too + mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); + } + } + } + if (atoi(r->fields[9])!=atoi(r->fields[19])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing use_ssl for server %d:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[9]) , mysrvc->use_ssl , atoi(r->fields[19])); + mysrvc->use_ssl=atoi(r->fields[19]); + } + if (atoi(r->fields[10])!=atoi(r->fields[20])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_latency_ms for server %d:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[10]) , mysrvc->max_latency_us/1000 , atoi(r->fields[20])); + mysrvc->max_latency_us=1000*atoi(r->fields[20]); + } + if (strcmp(r->fields[11],r->fields[21])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing comment for server %d:%s:%d (%s:%d) from '%s' to '%s'\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[11], r->fields[21]); + free(mysrvc->comment); + mysrvc->comment=strdup(r->fields[21]); + } + if (run_update) { + rc=(*proxy_sqlite3_bind_int64)(statement2, 1, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 2, (int)mysrvc->get_status()); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 3, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 4, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 5, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 6, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 7, mysrvc->max_latency_us/1000); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement2, 8, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 9, mysrvc->gtid_port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 10, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement2, 11, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 12, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement2); + rc=(*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, mydb); + } + if (mysrvc->gtid_port) { + // this server has gtid_port configured, we set use_gtid + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 6, "Server %u:%s:%d has gtid_port enabled, setting use_gitd=true if not already set\n", mysrvc->myhgc->hid , mysrvc->address, mysrvc->port); + use_gtid = true; + } + } + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement2); + } + if (use_gtid) { + has_gtid_port = true; + } else { + has_gtid_port = false; + } + if (resultset) { delete resultset; resultset=NULL; } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); + mydb->execute("DELETE FROM mysql_servers_incoming"); + + string global_checksum_v2 {}; + if (only_commit_runtime_mysql_servers == false) { + // replication + if (incoming_replication_hostgroups) { // this IF is extremely important, otherwise replication hostgroups may disappear + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_replication_hostgroups\n"); + mydb->execute("DELETE FROM mysql_replication_hostgroups"); + generate_mysql_replication_hostgroups_table(); + } + + // group replication + if (incoming_group_replication_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_group_replication_hostgroups\n"); + mydb->execute("DELETE FROM mysql_group_replication_hostgroups"); + generate_mysql_group_replication_hostgroups_table(); + } + + // galera + if (incoming_galera_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_galera_hostgroups\n"); + mydb->execute("DELETE FROM mysql_galera_hostgroups"); + generate_mysql_galera_hostgroups_table(); + } + + // AWS Aurora + if (incoming_aws_aurora_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_aws_aurora_hostgroups\n"); + mydb->execute("DELETE FROM mysql_aws_aurora_hostgroups"); + generate_mysql_aws_aurora_hostgroups_table(); + } + + // hostgroup attributes + if (incoming_hostgroup_attributes) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_hostgroup_attributes\n"); + mydb->execute("DELETE FROM mysql_hostgroup_attributes"); + generate_mysql_hostgroup_attributes_table(); + } + + // SSL params + if (incoming_mysql_servers_ssl_params) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_ssl_params\n"); + mydb->execute("DELETE FROM mysql_servers_ssl_params"); + generate_mysql_servers_ssl_params_table(); + } + + uint64_t new_hash = commit_update_checksum_from_mysql_servers_v2(peer_mysql_servers_v2.resultset); + + { + const string new_checksum { get_checksum_from_hash(new_hash) }; + proxy_info("Checksum for table %s is %s\n", "mysql_servers_v2", new_checksum.c_str()); + } + + global_checksum_v2 = gen_global_mysql_servers_v2_checksum(new_hash); + proxy_info("New computed global checksum for 'mysql_servers_v2' is '%s'\n", global_checksum_v2.c_str()); + } + + // Update 'mysql_servers' and global checksums + { + uint64_t new_hash = commit_update_checksum_from_mysql_servers(peer_runtime_mysql_servers.resultset); + const string new_checksum { get_checksum_from_hash(new_hash) }; + proxy_info("Checksum for table %s is %s\n", "mysql_servers", new_checksum.c_str()); + + pthread_mutex_lock(&GloVars.checksum_mutex); + if (only_commit_runtime_mysql_servers == false) { + update_glovars_mysql_servers_v2_checksum(global_checksum_v2, peer_mysql_servers_v2.checksum, true); + } + update_glovars_mysql_servers_checksum(new_checksum, peer_runtime_mysql_servers.checksum, update_version); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + + // fill Hostgroup_Manager_Mapping with latest records + update_hostgroup_manager_mappings(); + + + ev_async_send(gtid_ev_loop, gtid_ev_async); + + __sync_fetch_and_add(&status.servers_table_version,1); + + // We completely reset read_only_set1. It will generated (completely) again in read_only_action() + // Note: read_only_set1 will be regenerated all at once + read_only_set1.erase(read_only_set1.begin(), read_only_set1.end()); + // We completely reset read_only_set2. It will be again written in read_only_action() + // Note: read_only_set2 will be regenerated one server at the time + read_only_set2.erase(read_only_set2.begin(), read_only_set2.end()); + + this->status.p_counter_array[p_hg_counter::servers_table_version]->Increment(); + pthread_cond_broadcast(&status.servers_table_version_cond); + pthread_mutex_unlock(&status.servers_table_version_lock); + + // NOTE: In order to guarantee the latest generated version, this should be kept after all the + // calls to 'generate_mysql_servers'. + update_table_mysql_servers_for_monitor(false); + + wrunlock(); + unsigned long long curtime2=monotonic_time(); + curtime1 = curtime1/1000; + curtime2 = curtime2/1000; + proxy_info("MySQL_HostGroups_Manager::commit() locked for %llums\n", curtime2-curtime1); + + if (GloMTH) { + GloMTH->signal_all_threads(1); + } + + return true; +} + +/** + * @brief Calculate the checksum for the runtime mysql_servers record, after excluding all the rows + * with the status OFFLINE_HARD from the result set + * + * @details The runtime mysql_servers is now considered as a distinct module and have a separate checksum calculation. + * This is because the records in the runtime module may differ from those in the admin mysql_servers module, which + * can cause synchronization issues within the cluster. + * + * @param runtime_mysql_servers resultset of runtime mysql_servers or can be a nullptr. +*/ +uint64_t MySQL_HostGroups_Manager::get_mysql_servers_checksum(SQLite3_result* runtime_mysql_servers) { + + //Note: GloVars.checksum_mutex needs to be locked + SQLite3_result* resultset = nullptr; + + if (runtime_mysql_servers == nullptr) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + + mydb->execute_statement(MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + + if (resultset) { + save_runtime_mysql_servers(resultset); + } else { + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", (long unsigned int)0); + } + } else { + resultset = runtime_mysql_servers; + save_runtime_mysql_servers(runtime_mysql_servers); + } + + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = resultset != nullptr ? resultset->raw_checksum() : 0; + proxy_info("Checksum for table %s is 0x%lX\n", "mysql_servers", table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]); + + return table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS]; +} + + +/** + * @brief Purge the MySQL servers table by removing offline hard servers with no active connections. + * + * This function iterates through each host group in the host groups manager and examines each server within the host group. + * For each server that is marked as offline hard and has no active connections (both used and free), it removes the server from the host group. + * After removing the server, it deletes the server object to free up memory. + * This process ensures that offline hard servers with no connections are properly removed from the MySQL servers table. + */ +void MySQL_HostGroups_Manager::purge_mysql_servers_table() { + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + MySrvC *mysrvc=NULL; + for (unsigned int j=0; jmysrvs->servers->len; j++) { + mysrvc=myhgc->mysrvs->idx(j); + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + if (mysrvc->ConnectionsUsed->conns_length()==0 && mysrvc->ConnectionsFree->conns_length()==0) { + // no more connections for OFFLINE_HARD server, removing it + mysrvc=(MySrvC *)myhgc->mysrvs->servers->remove_index_fast(j); + // already being refreshed in MySrvC destructor + //myhgc->refresh_online_server_count(); + j--; + delete mysrvc; + } + } + } + } +} + + + +void MySQL_HostGroups_Manager::generate_mysql_servers_table(int *_onlyhg) { + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + + PtrArray *lst=new PtrArray(); + //sqlite3 *mydb3=mydb->get_db(); + char *query1=(char *)"INSERT INTO mysql_servers VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = mydb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, mydb); + std::string query32s = "INSERT INTO mysql_servers VALUES " + generate_multi_rows_query(32,13); + char *query32 = (char *)query32s.c_str(); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = mydb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, mydb); + + if (mysql_thread___hostgroup_manager_verbose) { + if (_onlyhg==NULL) { + proxy_info("Dumping current MySQL Servers structures for hostgroup ALL\n"); + } else { + int hidonly=*_onlyhg; + proxy_info("Dumping current MySQL Servers structures for hostgroup %d\n", hidonly); + } + } + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + if (_onlyhg) { + int hidonly=*_onlyhg; + if (myhgc->hid!=(unsigned int)hidonly) { + // skipping this HG + continue; + } + } + MySrvC *mysrvc=NULL; + for (unsigned int j=0; jmysrvs->servers->len; j++) { + mysrvc=myhgc->mysrvs->idx(j); + if (mysql_thread___hostgroup_manager_verbose) { + char *st; + switch ((int)mysrvc->get_status()) { + case 0: + st=(char *)"ONLINE"; + break; + case 2: + st=(char *)"OFFLINE_SOFT"; + break; + case 3: + st=(char *)"OFFLINE_HARD"; + break; + default: + case 1: + case 4: + st=(char *)"SHUNNED"; + break; + } + fprintf(stderr,"HID: %d , address: %s , port: %d , gtid_port: %d , weight: %ld , status: %s , max_connections: %ld , max_replication_lag: %u , use_ssl: %u , max_latency_ms: %u , comment: %s\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->gtid_port, mysrvc->weight, st, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us*1000, mysrvc->comment); + } + lst->add(mysrvc); + if (lst->len==32) { + while (lst->len) { + int i=lst->len; + i--; + MySrvC *mysrvc=(MySrvC *)lst->remove_index_fast(0); + uintptr_t ptr=(uintptr_t)mysrvc; + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+1, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement32, (i*13)+2, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+3, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+4, mysrvc->gtid_port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+5, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+6, (int)mysrvc->get_status()); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+7, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+8, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+9, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+10, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+11, mysrvc->max_latency_us/1000); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement32, (i*13)+12, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*13)+13, ptr); ASSERT_SQLITE_OK(rc, mydb); + } + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, mydb); + } + } + } + while (lst->len) { + MySrvC *mysrvc=(MySrvC *)lst->remove_index_fast(0); + uintptr_t ptr=(uintptr_t)mysrvc; + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, mysrvc->gtid_port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, (int)mysrvc->get_status()); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, mysrvc->max_latency_us/1000); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 12, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 13, ptr); ASSERT_SQLITE_OK(rc, mydb); + + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + if (mysql_thread___hostgroup_manager_verbose) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (_onlyhg==NULL) { + mydb->execute_statement((char *)"SELECT hostgroup_id hid, hostname, port, gtid_port gtid, weight, status, compression cmp, max_connections max_conns, max_replication_lag max_lag, use_ssl ssl, max_latency_ms max_lat, comment, mem_pointer FROM mysql_servers", &error , &cols , &affected_rows , &resultset); + } else { + int hidonly=*_onlyhg; + char *q1 = (char *)malloc(256); + sprintf(q1,"SELECT hostgroup_id hid, hostname, port, gtid_port gtid, weight, status, compression cmp, max_connections max_conns, max_replication_lag max_lag, use_ssl ssl, max_latency_ms max_lat, comment, mem_pointer FROM mysql_servers WHERE hostgroup_id=%d" , hidonly); + mydb->execute_statement(q1, &error , &cols , &affected_rows , &resultset); + free(q1); + } + if (error) { + proxy_error("Error on read from mysql_servers : %s\n", error); + } else { + if (resultset) { + if (_onlyhg==NULL) { + proxy_info("Dumping mysql_servers: ALL\n"); + } else { + int hidonly=*_onlyhg; + proxy_info("Dumping mysql_servers: HG %d\n", hidonly); + } + resultset->dump_to_stderr(); + } + } + if (resultset) { delete resultset; resultset=NULL; } + } + delete lst; +} + +/** + * @brief Generate the mysql_replication_hostgroups table based on incoming data. + * + * This function populates the mysql_replication_hostgroups table in the host groups manager database + * using the incoming replication hostgroups data. It iterates through each row of the incoming data, + * constructs an SQL INSERT query to insert the data into the table, and executes the query. + * If verbose mode is enabled, it logs information about each row being processed. + * + * @note This function assumes that the incoming_replication_hostgroups member variable is not NULL. + * If it is NULL, the function returns without performing any action. + */ +void MySQL_HostGroups_Manager::generate_mysql_replication_hostgroups_table() { + if (incoming_replication_hostgroups==NULL) + return; + if (mysql_thread___hostgroup_manager_verbose) { + proxy_info("New mysql_replication_hostgroups table\n"); + } + for (std::vector::iterator it = incoming_replication_hostgroups->rows.begin() ; it != incoming_replication_hostgroups->rows.end(); ++it) { + SQLite3_row *r=*it; + char *o=NULL; + int comment_length=0; // #issue #643 + //if (r->fields[3]) { // comment is not null + o=escape_string_single_quotes(r->fields[3],false); + comment_length=strlen(o); + //} + char *query=(char *)malloc(256+comment_length); + //if (r->fields[3]) { // comment is not null + sprintf(query,"INSERT INTO mysql_replication_hostgroups VALUES(%s,%s,'%s','%s')",r->fields[0], r->fields[1], r->fields[2], o); + if (o!=r->fields[3]) { // there was a copy + free(o); + } + //} else { + //sprintf(query,"INSERT INTO mysql_replication_hostgroups VALUES(%s,%s,NULL)",r->fields[0],r->fields[1]); + //} + mydb->execute(query); + if (mysql_thread___hostgroup_manager_verbose) { + fprintf(stderr,"writer_hostgroup: %s , reader_hostgroup: %s, check_type %s, comment: %s\n", r->fields[0],r->fields[1], r->fields[2], r->fields[3]); + } + free(query); + } + incoming_replication_hostgroups=NULL; +} + + +void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) { + if (lock) { + wrlock(); + } + + std::lock_guard mysql_servers_lock(this->mysql_servers_to_monitor_mutex); + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + const char* query = "SELECT hostname, port, status, use_ssl FROM mysql_servers WHERE status != 3 GROUP BY hostname, port"; + + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + + if (error != nullptr) { + proxy_error("Error on read from mysql_servers : %s\n", error); + } else { + if (resultset != nullptr) { + delete this->mysql_servers_to_monitor; + this->mysql_servers_to_monitor = resultset; + } + } + + if (lock) { + wrunlock(); + } + + MySQL_Monitor::trigger_dns_cache_update(); +} + +/** + * @brief Dump data from a specified MySQL table. + * + * This function retrieves data from the specified MySQL table and returns it as a result set. + * The table name determines the SQL query to be executed to fetch the data. If the table is + * one of the predefined tables with special handling (e.g., mysql_servers), additional actions + * such as purging and generating the table may be performed before fetching the data. + * + * @param name The name of the MySQL table from which to dump data. + * @return A SQLite3_result pointer representing the result set containing the dumped data. + * The caller is responsible for managing the memory of the result set. + * @note If the provided table name is not recognized, the function assertion fails. + */ +SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) { + char * query = (char *)""; + if (name == "mysql_aws_aurora_hostgroups") { + query=(char *)"SELECT writer_hostgroup,reader_hostgroup,active,aurora_port,domain_name,max_lag_ms," + "check_interval_ms,check_timeout_ms,writer_is_also_reader,new_reader_weight,add_lag_ms,min_lag_ms,lag_num_checks,comment FROM mysql_aws_aurora_hostgroups"; + } else if (name == "mysql_galera_hostgroups") { + query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; + } else if (name == "mysql_group_replication_hostgroups") { + query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_group_replication_hostgroups"; + } else if (name == "mysql_replication_hostgroups") { + query=(char *)"SELECT writer_hostgroup, reader_hostgroup, check_type, comment FROM mysql_replication_hostgroups"; + } else if (name == "mysql_hostgroup_attributes") { + query=(char *)"SELECT hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, hostgroup_settings, servers_defaults, comment FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; + } else if (name == "mysql_servers_ssl_params") { + query=(char *)"SELECT hostname, port, username, ssl_ca, ssl_cert, ssl_key, ssl_capath, ssl_crl, ssl_crlpath, ssl_cipher, tls_version, comment FROM mysql_servers_ssl_params ORDER BY hostname, port, username"; + } else if (name == "mysql_servers") { + query = (char *)MYHGM_GEN_ADMIN_RUNTIME_SERVERS; + } else if (name == "cluster_mysql_servers") { + query = (char *)MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS; + } else { + assert(0); + } + wrlock(); + if (name == "mysql_servers") { + purge_mysql_servers_table(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + } + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + wrunlock(); + return resultset; +} + +#endif // 0 +/** + * @brief Create a new MySQL host group container. + * + * This function creates a new instance of the MySQL host group container (`MyHGC`) with + * the specified host group ID and returns a pointer to it. + * + * @param _hid The host group ID for the new container. + * @return A pointer to the newly created `MyHGC` instance. + */ +template +HGC * Base_HostGroups_Manager::MyHGC_create(unsigned int _hid) { + HGC *myhgc=new HGC(_hid); + return myhgc; +} + +/** + * @brief Find a MySQL host group container by host group ID. + * + * This function searches for a MySQL host group container with the specified host group ID + * in the list of host groups. If found, it returns a pointer to the container; otherwise, + * it returns a null pointer. + * + * @param _hid The host group ID to search for. + * @return A pointer to the found `MyHGC` instance if found; otherwise, a null pointer. + */ +template +HGC * Base_HostGroups_Manager::MyHGC_find(unsigned int _hid) { + if (MyHostGroups->len < 100) { + // for few HGs, we use the legacy search + for (unsigned int i=0; ilen; i++) { + HGC *myhgc=(HGC *)MyHostGroups->index(i); + if (myhgc->hid==_hid) { + return myhgc; + } + } + } else { + // for a large number of HGs, we use the unordered_map + // this search is slower for a small number of HGs, therefore we use + // it only for large number of HGs + typename std::unordered_map::const_iterator it = MyHostGroups_map.find(_hid); + if (it != MyHostGroups_map.end()) { + HGC *myhgc = it->second; + return myhgc; + } + } + return NULL; +} + +/** + * @brief Lookup or create a MySQL host group container by host group ID. + * + * This function looks up a MySQL host group container with the specified host group ID. If + * found, it returns a pointer to the existing container; otherwise, it creates a new container + * with the specified host group ID, adds it to the list of host groups, and returns a pointer + * to it. + * + * @param _hid The host group ID to lookup or create. + * @return A pointer to the found or newly created `MyHGC` instance. + * @note The function assertion fails if a newly created container is not found. + */ +template +HGC * Base_HostGroups_Manager::MyHGC_lookup(unsigned int _hid) { + HGC *myhgc=NULL; + myhgc=MyHGC_find(_hid); + if (myhgc==NULL) { + myhgc=MyHGC_create(_hid); + } else { + return myhgc; + } + assert(myhgc); + MyHostGroups->add(myhgc); + MyHostGroups_map.emplace(_hid,myhgc); + return myhgc; +} + +#if 0 +void MySQL_HostGroups_Manager::increase_reset_counter() { + wrlock(); + status.myconnpoll_reset++; + wrunlock(); +} + +/** + * @brief Pushes a MySQL_Connection back to the connection pool. + * + * This method is responsible for returning a MySQL_Connection object back to its associated connection pool + * after it has been used. It performs various checks and optimizations before deciding whether to return + * the connection to the pool or destroy it. + * + * @param c The MySQL_Connection object to be pushed back to the pool. + * @param _lock Boolean flag indicating whether to acquire a lock before performing the operation. Default is true. + * + * @note The method assumes that the provided MySQL_Connection object has a valid parent server (MySrvC). + * If the parent server is not valid, unexpected behavior may occur. + * + * @note The method also assumes that the global thread handler (GloMTH) is available and initialized properly. + * If the global thread handler is not initialized, certain checks may fail, leading to unexpected behavior. + */ +void MySQL_HostGroups_Manager::push_MyConn_to_pool(MySQL_Connection *c, bool _lock) { + // Ensure that the provided connection has a valid parent server + assert(c->parent); + + MySrvC *mysrvc = nullptr; // Pointer to the parent server object + + // Acquire a lock if specified + if (_lock) + wrlock(); + + // Reset the auto-increment delay token associated with the connection + c->auto_increment_delay_token = 0; + + // Increment the counter tracking the number of connections pushed back to the pool + status.myconnpoll_push++; + + // Obtain a pointer to the parent server (MySrvC) + mysrvc = static_cast(c->parent); + + // Log debug information about the connection being returned to the pool + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL_Connection %p, server %s:%d with status %d\n", c, mysrvc->address, mysrvc->port, (int)mysrvc->get_status()); + + // Remove the connection from the list of used connections for the parent server + mysrvc->ConnectionsUsed->remove(c); + + // If the global thread handler (GloMTH) is not available, skip further processing + if (GloMTH == nullptr) { + goto __exit_push_MyConn_to_pool; + } + + // If the largest query length exceeds the threshold, destroy the connection + if (c->largest_query_length > (unsigned int)GloMTH->variables.threshold_query_length) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying MySQL_Connection %p, server %s:%d with status %d . largest_query_length = %lu\n", c, mysrvc->address, mysrvc->port, (int)mysrvc->get_status(), c->largest_query_length); + delete c; + goto __exit_push_MyConn_to_pool; + } + + // If the server is online and the connection is in the idle state + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { + if (c->async_state_machine==ASYNC_IDLE) { + if (GloMTH == NULL) { goto __exit_push_MyConn_to_pool; } + if (c->local_stmts->get_num_backend_stmts() > (unsigned int)GloMTH->variables.max_stmts_per_connection) { // Check if the connection has too many prepared statements + // Log debug information about destroying the connection due to too many prepared statements + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying MySQL_Connection %p, server %s:%d with status %d because has too many prepared statements\n", c, mysrvc->address, mysrvc->port, (int)mysrvc->get_status()); +// delete c; + mysrvc->ConnectionsUsed->add(c); // Add the connection back to the list of used connections + destroy_MyConn_from_pool(c, false); // Destroy the connection from the pool + } else { + c->optimize(); // Optimize the connection + mysrvc->ConnectionsFree->add(c); // Add the connection to the list of free connections + } + } else { + // Log debug information about destroying the connection + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying MySQL_Connection %p, server %s:%d with status %d\n", c, mysrvc->address, mysrvc->port, (int)mysrvc->get_status()); + delete c; // Destroy the connection + } + } else { + // Log debug information about destroying the connection + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying MySQL_Connection %p, server %s:%d with status %d\n", c, mysrvc->address, mysrvc->port, (int)mysrvc->get_status()); + delete c; // Destroy the connection + } + +// Exit point for releasing the lock +__exit_push_MyConn_to_pool: + if (_lock) + wrunlock(); // Release the lock if acquired +} + +/** + * @brief Pushes an array of MySQL_Connection objects back to the connection pool. + * + * This method is responsible for returning an array of MySQL_Connection objects back to their associated + * connection pool after they have been used. It iterates through the array and calls the push_MyConn_to_pool + * method for each connection without acquiring a lock for each individual push operation. + * + * @param ca An array of MySQL_Connection pointers representing the connections to be pushed back to the pool. + * @param cnt The number of connections in the array. + * + * @note This method assumes that the array of connections is valid and does not contain any nullptr entries. + * Unexpected behavior may occur if the array contains invalid pointers. + */ +void MySQL_HostGroups_Manager::push_MyConn_to_pool_array(MySQL_Connection **ca, unsigned int cnt) { + unsigned int i=0; // Index variable for iterating through the array + MySQL_Connection *c = nullptr; // Pointer to hold the current connection from the array + c=ca[i]; + + // Acquire a write lock to perform the operations atomically + wrlock(); + + // Iterate through the array of connections + while (ivariables.hostgroup_manager_verbose >= 3) { + char buf[64]; + if (skip_hid == NULL) { + sprintf(buf,"NULL"); + } else { + sprintf(buf,"%u", *skip_hid); + } + proxy_info("Calling unshun_server_all_hostgroups() for server %s:%d . Arguments: %lu , %d , %s\n" , address, port, t, max_wait_sec, buf); + } + int i, j; + for (i=0; i<(int)MyHostGroups->len; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + if (skip_hid != NULL && myhgc->hid == *skip_hid) { + // if skip_hid is not NULL, we skip that specific hostgroup + continue; + } + bool found = false; // was this server already found in this hostgroup? + for (j=0; found==false && j<(int)myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j); + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED) { + // we only care for SHUNNED nodes + // Note that we check for address and port only for status==MYSQL_SERVER_STATUS_SHUNNED , + // that means that potentially we will pass by the matching node and still looping . + // This is potentially an optimization because we only check status and do not perform any strcmp() + if (strcmp(mysrvc->address,address)==0 && mysrvc->port==port) { + // we found the server in this hostgroup + // no need to process more servers in the same hostgroup + found = true; + if (t > mysrvc->time_last_detected_error && (t - mysrvc->time_last_detected_error) > max_wait_sec) { + if ( + (mysrvc->shunned_and_kill_all_connections==false) // it is safe to bring it back online + || + (mysrvc->shunned_and_kill_all_connections==true && mysrvc->ConnectionsUsed->conns_length()==0 && mysrvc->ConnectionsFree->conns_length()==0) // if shunned_and_kill_all_connections is set, ensure all connections are already dropped + ) { + if (GloMTH->variables.hostgroup_manager_verbose >= 3) { + proxy_info("Unshunning server %d:%s:%d . time_last_detected_error=%lu\n", mysrvc->myhgc->hid, address, port, mysrvc->time_last_detected_error); + } + mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); + mysrvc->shunned_automatic=false; + mysrvc->shunned_and_kill_all_connections=false; + mysrvc->connect_ERR_at_time_last_detected_error=0; + mysrvc->time_last_detected_error=0; + } + } + } + } + } + } +} + +/** + * @brief Retrieves a MySQL_Connection from the connection pool for a given hostgroup. + * + * This method is responsible for retrieving a MySQL_Connection from the connection pool associated + * with the specified hostgroup. It selects a random MySQL server within the hostgroup based on various + * criteria such as GTID information, maximum lag, and session attributes. If a suitable connection is found, + * it is marked as used and returned to the caller. + * + * @param _hid The ID of the hostgroup from which to retrieve the connection. + * @param sess A pointer to the MySQL_Session object associated with the connection. + * @param ff A boolean flag indicating whether to prioritize failover connections. + * @param gtid_uuid The GTID UUID used for GTID-based routing. + * @param gtid_trxid The GTID transaction ID used for GTID-based routing. + * @param max_lag_ms The maximum allowed lag in milliseconds. + * + * @return A pointer to the retrieved MySQL_Connection object if successful, or nullptr if no suitable connection + * is available in the pool. + * + * @note This method locks the connection pool to ensure thread safety during access. It releases the lock once + * the operation is completed. + */ +MySQL_Connection * MySQL_HostGroups_Manager::get_MyConn_from_pool(unsigned int _hid, MySQL_Session *sess, bool ff, char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms) { + MySQL_Connection * conn = nullptr; // Pointer to hold the retrieved MySQL_Connection + + // Acquire a write lock to access the connection pool + wrlock(); + + // Increment the counter for connection pool retrieval attempts + status.myconnpoll_get++; + + // Look up the hostgroup by ID and retrieve a random MySQL server from it based on specified criteria + MyHGC *myhgc=MyHGC_lookup(_hid); + MySrvC *mysrvc = NULL; +#ifdef TEST_AURORA + for (int i=0; i<10; i++) +#endif // TEST_AURORA + mysrvc = myhgc->get_random_MySrvC(gtid_uuid, gtid_trxid, max_lag_ms, sess); + if (mysrvc) { // a MySrvC exists. If not, we return NULL = no targets + // Attempt to get a random MySQL_Connection from the server's free connection pool + conn=mysrvc->ConnectionsFree->get_random_MyConn(sess, ff); + + // If a connection is obtained, mark it as used and update connection pool statistics + if (conn) { + mysrvc->ConnectionsUsed->add(conn); + status.myconnpoll_get_ok++; + mysrvc->update_max_connections_used(); + } + } + + // Release the write lock after accessing the connection pool + wrunlock(); + + // Debug message indicating the retrieved MySQL_Connection and its server details + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, (conn ? conn->parent->address : "") , (conn ? conn->parent->port : 0 )); + + // Return the retrieved MySQL_Connection (or nullptr if none available) + return conn; +} + +void MySQL_HostGroups_Manager::destroy_MyConn_from_pool(MySQL_Connection *c, bool _lock) { + bool to_del=true; // the default, legacy behavior + MySrvC *mysrvc=(MySrvC *)c->parent; + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE && c->send_quit && queue.size() < __sync_fetch_and_add(&GloMTH->variables.connpoll_reset_queue_length, 0)) { + if (c->async_state_machine==ASYNC_IDLE) { + // overall, the backend seems healthy and so it is the connection. Try to reset it + int myerr=mysql_errno(c->mysql); + if (myerr >= 2000 && myerr < 3000) { + // client library error . We must not try to save the connection + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Not trying to reset MySQL_Connection %p, server %s:%d . Error code %d\n", c, mysrvc->address, mysrvc->port, myerr); + } else { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Trying to reset MySQL_Connection %p, server %s:%d\n", c, mysrvc->address, mysrvc->port); + to_del=false; + queue.add(c); + } + } else { + // the connection seems health, but we are trying to destroy it + // probably because there is a long running query + // therefore we will try to kill the connection + if (mysql_thread___kill_backend_connection_when_disconnect) { + int myerr=mysql_errno(c->mysql); + switch (myerr) { + case 1231: + break; + default: + if (c->mysql->thread_id) { + MySQL_Connection_userinfo *ui=c->userinfo; + char *auth_password=NULL; + if (ui->password) { + if (ui->password[0]=='*') { // we don't have the real password, let's pass sha1 + auth_password=ui->sha1_pass; + } else { + auth_password=ui->password; + } + } + KillArgs *ka = new KillArgs(ui->username, auth_password, c->parent->address, c->parent->port, c->parent->myhgc->hid, c->mysql->thread_id, KILL_CONNECTION, c->parent->use_ssl, NULL, c->connected_host_details.ip); + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + pthread_attr_setstacksize (&attr, 256*1024); + pthread_t pt; + if (pthread_create(&pt, &attr, &kill_query_thread, ka) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + break; + } + } + } + } + if (to_del) { + // we lock only this part of the code because we need to remove the connection from ConnectionsUsed + if (_lock) { + wrlock(); + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying MySQL_Connection %p, server %s:%d\n", c, mysrvc->address, mysrvc->port); + mysrvc->ConnectionsUsed->remove(c); + status.myconnpoll_destroy++; + if (_lock) { + wrunlock(); + } + delete c; + } +} + +inline double get_prometheus_counter_val( + std::map& counter_map, const std::string& endpoint_id +) { + const auto& counter_entry = counter_map.find(endpoint_id); + double current_val = 0; + + if (counter_entry != counter_map.end()) { + current_val = counter_entry->second->Value(); + } + + return current_val; +} + +void reset_hg_attrs_server_defaults(MySrvC* mysrvc) { + mysrvc->weight = -1; + mysrvc->max_connections = -1; + mysrvc->use_ssl = -1; +} + +void update_hg_attrs_server_defaults(MySrvC* mysrvc, MyHGC* myhgc) { + if (mysrvc->weight == -1) { + if (myhgc->servers_defaults.weight != -1) { + mysrvc->weight = myhgc->servers_defaults.weight; + } else { + // Same harcoded default as in 'CREATE TABLE mysql_servers ...' + mysrvc->weight = 1; + } + } + if (mysrvc->max_connections == -1) { + if (myhgc->servers_defaults.max_connections != -1) { + mysrvc->max_connections = myhgc->servers_defaults.max_connections; + } else { + // Same harcoded default as in 'CREATE TABLE mysql_servers ...' + mysrvc->max_connections = 1000; + } + } + if (mysrvc->use_ssl == -1) { + if (myhgc->servers_defaults.use_ssl != -1) { + mysrvc->use_ssl = myhgc->servers_defaults.use_ssl; + } else { + // Same harcoded default as in 'CREATE TABLE mysql_servers ...' + mysrvc->use_ssl = 0; + } + } +} + +/** + * @brief Adds a MySQL server connection (MySrvC) to the specified hostgroup. + * + * This method adds a MySQL server connection (MySrvC) to the hostgroup identified by the given hostgroup ID (_hid). + * It performs necessary updates to the server metrics and attributes associated with the hostgroup. Additionally, it + * updates the endpoint metrics for the server based on its address and port. + * + * @param mysrvc A pointer to the MySQL server connection (MySrvC) to be added to the hostgroup. + * @param _hid The ID of the hostgroup to which the server connection is being added. + * + * @note The method updates various metrics and attributes associated with the server and hostgroup. It also ensures + * that endpoint metrics are updated to reflect the addition of the server to the hostgroup. + */ +void MySQL_HostGroups_Manager::add(MySrvC *mysrvc, unsigned int _hid) { + + // Debug message indicating the addition of the MySQL server connection to the hostgroup + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Adding MySrvC %p (%s:%d) for hostgroup %d\n", mysrvc, mysrvc->address, mysrvc->port, _hid); + + // Construct the endpoint ID using the hostgroup ID, server address, and port + std::string endpoint_id { std::to_string(_hid) + ":" + string { mysrvc->address } + ":" + std::to_string(mysrvc->port) }; + + // Since metrics for servers are stored per-endpoint; the metrics for a particular endpoint can live longer than the + // 'MySrvC' itself. For example, a failover or a server config change could remove the server from a particular + // hostgroup, and a subsequent one bring it back to the original hostgroup. For this reason, everytime a 'mysrvc' is + // created and added to a particular hostgroup, we update the endpoint metrics for it. + + // Update server metrics based on endpoint ID + mysrvc->bytes_recv = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_recv_map, endpoint_id); + mysrvc->bytes_sent = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_sent_map, endpoint_id); + mysrvc->connect_ERR = get_prometheus_counter_val(this->status.p_connection_pool_conn_err_map, endpoint_id); + mysrvc->connect_OK = get_prometheus_counter_val(this->status.p_connection_pool_conn_ok_map, endpoint_id); + mysrvc->queries_sent = get_prometheus_counter_val(this->status.p_connection_pool_queries_map, endpoint_id); + + // Lookup the hostgroup by ID and add the server connection to it + MyHGC *myhgc=MyHGC_lookup(_hid); + + // Update server defaults with hostgroup attributes + update_hg_attrs_server_defaults(mysrvc, myhgc); + + // Add the server to the hostgroup's servers list + myhgc->mysrvs->add(mysrvc); +} + +void MySQL_HostGroups_Manager::replication_lag_action_inner(MyHGC *myhgc, const char *address, unsigned int port, + int current_replication_lag, bool override_repl_lag) { + + if (current_replication_lag == -1 && override_repl_lag == true) { + current_replication_lag = myhgc->get_monitor_slave_lag_when_null(); + override_repl_lag = false; + proxy_error("Replication lag on server %s:%d is NULL, using value %d\n", address, port, current_replication_lag); + } + + for (int j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j); + if (strcmp(mysrvc->address,address)==0 && mysrvc->port==port) { + mysrvc->cur_replication_lag = current_replication_lag; + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { + if ( +// (current_replication_lag==-1 ) +// || + ( + current_replication_lag >= 0 && + mysrvc->max_replication_lag > 0 && // see issue #4018 + (current_replication_lag > (int)mysrvc->max_replication_lag) + ) + ) { + // always increase the counter + mysrvc->cur_replication_lag_count += 1; + if (mysrvc->cur_replication_lag_count >= (unsigned int)mysql_thread___monitor_replication_lag_count) { + proxy_warning("Shunning server %s:%d from HG %u with replication lag of %d second, count number: '%d'\n", address, port, myhgc->hid, current_replication_lag, mysrvc->cur_replication_lag_count); + mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG); + } else { + proxy_info( + "Not shunning server %s:%d from HG %u with replication lag of %d second, count number: '%d' < replication_lag_count: '%d'\n", + address, + port, + myhgc->hid, + current_replication_lag, + mysrvc->cur_replication_lag_count, + mysql_thread___monitor_replication_lag_count + ); + } + } else { + mysrvc->cur_replication_lag_count = 0; + } + } else { + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { + if ( + (/*current_replication_lag >= 0 &&*/override_repl_lag == false && + (current_replication_lag <= (int)mysrvc->max_replication_lag)) + || + (current_replication_lag==-2 && override_repl_lag == true) // see issue 959 + ) { + mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); + proxy_warning("Re-enabling server %s:%d from HG %u with replication lag of %d second\n", address, port, myhgc->hid, current_replication_lag); + mysrvc->cur_replication_lag_count = 0; + } + } + } + return; + } + } +} + +void MySQL_HostGroups_Manager::replication_lag_action(const std::list& mysql_servers) { + + //this method does not use admin table, so this lock is not needed. + //GloAdmin->mysql_servers_wrlock(); + unsigned long long curtime1 = monotonic_time(); + wrlock(); + + for (const auto& server : mysql_servers) { + + const int hid = std::get(server); + const std::string& address = std::get(server); + const unsigned int port = std::get(server); + const int current_replication_lag = std::get(server); + const bool override_repl_lag = std::get(server); + + if (mysql_thread___monitor_replication_lag_group_by_host == false) { + // legacy check. 1 check per server per hostgroup + MyHGC *myhgc = MyHGC_find(hid); + replication_lag_action_inner(myhgc,address.c_str(),port,current_replication_lag,override_repl_lag); + } + else { + // only 1 check per server, no matter the hostgroup + // all hostgroups must be searched + for (unsigned int i=0; ilen; i++) { + MyHGC*myhgc=(MyHGC*)MyHostGroups->index(i); + replication_lag_action_inner(myhgc,address.c_str(),port,current_replication_lag,override_repl_lag); + } + } + } + + wrunlock(); + //GloAdmin->mysql_servers_wrunlock(); + + unsigned long long curtime2 = monotonic_time(); + curtime1 = curtime1 / 1000; + curtime2 = curtime2 / 1000; + proxy_debug(PROXY_DEBUG_MONITOR, 7, "MySQL_HostGroups_Manager::replication_lag_action() locked for %llums (server count:%ld)\n", curtime2 - curtime1, mysql_servers.size()); +} + +void MySQL_HostGroups_Manager::drop_all_idle_connections() { + // NOTE: the caller should hold wrlock + int i, j; + for (i=0; i<(int)MyHostGroups->len; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j); + if (mysrvc->get_status()!=MYSQL_SERVER_STATUS_ONLINE) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d is not online\n", mysrvc->address, mysrvc->port); + //__sync_fetch_and_sub(&status.server_connections_connected, mysrvc->ConnectionsFree->conns->len); + mysrvc->ConnectionsFree->drop_all_connections(); + } + + // Drop idle connections if beyond max_connection + while (mysrvc->ConnectionsFree->conns_length() && mysrvc->ConnectionsUsed->conns_length()+mysrvc->ConnectionsFree->conns_length() > mysrvc->max_connections) { + MySQL_Connection *conn=mysrvc->ConnectionsFree->remove(0); + delete conn; + } + + //PtrArray *pa=mysrvc->ConnectionsFree->conns; + MySrvConnList *mscl=mysrvc->ConnectionsFree; + int free_connections_pct = mysql_thread___free_connections_pct; + if (mysrvc->myhgc->attributes.configured == true) { + // mysql_hostgroup_attributes takes priority + free_connections_pct = mysrvc->myhgc->attributes.free_connections_pct; + } + while (mscl->conns_length() > free_connections_pct*mysrvc->max_connections/100) { + MySQL_Connection *mc=mscl->remove(0); + delete mc; + } + + // drop all connections with life exceeding mysql-connection_max_age + if (mysql_thread___connection_max_age_ms) { + unsigned long long curtime=monotonic_time(); + int i=0; + for (i=0; i<(int)mscl->conns_length() ; i++) { + MySQL_Connection *mc=mscl->index(i); + unsigned long long intv = mysql_thread___connection_max_age_ms; + intv *= 1000; + if (curtime > mc->creation_time + intv) { + mc=mscl->remove(0); + delete mc; + i--; + } + } + } + + } + } +} + +/* + * Prepares at most num_conn idle connections in the given hostgroup for + * pinging. When -1 is passed as a hostgroup, all hostgroups are examined. + * + * The resulting idle connections are returned in conn_list. Note that not all + * currently idle connections will be returned (some might be purged). + * + * Connections are purged according to 2 criteria: + * - whenever the maximal number of connections for a server is hit, free + * connections will be purged + * - also, idle connections that cause the number of free connections to rise + * above a certain percentage of the maximal number of connections will be + * dropped as well + */ +int MySQL_HostGroups_Manager::get_multiple_idle_connections(int _hid, unsigned long long _max_last_time_used, MySQL_Connection **conn_list, int num_conn) { + wrlock(); + drop_all_idle_connections(); + int num_conn_current=0; + int j,k; + MyHGC* myhgc = NULL; + // Multimap holding the required info for accesing the oldest idle connections found. + std::multimap> oldest_idle_connections {}; + + for (int i=0; i<(int)MyHostGroups->len; i++) { + if (_hid == -1) { + // all hostgroups must be examined + // as of version 2.3.2 , this is always the case + myhgc=(MyHGC *)MyHostGroups->index(i); + } else { + // only one hostgroup is examined + // as of version 2.3.2 , this never happen + // but the code support this functionality + myhgc = MyHGC_find(_hid); + i = (int)MyHostGroups->len; // to exit from this "for" loop + if (myhgc == NULL) + continue; // immediately exit + } + if (_hid >= 0 && _hid!=(int)myhgc->hid) continue; + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j); + //PtrArray *pa=mysrvc->ConnectionsFree->conns; + MySrvConnList *mscl=mysrvc->ConnectionsFree; + for (k=0; k<(int)mscl->conns_length(); k++) { + MySQL_Connection *mc=mscl->index(k); + // If the connection is idle ... + if (mc->last_time_used && mc->last_time_used < _max_last_time_used) { + if ((int)oldest_idle_connections.size() < num_conn) { + oldest_idle_connections.insert({mc->last_time_used, { mysrvc, k }}); + } else if (num_conn != 0) { + auto last_elem_it = std::prev(oldest_idle_connections.end()); + + if (mc->last_time_used < last_elem_it->first) { + oldest_idle_connections.erase(last_elem_it); + oldest_idle_connections.insert({mc->last_time_used, { mysrvc, k }}); + } + } + } + } + } + } + + // In order to extract the found connections, the following actions must be performed: + // + // 1. Filter the found connections by 'MySrvC'. + // 2. Order by indexes on 'ConnectionsFree' in desc order. + // 3. Move the conns from 'ConnectionsFree' into 'ConnectionsUsed'. + std::unordered_map> mysrvcs_conns_idxs {}; + + // 1. Filter the connections by 'MySrvC'. + // + // We extract this for being able to later iterate through the obtained 'MySrvC' using the conn indexes. + for (const auto& conn_info : oldest_idle_connections) { + MySrvC* mysrvc = conn_info.second.first; + int32_t mc_idx = conn_info.second.second; + auto mysrcv_it = mysrvcs_conns_idxs.find(mysrvc); + + if (mysrcv_it == mysrvcs_conns_idxs.end()) { + mysrvcs_conns_idxs.insert({ mysrvc, { mc_idx }}); + } else { + mysrcv_it->second.push_back(mc_idx); + } + } + + // 2. Order by indexes on FreeConns in desc order. + // + // Since the conns are stored in 'ConnectionsFree', which holds the conns in a 'PtrArray', and we plan + // to remove multiple connections using the pre-stored indexes. We need to reorder the indexes in 'desc' + // order, otherwise we could be trashing the array while consuming it. See 'PtrArray::remove_index_fast'. + for (auto& mysrvc_conns_idxs : mysrvcs_conns_idxs) { + std::sort(std::begin(mysrvc_conns_idxs.second), std::end(mysrvc_conns_idxs.second), std::greater()); + } + + // 3. Move the conns from 'ConnectionsFree' into 'ConnectionsUsed'. + for (auto& conn_info : mysrvcs_conns_idxs) { + MySrvC* mysrvc = conn_info.first; + + for (const int conn_idx : conn_info.second) { + MySrvConnList* mscl = mysrvc->ConnectionsFree; + MySQL_Connection* mc = mscl->remove(conn_idx); + mysrvc->ConnectionsUsed->add(mc); + + conn_list[num_conn_current] = mc; + num_conn_current++; + + // Left here as a safeguard + if (num_conn_current >= num_conn) { + goto __exit_get_multiple_idle_connections; + } + } + } + +__exit_get_multiple_idle_connections: + status.myconnpoll_get_ping+=num_conn_current; + wrunlock(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning %d idle connections\n", num_conn_current); + return num_conn_current; +} + +void MySQL_HostGroups_Manager::save_incoming_mysql_table(SQLite3_result *s, const string& name) { + SQLite3_result ** inc = NULL; + if (name == "mysql_aws_aurora_hostgroups") { + inc = &incoming_aws_aurora_hostgroups; + } else if (name == "mysql_galera_hostgroups") { + inc = &incoming_galera_hostgroups; + } else if (name == "mysql_group_replication_hostgroups") { + inc = &incoming_group_replication_hostgroups; + } else if (name == "mysql_replication_hostgroups") { + inc = &incoming_replication_hostgroups; + } else if (name == "mysql_hostgroup_attributes") { + inc = &incoming_hostgroup_attributes; + } else if (name == "mysql_servers_ssl_params") { + inc = &incoming_mysql_servers_ssl_params; + } else { + assert(0); + } + if (*inc != nullptr) { + delete *inc; + *inc = nullptr; + } + *inc = s; +} + +void MySQL_HostGroups_Manager::save_runtime_mysql_servers(SQLite3_result *s) { + if (runtime_mysql_servers) { + delete runtime_mysql_servers; + runtime_mysql_servers = nullptr; + } + runtime_mysql_servers=s; +} + +void MySQL_HostGroups_Manager::save_mysql_servers_v2(SQLite3_result* s) { + if (incoming_mysql_servers_v2) { + delete incoming_mysql_servers_v2; + incoming_mysql_servers_v2 = nullptr; + } + incoming_mysql_servers_v2 = s; +} + +/** + * @brief Retrieves the current SQLite3 result set associated with the specified MySQL table name. + * + * This method retrieves the current SQLite3 result set corresponding to the specified MySQL table name. + * The method is used to obtain the result set for various MySQL tables, such as hostgroups, replication configurations, + * SSL parameters, and runtime server information. + * + * @param name The name of the MySQL table for which the current SQLite3 result set is to be retrieved. + * Supported table names include: + * - "mysql_aws_aurora_hostgroups" + * - "mysql_galera_hostgroups" + * - "mysql_group_replication_hostgroups" + * - "mysql_replication_hostgroups" + * - "mysql_hostgroup_attributes" + * - "mysql_servers_ssl_params" + * - "cluster_mysql_servers" + * - "mysql_servers_v2" + * + * @return A pointer to the current SQLite3 result set associated with the specified MySQL table name. + * If the table name is not recognized or no result set is available for the specified table, NULL is returned. + * + * @note The method assumes that the result sets are stored in class member variables, and it returns the pointer to + * the appropriate result set based on the provided table name. If the table name is not recognized, an assertion + * failure occurs, indicating an invalid table name. + */ +SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_table(const string& name) { + if (name == "mysql_aws_aurora_hostgroups") { + return this->incoming_aws_aurora_hostgroups; + } else if (name == "mysql_galera_hostgroups") { + return this->incoming_galera_hostgroups; + } else if (name == "mysql_group_replication_hostgroups") { + return this->incoming_group_replication_hostgroups; + } else if (name == "mysql_replication_hostgroups") { + return this->incoming_replication_hostgroups; + } else if (name == "mysql_hostgroup_attributes") { + return this->incoming_hostgroup_attributes; + } else if (name == "mysql_servers_ssl_params") { + return this->incoming_mysql_servers_ssl_params; + } else if (name == "cluster_mysql_servers") { + return this->runtime_mysql_servers; + } else if (name == "mysql_servers_v2") { + return this->incoming_mysql_servers_v2; + } else { + assert(0); // Assertion failure for unrecognized table name + } + return NULL; +} + + + +SQLite3_result * MySQL_HostGroups_Manager::SQL3_Free_Connections() { + const int colnum=13; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Free Connections in Pool\n"); + SQLite3_result *result=new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"fd"); + result->add_column_definition(SQLITE_TEXT,"hostgroup"); + result->add_column_definition(SQLITE_TEXT,"srv_host"); + result->add_column_definition(SQLITE_TEXT,"srv_port"); + result->add_column_definition(SQLITE_TEXT,"user"); + result->add_column_definition(SQLITE_TEXT,"schema"); + result->add_column_definition(SQLITE_TEXT,"init_connect"); + result->add_column_definition(SQLITE_TEXT,"time_zone"); + result->add_column_definition(SQLITE_TEXT,"sql_mode"); + result->add_column_definition(SQLITE_TEXT,"autocommit"); + result->add_column_definition(SQLITE_TEXT,"idle_ms"); + result->add_column_definition(SQLITE_TEXT,"statistics"); + result->add_column_definition(SQLITE_TEXT,"mysql_info"); + unsigned long long curtime = monotonic_time(); + wrlock(); + int i,j, k, l; + for (i=0; i<(int)MyHostGroups->len; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j); + if (mysrvc->get_status()!=MYSQL_SERVER_STATUS_ONLINE) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d is not online\n", mysrvc->address, mysrvc->port); + mysrvc->ConnectionsFree->drop_all_connections(); + } + // drop idle connections if beyond max_connection + while (mysrvc->ConnectionsFree->conns_length() && mysrvc->ConnectionsUsed->conns_length()+mysrvc->ConnectionsFree->conns_length() > mysrvc->max_connections) { + //MySQL_Connection *conn=(MySQL_Connection *)mysrvc->ConnectionsFree->conns->remove_index_fast(0); + MySQL_Connection *conn=mysrvc->ConnectionsFree->remove(0); + delete conn; + } + char buf[1024]; + for (l=0; l < (int) mysrvc->ConnectionsFree->conns_length(); l++) { + char **pta=(char **)malloc(sizeof(char *)*colnum); + MySQL_Connection *conn = mysrvc->ConnectionsFree->index(l); + sprintf(buf,"%d", conn->fd); + pta[0]=strdup(buf); + sprintf(buf,"%d", (int)myhgc->hid); + pta[1]=strdup(buf); + pta[2]=strdup(mysrvc->address); + sprintf(buf,"%d", mysrvc->port); + pta[3]=strdup(buf); + pta[4] = strdup(conn->userinfo->username); + pta[5] = strdup(conn->userinfo->schemaname); + pta[6] = NULL; + if (conn->options.init_connect) { + pta[6] = strdup(conn->options.init_connect); + } + pta[7] = NULL; + if (conn->variables[SQL_TIME_ZONE].value) { + pta[7] = strdup(conn->variables[SQL_TIME_ZONE].value); + } + pta[8] = NULL; + if (conn->variables[SQL_SQL_MODE].value) { + pta[8] = strdup(conn->variables[SQL_SQL_MODE].value); + } + sprintf(buf,"%d", conn->options.autocommit); + pta[9]=strdup(buf); + sprintf(buf,"%llu", (curtime-conn->last_time_used)/1000); + pta[10]=strdup(buf); + { + json j; + char buff[32]; + sprintf(buff,"%p",conn); + j["address"] = buff; + uint64_t age_ms = (curtime - conn->creation_time)/1000; + j["age_ms"] = age_ms; + j["bytes_recv"] = conn->bytes_info.bytes_recv; + j["bytes_sent"] = conn->bytes_info.bytes_sent; + j["myconnpoll_get"] = conn->statuses.myconnpoll_get; + j["myconnpoll_put"] = conn->statuses.myconnpoll_put; + j["questions"] = conn->statuses.questions; + string s = j.dump(); + pta[11] = strdup(s.c_str()); + } + { + MYSQL *_my = conn->mysql; + json j; + char buff[32]; + sprintf(buff,"%p",_my); + j["address"] = buff; + j["host"] = _my->host; + j["host_info"] = _my->host_info; + j["port"] = _my->port; + j["server_version"] = _my->server_version; + j["user"] = _my->user; + j["unix_socket"] = (_my->unix_socket ? _my->unix_socket : ""); + j["db"] = (_my->db ? _my->db : ""); + j["affected_rows"] = _my->affected_rows; + j["insert_id"] = _my->insert_id; + j["thread_id"] = _my->thread_id; + j["server_status"] = _my->server_status; + j["charset"] = _my->charset->nr; + j["charset_name"] = _my->charset->csname; + + j["options"]["charset_name"] = ( _my->options.charset_name ? _my->options.charset_name : "" ); + j["options"]["use_ssl"] = _my->options.use_ssl; + j["client_flag"]["client_found_rows"] = (_my->client_flag & CLIENT_FOUND_ROWS ? 1 : 0); + j["client_flag"]["client_multi_statements"] = (_my->client_flag & CLIENT_MULTI_STATEMENTS ? 1 : 0); + j["client_flag"]["client_multi_results"] = (_my->client_flag & CLIENT_MULTI_RESULTS ? 1 : 0); + j["net"]["last_errno"] = _my->net.last_errno; + j["net"]["fd"] = _my->net.fd; + j["net"]["max_packet_size"] = _my->net.max_packet_size; + j["net"]["sqlstate"] = _my->net.sqlstate; + string s = j.dump(); + pta[12] = strdup(s.c_str()); + } + result->add_row(pta); + for (k=0; k& labels, std::map& m_map, unsigned long long value, p_hg_dyn_counter::metric idx +) { + const auto& counter_id = m_map.find(endpoint_id); + if (counter_id != m_map.end()) { + const auto& cur_val = counter_id->second->Value(); + counter_id->second->Increment(value - cur_val); + } else { + auto& new_counter = status.p_dyn_counter_array[idx]; + m_map.insert( + { + endpoint_id, + std::addressof(new_counter->Add(labels)) + } + ); + } +} + +void MySQL_HostGroups_Manager::p_update_connection_pool_update_gauge( + const std::string& endpoint_id, const std::map& labels, + std::map& m_map, unsigned long long value, p_hg_dyn_gauge::metric idx +) { + const auto& counter_id = m_map.find(endpoint_id); + if (counter_id != m_map.end()) { + counter_id->second->Set(value); + } else { + auto& new_counter = status.p_dyn_gauge_array[idx]; + m_map.insert( + { + endpoint_id, + std::addressof(new_counter->Add(labels)) + } + ); + } +} + +void MySQL_HostGroups_Manager::p_update_connection_pool() { + std::vector cur_servers_ids {}; + wrlock(); + for (int i = 0; i < static_cast(MyHostGroups->len); i++) { + MyHGC *myhgc = static_cast(MyHostGroups->index(i)); + for (int j = 0; j < static_cast(myhgc->mysrvs->cnt()); j++) { + MySrvC *mysrvc = static_cast(myhgc->mysrvs->servers->index(j)); + std::string endpoint_addr = mysrvc->address; + std::string endpoint_port = std::to_string(mysrvc->port); + std::string hostgroup_id = std::to_string(myhgc->hid); + std::string endpoint_id = hostgroup_id + ":" + endpoint_addr + ":" + endpoint_port; + const std::map common_labels { + {"endpoint", endpoint_addr + ":" + endpoint_port}, + {"hostgroup", hostgroup_id } + }; + cur_servers_ids.push_back(endpoint_id); + + // proxysql_connection_pool_bytes_data_recv metric + std::map recv_pool_bytes_labels = common_labels; + recv_pool_bytes_labels.insert({"traffic_flow", "recv"}); + p_update_connection_pool_update_counter(endpoint_id, recv_pool_bytes_labels, + status.p_conn_pool_bytes_data_recv_map, mysrvc->bytes_recv, p_hg_dyn_counter::conn_pool_bytes_data_recv); + + // proxysql_connection_pool_bytes_data_sent metric + std::map sent_pool_bytes_labels = common_labels; + sent_pool_bytes_labels.insert({"traffic_flow", "sent"}); + p_update_connection_pool_update_counter(endpoint_id, sent_pool_bytes_labels, + status.p_conn_pool_bytes_data_sent_map, mysrvc->bytes_sent, p_hg_dyn_counter::conn_pool_bytes_data_sent); + + // proxysql_connection_pool_conn_err metric + std::map pool_conn_err_labels = common_labels; + pool_conn_err_labels.insert({"status", "err"}); + p_update_connection_pool_update_counter(endpoint_id, pool_conn_err_labels, + status.p_connection_pool_conn_err_map, mysrvc->connect_ERR, p_hg_dyn_counter::connection_pool_conn_err); + + // proxysql_connection_pool_conn_ok metric + std::map pool_conn_ok_labels = common_labels; + pool_conn_ok_labels.insert({"status", "ok"}); + p_update_connection_pool_update_counter(endpoint_id, pool_conn_ok_labels, + status.p_connection_pool_conn_ok_map, mysrvc->connect_OK, p_hg_dyn_counter::connection_pool_conn_ok); + + // proxysql_connection_pool_conn_free metric + std::map pool_conn_free_labels = common_labels; + pool_conn_free_labels.insert({"status", "free"}); + p_update_connection_pool_update_gauge(endpoint_id, pool_conn_free_labels, + status.p_connection_pool_conn_free_map, mysrvc->ConnectionsFree->conns_length(), p_hg_dyn_gauge::connection_pool_conn_free); + + // proxysql_connection_pool_conn_used metric + std::map pool_conn_used_labels = common_labels; + pool_conn_used_labels.insert({"status", "used"}); + p_update_connection_pool_update_gauge(endpoint_id, pool_conn_used_labels, + status.p_connection_pool_conn_used_map, mysrvc->ConnectionsUsed->conns_length(), p_hg_dyn_gauge::connection_pool_conn_used); + + // proxysql_connection_pool_latency_us metric + p_update_connection_pool_update_gauge(endpoint_id, common_labels, + status.p_connection_pool_latency_us_map, mysrvc->current_latency_us, p_hg_dyn_gauge::connection_pool_latency_us); + + // proxysql_connection_pool_queries metric + p_update_connection_pool_update_counter(endpoint_id, common_labels, + status.p_connection_pool_queries_map, mysrvc->queries_sent, p_hg_dyn_counter::connection_pool_queries); + + // proxysql_connection_pool_status metric + p_update_connection_pool_update_gauge(endpoint_id, common_labels, + status.p_connection_pool_status_map, ((int)mysrvc->get_status()) + 1, p_hg_dyn_gauge::connection_pool_status); + } + } + + // Remove the non-present servers for the gauge metrics + vector missing_server_keys {}; + + for (const auto& key : status.p_connection_pool_status_map) { + if (std::find(cur_servers_ids.begin(), cur_servers_ids.end(), key.first) == cur_servers_ids.end()) { + missing_server_keys.push_back(key.first); + } + } + + for (const auto& key : missing_server_keys) { + auto gauge = status.p_connection_pool_status_map[key]; + status.p_dyn_gauge_array[p_hg_dyn_gauge::connection_pool_status]->Remove(gauge); + status.p_connection_pool_status_map.erase(key); + + gauge = status.p_connection_pool_conn_free_map[key]; + status.p_dyn_gauge_array[p_hg_dyn_gauge::connection_pool_conn_free]->Remove(gauge); + status.p_connection_pool_conn_free_map.erase(key); + + gauge = status.p_connection_pool_conn_used_map[key]; + status.p_dyn_gauge_array[p_hg_dyn_gauge::connection_pool_conn_used]->Remove(gauge); + status.p_connection_pool_conn_used_map.erase(key); + + gauge = status.p_connection_pool_latency_us_map[key]; + status.p_dyn_gauge_array[p_hg_dyn_gauge::connection_pool_latency_us]->Remove(gauge); + status.p_connection_pool_latency_us_map.erase(key); + } + + wrunlock(); +} + +SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int *hid) { + const int colnum=14; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Connection Pool\n"); + SQLite3_result *result=new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"hostgroup"); + result->add_column_definition(SQLITE_TEXT,"srv_host"); + result->add_column_definition(SQLITE_TEXT,"srv_port"); + result->add_column_definition(SQLITE_TEXT,"status"); + result->add_column_definition(SQLITE_TEXT,"ConnUsed"); + result->add_column_definition(SQLITE_TEXT,"ConnFree"); + result->add_column_definition(SQLITE_TEXT,"ConnOK"); + result->add_column_definition(SQLITE_TEXT,"ConnERR"); + result->add_column_definition(SQLITE_TEXT,"MaxConnUsed"); + result->add_column_definition(SQLITE_TEXT,"Queries"); + result->add_column_definition(SQLITE_TEXT,"Queries_GTID_sync"); + result->add_column_definition(SQLITE_TEXT,"Bytes_sent"); + result->add_column_definition(SQLITE_TEXT,"Bytes_recv"); + result->add_column_definition(SQLITE_TEXT,"Latency_us"); + wrlock(); + int i,j, k; + for (i=0; i<(int)MyHostGroups->len; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j); + if (hid == NULL) { + if (mysrvc->get_status()!=MYSQL_SERVER_STATUS_ONLINE) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d is not online\n", mysrvc->address, mysrvc->port); + //__sync_fetch_and_sub(&status.server_connections_connected, mysrvc->ConnectionsFree->conns->len); + mysrvc->ConnectionsFree->drop_all_connections(); + } + // drop idle connections if beyond max_connection + while (mysrvc->ConnectionsFree->conns_length() && mysrvc->ConnectionsUsed->conns_length()+mysrvc->ConnectionsFree->conns_length() > mysrvc->max_connections) { + //MySQL_Connection *conn=(MySQL_Connection *)mysrvc->ConnectionsFree->conns->remove_index_fast(0); + MySQL_Connection *conn=mysrvc->ConnectionsFree->remove(0); + delete conn; + //__sync_fetch_and_sub(&status.server_connections_connected, 1); + } + } else { + if (*hid != (int)myhgc->hid) { + continue; + } + } + char buf[1024]; + char **pta=(char **)malloc(sizeof(char *)*colnum); + sprintf(buf,"%d", (int)myhgc->hid); + pta[0]=strdup(buf); + pta[1]=strdup(mysrvc->address); + sprintf(buf,"%d", mysrvc->port); + pta[2]=strdup(buf); + switch ((int)mysrvc->get_status()) { + case 0: + pta[3]=strdup("ONLINE"); + break; + case 1: + pta[3]=strdup("SHUNNED"); + break; + case 2: + pta[3]=strdup("OFFLINE_SOFT"); + break; + case 3: + pta[3]=strdup("OFFLINE_HARD"); + break; + case 4: + pta[3]=strdup("SHUNNED_REPLICATION_LAG"); + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + sprintf(buf,"%u", mysrvc->ConnectionsUsed->conns_length()); + pta[4]=strdup(buf); + sprintf(buf,"%u", mysrvc->ConnectionsFree->conns_length()); + pta[5]=strdup(buf); + sprintf(buf,"%u", mysrvc->connect_OK); + pta[6]=strdup(buf); + if (_reset) { + mysrvc->connect_OK=0; + } + sprintf(buf,"%u", mysrvc->connect_ERR); + pta[7]=strdup(buf); + if (_reset) { + mysrvc->connect_ERR=0; + } + sprintf(buf,"%u", mysrvc->max_connections_used); + pta[8]=strdup(buf); + if (_reset) { + mysrvc->max_connections_used=0; + } + sprintf(buf,"%llu", mysrvc->queries_sent); + pta[9]=strdup(buf); + if (_reset) { + mysrvc->queries_sent=0; + } + sprintf(buf,"%llu", mysrvc->queries_gtid_sync); + pta[10]=strdup(buf); + if (_reset) { + mysrvc->queries_gtid_sync=0; + } + sprintf(buf,"%llu", mysrvc->bytes_sent); + pta[11]=strdup(buf); + if (_reset) { + mysrvc->bytes_sent=0; + } + sprintf(buf,"%llu", mysrvc->bytes_recv); + pta[12]=strdup(buf); + if (_reset) { + mysrvc->bytes_recv=0; + } + sprintf(buf,"%u", mysrvc->current_latency_us); + pta[13]=strdup(buf); + result->add_row(pta); + for (k=0; k3"; + mydb->execute_statement((char *)q1, &error , &cols , &affected_rows , &res_set1); + for (std::vector::iterator it = res_set1->rows.begin() ; it != res_set1->rows.end(); ++it) { + SQLite3_row *r=*it; + std::string s = r->fields[0]; + s += ":::"; + s += r->fields[1]; + read_only_set1.insert(s); + } + proxy_info("Regenerating read_only_set1 with %lu servers\n", read_only_set1.size()); + if (read_only_set1.empty()) { + // to avoid regenerating this set always with 0 entries, we generate a fake entry + read_only_set1.insert("----:::----"); + } + delete res_set1; + } + wrunlock(); + std::string ser = hostname; + ser += ":::"; + ser += std::to_string(port); + std::set::iterator it; + it = read_only_set1.find(ser); + if (it != read_only_set1.end()) { + num_rows=1; + } + + if (admindb==NULL) { // we initialize admindb only if needed + admindb=new SQLite3DB(); + admindb->open((char *)"file:mem_admindb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + } + + switch (read_only) { + case 0: + if (num_rows==0) { + // the server has read_only=0 , but we can't find any writer, so we perform a swap + GloAdmin->mysql_servers_wrlock(); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 phase 1 : Dumping mysql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->save_mysql_servers_runtime_to_database(false); // SAVE MYSQL SERVERS FROM RUNTIME + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 phase 2 : Dumping mysql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + sprintf(query,Q2A,hostname,port); + admindb->execute(query); + sprintf(query,Q2B,hostname,port); + admindb->execute(query); + if (mysql_thread___monitor_writer_is_also_reader) { + sprintf(query,Q3A,hostname,port); + } else { + sprintf(query,Q3B,hostname,port); + } + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 phase 3 : Dumping mysql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->load_mysql_servers_to_runtime(); // LOAD MYSQL SERVERS TO RUNTIME + GloAdmin->mysql_servers_wrunlock(); + } else { + // there is a server in writer hostgroup, let check the status of present and not present hosts + bool act=false; + wrlock(); + std::set::iterator it; + // read_only_set2 acts as a cache + // if the server was RO=0 on the previous check and no action was needed, + // it will be here + it = read_only_set2.find(ser); + if (it != read_only_set2.end()) { + // the server was already detected as RO=0 + // no action required + } else { + // it is the first time that we detect RO on this server + sprintf(query,Q1B,hostname,port,hostname,port,hostname,port); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int status=MYSQL_SERVER_STATUS_OFFLINE_HARD; // default status, even for missing + if (r->fields[1]) { // has status + status=atoi(r->fields[1]); + } + if (status==MYSQL_SERVER_STATUS_OFFLINE_HARD) { + act=true; + } + } + if (act == false) { + // no action required, therefore we write in read_only_set2 + proxy_info("read_only_action() detected RO=0 on server %s:%d for the first time after commit(), but no need to reconfigure\n", hostname, port); + read_only_set2.insert(ser); + } + } + wrunlock(); + if (act==true) { // there are servers either missing, or with stats=OFFLINE_HARD + GloAdmin->mysql_servers_wrlock(); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 , rows=%d , phase 1 : Dumping mysql_servers for %s:%d\n", num_rows, hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->save_mysql_servers_runtime_to_database(false); // SAVE MYSQL SERVERS FROM RUNTIME + sprintf(query,Q2A,hostname,port); + admindb->execute(query); + sprintf(query,Q2B,hostname,port); + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 , rows=%d , phase 2 : Dumping mysql_servers for %s:%d\n", num_rows, hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + if (mysql_thread___monitor_writer_is_also_reader) { + sprintf(query,Q3A,hostname,port); + } else { + sprintf(query,Q3B,hostname,port); + } + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 , rows=%d , phase 3 : Dumping mysql_servers for %s:%d\n", num_rows, hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->load_mysql_servers_to_runtime(); // LOAD MYSQL SERVERS TO RUNTIME + GloAdmin->mysql_servers_wrunlock(); + } + } + break; + case 1: + if (num_rows) { + // the server has read_only=1 , but we find it as writer, so we perform a swap + GloAdmin->mysql_servers_wrlock(); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=1 phase 1 : Dumping mysql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->save_mysql_servers_runtime_to_database(false); // SAVE MYSQL SERVERS FROM RUNTIME + sprintf(query,Q4,hostname,port); + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=1 phase 2 : Dumping mysql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + sprintf(query,Q5,hostname,port); + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM mysql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from mysql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=1 phase 3 : Dumping mysql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->load_mysql_servers_to_runtime(); // LOAD MYSQL SERVERS TO RUNTIME + GloAdmin->mysql_servers_wrunlock(); + } + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + + pthread_mutex_unlock(&readonly_mutex); + if (resultset) { + delete resultset; + } + free(query); +} + +/** + * @brief New implementation of the read_only_action method that does not depend on the admin table. + * The method checks each server in the provided list and adjusts the servers according to their corresponding read_only value. + * If any change has occured, checksum is calculated. + * + * @param mysql_servers List of servers having hostname, port and read only value. + * + */ +void MySQL_HostGroups_Manager::read_only_action_v2(const std::list& mysql_servers) { + + bool update_mysql_servers_table = false; + + unsigned long long curtime1 = monotonic_time(); + wrlock(); + for (const auto& server : mysql_servers) { + bool is_writer = false; + const std::string& hostname = std::get(server); + const int port = std::get(server); + const int read_only = std::get(server); + const std::string& srv_id = hostname + ":::" + std::to_string(port); + + auto itr = hostgroup_server_mapping.find(srv_id); + + if (itr == hostgroup_server_mapping.end()) { + proxy_warning("Server %s:%d not found\n", hostname.c_str(), port); + continue; + } + + HostGroup_Server_Mapping* host_server_mapping = itr->second.get(); + + if (!host_server_mapping) + assert(0); + + const std::vector& writer_map = host_server_mapping->get(HostGroup_Server_Mapping::Type::WRITER); + + is_writer = !writer_map.empty(); + + if (read_only == 0) { + if (is_writer == false) { + // the server has read_only=0 (writer), but we can't find any writer, + // so we copy all reader nodes to writer + proxy_info("Server '%s:%d' found with 'read_only=0', but not found as writer\n", hostname.c_str(), port); + proxy_debug(PROXY_DEBUG_MONITOR, 5, "Server '%s:%d' found with 'read_only=0', but not found as writer\n", hostname.c_str(), port); + host_server_mapping->copy_if_not_exists(HostGroup_Server_Mapping::Type::WRITER, HostGroup_Server_Mapping::Type::READER); + + if (mysql_thread___monitor_writer_is_also_reader == false) { + // remove node from reader + host_server_mapping->clear(HostGroup_Server_Mapping::Type::READER); + } + + update_mysql_servers_table = true; + proxy_info("Regenerating table 'mysql_servers' due to actions on server '%s:%d'\n", hostname.c_str(), port); + } else { + bool act = false; + + // if the server was RO=0 on the previous check then no action is needed + if (host_server_mapping->get_readonly_flag() != 0) { + // it is the first time that we detect RO on this server + const std::vector& reader_map = host_server_mapping->get(HostGroup_Server_Mapping::Type::READER); + + for (const auto& reader_node : reader_map) { + for (const auto& writer_node : writer_map) { + + if (reader_node.writer_hostgroup_id == writer_node.writer_hostgroup_id) { + goto __writer_found; + } + } + act = true; + break; + __writer_found: + continue; + } + + if (act == false) { + // no action required, therefore we set readonly_flag to 0 + proxy_info("read_only_action_v2() detected RO=0 on server %s:%d for the first time after commit(), but no need to reconfigure\n", hostname.c_str(), port); + host_server_mapping->set_readonly_flag(0); + } + } else { + // the server was already detected as RO=0 + // no action required + } + + if (act == true) { // there are servers either missing, or with stats=OFFLINE_HARD + + proxy_info("Server '%s:%d' with 'read_only=0' found missing at some 'writer_hostgroup'\n", hostname.c_str(), port); + proxy_debug(PROXY_DEBUG_MONITOR, 5, "Server '%s:%d' with 'read_only=0' found missing at some 'writer_hostgroup'\n", hostname.c_str(), port); + + // copy all reader nodes to writer + host_server_mapping->copy_if_not_exists(HostGroup_Server_Mapping::Type::WRITER, HostGroup_Server_Mapping::Type::READER); + + if (mysql_thread___monitor_writer_is_also_reader == false) { + // remove node from reader + host_server_mapping->clear(HostGroup_Server_Mapping::Type::READER); + } + + update_mysql_servers_table = true; + proxy_info("Regenerating table 'mysql_servers' due to actions on server '%s:%d'\n", hostname.c_str(), port); + } + } + } else if (read_only == 1) { + if (is_writer) { + // the server has read_only=1 (reader), but we find it as writer, so we copy all writer nodes to reader (previous reader nodes will be reused) + proxy_info("Server '%s:%d' found with 'read_only=1', but not found as reader\n", hostname.c_str(), port); + proxy_debug(PROXY_DEBUG_MONITOR, 5, "Server '%s:%d' found with 'read_only=1', but not found as reader\n", hostname.c_str(), port); + host_server_mapping->copy_if_not_exists(HostGroup_Server_Mapping::Type::READER, HostGroup_Server_Mapping::Type::WRITER); + + // clearing all writer nodes + host_server_mapping->clear(HostGroup_Server_Mapping::Type::WRITER); + + update_mysql_servers_table = true; + proxy_info("Regenerating table 'mysql_servers' due to actions on server '%s:%d'\n", hostname.c_str(), port); + } + } else { + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + } + + if (update_mysql_servers_table) { + purge_mysql_servers_table(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + mydb->execute("DELETE FROM mysql_servers"); + generate_mysql_servers_table(); + + // Update the global checksums after 'mysql_servers' regeneration + { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + uint64_t raw_checksum = resultset ? resultset->raw_checksum() : 0; + + // This is required to be updated to avoid extra rebuilding member 'hostgroup_server_mapping' + // during 'commit'. For extra details see 'hgsm_mysql_servers_checksum' @details. + hgsm_mysql_servers_checksum = raw_checksum; + + string mysrvs_checksum { get_checksum_from_hash(raw_checksum) }; + save_runtime_mysql_servers(resultset.release()); + proxy_info("Checksum for table %s is %s\n", "mysql_servers", mysrvs_checksum.c_str()); + + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_mysql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + } + wrunlock(); + unsigned long long curtime2 = monotonic_time(); + curtime1 = curtime1 / 1000; + curtime2 = curtime2 / 1000; + proxy_debug(PROXY_DEBUG_MONITOR, 7, "MySQL_HostGroups_Manager::read_only_action_v2() locked for %llums (server count:%ld)\n", curtime2 - curtime1, mysql_servers.size()); +} + +// shun_and_killall +// this function is called only from MySQL_Monitor::monitor_ping() +// it temporary disables a host that is not responding to pings, and mark the host in a way that when used the connection will be dropped +// return true if the status was changed +bool MySQL_HostGroups_Manager::shun_and_killall(char *hostname, int port) { + time_t t = time(NULL); + bool ret = false; + wrlock(); + MySrvC *mysrvc=NULL; + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + unsigned int j; + unsigned int l=myhgc->mysrvs->cnt(); + if (l) { + for (j=0; jmysrvs->idx(j); + if (mysrvc->port==port && strcmp(mysrvc->address,hostname)==0) { + switch ((MySerStatus)mysrvc->get_status()) { + case MYSQL_SERVER_STATUS_SHUNNED: + if (mysrvc->shunned_automatic==false) { + break; + } + case MYSQL_SERVER_STATUS_ONLINE: + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { + ret = true; + } + mysrvc->set_status(MYSQL_SERVER_STATUS_SHUNNED); + case MYSQL_SERVER_STATUS_OFFLINE_SOFT: + mysrvc->shunned_automatic=true; + mysrvc->shunned_and_kill_all_connections=true; + mysrvc->ConnectionsFree->drop_all_connections(); + break; + default: + break; + } + // if Monitor is enabled and mysql-monitor_ping_interval is + // set too high, ProxySQL will unshun hosts that are not + // available. For this reason time_last_detected_error will + // be tuned in the future + if (mysql_thread___monitor_enabled) { + int a = mysql_thread___shun_recovery_time_sec; + int b = mysql_thread___monitor_ping_interval; + b = b/1000; + if (b > a) { + t = t + (b - a); + } + } + mysrvc->time_last_detected_error = t; + } + } + } + } + wrunlock(); + return ret; +} + +// set_server_current_latency_us +// this function is called only from MySQL_Monitor::monitor_ping() +// it set the average latency for a host in the last 3 pings +// the connection pool will use this information to evaluate or exclude a specific hosts +// note that this variable is in microsecond, while user defines it in millisecond +void MySQL_HostGroups_Manager::set_server_current_latency_us(char *hostname, int port, unsigned int _current_latency_us) { + wrlock(); + MySrvC *mysrvc=NULL; + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + unsigned int j; + unsigned int l=myhgc->mysrvs->cnt(); + if (l) { + for (j=0; jmysrvs->idx(j); + if (mysrvc->port==port && strcmp(mysrvc->address,hostname)==0) { + mysrvc->current_latency_us=_current_latency_us; + } + } + } + } + wrunlock(); +} + +void MySQL_HostGroups_Manager::p_update_metrics() { + p_update_counter(status.p_counter_array[p_hg_counter::servers_table_version], status.servers_table_version); + // Update *server_connections* related metrics + status.p_gauge_array[p_hg_gauge::server_connections_connected]->Set(status.server_connections_connected); + p_update_counter(status.p_counter_array[p_hg_counter::server_connections_aborted], status.server_connections_aborted); + p_update_counter(status.p_counter_array[p_hg_counter::server_connections_created], status.server_connections_created); + p_update_counter(status.p_counter_array[p_hg_counter::server_connections_delayed], status.server_connections_delayed); + + // Update *client_connections* related metrics + p_update_counter(status.p_counter_array[p_hg_counter::client_connections_created], status.client_connections_created); + p_update_counter(status.p_counter_array[p_hg_counter::client_connections_aborted], status.client_connections_aborted); + status.p_gauge_array[p_hg_gauge::client_connections_connected]->Set(status.client_connections); + + // Update *acess_denied* related metrics + p_update_counter(status.p_counter_array[p_hg_counter::access_denied_wrong_password], status.access_denied_wrong_password); + p_update_counter(status.p_counter_array[p_hg_counter::access_denied_max_connections], status.access_denied_max_connections); + p_update_counter(status.p_counter_array[p_hg_counter::access_denied_max_user_connections], status.access_denied_max_user_connections); + + p_update_counter(status.p_counter_array[p_hg_counter::selects_for_update__autocommit0], status.select_for_update_or_equivalent); + + // Update *com_* related metrics + p_update_counter(status.p_counter_array[p_hg_counter::com_autocommit], status.autocommit_cnt); + p_update_counter(status.p_counter_array[p_hg_counter::com_autocommit_filtered], status.autocommit_cnt_filtered); + p_update_counter(status.p_counter_array[p_hg_counter::com_commit_cnt], status.commit_cnt); + p_update_counter(status.p_counter_array[p_hg_counter::com_commit_cnt_filtered], status.commit_cnt_filtered); + p_update_counter(status.p_counter_array[p_hg_counter::com_rollback], status.rollback_cnt); + p_update_counter(status.p_counter_array[p_hg_counter::com_rollback_filtered], status.rollback_cnt_filtered); + p_update_counter(status.p_counter_array[p_hg_counter::com_backend_init_db], status.backend_init_db); + p_update_counter(status.p_counter_array[p_hg_counter::com_backend_change_user], status.backend_change_user); + p_update_counter(status.p_counter_array[p_hg_counter::com_backend_set_names], status.backend_set_names); + p_update_counter(status.p_counter_array[p_hg_counter::com_frontend_init_db], status.frontend_init_db); + p_update_counter(status.p_counter_array[p_hg_counter::com_frontend_set_names], status.frontend_set_names); + p_update_counter(status.p_counter_array[p_hg_counter::com_frontend_use_db], status.frontend_use_db); + + // Update *myconnpoll* related metrics + p_update_counter(status.p_counter_array[p_hg_counter::myhgm_myconnpool_get], status.myconnpoll_get); + p_update_counter(status.p_counter_array[p_hg_counter::myhgm_myconnpool_get_ok], status.myconnpoll_get_ok); + p_update_counter(status.p_counter_array[p_hg_counter::myhgm_myconnpool_get_ping], status.myconnpoll_get_ping); + p_update_counter(status.p_counter_array[p_hg_counter::myhgm_myconnpool_push], status.myconnpoll_push); + p_update_counter(status.p_counter_array[p_hg_counter::myhgm_myconnpool_reset], status.myconnpoll_reset); + p_update_counter(status.p_counter_array[p_hg_counter::myhgm_myconnpool_destroy], status.myconnpoll_destroy); + + p_update_counter(status.p_counter_array[p_hg_counter::auto_increment_delay_multiplex], status.auto_increment_delay_multiplex); + + // Update the *connection_pool* metrics + this->p_update_connection_pool(); + // Update the *gtid_executed* metrics + this->p_update_mysql_gtid_executed(); +} + +SQLite3_result * MySQL_HostGroups_Manager::SQL3_Get_ConnPool_Stats() { + const int colnum=2; + char buf[256]; + char **pta=(char **)malloc(sizeof(char *)*colnum); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping MySQL Global Status\n"); + SQLite3_result *result=new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"Variable_Name"); + result->add_column_definition(SQLITE_TEXT,"Variable_Value"); + wrlock(); + // NOTE: as there is no string copy, we do NOT free pta[0] and pta[1] + { + pta[0]=(char *)"MyHGM_myconnpoll_get"; + sprintf(buf,"%lu",status.myconnpoll_get); + pta[1]=buf; + result->add_row(pta); + } + { + pta[0]=(char *)"MyHGM_myconnpoll_get_ok"; + sprintf(buf,"%lu",status.myconnpoll_get_ok); + pta[1]=buf; + result->add_row(pta); + } + { + pta[0]=(char *)"MyHGM_myconnpoll_push"; + sprintf(buf,"%lu",status.myconnpoll_push); + pta[1]=buf; + result->add_row(pta); + } + { + pta[0]=(char *)"MyHGM_myconnpoll_destroy"; + sprintf(buf,"%lu",status.myconnpoll_destroy); + pta[1]=buf; + result->add_row(pta); + } + { + pta[0]=(char *)"MyHGM_myconnpoll_reset"; + sprintf(buf,"%lu",status.myconnpoll_reset); + pta[1]=buf; + result->add_row(pta); + } + wrunlock(); + free(pta); + return result; +} + +/** + * @brief Retrieves memory usage statistics for the MySQL host groups manager. + * + * This method calculates the total memory usage of the MySQL host groups manager, including memory allocated for + * host groups, server connections, and MySQL connections. It iterates over all host groups and their associated + * server connections to compute the memory usage. + * + * @return The total memory usage of the MySQL host groups manager in bytes. + */ +unsigned long long MySQL_HostGroups_Manager::Get_Memory_Stats() { + // Initialize the memory size counter + unsigned long long intsize=0; + // Acquire write lock to ensure thread safety during memory calculation + wrlock(); + MySrvC *mysrvc=NULL; // Pointer to a MySQL server connection + + // Iterate over all hostgroups + for (unsigned int i=0; ilen; i++) { + // Add memory size for the hostgroup object + intsize+=sizeof(MyHGC); + // Get the hostgroup object + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + unsigned int j,k; + // Get the number of server connections in the hostgroup + unsigned int l=myhgc->mysrvs->cnt(); + // Iterate over all server connections in the hostgroup + if (l) { + for (j=0; jmysrvs->idx(j); + // Calculate memory usage for each connection in the "ConnectionsFree" list + intsize+=((mysrvc->ConnectionsUsed->conns_length())*sizeof(MySQL_Connection *)); + for (k=0; kConnectionsFree->conns_length(); k++) { + // Get a MySQL connection + MySQL_Connection *myconn=mysrvc->ConnectionsFree->index(k); + // Add memory size for MySQL connection object and MYSQL struct + intsize+=sizeof(MySQL_Connection)+sizeof(MYSQL); + // Add memory size for the MySQL packet buffer + intsize+=myconn->mysql->net.max_packet; + // Add memory size for the default stack size of the asynchronous context + intsize+=(4096*15); // ASYNC_CONTEXT_DEFAULT_STACK_SIZE + // Add memory size for result set object if present + if (myconn->MyRS) { + intsize+=myconn->MyRS->current_size(); + } + } + intsize+=((mysrvc->ConnectionsUsed->conns_length())*sizeof(MySQL_Connection *)); + } + } + } + // Release the write lock + wrunlock(); + // Return the total memory usage + return intsize; +} + +Group_Replication_Info::Group_Replication_Info(int w, int b, int r, int o, int mw, int mtb, bool _a, int _w, char *c) { + comment=NULL; + if (c) { + comment=strdup(c); + } + writer_hostgroup=w; + backup_writer_hostgroup=b; + reader_hostgroup=r; + offline_hostgroup=o; + max_writers=mw; + max_transactions_behind=mtb; + active=_a; + writer_is_also_reader=_w; + current_num_writers=0; + current_num_backup_writers=0; + current_num_readers=0; + current_num_offline=0; + __active=true; + need_converge=true; +} + +Group_Replication_Info::~Group_Replication_Info() { + if (comment) { + free(comment); + comment=NULL; + } +} + +bool Group_Replication_Info::update(int b, int r, int o, int mw, int mtb, bool _a, int _w, char *c) { + bool ret=false; + __active=true; + if (backup_writer_hostgroup!=b) { + backup_writer_hostgroup=b; + ret=true; + } + if (reader_hostgroup!=r) { + reader_hostgroup=r; + ret=true; + } + if (offline_hostgroup!=o) { + offline_hostgroup=o; + ret=true; + } + if (max_writers!=mw) { + max_writers=mw; + ret=true; + } + if (max_transactions_behind!=mtb) { + max_transactions_behind=mtb; + ret=true; + } + if (active!=_a) { + active=_a; + ret=true; + } + if (writer_is_also_reader!=_w) { + writer_is_also_reader=_w; + ret=true; + } + // for comment we don't change return value + if (comment) { + if (c) { + if (strcmp(comment,c)) { + free(comment); + comment=strdup(c); + } + } else { + free(comment); + comment=NULL; + } + } else { + if (c) { + comment=strdup(c); + } + } + return ret; +} + +class MySQL_Errors_stats { + public: + int hostgroup; + char *hostname; + int port; + char *username; + char *client_address; + char *schemaname; + int err_no; + char *last_error; + time_t first_seen; + time_t last_seen; + unsigned long long count_star; + MySQL_Errors_stats(int hostgroup_, char *hostname_, int port_, char *username_, char *address_, char *schemaname_, int err_no_, char *last_error_, time_t tn) { + hostgroup = hostgroup_; + if (hostname_) { + hostname = strdup(hostname_); + } else { + hostname = strdup((char *)""); + } + port = port_; + if (username_) { + username = strdup(username_); + } else { + username = strdup((char *)""); + } + if (address_) { + client_address = strdup(address_); + } else { + client_address = strdup((char *)""); + } + if (schemaname_) { + schemaname = strdup(schemaname_); + } else { + schemaname = strdup((char *)""); + } + err_no = err_no_; + if (last_error_) { + last_error = strdup(last_error_); + } else { + last_error = strdup((char *)""); + } + last_seen = tn; + first_seen = tn; + count_star = 1; + } + ~MySQL_Errors_stats() { + if (hostname) { + free(hostname); + hostname=NULL; + } + if (username) { + free(username); + username=NULL; + } + if (client_address) { + free(client_address); + client_address=NULL; + } + if (schemaname) { + free(schemaname); + schemaname=NULL; + } + if (last_error) { + free(last_error); + last_error=NULL; + } + } + char **get_row() { + char buf[128]; + char **pta=(char **)malloc(sizeof(char *)*MYSQL_ERRORS_STATS_FIELD_NUM); + sprintf(buf,"%d",hostgroup); + pta[0]=strdup(buf); + assert(hostname); + pta[1]=strdup(hostname); + sprintf(buf,"%d",port); + pta[2]=strdup(buf); + assert(username); + pta[3]=strdup(username); + assert(client_address); + pta[4]=strdup(client_address); + assert(schemaname); + pta[5]=strdup(schemaname); + sprintf(buf,"%d",err_no); + pta[6]=strdup(buf); + + sprintf(buf,"%llu",count_star); + pta[7]=strdup(buf); + + sprintf(buf,"%ld", first_seen); + pta[8]=strdup(buf); + + sprintf(buf,"%ld", last_seen); + pta[9]=strdup(buf); + + assert(last_error); + pta[10]=strdup(last_error); + return pta; + } + void add_time(unsigned long long n, char *le) { + count_star++; + if (first_seen==0) { + first_seen=n; + } + last_seen=n; + if (strcmp(last_error,le)){ + free(last_error); + last_error=strdup(le); + } + } + void free_row(char **pta) { + int i; + for (i=0;i::iterator it; + pthread_mutex_lock(&mysql_errors_mutex); + + it=mysql_errors_umap.find(hash1); + + if (it != mysql_errors_umap.end()) { + // found + mes=(MySQL_Errors_stats *)it->second; + mes->add_time(tn, last_error); +/* + mes->last_seen = tn; + if (strcmp(mes->last_error,last_error)) { + free(mes->last_error); + mes->last_error = strdup(last_error); + mes->count_star++; + } +*/ + } else { + mes = new MySQL_Errors_stats(hostgroup, hostname, port, username, address, schemaname, err_no, last_error, tn); + mysql_errors_umap.insert(std::make_pair(hash1,(void *)mes)); + } + pthread_mutex_unlock(&mysql_errors_mutex); +} + +SQLite3_result * MySQL_HostGroups_Manager::get_mysql_errors(bool reset) { + SQLite3_result *result=new SQLite3_result(MYSQL_ERRORS_STATS_FIELD_NUM); + pthread_mutex_lock(&mysql_errors_mutex); + result->add_column_definition(SQLITE_TEXT,"hid"); + result->add_column_definition(SQLITE_TEXT,"hostname"); + result->add_column_definition(SQLITE_TEXT,"port"); + result->add_column_definition(SQLITE_TEXT,"username"); + result->add_column_definition(SQLITE_TEXT,"client_address"); + result->add_column_definition(SQLITE_TEXT,"schemaname"); + result->add_column_definition(SQLITE_TEXT,"err_no"); + result->add_column_definition(SQLITE_TEXT,"count_star"); + result->add_column_definition(SQLITE_TEXT,"first_seen"); + result->add_column_definition(SQLITE_TEXT,"last_seen"); + result->add_column_definition(SQLITE_TEXT,"last_error"); + for (std::unordered_map::iterator it=mysql_errors_umap.begin(); it!=mysql_errors_umap.end(); ++it) { + MySQL_Errors_stats *mes=(MySQL_Errors_stats *)it->second; + char **pta=mes->get_row(); + result->add_row(pta); + mes->free_row(pta); + if (reset) { + delete mes; + } + } + if (reset) { + mysql_errors_umap.erase(mysql_errors_umap.begin(),mysql_errors_umap.end()); + } + pthread_mutex_unlock(&mysql_errors_mutex); + return result; +} + +/** + * @brief Initializes the supplied 'MyHGC' with the specified 'hostgroup_settings'. + * @details Input verification is performed in the supplied 'hostgroup_settings'. It's expected to be a valid + * JSON that may contain the following fields: + * - handle_warnings: Value must be >= 0. + * + * In case input verification fails for a field, supplied 'MyHGC' is NOT updated for that field. An error + * message is logged specifying the source of the error. + * + * @param hostgroup_settings String containing a JSON defined in 'mysql_hostgroup_attributes'. + * @param myhgc The 'MyHGC' of the target hostgroup of the supplied 'hostgroup_settings'. + */ +void init_myhgc_hostgroup_settings(const char* hostgroup_settings, MyHGC* myhgc) { + const uint32_t hid = myhgc->hid; + + if (hostgroup_settings[0] != '\0') { + try { + nlohmann::json j = nlohmann::json::parse(hostgroup_settings); + + const auto handle_warnings_check = [](int8_t handle_warnings) -> bool { return handle_warnings == 0 || handle_warnings == 1; }; + const int8_t handle_warnings = j_get_srv_default_int_val(j, hid, "handle_warnings", handle_warnings_check); + myhgc->attributes.handle_warnings = handle_warnings; + + const auto monitor_slave_lag_when_null_check = [](int32_t monitor_slave_lag_when_null) -> bool + { return (monitor_slave_lag_when_null >= 0 && monitor_slave_lag_when_null <= 604800); }; + const int32_t monitor_slave_lag_when_null = j_get_srv_default_int_val(j, hid, "monitor_slave_lag_when_null", monitor_slave_lag_when_null_check); + myhgc->attributes.monitor_slave_lag_when_null = monitor_slave_lag_when_null; + } + catch (const json::exception& e) { + proxy_error( + "JSON parsing for 'mysql_hostgroup_attributes.hostgroup_settings' for hostgroup %d failed with exception `%s`.\n", + hid, e.what() + ); + } + } +} + +/** + * @brief Initializes the supplied 'MyHGC' with the specified 'servers_defaults'. + * @details Input verification is performed in the supplied 'server_defaults'. It's expected to be a valid + * JSON that may contain the following fields: + * - weight: Must be an unsigned integer >= 0. + * - max_connections: Must be an unsigned integer >= 0. + * - use_ssl: Must be a integer with either value 0 or 1. + * + * In case input verification fails for a field, supplied 'MyHGC' is NOT updated for that field. An error + * message is logged specifying the source of the error. + * + * @param servers_defaults String containing a JSON defined in 'mysql_hostgroup_attributes'. + * @param myhgc The 'MyHGC' of the target hostgroup of the supplied 'servers_defaults'. + */ +void init_myhgc_servers_defaults(char* servers_defaults, MyHGC* myhgc) { + uint32_t hid = myhgc->hid; + + if (strcmp(servers_defaults, "") != 0) { + try { + nlohmann::json j = nlohmann::json::parse(servers_defaults); + + const auto weight_check = [] (int64_t weight) -> bool { return weight >= 0; }; + int64_t weight = j_get_srv_default_int_val(j, hid, "weight", weight_check); + + myhgc->servers_defaults.weight = weight; + + const auto max_conns_check = [] (int64_t max_conns) -> bool { return max_conns >= 0; }; + int64_t max_conns = j_get_srv_default_int_val(j, hid, "max_connections", max_conns_check); + + myhgc->servers_defaults.max_connections = max_conns; + + const auto use_ssl_check = [] (int32_t use_ssl) -> bool { return use_ssl == 0 || use_ssl == 1; }; + int32_t use_ssl = j_get_srv_default_int_val(j, hid, "use_ssl", use_ssl_check); + + myhgc->servers_defaults.use_ssl = use_ssl; + } catch (const json::exception& e) { + proxy_error( + "JSON parsing for 'mysql_hostgroup_attributes.servers_defaults' for hostgroup %d failed with exception `%s`.\n", + hid, e.what() + ); + } + } +} + +void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() { + if (incoming_hostgroup_attributes==NULL) { + return; + } + int rc; + sqlite3_stmt *statement=NULL; + + const char * query=(const char *)"INSERT INTO mysql_hostgroup_attributes ( " + "hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, " + "init_connect, multiplex, connection_warming, throttle_connections_per_sec, " + "ignore_session_variables, hostgroup_settings, servers_defaults, comment) VALUES " + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query, -1, &statement, 0); + rc = mydb->prepare_v2(query, &statement); + ASSERT_SQLITE_OK(rc, mydb); + proxy_info("New mysql_hostgroup_attributes table\n"); + bool current_configured[MyHostGroups->len]; + // set configured = false to all + // in this way later we can known which HG were updated + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + current_configured[i] = myhgc->attributes.configured; + myhgc->attributes.configured = false; + } + + /** + * @brief We iterate the whole resultset incoming_hostgroup_attributes and configure + * both the hostgroup in memory, but also pupulate table mysql_hostgroup_attributes + * connection errors. + * @details for each row in incoming_hostgroup_attributes: + * 1. it finds (or create) the hostgroup + * 2. it writes the in mysql_hostgroup_attributes + * 3. it finds (or create) the attributes of the hostgroup + */ + for (std::vector::iterator it = incoming_hostgroup_attributes->rows.begin() ; it != incoming_hostgroup_attributes->rows.end(); ++it) { + SQLite3_row *r=*it; + unsigned int hid = (unsigned int)atoi(r->fields[0]); + MyHGC *myhgc = MyHGC_lookup(hid); // note: MyHGC_lookup() will create the HG if doesn't exist! + int max_num_online_servers = atoi(r->fields[1]); + int autocommit = atoi(r->fields[2]); + int free_connections_pct = atoi(r->fields[3]); + char * init_connect = r->fields[4]; + int multiplex = atoi(r->fields[5]); + int connection_warming = atoi(r->fields[6]); + int throttle_connections_per_sec = atoi(r->fields[7]); + char * ignore_session_variables = r->fields[8]; + char * hostgroup_settings = r->fields[9]; + char * servers_defaults = r->fields[10]; + char * comment = r->fields[11]; + proxy_info("Loading MySQL Hostgroup Attributes info for (%d,%d,%d,%d,\"%s\",%d,%d,%d,\"%s\",\"%s\",\"%s\",\"%s\")\n", + hid, max_num_online_servers, autocommit, free_connections_pct, + init_connect, multiplex, connection_warming, throttle_connections_per_sec, + ignore_session_variables, hostgroup_settings, servers_defaults, comment + ); + rc=(*proxy_sqlite3_bind_int64)(statement, 1, hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, max_num_online_servers); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 3, autocommit); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 4, free_connections_pct); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 5, init_connect, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, multiplex); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7, connection_warming); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, throttle_connections_per_sec); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 9, ignore_session_variables, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 10, hostgroup_settings, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 11, servers_defaults, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 12, comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mydb); + myhgc->attributes.configured = true; + myhgc->attributes.max_num_online_servers = max_num_online_servers; + myhgc->attributes.autocommit = autocommit; + myhgc->attributes.free_connections_pct = free_connections_pct; + myhgc->attributes.multiplex = multiplex; + myhgc->attributes.connection_warming = connection_warming; + myhgc->attributes.throttle_connections_per_sec = throttle_connections_per_sec; + if (myhgc->attributes.init_connect != NULL) + free(myhgc->attributes.init_connect); + myhgc->attributes.init_connect = strdup(init_connect); + if (myhgc->attributes.comment != NULL) + free(myhgc->attributes.comment); + myhgc->attributes.comment = strdup(comment); + // for ignore_session_variables we store 2 versions: + // 1. the text + // 2. the JSON + // Because calling JSON functions is expensive, we first verify if it changes + if (myhgc->attributes.ignore_session_variables_text == NULL) { + myhgc->attributes.ignore_session_variables_text = strdup(ignore_session_variables); + if (strlen(ignore_session_variables) != 0) { // only if there is a valid JSON + if (myhgc->attributes.ignore_session_variables_json != nullptr) { delete myhgc->attributes.ignore_session_variables_json; } + myhgc->attributes.ignore_session_variables_json = new json(json::parse(ignore_session_variables)); + } + } else { + if (strcmp(myhgc->attributes.ignore_session_variables_text, ignore_session_variables) != 0) { + free(myhgc->attributes.ignore_session_variables_text); + myhgc->attributes.ignore_session_variables_text = strdup(ignore_session_variables); + if (strlen(ignore_session_variables) != 0) { // only if there is a valid JSON + if (myhgc->attributes.ignore_session_variables_json != nullptr) { delete myhgc->attributes.ignore_session_variables_json; } + myhgc->attributes.ignore_session_variables_json = new json(json::parse(ignore_session_variables)); + } + // TODO: assign the variables + } + } + init_myhgc_hostgroup_settings(hostgroup_settings, myhgc); + init_myhgc_servers_defaults(servers_defaults, myhgc); + } + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + if (myhgc->attributes.configured == false) { + if (current_configured[i] == true) { + // if configured == false and previously it was configured == true , reset to defaults + proxy_info("Resetting hostgroup attributes for hostgroup %u\n", myhgc->hid); + myhgc->reset_attributes(); + } + } + } + + (*proxy_sqlite3_finalize)(statement); + delete incoming_hostgroup_attributes; + incoming_hostgroup_attributes=NULL; +} + +void MySQL_HostGroups_Manager::generate_mysql_servers_ssl_params_table() { + if (incoming_mysql_servers_ssl_params==NULL) { + return; + } + int rc; + sqlite3_stmt *statement=NULL; + + const char * query = (const char *)"INSERT INTO mysql_servers_ssl_params (" + "hostname, port, username, ssl_ca, ssl_cert, ssl_key, ssl_capath, " + "ssl_crl, ssl_crlpath, ssl_cipher, tls_version, comment) VALUES " + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + + rc = mydb->prepare_v2(query, &statement); + ASSERT_SQLITE_OK(rc, mydb); + proxy_info("New mysql_servers_ssl_params table\n"); + std::lock_guard lock(Servers_SSL_Params_map_mutex); + Servers_SSL_Params_map.clear(); + + for (std::vector::iterator it = incoming_mysql_servers_ssl_params->rows.begin() ; it != incoming_mysql_servers_ssl_params->rows.end(); ++it) { + SQLite3_row *r=*it; + proxy_info("Loading MySQL Server SSL Params for (%s,%s,%s)\n", + r->fields[0], r->fields[1], r->fields[2] + ); + + rc=(*proxy_sqlite3_bind_text)(statement, 1, r->fields[0] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // hostname + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoi(r->fields[1])); ASSERT_SQLITE_OK(rc, mydb); // port + rc=(*proxy_sqlite3_bind_text)(statement, 3, r->fields[2] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // username + rc=(*proxy_sqlite3_bind_text)(statement, 4, r->fields[3] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // ssl_ca + rc=(*proxy_sqlite3_bind_text)(statement, 5, r->fields[4] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // ssl_cert + rc=(*proxy_sqlite3_bind_text)(statement, 6, r->fields[5] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // ssl_key + rc=(*proxy_sqlite3_bind_text)(statement, 7, r->fields[6] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // ssl_capath + rc=(*proxy_sqlite3_bind_text)(statement, 8, r->fields[7] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // ssl_crl + rc=(*proxy_sqlite3_bind_text)(statement, 9, r->fields[8] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // ssl_crlpath + rc=(*proxy_sqlite3_bind_text)(statement, 10, r->fields[9] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // ssl_cipher + rc=(*proxy_sqlite3_bind_text)(statement, 11, r->fields[10] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // tls_version + rc=(*proxy_sqlite3_bind_text)(statement, 12, r->fields[11] , -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); // comment + + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mydb); + + MySQLServers_SslParams MSSP( + r->fields[0], atoi(r->fields[1]), r->fields[2], + r->fields[3], r->fields[4], r->fields[5], + r->fields[6], r->fields[7], r->fields[8], + r->fields[9], r->fields[10], r->fields[11] + ); + string MapKey = MSSP.getMapKey(rand_del); + Servers_SSL_Params_map.emplace(MapKey, MSSP); + } + (*proxy_sqlite3_finalize)(statement); + delete incoming_mysql_servers_ssl_params; + incoming_mysql_servers_ssl_params=NULL; +} + +int MySQL_HostGroups_Manager::create_new_server_in_hg( + uint32_t hid, const srv_info_t& srv_info, const srv_opts_t& srv_opts +) { + int32_t res = -1; + MySrvC* mysrvc = find_server_in_hg(hid, srv_info.addr, srv_info.port); + + if (mysrvc == nullptr) { + char* c_hostname { const_cast(srv_info.addr.c_str()) }; + MySrvC* mysrvc = new MySrvC( + c_hostname, srv_info.port, 0, srv_opts.weigth, MYSQL_SERVER_STATUS_ONLINE, 0, srv_opts.max_conns, 0, + srv_opts.use_ssl, 0, const_cast("") + ); + add(mysrvc,hid); + proxy_info( + "Adding new discovered %s node %s:%d with: hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", + srv_info.kind.c_str(), c_hostname, srv_info.port, hid, mysrvc->weight, mysrvc->max_connections, + mysrvc->use_ssl + ); + + res = 0; + } else { + // If the server is found as 'OFFLINE_HARD' we reset the 'MySrvC' values corresponding with the + // 'servers_defaults' (as in a new 'MySrvC' creation). We then later update these values with the + // 'servers_defaults' attributes from its corresponding 'MyHGC'. This way we ensure uniform behavior + // of new servers, and 'OFFLINE_HARD' ones when a user update 'servers_defaults' values, and reloads + // the servers to runtime. + if (mysrvc && mysrvc->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + reset_hg_attrs_server_defaults(mysrvc); + update_hg_attrs_server_defaults(mysrvc, mysrvc->myhgc); + mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); + + proxy_info( + "Found healthy previously discovered %s node %s:%d as 'OFFLINE_HARD', setting back as 'ONLINE' with:" + " hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", + srv_info.kind.c_str(), srv_info.addr.c_str(), srv_info.port, hid, mysrvc->weight, + mysrvc->max_connections, mysrvc->use_ssl + ); + + res = 0; + } + } + + return res; +} + +int MySQL_HostGroups_Manager::remove_server_in_hg(uint32_t hid, const string& addr, uint16_t port) { + MySrvC* mysrvc = find_server_in_hg(hid, addr, port); + if (mysrvc == nullptr) { + return -1; + } + + uint64_t mysrvc_addr = reinterpret_cast(mysrvc); + + proxy_warning( + "Removed server at address %ld, hostgroup %d, address %s port %d." + " Setting status OFFLINE HARD and immediately dropping all free connections." + " Used connections will be dropped when trying to use them\n", + mysrvc_addr, hid, mysrvc->address, mysrvc->port + ); + + // Set the server status + mysrvc->set_status(MYSQL_SERVER_STATUS_OFFLINE_HARD); + mysrvc->ConnectionsFree->drop_all_connections(); + + // TODO-NOTE: This is only required in case the caller isn't going to perform: + // - Full deletion of servers in the target 'hid'. + // - Table regeneration for the servers in the target 'hid'. + // This is a very common pattern when further operations have been performed over the + // servers, e.g. a set of servers additions and deletions over the target hostgroups. + // //////////////////////////////////////////////////////////////////////// + + // Remove the server from the table + const string del_srv_query { "DELETE FROM mysql_servers WHERE mem_pointer=" + std::to_string(mysrvc_addr) }; + mydb->execute(del_srv_query.c_str()); + + // //////////////////////////////////////////////////////////////////////// + + return 0; +} + +void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::copy_if_not_exists(Type dest_type, Type src_type) { + + assert(dest_type != src_type); + + const std::vector& src_nodes = mapping[src_type]; + + if (src_nodes.empty()) return; + + std::vector& dest_nodes = mapping[dest_type]; + std::list append; + + for (const auto& src_node : src_nodes) { + + for (const auto& dest_node : dest_nodes) { + + if (src_node.reader_hostgroup_id == dest_node.reader_hostgroup_id && + src_node.writer_hostgroup_id == dest_node.writer_hostgroup_id) { + goto __skip; + } + } + + append.push_back(src_node); + + __skip: + continue; + } + + if (append.empty()) { + return; + } + + if (dest_nodes.capacity() < (dest_nodes.size() + append.size())) + dest_nodes.reserve(dest_nodes.size() + append.size()); + + for (auto& node : append) { + + if (node.srv->get_status() == MYSQL_SERVER_STATUS_SHUNNED || + node.srv->get_status() == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { + // Status updated from "*SHUNNED" to "ONLINE" as "read_only" value was successfully + // retrieved from the backend server, indicating server is now online. + node.srv->set_status(MYSQL_SERVER_STATUS_ONLINE); + } + + MySrvC* new_srv = insert_HGM(get_hostgroup_id(dest_type, node), node.srv); + + if (!new_srv) assert(0); + + node.srv = new_srv; + dest_nodes.push_back(node); + } +} + +void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::remove(Type type, size_t index) { + + std::vector& nodes = mapping[type]; + + // ensure that we're not attempting to access out of the bounds of the container. + assert(index < nodes.size()); + + remove_HGM(nodes[index].srv); + + //Swap the element with the back element, except in the case when we're the last element. + if (index + 1 != nodes.size()) + std::swap(nodes[index], nodes.back()); + + //Pop the back of the container, deleting our old element. + nodes.pop_back(); +} + +void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::clear(Type type) { + + for (const auto& node : mapping[type]) { + remove_HGM(node.srv); + } + + mapping[type].clear(); +} + +unsigned int MySQL_HostGroups_Manager::HostGroup_Server_Mapping::get_hostgroup_id(Type type, const Node& node) const { + + if (type == Type::WRITER) + return node.writer_hostgroup_id; + else if (type == Type::READER) + return node.reader_hostgroup_id; + else + assert(0); +} + +MySrvC* MySQL_HostGroups_Manager::HostGroup_Server_Mapping::insert_HGM(unsigned int hostgroup_id, const MySrvC* srv) { + + MyHGC* myhgc = myHGM->MyHGC_lookup(hostgroup_id); + + if (!myhgc) + return NULL; + + MySrvC* ret_srv = NULL; + + for (uint32_t j = 0; j < myhgc->mysrvs->cnt(); j++) { + MySrvC* mysrvc = static_cast(myhgc->mysrvs->servers->index(j)); + if (strcmp(mysrvc->address, srv->address) == 0 && mysrvc->port == srv->port) { + if (mysrvc->get_status() == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + + mysrvc->gtid_port = srv->gtid_port; + mysrvc->weight = srv->weight; + mysrvc->compression = srv->compression; + mysrvc->max_connections = srv->max_connections; + mysrvc->max_replication_lag = srv->max_replication_lag; + mysrvc->use_ssl = srv->use_ssl; + mysrvc->max_latency_us = srv->max_latency_us; + mysrvc->comment = strdup(srv->comment); + mysrvc->set_status(MYSQL_SERVER_STATUS_ONLINE); + + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info( + "Found server node in Host Group Container %s:%d as 'OFFLINE_HARD', setting back as 'ONLINE' with:" + " hostgroup_id=%d, gtid_port=%d, weight=%ld, compression=%d, max_connections=%ld, use_ssl=%d," + " max_replication_lag=%d, max_latency_ms=%d, comment=%s\n", + mysrvc->address, mysrvc->port, hostgroup_id, mysrvc->gtid_port, mysrvc->weight, mysrvc->compression, + mysrvc->max_connections, mysrvc->use_ssl, mysrvc->max_replication_lag, (mysrvc->max_latency_us / 1000), + mysrvc->comment + ); + } + ret_srv = mysrvc; + break; + } + } + } + + if (!ret_srv) { + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%ld, status=%d\n", hostgroup_id, srv->address, srv->port, srv->gtid_port, srv->weight, (int)srv->get_status()); + } + + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%ld, status=%d, mem_ptr=%p into hostgroup=%d\n", srv->address, srv->port, srv->weight, (int)srv->get_status(), srv, hostgroup_id); + + ret_srv = new MySrvC(srv->address, srv->port, srv->gtid_port, srv->weight, srv->get_status(), srv->compression, + srv->max_connections, srv->max_replication_lag, srv->use_ssl, (srv->max_latency_us / 1000), srv->comment); + + myhgc->mysrvs->add(ret_srv); + } + + return ret_srv; +} + +void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::remove_HGM(MySrvC* srv) { + proxy_warning("Removed server at address %p, hostgroup %d, address %s port %d. Setting status OFFLINE HARD and immediately dropping all free connections. Used connections will be dropped when trying to use them\n", (void*)srv, srv->myhgc->hid, srv->address, srv->port); + srv->set_status(MYSQL_SERVER_STATUS_OFFLINE_HARD); + srv->ConnectionsFree->drop_all_connections(); +} + +MySQLServers_SslParams * MySQL_HostGroups_Manager::get_Server_SSL_Params(char *hostname, int port, char *username) { + string MapKey = string(hostname) + string(rand_del) + to_string(port) + string(rand_del) + string(username); + std::lock_guard lock(Servers_SSL_Params_map_mutex); + auto it = Servers_SSL_Params_map.find(MapKey); + if (it != Servers_SSL_Params_map.end()) { + MySQLServers_SslParams * MSSP = new MySQLServers_SslParams(it->second); + return MSSP; + } else { + MapKey = string(hostname) + string(rand_del) + to_string(port) + string(rand_del) + ""; // search for empty username + it = Servers_SSL_Params_map.find(MapKey); + if (it != Servers_SSL_Params_map.end()) { + MySQLServers_SslParams * MSSP = new MySQLServers_SslParams(it->second); + return MSSP; + } + } + return NULL; +} + +/** +* @brief Updates replication hostgroups by adding autodiscovered mysql servers. +* @details Adds each server from 'new_servers' to the 'runtime_mysql_servers' table. +* We then rebuild the 'mysql_servers' table as well as the internal 'hostname_hostgroup_mapping'. +* @param new_servers A vector of tuples where each tuple contains the values needed to add each new server. +*/ +void MySQL_HostGroups_Manager::add_discovered_servers_to_mysql_servers_and_replication_hostgroups( + const vector>& new_servers +) { + int added_new_server = -1; + + GloAdmin->mysql_servers_wrlock(); + wrlock(); + + // Add the discovered server with default values + for (const tuple& s : new_servers) { + string host = std::get<0>(s); + uint16_t port = std::get<1>(s); + long int hostgroup_id = std::get<2>(s); + + srv_info_t srv_info { host.c_str(), port, "AWS RDS" }; + srv_opts_t srv_opts { -1, -1, -1 }; + + added_new_server = create_new_server_in_hg(hostgroup_id, srv_info, srv_opts); + } + + // If servers were added, perform necessary updates to internal structures + if (added_new_server > -1) { + purge_mysql_servers_table(); + mydb->execute("DELETE FROM mysql_servers"); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + generate_mysql_servers_table(); + + // Update the global checksums after 'mysql_servers' regeneration + { + unique_ptr resultset { get_admin_runtime_mysql_servers(mydb) }; + string mysrvs_checksum { get_checksum_from_hash(resultset ? resultset->raw_checksum() : 0) }; + save_runtime_mysql_servers(resultset.release()); + + // Update the runtime_mysql_servers checksum with the new checksum + uint64_t raw_checksum = this->runtime_mysql_servers ? this->runtime_mysql_servers->raw_checksum() : 0; + table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS] = raw_checksum; + + // This is required for preserving coherence in the checksums, otherwise they would be inconsistent with `commit` generated checksums + SpookyHash rep_hgs_hash {}; + bool init = false; + uint64_t servers_v2_hash = table_resultset_checksum[HGM_TABLES::MYSQL_SERVERS_V2]; + + if (servers_v2_hash) { + if (init == false) { + init = true; + rep_hgs_hash.Init(19, 3); + } + + rep_hgs_hash.Update(&servers_v2_hash, sizeof(servers_v2_hash)); + } + + CUCFT1( + rep_hgs_hash, init, "mysql_replication_hostgroups", "writer_hostgroup", + table_resultset_checksum[HGM_TABLES::MYSQL_REPLICATION_HOSTGROUPS] + ); + + proxy_info("Checksum for table %s is %s\n", "mysql_servers", mysrvs_checksum.c_str()); + + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_mysql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + + update_table_mysql_servers_for_monitor(false); + update_hostgroup_manager_mappings(); + } + + wrunlock(); + GloAdmin->mysql_servers_wrunlock(); +} +#endif // 0 diff --git a/lib/Base_Session.cpp b/lib/Base_Session.cpp new file mode 100644 index 0000000000..ffc20a613a --- /dev/null +++ b/lib/Base_Session.cpp @@ -0,0 +1,735 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include "Base_Session.h" + +#include "MySQL_PreparedStatement.h" +#include "MySQL_Data_Stream.h" +#include "PgSQL_Data_Stream.h" + +#define SELECT_DB_USER "select DATABASE(), USER() limit 1" +#define SELECT_DB_USER_LEN 33 +#define SELECT_CHARSET_STATUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" +#define SELECT_CHARSET_STATUS_LEN 115 + +using json = nlohmann::json; + +// Explicitly instantiate the required template class and member functions +template void Base_Session::init(); +template void Base_Session::init(); + +template Base_Session::Base_Session(); +template Base_Session::Base_Session(); +template Base_Session::~Base_Session(); +template Base_Session::~Base_Session(); + +template MySQL_Backend * Base_Session::find_backend(int); +template PgSQL_Backend * Base_Session::find_backend(int); + +template MySQL_Backend * Base_Session::find_or_create_backend(int, MySQL_Data_Stream *); +template PgSQL_Backend * Base_Session::find_or_create_backend(int, PgSQL_Data_Stream *); + +template void Base_Session::writeout(); +template void Base_Session::writeout(); + +template void Base_Session::return_proxysql_internal(_PtrSize_t*); +template void Base_Session::return_proxysql_internal(_PtrSize_t*); + +template bool Base_Session::has_any_backend(); +template bool Base_Session::has_any_backend(); + +template void Base_Session::reset_all_backends(); +template void Base_Session::reset_all_backends(); + +template bool Base_Session::handler_special_queries_STATUS(_PtrSize_t*); +template bool Base_Session::handler_special_queries_STATUS(_PtrSize_t*); + +template void Base_Session::housekeeping_before_pkts(); +template void Base_Session::housekeeping_before_pkts(); + + +template void Base_Session::update_expired_conns(std::vector, std::allocator > > const&); +template void Base_Session::update_expired_conns(std::vector, std::allocator > > const&); + +template unsigned int Base_Session::NumActiveTransactions(bool); +template unsigned int Base_Session::NumActiveTransactions(bool); + +template void Base_Session::set_unhealthy(); +template void Base_Session::set_unhealthy(); + +template int Base_Session::FindOneActiveTransaction(bool); +template int Base_Session::FindOneActiveTransaction(bool); + +template bool Base_Session::HasOfflineBackends(); +template bool Base_Session::HasOfflineBackends(); + +template bool Base_Session::SetEventInOfflineBackends(); +template bool Base_Session::SetEventInOfflineBackends(); + +template +Base_Session::Base_Session() { +}; + +template +Base_Session::~Base_Session() { +}; + +template +void Base_Session::init() { + transaction_persistent_hostgroup = -1; + transaction_persistent = false; + mybes = new PtrArray(4); + // Conditional initialization based on derived class + if constexpr (std::is_same_v) { + sess_STMTs_meta = new MySQL_STMTs_meta(); + SLDH = new StmtLongDataHandler(); + } else if constexpr (std::is_same_v) { + sess_STMTs_meta = NULL; + SLDH = NULL; + } else { + assert(0); + } +}; + + +template +B * Base_Session::find_backend(int hostgroup_id) { + B *_mybe; + unsigned int i; + for (i=0; i < mybes->len; i++) { + _mybe=(B *)mybes->index(i); + if (_mybe->hostgroup_id==hostgroup_id) { + return _mybe; + } + } + return NULL; // NULL = backend not found +}; + +/** + * @brief Create a new MySQL backend associated with the specified hostgroup ID and data stream. + * + * This function creates a new MySQL backend object and associates it with the provided hostgroup ID + * and data stream. If the data stream is not provided (_myds is nullptr), a new MySQL_Data_Stream + * object is created and initialized. + * + * @param hostgroup_id The ID of the hostgroup to which the backend belongs. + * @param _myds The MySQL data stream associated with the backend. + * @return A pointer to the newly created MySQL_Backend object. + */ +template +B * Base_Session::create_backend(int hostgroup_id, DS *_myds) { + B *_mybe = new B(); + proxy_debug(PROXY_DEBUG_NET,4,"HID=%d, _myds=%p, _mybe=%p\n" , hostgroup_id, _myds, _mybe); + _mybe->hostgroup_id=hostgroup_id; + if (_myds) { + _mybe->server_myds=_myds; + } else { + _mybe->server_myds = new DS(); + _mybe->server_myds->DSS=STATE_NOT_INITIALIZED; + _mybe->server_myds->init(MYDS_BACKEND_NOT_CONNECTED, static_cast(this), 0); + } + // the newly created backend is added to the session's list of backends (mybes) and a pointer to it is returned. + mybes->add(_mybe); + return _mybe; +}; + +/** + * @brief Find or create a MySQL backend associated with the specified hostgroup ID and data stream. + * + * This function first attempts to find an existing MySQL backend associated with the provided + * hostgroup ID. If a backend is found, its pointer is returned. Otherwise, a new MySQL backend + * is created and associated with the hostgroup ID and data stream. If the data stream is not provided + * (_myds is nullptr), a new MySQL_Data_Stream object is created and initialized for the new backend. + * + * @param hostgroup_id The ID of the hostgroup to which the backend belongs. + * @param _myds The MySQL data stream associated with the backend. + * @return A pointer to the MySQL_Backend object found or created. + */ +template +B * Base_Session::find_or_create_backend(int hostgroup_id, DS *_myds) { + B * _mybe = find_backend(hostgroup_id); + proxy_debug(PROXY_DEBUG_NET,4,"HID=%d, _myds=%p, _mybe=%p\n" , hostgroup_id, _myds, _mybe); + // The pointer to the found or newly created backend is returned. + return ( _mybe ? _mybe : create_backend(hostgroup_id, _myds) ); +}; + +/** + * @brief Writes data from the session to the network with optional throttling and flow control. + * + * The writeout() function in the MySQL_Session class is responsible for writing data from the session to the network. + * It supports throttling, which limits the rate at which data is sent to the client. Throttling is controlled by the + * mysql_thread___throttle_max_bytes_per_second_to_client configuration parameter. If throttling is disabled (the parameter + * is set to 0), the function bypasses throttling. + * + * This function first ensures that any pending data in the session's data stream (client_myds) is written to the network. + * This ensures that the network buffers are emptied, allowing new data to be sent. + * + * After writing data to the network, the function checks if flow control is necessary. If the total amount of data written + * exceeds the maximum allowed per call (mwpl), or if the data is sent too quickly, the function pauses writing for a brief + * period to control the flow of data. + * + * If throttling is enabled, the function adjusts the throttle based on the amount of data written and the configured maximum + * bytes per second. If the current throughput exceeds the configured limit, the function increases the pause duration to + * regulate the flow of data. + * + * Finally, if the session has a backend associated with it (mybe), and the backend has a server data stream (server_myds), + * the function also writes data from the server data stream to the network. + * + * @note This function assumes that necessary session and network structures are properly initialized. + * + * @see mysql_thread___throttle_max_bytes_per_second_to_client + * @see MySQL_Session::client_myds + * @see MySQL_Session::mybe + * @see MySQL_Backend::server_myds + */ + +template +void Base_Session::writeout() { + int tps = 10; // throttling per second , by default every 100ms + int total_written = 0; + unsigned long long last_sent_=0; + int tmbpstc = 0; // throttle_max_bytes_per_second_to_client + enum proxysql_session_type _tmp_session_type_cmp1; + if constexpr (std::is_same_v) { + tmbpstc = mysql_thread___throttle_max_bytes_per_second_to_client; + _tmp_session_type_cmp1 = PROXYSQL_SESSION_MYSQL; + } else if constexpr (std::is_same_v) { + tmbpstc = pgsql_thread___throttle_max_bytes_per_second_to_client; + _tmp_session_type_cmp1 = PROXYSQL_SESSION_PGSQL; + } else { + assert(0); + } + bool disable_throttle = tmbpstc == 0; + int mwpl = tmbpstc; // max writes per call + mwpl = mwpl/tps; + // logic to disable throttling + + if (session_type != _tmp_session_type_cmp1) { + disable_throttle = true; + } + + if (client_myds) client_myds->array2buffer_full(); + if (mybe && mybe->server_myds && mybe->server_myds->myds_type == MYDS_BACKEND) { + if (session_type == _tmp_session_type_cmp1) { + if (mybe->server_myds->net_failure == false) { + if (mybe->server_myds->poll_fds_idx > -1) { // NOTE: attempt to force writes + mybe->server_myds->array2buffer_full(); + } + } + } + else { + mybe->server_myds->array2buffer_full(); + } + } + + if (client_myds && thread->curtime >= client_myds->pause_until) { + if (mirror==false) { + bool runloop=false; + if (client_myds->mypolls) { + last_sent_ = client_myds->mypolls->last_sent[client_myds->poll_fds_idx]; + } + int retbytes=client_myds->write_to_net_poll(); + total_written+=retbytes; + if (retbytes==QUEUE_T_DEFAULT_SIZE) { // optimization to solve memory bloat + runloop=true; + } + while (runloop && (disable_throttle || total_written < mwpl)) { + runloop=false; // the default + client_myds->array2buffer_full(); + struct pollfd fds; + fds.fd=client_myds->fd; + fds.events=POLLOUT; + fds.revents=0; + int retpoll=poll(&fds, 1, 0); + if (retpoll>0) { + if (fds.revents==POLLOUT) { + retbytes=client_myds->write_to_net_poll(); + total_written+=retbytes; + if (retbytes==QUEUE_T_DEFAULT_SIZE) { // optimization to solve memory bloat + runloop=true; + } + } + } + } + } + } + + // flow control + if (!disable_throttle && total_written > 0) { + if (total_written > mwpl) { + unsigned long long add_ = 1000000 / tps + 1000000 / tps * ((unsigned long long)total_written - (unsigned long long)mwpl) / mwpl; + pause_until = thread->curtime + add_; + client_myds->remove_pollout(); + client_myds->pause_until = thread->curtime + add_; + } + else { + if (total_written >= QUEUE_T_DEFAULT_SIZE) { + unsigned long long time_diff = thread->curtime - last_sent_; + if (time_diff == 0) { // sending data really too fast! + unsigned long long add_ = 1000000 / tps + 1000000 / tps * ((unsigned long long)total_written - (unsigned long long)mwpl) / mwpl; + pause_until = thread->curtime + add_; + client_myds->remove_pollout(); + client_myds->pause_until = thread->curtime + add_; + } + else { + float current_Bps = (float)total_written * 1000 * 1000 / time_diff; + if (current_Bps > tmbpstc) { + unsigned long long add_ = 1000000 / tps; + pause_until = thread->curtime + add_; + assert(pause_until > thread->curtime); + client_myds->remove_pollout(); + client_myds->pause_until = thread->curtime + add_; + } + } + } + } + } + if (mybe) { + if (mybe->server_myds) mybe->server_myds->write_to_net_poll(); + } + proxy_debug(PROXY_DEBUG_NET,1,"Thread=%p, Session=%p -- Writeout Session %p\n" , this->thread, this, this); +} + +template +void Base_Session::return_proxysql_internal(PtrSize_t* pkt) { + unsigned int l = 0; + l = strlen((char*)"PROXYSQL INTERNAL SESSION"); + if constexpr (std::is_same_v) { + if (pkt->size == (5 + l) && strncasecmp((char*)"PROXYSQL INTERNAL SESSION", (char*)pkt->ptr + 5, l) == 0) { + json j; + generate_proxysql_internal_session_json(j); + std::string s = j.dump(4, ' ', false, json::error_handler_t::replace); + SQLite3_result* resultset = new SQLite3_result(1); + resultset->add_column_definition(SQLITE_TEXT, "session_info"); + char* pta[1]; + pta[0] = (char*)s.c_str(); + resultset->add_row(pta); + bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); + delete resultset; + l_free(pkt->size, pkt->ptr); + return; + } + // default + client_myds->DSS = STATE_QUERY_SENT_NET; + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, 1064, (char*)"42000", (char*)"Unknown PROXYSQL INTERNAL command", true); + } + else if constexpr (std::is_same_v) { + if (pkt->size >= (5 + 1 + l) && strncasecmp((char*)"PROXYSQL INTERNAL SESSION", (char*)pkt->ptr + 5, l) == 0) { + json j; + generate_proxysql_internal_session_json(j); + std::string s = j.dump(4, ' ', false, json::error_handler_t::replace); + SQLite3_result* resultset = new SQLite3_result(1); + resultset->add_column_definition(SQLITE_TEXT, "session_info"); + char* pta[1]; + pta[0] = (char*)s.c_str(); + resultset->add_row(pta); + SQLite3_to_Postgres(client_myds->PSarrayOUT, resultset, nullptr, 0, (const char*)pkt->ptr + 5); + delete resultset; + l_free(pkt->size, pkt->ptr); + return; + } + client_myds->DSS = STATE_QUERY_SENT_NET; + client_myds->myprot.generate_error_packet(true, true, "Unknown PROXYSQL INTERNAL command", PGSQL_ERROR_CODES::ERRCODE_SYNTAX_ERROR, false, true); + } + else { + assert(0); + } + if (mirror == false) { + RequestEnd(NULL); + } + else { + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + } + l_free(pkt->size, pkt->ptr); +} + +/** + * @brief Check if any backend has an active MySQL connection. + * + * This function iterates through all backends associated with the session and checks if any backend has an + * active MySQL connection. If any backend has an active connection, it returns true; otherwise, it returns false. + * + * @return true if any backend has an active MySQL connection, otherwise false. + */ +template +bool Base_Session::has_any_backend() { + for (unsigned int j=0;j < mybes->len;j++) { + B * tmp_mybe=(B *)mybes->index(j); + DS *__myds=tmp_mybe->server_myds; + if (__myds->myconn) { + return true; + } + } + return false; +} + + + + +/** + * @brief Reset all MySQL backends associated with this session. + * + * This function resets all MySQL backends associated with the current session. + * It iterates over all backends stored in the session, resets each backend, and then deletes it. + * + */ +template +void Base_Session::reset_all_backends() { + B *mybe; + while(mybes->len) { + mybe=(B *)mybes->remove_index_fast(0); + mybe->reset(); + delete mybe; + } +}; + + + +/** + * @brief Handles special queries executed by the STATUS command in mysql cli . + * Specifically: + * "select DATABASE(), USER() limit 1" + * "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" + * See Github issues 4396 and 4426 + * + * @param PtrSize_t The packet from the client + * + * @return True if the queries are handled + * + * @note even if this function uses templates, perhaps is relevant only for MySQL client and not PostgreSQL + */ +template +bool Base_Session::handler_special_queries_STATUS(PtrSize_t* pkt) { + if (pkt->size == (SELECT_DB_USER_LEN + 5)) { + if (strncasecmp(SELECT_DB_USER, (char*)pkt->ptr + 5, SELECT_DB_USER_LEN) == 0) { + SQLite3_result* resultset = new SQLite3_result(2); + resultset->add_column_definition(SQLITE_TEXT, "DATABASE()"); + resultset->add_column_definition(SQLITE_TEXT, "USER()"); + char* pta[2]; + pta[0] = client_myds->myconn->userinfo->username; + pta[1] = client_myds->myconn->userinfo->schemaname; + resultset->add_row(pta); + bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); + delete resultset; + l_free(pkt->size, pkt->ptr); + return true; + } + } + + if (pkt->size == (SELECT_CHARSET_STATUS_LEN + 5)) { + if (strncasecmp(SELECT_CHARSET_STATUS, (char*)pkt->ptr + 5, SELECT_CHARSET_STATUS_LEN) == 0) { + SQLite3_result* resultset = new SQLite3_result(4); + resultset->add_column_definition(SQLITE_TEXT, "@@character_set_client"); + resultset->add_column_definition(SQLITE_TEXT, "@@character_set_connection"); + resultset->add_column_definition(SQLITE_TEXT, "@@character_set_server"); + resultset->add_column_definition(SQLITE_TEXT, "@@character_set_database"); + + // here we do a bit back and forth to and from JSON to reuse existing code instead of writing new code. + // This is not great for performance, but this query is rarely executed. + string vals[4]; + json j = {}; + json& jc = j["conn"]; + if constexpr (std::is_same_v) { + MySQL_Connection * conn = client_myds->myconn; + conn->variables[SQL_CHARACTER_SET_CLIENT].fill_client_internal_session(jc, SQL_CHARACTER_SET_CLIENT); + conn->variables[SQL_CHARACTER_SET_CONNECTION].fill_client_internal_session(jc, SQL_CHARACTER_SET_CONNECTION); + conn->variables[SQL_CHARACTER_SET_DATABASE].fill_client_internal_session(jc, SQL_CHARACTER_SET_DATABASE); + } else if constexpr (std::is_same_v) { + PgSQL_Connection * conn = client_myds->myconn; + conn->variables[SQL_CHARACTER_SET_CLIENT].fill_client_internal_session(jc, SQL_CHARACTER_SET_CLIENT); + conn->variables[SQL_CHARACTER_SET_CONNECTION].fill_client_internal_session(jc, SQL_CHARACTER_SET_CONNECTION); + conn->variables[SQL_CHARACTER_SET_DATABASE].fill_client_internal_session(jc, SQL_CHARACTER_SET_DATABASE); + } else { + assert(0); + } + + // @@character_set_client + vals[0] = jc[mysql_tracked_variables[SQL_CHARACTER_SET_CLIENT].internal_variable_name]; + // @@character_set_connection + vals[1] = jc[mysql_tracked_variables[SQL_CHARACTER_SET_CONNECTION].internal_variable_name]; + // @@character_set_server + if constexpr (std::is_same_v) { + vals[2] = string(mysql_thread___default_variables[SQL_CHARACTER_SET]); + } else if constexpr (std::is_same_v) { + vals[2] = string(mysql_thread___default_variables[SQL_CHARACTER_SET]); + } else { + assert(0); + } + // @@character_set_database + vals[3] = jc[mysql_tracked_variables[SQL_CHARACTER_SET_DATABASE].internal_variable_name]; + + const char* pta[4]; + for (int i = 0; i < 4; i++) { + pta[i] = vals[i].c_str(); + } + resultset->add_row(pta); + bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); + delete resultset; + l_free(pkt->size, pkt->ptr); + return true; + } + } + return false; +} + + + +/** + * @brief Perform housekeeping tasks before processing packets. + * + * This function is responsible for performing necessary housekeeping tasks + * before processing packets. These tasks include handling expired connections + * for multiplexing scenarios. If multiplexing is enabled, it iterates over + * the list of expired backend connections and either returns them to the connection pool + * or destroys them based on certain conditions. + * + * @note This function assumes that the `hgs_expired_conns` vector contains the IDs + * of the backend connections that have expired. + * + * @return None. + */ +template +void Base_Session::housekeeping_before_pkts() { + bool thread___multiplexing = true; + if constexpr (std::is_same_v) { + thread___multiplexing = mysql_thread___multiplexing; + } else if constexpr (std::is_same_v) { + thread___multiplexing = pgsql_thread___multiplexing; + } else { + assert(0); + } + if (thread___multiplexing) { + for (const int hg_id : hgs_expired_conns) { + B * mybe = find_backend(hg_id); + + if (mybe != nullptr) { + DS * myds = mybe->server_myds; + // FIXME: NOTE: the logic for autocommit is relevant only for MYSQL + if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit()==false) { + if constexpr (std::is_same_v) { + if (mysql_thread___reset_connection_algorithm == 2) { + create_new_session_and_reset_connection(myds); + } else { + myds->destroy_MySQL_Connection_From_Pool(true); + } + } else if constexpr (std::is_same_v) { + create_new_session_and_reset_connection(myds); + } else { + assert(0); + } + } else { + myds->return_MySQL_Connection_To_Pool(); + } + } + } + // We are required to perform a cleanup after consuming the elements, thus preventing any subsequent + // 'handler' call to perform recomputing of the already processed elements. + if (hgs_expired_conns.empty() == false) { + hgs_expired_conns.clear(); + } + } +} + +/** + * @brief Update expired connections based on specified checks. + * + * This function iterates through the list of backends and their connections + * to determine if any connections have expired based on the provided checks. + * If a connection is found to be expired, its hostgroup ID is added to the + * list of expired connections for further processing. + * + * @param checks A vector of function objects representing checks to determine if a connection has expired. + */ +template +using TypeConn = typename std::conditional< + std::is_same_v, MySQL_Connection, PgSQL_Connection +>::type; + +template +void Base_Session::update_expired_conns(const vector>& checks) { + for (uint32_t i = 0; i < mybes->len; i++) { // iterate through the list of backends + B * mybe = static_cast(mybes->index(i)); + DS * myds = mybe != nullptr ? mybe->server_myds : nullptr; + + + TypeConn * myconn = myds != nullptr ? myds->myconn : nullptr; + + //! it performs a series of checks to determine if it has expired + if (myconn != nullptr) { + const bool is_active_transaction = myconn->IsActiveTransaction(); + const bool multiplex_disabled = myconn->MultiplexDisabled(false); + const bool is_idle = myconn->async_state_machine == ASYNC_IDLE; + + // Make sure the connection is reusable before performing any check + if (myconn->reusable == true && is_active_transaction == false && multiplex_disabled == false && is_idle) { + for (const function& check : checks) { + if (check(myconn)) { + // If a connection is found to be expired based on the provided checks, + // its hostgroup ID is added to the list of expired connections (hgs_expired_conns) + // for further processing. + this->hgs_expired_conns.push_back(mybe->hostgroup_id); + break; + } + } + } + } + } +} + + +template +void Base_Session::set_unhealthy() { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess:%p\n", this); + healthy=0; +} + + +template +unsigned int Base_Session::NumActiveTransactions(bool check_savepoint) { + unsigned int ret=0; + if (mybes==0) return ret; + B *_mybe; + unsigned int i; + for (i=0; i < mybes->len; i++) { + _mybe=(B *)mybes->index(i); + if (_mybe->server_myds) { + if (_mybe->server_myds->myconn) { + if (_mybe->server_myds->myconn->IsActiveTransaction()) { + ret++; + } else { + // we use check_savepoint to check if we shouldn't ignore COMMIT or ROLLBACK due + // to MySQL bug https://bugs.mysql.com/bug.php?id=107875 related to + // SAVEPOINT and autocommit=0 + if (check_savepoint) { + if (_mybe->server_myds->myconn->AutocommitFalse_AndSavepoint() == true) { + ret++; + } + } + } + } + } + } + return ret; +} + +template +bool Base_Session::HasOfflineBackends() { + bool ret=false; + if (mybes==0) return ret; + B * _mybe; + unsigned int i; + for (i=0; i < mybes->len; i++) { + _mybe=(B *)mybes->index(i); + if (_mybe->server_myds) + if (_mybe->server_myds->myconn) + if (_mybe->server_myds->myconn->IsServerOffline()) { + ret=true; + return ret; + } + } + return ret; +} + +template +bool Base_Session::SetEventInOfflineBackends() { + bool ret=false; + if (mybes==0) return ret; + B * _mybe; + unsigned int i; + for (i = 0; i < mybes->len; i++) { + _mybe = (B *) mybes->index(i); + if (_mybe->server_myds) + if (_mybe->server_myds->myconn) + if (_mybe->server_myds->myconn->IsServerOffline()) { + _mybe->server_myds->revents |= POLLIN; + ret = true; + } + } + return ret; +} + + +template +int Base_Session::FindOneActiveTransaction(bool check_savepoint) { + int ret=-1; + if (mybes==0) return ret; + B * _mybe; + unsigned int i; + for (i=0; i < mybes->len; i++) { + _mybe = (B *) mybes->index(i); + if (_mybe->server_myds) { + if (_mybe->server_myds->myconn) { + if (_mybe->server_myds->myconn->IsKnownActiveTransaction()) { + return (int)_mybe->server_myds->myconn->parent->myhgc->hid; + } + else if (_mybe->server_myds->myconn->IsActiveTransaction()) { + ret = (int)_mybe->server_myds->myconn->parent->myhgc->hid; + } + else { + // we use check_savepoint to check if we shouldn't ignore COMMIT or ROLLBACK due + // to MySQL bug https://bugs.mysql.com/bug.php?id=107875 related to + // SAVEPOINT and autocommit=0 + if (check_savepoint) { + if (_mybe->server_myds->myconn->AutocommitFalse_AndSavepoint() == true) { + return (int)_mybe->server_myds->myconn->parent->myhgc->hid; + } + } + } + } + } + } + return ret; +} + +Session_Regex::Session_Regex(char* p) { + s = strdup(p); + re2::RE2::Options* opt2 = new re2::RE2::Options(RE2::Quiet); + opt2->set_case_sensitive(false); + opt = (void*)opt2; + re = (RE2*)new RE2(s, *opt2); +} + +Session_Regex::~Session_Regex() { + free(s); + delete (RE2*)re; + delete (re2::RE2::Options*)opt; +} + +bool Session_Regex::match(char* m) { + bool rc = false; + rc = RE2::PartialMatch(m, *(RE2*)re); + return rc; +} + + +std::string proxysql_session_type_str(enum proxysql_session_type session_type) { + if (session_type == PROXYSQL_SESSION_MYSQL) { + return "PROXYSQL_SESSION_MYSQL"; + } + else if (session_type == PROXYSQL_SESSION_ADMIN) { + return "PROXYSQL_SESSION_ADMIN"; + } + else if (session_type == PROXYSQL_SESSION_STATS) { + return "PROXYSQL_SESSION_STATS"; + } + else if (session_type == PROXYSQL_SESSION_SQLITE) { + return "PROXYSQL_SESSION_SQLITE"; + } + else if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { + return "PROXYSQL_SESSION_CLICKHOUSE"; + } + else if (session_type == PROXYSQL_SESSION_MYSQL_EMU) { + return "PROXYSQL_SESSION_MYSQL_EMU"; + } + else if (session_type == PROXYSQL_SESSION_PGSQL) { + return "PROXYSQL_SESSION_PGSQL"; + } + else { + return "PROXYSQL_SESSION_NONE"; + } +}; diff --git a/lib/Base_Thread.cpp b/lib/Base_Thread.cpp new file mode 100644 index 0000000000..50cbb0c186 --- /dev/null +++ b/lib/Base_Thread.cpp @@ -0,0 +1,539 @@ +#include "Base_Thread.h" + +#include "cpp.h" + +#include +#include +#include "PgSQL_Data_Stream.h" +#include "MySQL_Data_Stream.h" + + +// Explicitly instantiate the required template class and member functions +template MySQL_Session* Base_Thread::create_new_session_and_client_data_stream(int); +template PgSQL_Session* Base_Thread::create_new_session_and_client_data_stream(int); +template void Base_Thread::ProcessAllSessions_SortingSessions(); +template void Base_Thread::ProcessAllSessions_SortingSessions(); +template void Base_Thread::ProcessAllMyDS_AfterPoll(); +template void Base_Thread::ProcessAllMyDS_AfterPoll(); +template void Base_Thread::ProcessAllMyDS_BeforePoll(); +template void Base_Thread::ProcessAllMyDS_BeforePoll(); +template void Base_Thread::register_session(MySQL_Thread*, MySQL_Session*, bool); +template void Base_Thread::register_session(PgSQL_Thread*, PgSQL_Session*, bool); +template void Base_Thread::run_SetAllSession_ToProcess0(); +template void Base_Thread::run_SetAllSession_ToProcess0(); + +Base_Thread::Base_Thread() { +}; + +Base_Thread::~Base_Thread() { +}; + +template +void Base_Thread::register_session(T thr, S _sess, bool up_start) { + if (mysql_sessions==NULL) { + mysql_sessions = new PtrArray(); + } + mysql_sessions->add(_sess); + + _sess->thread = thr; +// if (T a = dynamic_cast(thr)) { +// _sess->thread = a; +// } else { +// assert(0); +// } + _sess->match_regexes=match_regexes; + if (up_start) + _sess->start_time=curtime; + proxy_debug(PROXY_DEBUG_NET,1,"Thread=%p, Session=%p -- Registered new session\n", _sess->thread, _sess); +} + + +template +S Base_Thread::create_new_session_and_client_data_stream(int _fd) { + int arg_on = 1; + S sess = NULL; + bool use_tcp_keepalive = false; + int tcp_keepalive_time = 0; + if constexpr (std::is_same_v) { + sess = new PgSQL_Session(); + use_tcp_keepalive = pgsql_thread___use_tcp_keepalive; + tcp_keepalive_time = pgsql_thread___tcp_keepalive_time; + } else if constexpr (std::is_same_v) { + sess = new MySQL_Session(); + use_tcp_keepalive = mysql_thread___use_tcp_keepalive; + tcp_keepalive_time = mysql_thread___tcp_keepalive_time; + } else { + assert(0); + } + register_session(static_cast(this), sess); + if constexpr (std::is_same_v) { + sess->client_myds = new PgSQL_Data_Stream(); + } else if constexpr (std::is_same_v) { + sess->client_myds = new MySQL_Data_Stream(); + } else { + assert(0); + } + sess->client_myds->fd = _fd; + + // set not blocking for client connections too! + { + // PMC-10004 + // While implementing SSL and fast_forward it was noticed that all frontend connections + // are in blocking, although this was never a problem because we call poll() before reading. + // Although it became a problem with fast_forward, SSL and large packets because SSL handled + // data in chunks of 16KB and there may be data inside SSL even when there is no data + // received from the network. + // The only modules that seems to be affected by this issue are Admin, SQLite3 Server + // and Clickhouse Server + int prevflags = fcntl(_fd, F_GETFL, 0); + if (prevflags == -1) { + proxy_error("For FD %d fcntl() returned -1 errno %d\n", _fd, errno); + if (shutdown == 0) + assert(prevflags != -1); + } + int nb = fcntl(_fd, F_SETFL, prevflags | O_NONBLOCK); + if (nb == -1) { + proxy_error("For FD %d fcntl() returned -1 , previous flags %d , errno %d\n", _fd, prevflags, errno); + // previously we were asserting here. But it is possible that this->shutdown is still 0 during the + // shutdown itself: + // - the current thread is processing connections + // - the signal handler thread is still setting shutdown = 0 + //if (shutdown == 0) + // assert (nb != -1); + } + } + setsockopt(sess->client_myds->fd, IPPROTO_TCP, TCP_NODELAY, (char*)&arg_on, sizeof(arg_on)); + + if (use_tcp_keepalive) { + setsockopt(sess->client_myds->fd, SOL_SOCKET, SO_KEEPALIVE, (char*)&arg_on, sizeof(arg_on)); +#ifdef TCP_KEEPIDLE + if (tcp_keepalive_time > 0) { + int keepalive_time = tcp_keepalive_time; + setsockopt(sess->client_myds->fd, IPPROTO_TCP, TCP_KEEPIDLE, (char*)&keepalive_time, sizeof(keepalive_time)); + } +#endif + } + +#ifdef __APPLE__ + setsockopt(sess->client_myds->fd, SOL_SOCKET, SO_NOSIGPIPE, (char*)&arg_on, sizeof(int)); +#endif + sess->client_myds->init(MYDS_FRONTEND, sess, sess->client_myds->fd); + proxy_debug(PROXY_DEBUG_NET, 1, "Thread=%p, Session=%p, DataStream=%p -- Created new client Data Stream\n", sess->thread, sess, sess->client_myds); +#ifdef DEBUG + sess->client_myds->myprot.dump_pkt = true; +#endif + if constexpr (std::is_same_v) { + PgSQL_Connection* myconn = new PgSQL_Connection(); + sess->client_myds->attach_connection(myconn); + } else if constexpr (std::is_same_v) { + MySQL_Connection* myconn = new MySQL_Connection(); + sess->client_myds->attach_connection(myconn); + } else { + assert(0); + } + sess->client_myds->myconn->set_is_client(); // this is used for prepared statements + sess->client_myds->myconn->last_time_used = curtime; + sess->client_myds->myconn->myds = sess->client_myds; // 20141011 + sess->client_myds->myconn->fd = sess->client_myds->fd; // 20141011 + + sess->client_myds->myprot.init(&sess->client_myds, sess->client_myds->myconn->userinfo, sess); + + if constexpr (std::is_same_v) { + uint32_t session_track_gtids_int = SpookyHash::Hash32(mysql_thread___default_session_track_gtids, strlen(mysql_thread___default_session_track_gtids), 10); + sess->client_myds->myconn->options.session_track_gtids_int = session_track_gtids_int; + if (sess->client_myds->myconn->options.session_track_gtids) { + free(sess->client_myds->myconn->options.session_track_gtids); + } + sess->client_myds->myconn->options.session_track_gtids = strdup(mysql_thread___default_session_track_gtids); + } + return sess; +} + + +/** + * @brief Checks for timing out session and marks them for processing. + * + * This function checks for timing out sessions and marks them for processing. Although the logic for managing connection timeout + * was removed due to the addition of the MariaDB client library, this function remains as a placeholder. It checks if the session + * has reached its wait_until or pause_until time, and if so, marks the session for processing. + * + * @param n The index of the session in the MySQL_Data_Stream array. + */ +template +void Base_Thread::check_timing_out_session(unsigned int n) { + // FIXME: this logic was removed completely because we added mariadb client library. Yet, we need to implement a way to manage connection timeout + // check for timeout + // no events. This section is copied from process_data_on_data_stream() + T* thr = static_cast(this); + auto * _myds = thr->mypolls.myds[n]; + if (_myds && _myds->sess) { + if (_myds->wait_until && curtime > _myds->wait_until) { + // timeout + _myds->sess->to_process=1; + } else { + if (_myds->sess->pause_until && curtime > _myds->sess->pause_until) { + // timeout + _myds->sess->to_process=1; + } + } + } +} + + + + +/** + * @brief Checks for an invalid file descriptor (FD) and raises an error if found. + * + * This function checks if the file descriptor (FD) at the specified index in the `mypolls.fds` array is invalid (`POLLNVAL`). + * If an invalid FD is found, it raises an error and asserts to ensure that the program does not proceed with an invalid FD. + * + * @param n The index of the file descriptor in the `mypolls.fds` array. + */ +template +void Base_Thread::check_for_invalid_fd(unsigned int n) { + // check if the FD is valid + T* thr = static_cast(this); + if (thr->mypolls.fds[n].revents==POLLNVAL) { + // debugging output before assert + auto *_myds=thr->mypolls.myds[n]; + if (_myds) { + if (_myds->myconn) { + proxy_error("revents==POLLNVAL for FD=%d, events=%d, MyDSFD=%d, MyConnFD=%d\n", thr->mypolls.fds[n].fd, thr->mypolls.fds[n].events, _myds->fd, _myds->myconn->fd); + assert(thr->mypolls.fds[n].revents!=POLLNVAL); + } + } + // if we reached her, we didn't assert() yet + proxy_error("revents==POLLNVAL for FD=%d, events=%d, MyDSFD=%d\n", thr->mypolls.fds[n].fd, thr->mypolls.fds[n].events, _myds->fd); + assert(thr->mypolls.fds[n].revents!=POLLNVAL); + } +} + +// this function was inline in MySQL_Thread::process_all_sessions() +/** + * @brief Sort all sessions based on maximum connection time. + * + * This function iterates through all MySQL sessions and sorts them based on their maximum connection time. + * Sessions with a valid maximum connection time are compared, and if one session has a greater maximum connection + * time than another, their positions in the session list are swapped. The sorting is performed in-place. + * + * @note This function assumes that MySQL sessions and their associated data structures have been initialized + * and are accessible within the MySQL Thread. + */ +template +void Base_Thread::ProcessAllSessions_SortingSessions() { + unsigned int a=0; + for (unsigned int n=0; nlen; n++) { + S *sess=(S *)mysql_sessions->index(n); + if (sess->mybe && sess->mybe->server_myds) { + if (sess->mybe->server_myds->max_connect_time) { + S *sess2=(S *)mysql_sessions->index(a); + if (sess2->mybe && sess2->mybe->server_myds && sess2->mybe->server_myds->max_connect_time && sess2->mybe->server_myds->max_connect_time <= sess->mybe->server_myds->max_connect_time) { + // do nothing + } else { + void *p=mysql_sessions->pdata[a]; + mysql_sessions->pdata[a]=mysql_sessions->pdata[n]; + mysql_sessions->pdata[n]=p; + a++; + } + } + } + } +} + +// this function was inline in MySQL_Thread::run() +/** + * @brief Processes all MySQL Data Streams after polling. + * + * This function iterates through all MySQL polls and processes the associated data streams. + * For each poll, it prints debug information about the file descriptor and its events. + * If a MySQL Data Stream is associated with the poll, it checks for events on the file descriptor. + * If there are no events and a poll timeout is enabled, it checks for sessions timing out. + * If there are events, it checks for invalid file descriptors and handles new connections + * for listener type data streams. For other types of data streams, it processes data and + * handles any potential errors. + */ +template +void Base_Thread::ProcessAllMyDS_AfterPoll() { + T* thr = static_cast(this); + for (unsigned int n = 0; n < thr->mypolls.len; n++) { + proxy_debug(PROXY_DEBUG_NET,3, "poll for fd %d events %d revents %d\n", thr->mypolls.fds[n].fd , thr->mypolls.fds[n].events, thr->mypolls.fds[n].revents); + + auto * myds = thr->mypolls.myds[n]; + if (myds==NULL) { + read_one_byte_from_pipe(n); + continue; + } + if (thr->mypolls.fds[n].revents==0) { + if (thr->poll_timeout_bool) { + check_timing_out_session(n); + } + } else { + check_for_invalid_fd(n); // this is designed to assert in case of failure + switch(myds->myds_type) { + // Note: this logic that was here was removed completely because we added mariadb client library. + case MYDS_LISTENER: + // we got a new connection! + thr->listener_handle_new_connection(myds,n); + continue; + break; + default: + break; + } + // data on exiting connection + bool rc = thr->process_data_on_data_stream(myds, n); + if (rc==false) { + n--; + } + } + } +} + + +template +void Base_Thread::read_one_byte_from_pipe(unsigned int n) { + T* thr = static_cast(this); + if (thr->mypolls.fds[n].revents) { + unsigned char c; + if (read(thr->mypolls.fds[n].fd, &c, 1)==-1) {// read just one byte + proxy_error("Error during read from signal_all_threads()\n"); + } + proxy_debug(PROXY_DEBUG_GENERIC,3, "Got signal from admin , done nothing\n"); + //fprintf(stderr,"Got signal from admin , done nothing\n"); // FIXME: this is just the skeleton for issue #253 + if (c) { + // we are being signaled to sleep for some ms. Before going to sleep we also release the mutex + pthread_mutex_unlock(&thr->thread_mutex); + usleep(c*1000); + pthread_mutex_lock(&thr->thread_mutex); + // we enter in maintenance loop only if c is set + // when threads are signaling each other, there is no need to set maintenance_loop + maintenance_loop=true; + } + } +} + +template +void Base_Thread::tune_timeout_for_myds_needs_pause(DS * myds) { + T* thr = static_cast(this); + if (myds->wait_until > curtime) { + if (thr->mypolls.poll_timeout==0 || (myds->wait_until - curtime < thr->mypolls.poll_timeout) ) { + thr->mypolls.poll_timeout= myds->wait_until - curtime; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Session=%p , poll_timeout=%u , wait_until=%llu , curtime=%llu\n", myds->sess, thr->mypolls.poll_timeout, myds->wait_until, curtime); + } + } +} + +template +void Base_Thread::tune_timeout_for_session_needs_pause(DS * myds) { + T* thr = static_cast(this); + if (thr->mypolls.poll_timeout==0 || (myds->sess->pause_until - curtime < thr->mypolls.poll_timeout) ) { + thr->mypolls.poll_timeout= myds->sess->pause_until - curtime; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Session=%p , poll_timeout=%u , pause_until=%llu , curtime=%llu\n", myds->sess, thr->mypolls.poll_timeout, myds->sess->pause_until, curtime); + } +} + +template +void Base_Thread::configure_pollout(DS * myds, unsigned int n) { + T* thr = static_cast(this); + if (myds->myds_type==MYDS_FRONTEND && myds->DSS==STATE_SLEEP && myds->sess && myds->sess->status==WAITING_CLIENT_DATA) { + myds->set_pollout(); + } else { + if (myds->DSS > STATE_MARIADB_BEGIN && myds->DSS < STATE_MARIADB_END) { + thr->mypolls.fds[n].events = POLLIN; + if (thr->mypolls.myds[n]->myconn->async_exit_status & MYSQL_WAIT_WRITE) + thr->mypolls.fds[n].events |= POLLOUT; + } else { + myds->set_pollout(); + } + } + if (unlikely(myds->sess->pause_until > curtime)) { + if (myds->myds_type==MYDS_FRONTEND) { + myds->remove_pollout(); + } + if (myds->myds_type==MYDS_BACKEND) { + if constexpr (std::is_same_v) { + if (pgsql_thread___throttle_ratio_server_to_client) { + thr->mypolls.fds[n].events = 0; + } + } else if constexpr (std::is_same_v) { + if (mysql_thread___throttle_ratio_server_to_client) { + thr->mypolls.fds[n].events = 0; + } + } else { + assert(0); + } + } + } + if (myds->myds_type==MYDS_BACKEND) { + set_backend_to_be_skipped_if_frontend_is_slow(myds, n); + } +} + +template +bool Base_Thread::set_backend_to_be_skipped_if_frontend_is_slow(DS * myds, unsigned int n) { + T* thr = static_cast(this); + if (myds->sess && myds->sess->client_myds && myds->sess->mirror==false) { + // we pause receiving from backend at mysql_thread___threshold_resultset_size * 8 + // but assuming that client isn't completely blocked, we will stop checking for data + // only at mysql_thread___threshold_resultset_size * 4 + if constexpr (std::is_same_v) { + unsigned int buffered_data = 0; + buffered_data = myds->sess->client_myds->PSarrayOUT->len * PGSQL_RESULTSET_BUFLEN; + buffered_data += myds->sess->client_myds->resultset->len * PGSQL_RESULTSET_BUFLEN; + if (buffered_data > (unsigned int)pgsql_thread___threshold_resultset_size * 4) { + thr->mypolls.fds[n].events = 0; + return true; + } + } else if constexpr (std::is_same_v) { + unsigned int buffered_data = 0; + buffered_data = myds->sess->client_myds->PSarrayOUT->len * RESULTSET_BUFLEN; + buffered_data += myds->sess->client_myds->resultset->len * RESULTSET_BUFLEN; + if (buffered_data > (unsigned int)mysql_thread___threshold_resultset_size * 4) { + thr->mypolls.fds[n].events = 0; + return true; + } + } + else { + assert(0); + } + + } + return false; +} + +#ifdef IDLE_THREADS +/** + * @brief Moves a session to the idle session array if it meets the idle criteria. + * + * This function checks if a session should be moved to the idle session array based on its idle time + * and other conditions. If the session meets the idle criteria, it is moved to the idle session array. + * + * @param myds Pointer to the MySQL data stream associated with the session. + * @param n The index of the session in the poll array. + * @return True if the session is moved to the idle session array, false otherwise. + */ +template +bool Base_Thread::move_session_to_idle_mysql_sessions(DS * myds, unsigned int n) { + T* thr = static_cast(this); + unsigned long long _tmp_idle = thr->mypolls.last_recv[n] > thr->mypolls.last_sent[n] ? thr->mypolls.last_recv[n] : thr->mypolls.last_sent[n] ; + + int session_idle_ms = 0; + + if constexpr (std::is_same_v) { + session_idle_ms = pgsql_thread___session_idle_ms; + } else if constexpr (std::is_same_v) { + session_idle_ms = mysql_thread___session_idle_ms; + } else { + assert(0); + } + + if (_tmp_idle < ( (curtime > (unsigned int)session_idle_ms * 1000) ? (curtime - session_idle_ms * 1000) : 0)) { + // make sure data stream has no pending data out and session is not throttled (#1939) + // because epoll thread does not handle data stream with data out + if (myds->sess->client_myds == myds && !myds->available_data_out() && myds->sess->pause_until <= curtime) { + //unsigned int j; + bool has_backends = myds->sess->has_any_backend(); + if (has_backends==false) { + unsigned long long idle_since = curtime - myds->sess->IdleTime(); + thr->mypolls.remove_index_fast(n); + myds->mypolls=NULL; + unsigned int i = find_session_idx_in_mysql_sessions(myds->sess); + myds->sess->thread=NULL; + thr->unregister_session(i); + myds->sess->idle_since = idle_since; + thr->idle_mysql_sessions->add(myds->sess); + return true; + } + } + } + return false; +} +#endif // IDLE_THREADS + +template +unsigned int Base_Thread::find_session_idx_in_mysql_sessions(S * sess) { + T* thr = static_cast(this); + unsigned int i=0; + for (i=0;ilen;i++) { + S *mysess=(S *)thr->mysql_sessions->index(i); + if (mysess==sess) { + return i; + } + } + return i; +} + +template +void Base_Thread::ProcessAllMyDS_BeforePoll() { + T* thr = static_cast(this); + bool check_if_move_to_idle_thread = false; +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + + int session_idle_ms = 0; + + if constexpr (std::is_same_v) { + session_idle_ms = pgsql_thread___session_idle_ms; + } else if constexpr (std::is_same_v) { + session_idle_ms = mysql_thread___session_idle_ms; + } else { + assert(0); + } + + if (curtime > last_move_to_idle_thread_time + (unsigned long long)session_idle_ms * 1000) { + last_move_to_idle_thread_time=curtime; + check_if_move_to_idle_thread=true; + } + } +#endif + for (unsigned int n = 0; n < thr->mypolls.len; n++) { + auto * myds=thr->mypolls.myds[n]; + thr->mypolls.fds[n].revents=0; + if (myds) { +#ifdef IDLE_THREADS + if (check_if_move_to_idle_thread == true) { + // here we try to move it to the maintenance thread + if (myds->myds_type==MYDS_FRONTEND && myds->sess) { + if (myds->DSS==STATE_SLEEP && myds->sess->status==WAITING_CLIENT_DATA) { + if (move_session_to_idle_mysql_sessions(myds, n)) { + n--; // compensate mypolls.remove_index_fast(n) and n++ of loop + continue; + } + } + } + } +#endif // IDLE_THREADS + if (unlikely(myds->wait_until)) { + tune_timeout_for_myds_needs_pause(myds); + } + if (myds->sess) { + if (unlikely(myds->sess->pause_until > 0)) { + tune_timeout_for_session_needs_pause(myds); + } + } + myds->revents=0; + if (myds->myds_type!=MYDS_LISTENER) { + configure_pollout(myds, n); + } + } + proxy_debug(PROXY_DEBUG_NET,1,"Poll for DataStream=%p will be called with FD=%d and events=%d\n", thr->mypolls.myds[n], thr->mypolls.fds[n].fd, thr->mypolls.fds[n].events); + } +} + +template +void Base_Thread::run_SetAllSession_ToProcess0() { + T* thr = static_cast(this); + unsigned int n; +#ifdef IDLE_THREADS + // @note: in MySQL_Thread::run we have: bool idle_maintenance_thread=epoll_thread; + // Thus idle_maintenance_thread and epoll_thread are equivalent. + if (epoll_thread==false) { +#endif // IDLE_THREADS + for (n=0; nlen; n++) { + S *_sess=(S *)mysql_sessions->index(n); + _sess->to_process=0; + } +#ifdef IDLE_THREADS + } +#endif // IDLE_THREADS +} diff --git a/lib/ClickHouse_Authentication.cpp b/lib/ClickHouse_Authentication.cpp index 1ba0062072..ae41665f6f 100644 --- a/lib/ClickHouse_Authentication.cpp +++ b/lib/ClickHouse_Authentication.cpp @@ -382,8 +382,11 @@ bool ClickHouse_Authentication::exists(char * username) { } */ -char * ClickHouse_Authentication::lookup(char * username, enum cred_username_type usertype, bool *use_ssl, int *default_hostgroup, char **default_schema, bool *schema_locked, bool *transaction_persistent, bool *fast_forward, int *max_connections, void **sha1_pass) { - char *ret=NULL; +ch_account_details_t ClickHouse_Authentication::lookup( + char* username, enum cred_username_type usertype, const ch_dup_account_details_t& dup_details +) { + ch_account_details_t ret {}; + uint64_t hash1, hash2; SpookyHash myhash; myhash.Init(1,2); @@ -397,32 +400,39 @@ char * ClickHouse_Authentication::lookup(char * username, enum cred_username_typ #else spin_rdlock(&cg.lock); #endif - std::map::iterator lookup; - lookup = cg.bt_map.find(hash1); + std::map::iterator lookup = cg.bt_map.find(hash1); + if (lookup != cg.bt_map.end()) { - ch_account_details_t *ad=lookup->second; - ret=l_strdup(ad->password); - if (use_ssl) *use_ssl=ad->use_ssl; - if (default_hostgroup) *default_hostgroup=ad->default_hostgroup; - if (default_schema) *default_schema=l_strdup(ad->default_schema); - if (schema_locked) *schema_locked=ad->schema_locked; - if (transaction_persistent) *transaction_persistent=ad->transaction_persistent; - if (fast_forward) *fast_forward=ad->fast_forward; - if (max_connections) *max_connections=ad->max_connections; - if (sha1_pass) { + ch_account_details_t* ad = lookup->second; + + ret.password = strdup(ad->password); + ret.use_ssl = ad->use_ssl; + ret.default_hostgroup = ad->default_hostgroup; + + if (dup_details.default_schema) { + ret.default_schema = l_strdup(ad->default_schema); + } + + ret.schema_locked = ad->schema_locked; + ret.transaction_persistent = ad->transaction_persistent; + ret.fast_forward = ad->fast_forward; + ret.max_connections = ad->max_connections; + + if (dup_details.sha1_pass) { if (ad->sha1_pass) { - *sha1_pass=malloc(SHA_DIGEST_LENGTH); - memcpy(*sha1_pass,ad->sha1_pass,SHA_DIGEST_LENGTH); + ret.sha1_pass = malloc(SHA_DIGEST_LENGTH); + memcpy(ret.sha1_pass,ad->sha1_pass,SHA_DIGEST_LENGTH); } } } + #ifdef PROXYSQL_AUTH_PTHREAD_MUTEX pthread_rwlock_unlock(&cg.lock); #else spin_rdunlock(&cg.lock); #endif - return ret; + return ret; } bool ClickHouse_Authentication::_reset(enum cred_username_type usertype) { diff --git a/lib/ClickHouse_Server.cpp b/lib/ClickHouse_Server.cpp index fff779f0bc..28ef34171c 100644 --- a/lib/ClickHouse_Server.cpp +++ b/lib/ClickHouse_Server.cpp @@ -5,11 +5,12 @@ #include "re2/re2.h" #include "re2/regexp.h" #include "proxysql.h" +#include "clickhouse/client.h" #include "cpp.h" #include "MySQL_Logger.hpp" #include "MySQL_Data_Stream.h" -#include "query_processor.h" +#include "MySQL_Query_Processor.h" #include #include @@ -443,7 +444,7 @@ static int __ClickHouse_Server_refresh_interval=1000; extern Query_Cache *GloQC; extern ClickHouse_Authentication *GloClickHouseAuth; extern ProxySQL_Admin *GloAdmin; -extern Query_Processor *GloQPro; +extern MySQL_Query_Processor* GloMyQPro; extern MySQL_Threads_Handler *GloMTH; extern MySQL_Logger *GloMyLogger; extern MySQL_Monitor *GloMyMon; @@ -585,7 +586,7 @@ class sqlite3server_main_loop_listeners { static sqlite3server_main_loop_listeners S_amll; -void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { +void ClickHouse_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *pkt) { char *error=NULL; int cols; int affected_rows; @@ -612,7 +613,7 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t query_no_space[query_no_space_length]=0; } - proxy_debug(PROXY_DEBUG_SQLITE, 4, "Received query on Session %p , thread_session_id %u : %s\n", sess, sess->thread_session_id, query_no_space); + proxy_debug(PROXY_DEBUG_SQLITE, 4, "Received query on Session %p , thread_session_id %u : %s\n", (MySQL_Session*)sess, sess->thread_session_id, query_no_space); if (sess->session_type == PROXYSQL_SESSION_CLICKHOUSE) { @@ -1340,6 +1341,7 @@ bool ClickHouse_Session::init() { hostname = GloClickHouseServer->get_variable((char *)"hostname"); port = GloClickHouseServer->get_variable((char *)"port"); try { + clickhouse::ClientOptions co; co.SetHost(hostname); co.SetPort(atoi(port)); co.SetCompressionMethod(CompressionMethod::None); @@ -1393,9 +1395,9 @@ static void *child_mysql(void *arg) { sqlite_sess->init(); mysql_thr->gen_args = (void *)sqlite_sess; - GloQPro->init_thread(); + GloMyQPro->init_thread(); mysql_thr->refresh_variables(); - sess=mysql_thr->create_new_session_and_client_data_stream(client); + sess=mysql_thr->create_new_session_and_client_data_stream(client); sess->thread=mysql_thr; sess->session_type = PROXYSQL_SESSION_CLICKHOUSE; sess->handler_function=ClickHouse_Server_session_handler; diff --git a/lib/Makefile b/lib/Makefile index 88aeff654e..d80d97b986 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -57,10 +57,21 @@ PROMETHEUS_PATH := $(DEPS_PATH)/prometheus-cpp/prometheus-cpp PROMETHEUS_IDIR := $(PROMETHEUS_PATH)/pull/include -I$(PROMETHEUS_PATH)/core/include PROMETHEUS_LDIR := $(PROMETHEUS_PATH)/lib +LIBUSUAL_PATH=$(DEPS_PATH)/libusual/libusual +LIBUSUAL_IDIR=$(LIBUSUAL_PATH) +LIBUSUAL_LDIR=$(LIBUSUAL_PATH)/.libs/ + +LIBSCRAM_PATH=$(DEPS_PATH)/libscram/ +LIBSCRAM_IDIR=$(LIBSCRAM_PATH)/include/ +LIBSCRAM_LDIR=$(LIBSCRAM_PATH)/lib/ + +POSTGRES_PATH=$(DEPS_PATH)/postgresql/postgresql/src/ +POSTGRES_IFACE=$(POSTGRES_PATH)/interfaces/libpq/ -I$(POSTGRES_PATH)/include +POSTGRES_LDIR=$(POSTGRES_IFACES)/libpq/ IDIR := ../include -IDIRS := -I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(RE2_IDIR) -I$(SQLITE3_DIR) -I$(PCRE_PATH) -I/usr/local/include -I$(CLICKHOUSE_CPP_DIR) -I$(CLICKHOUSE_CPP_DIR)/contrib/ $(MICROHTTPD_IDIR) $(LIBHTTPSERVER_IDIR) $(LIBINJECTION_IDIR) -I$(CURL_IDIR) -I$(EV_DIR) -I$(SSL_IDIR) -I$(PROMETHEUS_IDIR) +IDIRS := -I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(RE2_IDIR) -I$(SQLITE3_DIR) -I$(PCRE_PATH) -I/usr/local/include -I$(CLICKHOUSE_CPP_DIR) -I$(CLICKHOUSE_CPP_DIR)/contrib/ $(MICROHTTPD_IDIR) $(LIBHTTPSERVER_IDIR) $(LIBINJECTION_IDIR) -I$(CURL_IDIR) -I$(EV_DIR) -I$(SSL_IDIR) -I$(PROMETHEUS_IDIR) -I$(LIBUSUAL_IDIR) -I$(LIBSCRAM_IDIR) -I$(POSTGRES_IFACE) ifeq ($(UNAME_S),Linux) IDIRS += -I$(COREDUMPER_IDIR) endif @@ -125,13 +136,19 @@ MYCXXFLAGS := $(STDCPP) $(MYCFLAGS) $(PSQLCH) $(ENABLE_EPOLL) default: libproxysql.a .PHONY: default -_OBJ_CXX := ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo ProxySQL_Config.oo ProxySQL_Restapi.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo MySQL_PreparedStatement.oo ProxySQL_Cluster.oo ClickHouse_Authentication.oo ClickHouse_Server.oo ProxySQL_Statistics.oo Chart_bundle_js.oo ProxySQL_HTTP_Server.oo ProxySQL_RESTAPI_Server.oo font-awesome.min.css.oo main-bundle.min.css.oo set_parser.oo MySQL_Variables.oo c_tokenizer.oo proxysql_utils.oo proxysql_coredump.oo proxysql_sslkeylog.oo \ +_OBJ_CXX := ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo MySQL_Query_Processor.oo PgSQL_Query_Processor.oo ProxySQL_Admin.oo ProxySQL_Config.oo ProxySQL_Restapi.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo MySQL_PreparedStatement.oo ProxySQL_Cluster.oo ClickHouse_Authentication.oo ClickHouse_Server.oo ProxySQL_Statistics.oo Chart_bundle_js.oo ProxySQL_HTTP_Server.oo ProxySQL_RESTAPI_Server.oo font-awesome.min.css.oo main-bundle.min.css.oo set_parser.oo MySQL_Variables.oo c_tokenizer.oo proxysql_utils.oo proxysql_coredump.oo proxysql_sslkeylog.oo \ sha256crypt.oo \ + BaseSrvList.oo BaseHGC.oo Base_HostGroups_Manager.oo \ QP_rule_text.oo QP_query_digest_stats.oo \ - GTID_Server_Data.oo MyHGC.oo MySrvConnList.oo MySrvList.oo MySrvC.oo \ + GTID_Server_Data.oo MyHGC.oo MySrvConnList.oo MySrvC.oo \ MySQL_encode.oo MySQL_ResultSet.oo \ + ProxySQL_Admin_Tests.oo ProxySQL_Admin_Tests2.oo ProxySQL_Admin_Scheduler.oo ProxySQL_Admin_Disk_Upgrade.oo ProxySQL_Admin_Stats.oo \ + Admin_Handler.oo Admin_FlushVariables.oo Admin_Bootstrap.oo \ + Base_Session.oo Base_Thread.oo \ proxy_protocol_info.oo \ - proxysql_find_charset.oo ProxySQL_Poll.oo + proxysql_find_charset.oo ProxySQL_Poll.oo \ + PgSQL_Protocol.oo PgSQL_Thread.oo PgSQL_Data_Stream.oo PgSQL_Session.oo PgSQL_Variables.oo PgSQL_HostGroups_Manager.oo PgSQL_Connection.oo PgSQL_Backend.oo PgSQL_Logger.oo PgSQL_Authentication.oo PgSQL_Error_Helper.oo + OBJ_CXX := $(patsubst %,$(ODIR)/%,$(_OBJ_CXX)) HEADERS := ../include/*.h ../include/*.hpp diff --git a/lib/MyHGC.cpp b/lib/MyHGC.cpp index 8742db240c..8af6956d17 100644 --- a/lib/MyHGC.cpp +++ b/lib/MyHGC.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include "MySQL_HostGroups_Manager.h" #ifdef TEST_AURORA @@ -7,51 +11,6 @@ static unsigned long long array_mysrvc_cands = 0; extern MySQL_Threads_Handler *GloMTH; -MyHGC::MyHGC(int _hid) { - hid=_hid; - mysrvs=new MySrvList(this); - current_time_now = 0; - new_connections_now = 0; - attributes.initialized = false; - reset_attributes(); - // Uninitialized server defaults. Should later be initialized via 'mysql_hostgroup_attributes'. - servers_defaults.weight = -1; - servers_defaults.max_connections = -1; - servers_defaults.use_ssl = -1; - num_online_servers.store(0, std::memory_order_relaxed);; - last_log_time_num_online_servers = 0; -} - -void MyHGC::reset_attributes() { - if (attributes.initialized == false) { - attributes.init_connect = NULL; - attributes.comment = NULL; - attributes.ignore_session_variables_text = NULL; - } - attributes.initialized = true; - attributes.configured = false; - attributes.max_num_online_servers = 1000000; - attributes.throttle_connections_per_sec = 1000000; - attributes.autocommit = -1; - attributes.free_connections_pct = 10; - attributes.handle_warnings = -1; - attributes.monitor_slave_lag_when_null = -1; - attributes.multiplex = true; - attributes.connection_warming = false; - free(attributes.init_connect); - attributes.init_connect = NULL; - free(attributes.comment); - attributes.comment = NULL; - free(attributes.ignore_session_variables_text); - attributes.ignore_session_variables_text = NULL; - attributes.ignore_session_variables_json = json(); -} - -MyHGC::~MyHGC() { - reset_attributes(); // free all memory - delete mysrvs; -} - MySrvC *MyHGC::get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms, MySQL_Session *sess) { MySrvC *mysrvc=NULL; unsigned int j; @@ -393,31 +352,3 @@ MySrvC *MyHGC::get_random_MySrvC(char * gtid_uuid, uint64_t gtid_trxid, int max_ #endif // TEST_AURORA return NULL; // if we reach here, we couldn't find any target } - -void MyHGC::refresh_online_server_count() { - if (__sync_fetch_and_add(&glovars.shutdown, 0) != 0) - return; -#ifdef DEBUG - assert(MyHGM->is_locked); -#endif - unsigned int online_servers_count = 0; - for (unsigned int i = 0; i < mysrvs->servers->len; i++) { - MySrvC* mysrvc = (MySrvC*)mysrvs->servers->index(i); - if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE) { - online_servers_count++; - } - } - num_online_servers.store(online_servers_count, std::memory_order_relaxed); -} - -void MyHGC::log_num_online_server_count_error() { - const time_t curtime = time(NULL); - // if this is the first time the method is called or if more than 10 seconds have passed since the last log - if (last_log_time_num_online_servers == 0 || - ((curtime - last_log_time_num_online_servers) > 10)) { - last_log_time_num_online_servers = curtime; - proxy_error( - "Number of online servers detected in a hostgroup exceeds the configured maximum online servers. hostgroup:%u, num_online_servers:%u, max_online_servers:%u\n", - hid, num_online_servers.load(std::memory_order_relaxed), attributes.max_num_online_servers); - } -} diff --git a/lib/MySQL_Authentication.cpp b/lib/MySQL_Authentication.cpp index 6ce2497d3e..59e4719dd1 100644 --- a/lib/MySQL_Authentication.cpp +++ b/lib/MySQL_Authentication.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +//using json = nlohmann::json; +#define PROXYJSON + //#include "btree_map.h" #include "proxysql.h" #include "cpp.h" @@ -10,6 +14,41 @@ #define SPOOKYV2 #endif +void free_account_details(account_details_t& ad) { + if (ad.password) { + free(ad.password); + ad.password = nullptr; + } + if (ad.sha1_pass) { + free(ad.sha1_pass); + ad.sha1_pass=NULL; + } + if (ad.password) { + free(ad.password); + ad.password = nullptr; + } + if (ad.clear_text_password[PASSWORD_TYPE::PRIMARY]) { + free(ad.clear_text_password[PASSWORD_TYPE::PRIMARY]); + ad.clear_text_password[PASSWORD_TYPE::PRIMARY] = nullptr; + } + if (ad.clear_text_password[PASSWORD_TYPE::ADDITIONAL]) { + free(ad.clear_text_password[PASSWORD_TYPE::ADDITIONAL]); + ad.clear_text_password[PASSWORD_TYPE::ADDITIONAL] = nullptr; + } + if (ad.default_schema) { + free(ad.default_schema); + ad.default_schema = nullptr; + } + if (ad.attributes) { + free(ad.attributes); + ad.attributes = nullptr; + } + if (ad.comment) { + free(ad.comment); + ad.comment = nullptr; + } +} + MySQL_Authentication::MySQL_Authentication() { #ifdef DEBUG if (glovars.has_debug==false) { @@ -111,9 +150,13 @@ bool MySQL_Authentication::add(char * username, char * password, enum cred_usern free(ad->sha1_pass); ad->sha1_pass=NULL; } - if (ad->clear_text_password) { - free(ad->clear_text_password); - ad->clear_text_password=NULL; + if (ad->clear_text_password[0]) { + free(ad->clear_text_password[0]); + ad->clear_text_password[0]=NULL; + } + if (ad->clear_text_password[1]) { + free(ad->clear_text_password[1]); + ad->clear_text_password[1]=NULL; } // FIXME: if the password is a clear text password, automatically generate sha1_pass and clear_text_password } @@ -201,7 +244,8 @@ bool MySQL_Authentication::add(char * username, char * password, enum cred_usern new_ad=true; ad->sha1_pass=NULL; ad->num_connections_used=0; - ad->clear_text_password = NULL; + ad->clear_text_password[0] = NULL; + ad->clear_text_password[1] = NULL; // FIXME: if the password is a clear text password, automatically generate sha1_pass and clear_text_password } @@ -241,7 +285,8 @@ unsigned int MySQL_Authentication::memory_usage() { if (ado->username) ret += strlen(ado->username) + 1; if (ado->password) ret += strlen(ado->password) + 1; if (ado->sha1_pass) ret += SHA_DIGEST_LENGTH; - if (ado->clear_text_password) ret += strlen(ado->clear_text_password) + 1; + if (ado->clear_text_password[0]) ret += strlen(ado->clear_text_password[0]) + 1; + if (ado->clear_text_password[1]) ret += strlen(ado->clear_text_password[1]) + 1; if (ado->default_schema) ret += strlen(ado->default_schema) + 1; if (ado->comment) ret += strlen(ado->comment) + 1; if (ado->attributes) ret += strlen(ado->attributes) + 1; @@ -255,7 +300,8 @@ unsigned int MySQL_Authentication::memory_usage() { if (ado->username) ret += strlen(ado->username) + 1; if (ado->password) ret += strlen(ado->password) + 1; if (ado->sha1_pass) ret += SHA_DIGEST_LENGTH; - if (ado->clear_text_password) ret += strlen(ado->clear_text_password) + 1; + if (ado->clear_text_password[0]) ret += strlen(ado->clear_text_password[0]) + 1; + if (ado->clear_text_password[1]) ret += strlen(ado->clear_text_password[1]) + 1; if (ado->default_schema) ret += strlen(ado->default_schema) + 1; if (ado->comment) ret += strlen(ado->comment) + 1; if (ado->attributes) ret += strlen(ado->attributes) + 1; @@ -307,7 +353,8 @@ int MySQL_Authentication::dump_all_users(account_details_t ***ads, bool _complet ad->num_connections_used=ado->num_connections_used; ad->password=strdup(ado->password); ad->sha1_pass=NULL; - ad->clear_text_password = NULL; + ad->clear_text_password[0] = NULL; + ad->clear_text_password[1] = NULL; ad->use_ssl=ado->use_ssl; ad->default_schema=strdup(ado->default_schema); ad->attributes=strdup(ado->attributes); @@ -329,7 +376,8 @@ int MySQL_Authentication::dump_all_users(account_details_t ***ads, bool _complet ad->username=strdup(ado->username); ad->password=strdup(ado->password); ad->sha1_pass=NULL; - ad->clear_text_password = NULL; + ad->clear_text_password[0] = NULL; + ad->clear_text_password[1] = NULL; ad->use_ssl=ado->use_ssl; ad->default_hostgroup=ado->default_hostgroup; ad->default_schema=strdup(ado->default_schema); @@ -358,7 +406,9 @@ int MySQL_Authentication::dump_all_users(account_details_t ***ads, bool _complet } -int MySQL_Authentication::increase_frontend_user_connections(char *username, int *mc) { +int MySQL_Authentication::increase_frontend_user_connections( + char *username, PASSWORD_TYPE::E passtype, int *mc +) { uint64_t hash1, hash2; SpookyHash *myhash=new SpookyHash(); myhash->Init(1,2); @@ -379,6 +429,10 @@ int MySQL_Authentication::increase_frontend_user_connections(char *username, int if (ad->max_connections > ad->num_connections_used) { ret=ad->max_connections-ad->num_connections_used; ad->num_connections_used++; + + if (passtype == PASSWORD_TYPE::ADDITIONAL) { + ad->num_connections_used_addl_pass += 1; + } } if (mc) { *mc=ad->max_connections; @@ -392,7 +446,7 @@ int MySQL_Authentication::increase_frontend_user_connections(char *username, int return ret; } -void MySQL_Authentication::decrease_frontend_user_connections(char *username) { +void MySQL_Authentication::decrease_frontend_user_connections(char *username, PASSWORD_TYPE::E passtype) { uint64_t hash1, hash2; SpookyHash *myhash=new SpookyHash(); myhash->Init(1,2); @@ -411,6 +465,10 @@ void MySQL_Authentication::decrease_frontend_user_connections(char *username) { account_details_t *ad=it->second; if (ad->num_connections_used > 0) { ad->num_connections_used--; + + if (passtype == PASSWORD_TYPE::ADDITIONAL) { + ad->num_connections_used_addl_pass -= 1; + } } } #ifdef PROXYSQL_AUTH_PTHREAD_MUTEX @@ -446,7 +504,8 @@ bool MySQL_Authentication::del(char * username, enum cred_username_type usertype free(ad->username); free(ad->password); if (ad->sha1_pass) { free(ad->sha1_pass); ad->sha1_pass=NULL; } - if (ad->clear_text_password) { free(ad->clear_text_password); ad->clear_text_password=NULL; } + if (ad->clear_text_password[0]) { free(ad->clear_text_password[0]); ad->clear_text_password[0]=NULL; } + if (ad->clear_text_password[1]) { free(ad->clear_text_password[1]); ad->clear_text_password[1]=NULL; } free(ad->default_schema); free(ad->attributes); free(ad->comment); @@ -497,7 +556,9 @@ bool MySQL_Authentication::set_SHA1(char * username, enum cred_username_type use return ret; }; -bool MySQL_Authentication::set_clear_text_password(char * username, enum cred_username_type usertype, const char *clear_text_password) { +bool MySQL_Authentication::set_clear_text_password( + char* username, enum cred_username_type usertype, const char* clear_text_password, PASSWORD_TYPE::E passtype +) { bool ret=false; uint64_t hash1, hash2; SpookyHash *myhash=new SpookyHash(); @@ -517,9 +578,22 @@ bool MySQL_Authentication::set_clear_text_password(char * username, enum cred_us lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; - if (ad->clear_text_password) { free(ad->clear_text_password); ad->clear_text_password=NULL; } - if (clear_text_password) { - ad->clear_text_password = strdup(clear_text_password); + if (passtype == PASSWORD_TYPE::PRIMARY) { + if (ad->clear_text_password[0]) { + free(ad->clear_text_password[0]); + ad->clear_text_password[0]=NULL; + } + if (clear_text_password) { + ad->clear_text_password[0] = strdup(clear_text_password); + } + } else { + if (ad->clear_text_password[1]) { + free(ad->clear_text_password[1]); + ad->clear_text_password[1]=NULL; + } + if (clear_text_password) { + ad->clear_text_password[1] = strdup(clear_text_password); + } } ret=true; } @@ -550,8 +624,11 @@ bool MySQL_Authentication::exists(char * username) { return ret; } -char * MySQL_Authentication::lookup(char * username, enum cred_username_type usertype, bool *use_ssl, int *default_hostgroup, char **default_schema, bool *schema_locked, bool *transaction_persistent, bool *fast_forward, int *max_connections, void **sha1_pass, char **attributes) { - char *ret=NULL; +account_details_t MySQL_Authentication::lookup( + char* username, enum cred_username_type usertype, const dup_account_details_t& dup_details +) { + account_details_t ret {}; + uint64_t hash1, hash2; SpookyHash myhash; myhash.Init(1,2); @@ -565,39 +642,53 @@ char * MySQL_Authentication::lookup(char * username, enum cred_username_type use #else spin_rdlock(&cg.lock); #endif - std::map::iterator lookup; - lookup = cg.bt_map.find(hash1); + std::map::iterator lookup = cg.bt_map.find(hash1); + if (lookup != cg.bt_map.end()) { - account_details_t *ad=lookup->second; - if (ad->clear_text_password == NULL) { - ret=strdup(ad->password); - } else { - // we return the best password we have - // if we were able to derive the clear text password, we provide that - ret=strdup(ad->clear_text_password); + account_details_t* ad = lookup->second; + + ret.password = strdup(ad->password); + + if (ad->clear_text_password[PASSWORD_TYPE::PRIMARY]) { + ret.clear_text_password[PASSWORD_TYPE::PRIMARY] = + strdup(ad->clear_text_password[PASSWORD_TYPE::PRIMARY]); + } + if (ad->clear_text_password[PASSWORD_TYPE::ADDITIONAL]) { + ret.clear_text_password[PASSWORD_TYPE::ADDITIONAL] = + strdup(ad->clear_text_password[PASSWORD_TYPE::ADDITIONAL]); } - if (use_ssl) *use_ssl=ad->use_ssl; - if (default_hostgroup) *default_hostgroup=ad->default_hostgroup; - if (default_schema) *default_schema=l_strdup(ad->default_schema); - if (schema_locked) *schema_locked=ad->schema_locked; - if (transaction_persistent) *transaction_persistent=ad->transaction_persistent; - if (fast_forward) *fast_forward=ad->fast_forward; - if (max_connections) *max_connections=ad->max_connections; - if (sha1_pass) { + + ret.use_ssl = ad->use_ssl; + ret.default_hostgroup = ad->default_hostgroup; + + if (dup_details.default_schema) { + ret.default_schema = l_strdup(ad->default_schema); + } + + ret.schema_locked = ad->schema_locked; + ret.transaction_persistent = ad->transaction_persistent; + ret.fast_forward = ad->fast_forward; + ret.max_connections = ad->max_connections; + + if (dup_details.sha1_pass) { if (ad->sha1_pass) { - *sha1_pass=malloc(SHA_DIGEST_LENGTH); - memcpy(*sha1_pass,ad->sha1_pass,SHA_DIGEST_LENGTH); + ret.sha1_pass = malloc(SHA_DIGEST_LENGTH); + memcpy(ret.sha1_pass,ad->sha1_pass,SHA_DIGEST_LENGTH); } } - if (attributes) *attributes=l_strdup(ad->attributes); + + if (dup_details.attributes) { + ret.attributes = l_strdup(ad->attributes); + } } + #ifdef PROXYSQL_AUTH_PTHREAD_MUTEX pthread_rwlock_unlock(&cg.lock); #else spin_rdunlock(&cg.lock); #endif - return ret; + return ret; } bool MySQL_Authentication::_reset(enum cred_username_type usertype) { @@ -618,7 +709,14 @@ bool MySQL_Authentication::_reset(enum cred_username_type usertype) { free(ad->username); free(ad->password); if (ad->sha1_pass) { free(ad->sha1_pass); ad->sha1_pass=NULL; } - if (ad->clear_text_password) { free(ad->clear_text_password); ad->clear_text_password=NULL; } + if (ad->clear_text_password[0]) { + free(ad->clear_text_password[0]); + ad->clear_text_password[0] = NULL; + } + if (ad->clear_text_password[1]) { + free(ad->clear_text_password[1]); + ad->clear_text_password[1] = NULL; + } free(ad->default_schema); free(ad->comment); free(ad->attributes); @@ -644,7 +742,7 @@ bool MySQL_Authentication::reset() { using std::map; -uint64_t compute_accounts_hash(const umap_auth& accs_map) { +static uint64_t compute_accounts_hash(const umap_auth& accs_map) { if (accs_map.size() == 0) { return 0; } @@ -698,7 +796,7 @@ uint64_t MySQL_Authentication::get_runtime_checksum() { return hashB+hashF; } -pair extract_accounts_details(MYSQL_RES* resultset, unique_ptr& all_users) { +static pair extract_accounts_details(MYSQL_RES* resultset, unique_ptr& all_users) { if (resultset == nullptr) { return { umap_auth {}, umap_auth {} }; } // The following order is assumed for the resulset received fields: diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index e0e90bf4d6..51f7531d9f 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include "MySQL_HostGroups_Manager.h" #include "proxysql.h" #include "cpp.h" @@ -53,10 +57,11 @@ class MySrvC; class MySrvList; class MyHGC; +const int MYSQL_ERRORS_STATS_FIELD_NUM = 11; + struct ev_io * new_connector(char *address, uint16_t gtid_port, uint16_t mysql_port); void * GTID_syncer_run(); - static int wait_for_mysql(MYSQL *mysql, int status) { struct pollfd pfd; int timeout, res; @@ -81,6 +86,49 @@ static int wait_for_mysql(MYSQL *mysql, int status) { } } +/** + * @brief Helper function used to try to extract a value from the JSON field 'servers_defaults'. + * + * @param j JSON object constructed from 'servers_defaults' field. + * @param hid Hostgroup for which the 'servers_defaults' is defined in 'mysql_hostgroup_attributes'. Used for + * error logging. + * @param key The key for the value to be extracted. + * @param val_check A validation function, checks if the value is within a expected range. + * + * @return The value extracted from the supplied JSON. In case of error '-1', and error cause is logged. + */ +template ::value, bool>::type = true> +T j_get_srv_default_int_val( + const json& j, uint32_t hid, const string& key, const function& val_check +) { + if (j.find(key) != j.end()) { + const json::value_t val_type = j[key].type(); + const char* type_name = j[key].type_name(); + + if (val_type == json::value_t::number_integer || val_type == json::value_t::number_unsigned) { + T val = j[key].get(); + + if (val_check(val)) { + return val; + } else { + proxy_error( + "Invalid value %ld supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." + " Value NOT UPDATED.\n", + static_cast(val), key.c_str(), hid + ); + } + } else { + proxy_error( + "Invalid type '%s'(%hhu) supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." + " Value NOT UPDATED.\n", + type_name, static_cast(val_type), key.c_str(), hid + ); + } + } + + return static_cast(-1); +} + //static void * HGCU_thread_run() { static void * HGCU_thread_run() { @@ -302,6 +350,12 @@ hg_metrics_map = std::make_tuple( { "status", "created" } } ), + std::make_tuple ( + p_hg_counter::client_connections_sha2cached, + "proxysql_client_connections_sha2cached_total", + "Total number of attempted client connections with known cached passwords.", + metric_tags {} + ), std::make_tuple ( p_hg_counter::client_connections_aborted, "proxysql_client_connections_total", @@ -471,6 +525,18 @@ hg_metrics_map = std::make_tuple( "proxysql_client_connections_connected", "Client connections that are currently connected.", metric_tags {} + ), + std::make_tuple ( + p_hg_gauge::client_connections_connected_prim, + "proxysql_client_connections_connected_primary", + "Client connections that are currently connected using primary password.", + metric_tags {} + ), + std::make_tuple ( + p_hg_gauge::client_connections_connected_addl, + "proxysql_client_connections_connected_additional", + "Client connections that are currently connected using additional password.", + metric_tags {} ) }, // prometheus dynamic counters @@ -578,8 +644,11 @@ hg_metrics_map = std::make_tuple( MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { status.client_connections=0; + status.client_connections_prim_pass=0; + status.client_connections_addl_pass=0; status.client_connections_aborted=0; status.client_connections_created=0; + status.client_connections_sha2cached=0; status.server_connections_connected=0; status.server_connections_aborted=0; status.server_connections_created=0; @@ -610,17 +679,17 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { status.access_denied_max_user_connections=0; status.select_for_update_or_equivalent=0; status.auto_increment_delay_multiplex=0; +#if 0 pthread_mutex_init(&readonly_mutex, NULL); +#endif // 0 pthread_mutex_init(&Group_Replication_Info_mutex, NULL); pthread_mutex_init(&Galera_Info_mutex, NULL); pthread_mutex_init(&AWS_Aurora_Info_mutex, NULL); -#ifdef MHM_PTHREAD_MUTEX +#if 0 pthread_mutex_init(&lock, NULL); -#else - spinlock_rwlock_init(&rwlock); -#endif admindb=NULL; // initialized only if needed mydb=new SQLite3DB(); +#endif // 0 #ifdef DEBUG mydb->open((char *)"file:mem_mydb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); #else @@ -709,21 +778,7 @@ MySQL_HostGroups_Manager::~MySQL_HostGroups_Manager() { ev_loop_destroy(gtid_ev_loop); if (gtid_ev_timer) free(gtid_ev_timer); -#ifdef MHM_PTHREAD_MUTEX pthread_mutex_destroy(&lock); -#endif -} - -// wrlock() is only required during commit() -void MySQL_HostGroups_Manager::wrlock() { -#ifdef MHM_PTHREAD_MUTEX - pthread_mutex_lock(&lock); -#else - spin_wrlock(&rwlock); -#endif -#ifdef DEBUG - is_locked = true; -#endif } void MySQL_HostGroups_Manager::p_update_mysql_error_counter(p_mysql_error_type err_type, unsigned int hid, char* address, uint16_t port, unsigned int code) { @@ -757,18 +812,6 @@ void MySQL_HostGroups_Manager::p_update_mysql_error_counter(p_mysql_error_type e pthread_mutex_unlock(&mysql_errors_mutex); } -void MySQL_HostGroups_Manager::wrunlock() { -#ifdef DEBUG - is_locked = false; -#endif -#ifdef MHM_PTHREAD_MUTEX - pthread_mutex_unlock(&lock); -#else - spin_wrunlock(&rwlock); -#endif -} - - void MySQL_HostGroups_Manager::wait_servers_table_version(unsigned v, unsigned w) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); @@ -870,29 +913,6 @@ int MySQL_HostGroups_Manager::servers_add(SQLite3_result *resultset) { return 0; } -/** - * @brief Execute a SQL query and retrieve the resultset. - * - * This function executes a SQL query using the provided query string and returns the resultset obtained from the - * database operation. It also provides an optional error parameter to capture any error messages encountered during - * query execution. - * - * @param query A pointer to a null-terminated string containing the SQL query to be executed. - * @param error A pointer to a char pointer where any error message encountered during query execution will be stored. - * Pass nullptr if error handling is not required. - * @return A pointer to a SQLite3_result object representing the resultset obtained from the query execution. This - * pointer may be nullptr if the query execution fails or returns an empty result. - */ -SQLite3_result * MySQL_HostGroups_Manager::execute_query(char *query, char **error) { - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - wrlock(); - mydb->execute_statement(query, error , &cols , &affected_rows , &resultset); - wrunlock(); - return resultset; -} - /** * @brief Calculate and update the checksum for a specified table in the database. * @@ -1076,7 +1096,7 @@ unique_ptr get_mysql_servers_v2() { return unique_ptr(resultset); } -void update_glovars_checksum_with_peers( +static void update_glovars_checksum_with_peers( ProxySQL_Checksum_Value& module_checksum, const string& new_checksum, const string& peer_checksum_value, @@ -1113,7 +1133,7 @@ void update_glovars_checksum_with_peers( * @param epoch The epoch to be preserved in case the supplied 'peer_checksum' matches the new computed * checksum. */ -void update_glovars_mysql_servers_checksum( +static void update_glovars_mysql_servers_checksum( const string& new_checksum, const runtime_mysql_servers_checksum_t& peer_checksum = {}, bool update_version = false @@ -1144,7 +1164,7 @@ void update_glovars_mysql_servers_checksum( * @param epoch The epoch to be preserved in case the supplied 'peer_checksum' matches the new computed * checksum. */ -void update_glovars_mysql_servers_v2_checksum( +static void update_glovars_mysql_servers_v2_checksum( const string& new_checksum, const mysql_servers_v2_checksum_t& peer_checksum = {}, bool update_version = false @@ -1238,6 +1258,10 @@ std::string MySQL_HostGroups_Manager::gen_global_mysql_servers_v2_checksum(uint6 return mysrvs_checksum; } +bool MySQL_HostGroups_Manager::commit() { + return commit({},{}); +} + bool MySQL_HostGroups_Manager::commit( const peer_runtime_mysql_servers_t& peer_runtime_mysql_servers, const peer_mysql_servers_v2_t& peer_mysql_servers_v2, @@ -2155,7 +2179,7 @@ void MySQL_HostGroups_Manager::update_table_mysql_servers_for_monitor(bool lock) int cols = 0; int affected_rows = 0; SQLite3_result* resultset = NULL; - char* query = const_cast("SELECT hostname, port, status, use_ssl FROM mysql_servers WHERE status != 3 GROUP BY hostname, port"); + const char* query = "SELECT hostname, port, status, use_ssl FROM mysql_servers WHERE status != 3 GROUP BY hostname, port"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); @@ -2228,78 +2252,6 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name) return resultset; } -/** - * @brief Create a new MySQL host group container. - * - * This function creates a new instance of the MySQL host group container (`MyHGC`) with - * the specified host group ID and returns a pointer to it. - * - * @param _hid The host group ID for the new container. - * @return A pointer to the newly created `MyHGC` instance. - */ -MyHGC * MySQL_HostGroups_Manager::MyHGC_create(unsigned int _hid) { - MyHGC *myhgc=new MyHGC(_hid); - return myhgc; -} - -/** - * @brief Find a MySQL host group container by host group ID. - * - * This function searches for a MySQL host group container with the specified host group ID - * in the list of host groups. If found, it returns a pointer to the container; otherwise, - * it returns a null pointer. - * - * @param _hid The host group ID to search for. - * @return A pointer to the found `MyHGC` instance if found; otherwise, a null pointer. - */ -MyHGC * MySQL_HostGroups_Manager::MyHGC_find(unsigned int _hid) { - if (MyHostGroups->len < 100) { - // for few HGs, we use the legacy search - for (unsigned int i=0; ilen; i++) { - MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); - if (myhgc->hid==_hid) { - return myhgc; - } - } - } else { - // for a large number of HGs, we use the unordered_map - // this search is slower for a small number of HGs, therefore we use - // it only for large number of HGs - std::unordered_map::const_iterator it = MyHostGroups_map.find(_hid); - if (it != MyHostGroups_map.end()) { - MyHGC *myhgc = it->second; - return myhgc; - } - } - return NULL; -} - -/** - * @brief Lookup or create a MySQL host group container by host group ID. - * - * This function looks up a MySQL host group container with the specified host group ID. If - * found, it returns a pointer to the existing container; otherwise, it creates a new container - * with the specified host group ID, adds it to the list of host groups, and returns a pointer - * to it. - * - * @param _hid The host group ID to lookup or create. - * @return A pointer to the found or newly created `MyHGC` instance. - * @note The function assertion fails if a newly created container is not found. - */ -MyHGC * MySQL_HostGroups_Manager::MyHGC_lookup(unsigned int _hid) { - MyHGC *myhgc=NULL; - myhgc=MyHGC_find(_hid); - if (myhgc==NULL) { - myhgc=MyHGC_create(_hid); - } else { - return myhgc; - } - assert(myhgc); - MyHostGroups->add(myhgc); - MyHostGroups_map.emplace(_hid,myhgc); - return myhgc; -} - void MySQL_HostGroups_Manager::increase_reset_counter() { wrlock(); status.myconnpoll_reset++; @@ -4152,7 +4104,10 @@ void MySQL_HostGroups_Manager::p_update_metrics() { // Update *client_connections* related metrics p_update_counter(status.p_counter_array[p_hg_counter::client_connections_created], status.client_connections_created); p_update_counter(status.p_counter_array[p_hg_counter::client_connections_aborted], status.client_connections_aborted); + p_update_counter(status.p_counter_array[p_hg_counter::client_connections_sha2cached], status.client_connections_sha2cached); status.p_gauge_array[p_hg_gauge::client_connections_connected]->Set(status.client_connections); + status.p_gauge_array[p_hg_gauge::client_connections_connected_prim]->Set(status.client_connections_prim_pass); + status.p_gauge_array[p_hg_gauge::client_connections_connected_addl]->Set(status.client_connections_addl_pass); // Update *acess_denied* related metrics p_update_counter(status.p_counter_array[p_hg_counter::access_denied_wrong_password], status.access_denied_wrong_password); @@ -5069,7 +5024,7 @@ bool Galera_Info::update(int b, int r, int o, int mw, int mtb, bool _a, int _w, * @brief Dumps to stderr the current info for the monitored Galera hosts ('Galera_Hosts_Map'). * @details No action if `mysql_thread___hostgroup_manager_verbose=0`. */ -void print_galera_nodes_last_status() { +static void print_galera_nodes_last_status() { if (!mysql_thread___hostgroup_manager_verbose) return; std::unique_ptr result { new SQLite3_result(13) }; @@ -5981,7 +5936,7 @@ class MySQL_Errors_stats { } char **get_row() { char buf[128]; - char **pta=(char **)malloc(sizeof(char *)*11); + char **pta=(char **)malloc(sizeof(char *)*MYSQL_ERRORS_STATS_FIELD_NUM); sprintf(buf,"%d",hostgroup); pta[0]=strdup(buf); assert(hostname); @@ -6023,7 +5978,7 @@ class MySQL_Errors_stats { } void free_row(char **pta) { int i; - for (i=0;i<11;i++) { + for (i=0;iadd_column_definition(SQLITE_TEXT,"hid"); result->add_column_definition(SQLITE_TEXT,"hostname"); @@ -6404,14 +6359,16 @@ void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() { if (myhgc->attributes.ignore_session_variables_text == NULL) { myhgc->attributes.ignore_session_variables_text = strdup(ignore_session_variables); if (strlen(ignore_session_variables) != 0) { // only if there is a valid JSON - myhgc->attributes.ignore_session_variables_json = json::parse(ignore_session_variables); + if (myhgc->attributes.ignore_session_variables_json != nullptr) { delete myhgc->attributes.ignore_session_variables_json; } + myhgc->attributes.ignore_session_variables_json = new json(json::parse(ignore_session_variables)); } } else { if (strcmp(myhgc->attributes.ignore_session_variables_text, ignore_session_variables) != 0) { free(myhgc->attributes.ignore_session_variables_text); myhgc->attributes.ignore_session_variables_text = strdup(ignore_session_variables); if (strlen(ignore_session_variables) != 0) { // only if there is a valid JSON - myhgc->attributes.ignore_session_variables_json = json::parse(ignore_session_variables); + if (myhgc->attributes.ignore_session_variables_json != nullptr) { delete myhgc->attributes.ignore_session_variables_json; } + myhgc->attributes.ignore_session_variables_json = new json(json::parse(ignore_session_variables)); } // TODO: assign the variables } diff --git a/lib/MySQL_Logger.cpp b/lib/MySQL_Logger.cpp index 6dd183846c..5208a79bbf 100644 --- a/lib/MySQL_Logger.cpp +++ b/lib/MySQL_Logger.cpp @@ -1,18 +1,19 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include #include "proxysql.h" #include "cpp.h" #include "MySQL_Data_Stream.h" -#include "query_processor.h" +#include "MySQL_Query_Processor.h" #include "MySQL_PreparedStatement.h" #include "MySQL_Logger.hpp" #include #include -#include "../deps/json/json.hpp" -using json = nlohmann::json; - #ifdef DEBUG #define DEB "_DEBUG" #else @@ -465,7 +466,7 @@ uint64_t MySQL_Event::write_query_format_2_json(std::fstream *f) { return total_bytes; // always 0 } -extern Query_Processor *GloQPro; +extern MySQL_Query_Processor* GloMyQPro; MySQL_Logger::MySQL_Logger() { events.enabled=false; @@ -724,7 +725,7 @@ void MySQL_Logger::log_request(MySQL_Session *sess, MySQL_Data_Stream *myds) { uint64_t query_digest = 0; if (sess->status != PROCESSING_STMT_EXECUTE) { - query_digest = GloQPro->get_digest(&sess->CurrentQuery.QueryParserArgs); + query_digest = GloMyQPro->get_digest(&sess->CurrentQuery.QueryParserArgs); } else { query_digest = sess->CurrentQuery.stmt_info->digest; } diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 3caf199e08..707ae7e7cc 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -7,6 +7,11 @@ 0.2.0902 * original implementation */ + +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include #include #include @@ -34,6 +39,15 @@ #endif /* DEBUG */ #define MYSQL_MONITOR_VERSION "2.0.1226" DEB +#ifdef DEBUG +//#define VALGRIND_ENABLE_ERROR_REPORTING +//#define VALGRIND_DISABLE_ERROR_REPORTING +#include "valgrind.h" +#else +#define VALGRIND_ENABLE_ERROR_REPORTING +#define VALGRIND_DISABLE_ERROR_REPORTING +#endif // DEBUG + extern ProxySQL_Admin *GloAdmin; extern MySQL_Threads_Handler *GloMTH; extern ProxySQL_Cluster* GloProxyCluster; @@ -1564,10 +1578,6 @@ bool MySQL_Monitor_State_Data::create_new_connection() { myrc=mysql_real_connect(mysql, "localhost", mysql_thread___monitor_username, mysql_thread___monitor_password, NULL, 0, hostname, 0); } if (myrc==NULL) { - // port == 0 means we are connecting to a unix socket - if (port) { - MySQL_Monitor::remove_dns_record_from_dns_cache(hostname); - } mysql_error_msg=strdup(mysql_error(mysql)); int myerrno=mysql_errno(mysql); MyHGM->p_update_mysql_error_counter(p_mysql_error_type::proxysql, hostgroup_id, hostname, port, myerrno); @@ -6674,15 +6684,6 @@ bool MySQL_Monitor::update_dns_cache_from_mysql_conn(const MYSQL* mysql) return result; } -void MySQL_Monitor::remove_dns_record_from_dns_cache(const std::string& hostname) { - - // if IP was provided, no need to update dns cache - if (hostname.empty() || validate_ip(hostname)) - return; - - _remove_dns_record_from_dns_cache(hostname); -} - bool MySQL_Monitor::_dns_cache_update(const std::string &hostname, std::vector&& ip_address) { static thread_local std::shared_ptr dns_cache_thread; @@ -6699,18 +6700,6 @@ bool MySQL_Monitor::_dns_cache_update(const std::string &hostname, std::vector dns_cache_thread; - - if (!dns_cache_thread && GloMyMon) - dns_cache_thread = GloMyMon->dns_cache; - - if (dns_cache_thread) { - dns_cache_thread->remove(trim(hostname)); - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Direct DNS cache record removed. (Hostname:[%s])\n", hostname.c_str()); - } -} - void MySQL_Monitor::trigger_dns_cache_update() { if (GloMyMon) { GloMyMon->force_dns_cache_update = true; @@ -6739,7 +6728,7 @@ bool DNS_Cache::add(const std::string& hostname, std::vector&& ips) bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vector&& ips) { if (!enabled) return false; - bool item_added = false; + int rc = pthread_rwlock_wrlock(&rwlock_); assert(rc == 0); if (records.find(hostname) == records.end()) { @@ -6747,12 +6736,11 @@ bool DNS_Cache::add_if_not_exist(const std::string& hostname, std::vectordns_cache_record_updated, 1); return true; diff --git a/lib/MySQL_Protocol.cpp b/lib/MySQL_Protocol.cpp index cd13ea68fe..1b23d5fd1d 100644 --- a/lib/MySQL_Protocol.cpp +++ b/lib/MySQL_Protocol.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include "openssl/rand.h" #include "proxysql.h" #include "cpp.h" @@ -32,6 +36,16 @@ extern ClickHouse_Authentication *GloClickHouseAuth; #include "proxysql_find_charset.h" +mf_unique_ptr get_masked_pass(const char* pass) { + char* tmp_pass = strdup(pass); + int lpass = strlen(tmp_pass); + + for (int i=2; i(static_cast(tmp_pass)); +} extern "C" char * sha256_crypt_r (const char *key, const char *salt, char *buffer, int buflen); @@ -43,6 +57,52 @@ static const char *plugins[3] = { #include "MySQL_encode.h" +std::string unhex(const std::string& hex) { + if (hex.size() % 2 || hex.size() == 0) { return {}; }; + + string result {}; + + for (size_t i = 0; i < hex.size() - 1; i += 2) { + string hex_char { string { hex[i] } + hex[i+1] }; + uint64_t char_val { 0 }; + + std::istringstream stream { hex_char }; + stream >> std::hex >> char_val; + + result += string { static_cast(char_val) }; + } + + return result; +} + +char* get_password(account_details_t& ad, PASSWORD_TYPE::E passtype) { + char* ret = nullptr; + + if (ad.clear_text_password[passtype] == NULL) { + if (passtype == PASSWORD_TYPE::PRIMARY) { + if (ad.password) { + ret = strdup(ad.password); + } + } else if (ad.attributes) { + nlohmann::json attrs = nlohmann::json::parse(ad.attributes, nullptr, false); + string addl_pass { get_nested_elem_val(attrs, { "additional_password" }, string {}) }; + string uh_addl_pass { unhex(addl_pass) }; + proxy_info("Password info length:%ld, val:`%s`, addl_val:`%s`\n", uh_addl_pass.length(), uh_addl_pass.c_str(), addl_pass.c_str()); + ret = reinterpret_cast(strdup(uh_addl_pass.c_str())); + } + } else { + // best password we have; if we were able to derive the clear text password, we provide that + ret = strdup(ad.clear_text_password[passtype]); + + // Only count one attempt using the cache per connection + if (passtype == PASSWORD_TYPE::PRIMARY) { + __sync_add_and_fetch(&MyHGM->status.client_connections_sha2cached, 1); + } + } + + return ret; +} + #ifdef DEBUG void debug_spiffe_id(const unsigned char *user, const char *attributes, int __line, const char *__func) { if (attributes!=NULL && strlen(attributes)) { @@ -1113,6 +1173,30 @@ bool MySQL_Protocol::generate_pkt_initial_handshake(bool send, void **ptr, unsig return true; } +void ch_account_to_my(account_details_t& account, ch_account_details_t& ch_account) { + account.username = ch_account.username; + account.password = ch_account.password; + account.sha1_pass = ch_account.sha1_pass; + account.use_ssl = ch_account.use_ssl; + account.default_hostgroup = ch_account.default_hostgroup; + account.default_schema = ch_account.default_schema; + account.schema_locked = ch_account.schema_locked; + account.transaction_persistent = ch_account.transaction_persistent; + account.fast_forward = ch_account.fast_forward; + account.max_connections = ch_account.max_connections; + account.num_connections_used = ch_account.num_connections_used; + + // Fields that are not present in `ch_account_details_t` + account.num_connections_used_addl_pass = 0; // Assuming no additional password used + account.clear_text_password[0] = nullptr; // No clear text passwords by default + account.clear_text_password[1] = nullptr; + account.__frontend = ch_account.__frontend; // Copy frontend flag + account.__backend = ch_account.__backend; // Copy backend flag + account.__active = ch_account.__active; // Copy active flag + account.attributes = nullptr; // No attributes by default + account.comment = nullptr; // No comment by default +} + bool MySQL_Protocol::process_pkt_auth_swich_response(unsigned char *pkt, unsigned int len) { bool ret=false; char *password=NULL; @@ -1126,29 +1210,41 @@ bool MySQL_Protocol::process_pkt_auth_swich_response(unsigned char *pkt, unsigne } mysql_hdr hdr; memcpy(&hdr,pkt,sizeof(mysql_hdr)); - int default_hostgroup=-1; - bool transaction_persistent; - bool _ret_use_ssl=false; unsigned char pass[128]; memset(pass,0,128); pkt+=sizeof(mysql_hdr); memcpy(pass, pkt, 20); - char reply[SHA_DIGEST_LENGTH+1]; - reply[SHA_DIGEST_LENGTH]='\0'; - void *sha1_pass=NULL; + + MyProt_tmp_auth_vars vars1; + account_details_t account_details {}; + dup_account_details_t dup_details {}; + dup_details.sha1_pass = true; + enum proxysql_session_type session_type = (*myds)->sess->session_type; if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { #ifdef PROXYSQLCLICKHOUSE - password=GloClickHouseAuth->lookup((char *)userinfo->username, USERNAME_FRONTEND, &_ret_use_ssl, &default_hostgroup, NULL, NULL, &transaction_persistent, NULL, NULL, &sha1_pass); + ch_dup_account_details_t ch_dup_details {}; + ch_dup_details.sha1_pass = true; + + ch_account_details_t ch_account { + GloClickHouseAuth->lookup((char*)userinfo->username, USERNAME_FRONTEND, ch_dup_details) + }; + + ch_account_to_my(account_details, ch_account); + password = ch_account.password; #endif /* PROXYSQLCLICKHOUSE */ } else { - password=GloMyAuth->lookup((char *)userinfo->username, USERNAME_FRONTEND, &_ret_use_ssl, &default_hostgroup, NULL, NULL, &transaction_persistent, NULL, NULL, &sha1_pass, NULL); + account_details = GloMyAuth->lookup((char*)userinfo->username, USERNAME_FRONTEND, dup_details); + password = account_details.password; } // FIXME: add support for default schema and fast forward , issues #255 and #256 // FIXME: not sure if we should also handle user_attributes *here* . For now we pass NULL (no change) if (password==NULL) { ret=false; } else { + char reply[SHA_DIGEST_LENGTH+1]; + reply[SHA_DIGEST_LENGTH]='\0'; + if (password[0]!='*') { // clear text password proxy_scramble(reply, (*myds)->myconn->scramble_buff, password); if (memcmp(reply, pass, SHA_DIGEST_LENGTH)==0) { @@ -1157,7 +1253,7 @@ bool MySQL_Protocol::process_pkt_auth_swich_response(unsigned char *pkt, unsigne } else { ret=proxy_scramble_sha1((char *)pass,(*myds)->myconn->scramble_buff,password+1, reply); if (ret) { - if (sha1_pass==NULL) { + if (account_details.sha1_pass==NULL) { // currently proxysql doesn't know any sha1_pass for that specific user, let's set it! GloMyAuth->set_SHA1((char *)userinfo->username, USERNAME_FRONTEND,reply); } @@ -1166,10 +1262,8 @@ bool MySQL_Protocol::process_pkt_auth_swich_response(unsigned char *pkt, unsigne } } } - if (sha1_pass) { - free(sha1_pass); - sha1_pass=NULL; - } + free_account_details(account_details); + return ret; } @@ -1268,14 +1362,9 @@ bool MySQL_Protocol::process_pkt_COM_CHANGE_USER(unsigned char *pkt, unsigned in bool ret=false; int cur=sizeof(mysql_hdr); unsigned char *user=NULL; - char *password=NULL; char *db=NULL; - char* user_attributes=NULL; mysql_hdr hdr; memcpy(&hdr,pkt,sizeof(mysql_hdr)); - int default_hostgroup=-1; - bool transaction_persistent = true; - bool _ret_use_ssl=false; cur++; user=pkt+cur; cur+=strlen((const char *)user); @@ -1284,7 +1373,6 @@ bool MySQL_Protocol::process_pkt_COM_CHANGE_USER(unsigned char *pkt, unsigned in cur++; unsigned char pass[128]; memset(pass,0,128); - //pkt+=sizeof(mysql_hdr); memcpy(pass, pkt+cur, pass_len); cur+=pass_len; db=(char *)pkt+cur; @@ -1310,24 +1398,36 @@ bool MySQL_Protocol::process_pkt_COM_CHANGE_USER(unsigned char *pkt, unsigned in } } - void *sha1_pass=NULL; + account_details_t account_details {}; + dup_account_details_t dup_details { false, true, true }; enum proxysql_session_type session_type = (*myds)->sess->session_type; + if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { #ifdef PROXYSQLCLICKHOUSE - password=GloClickHouseAuth->lookup((char *)user, USERNAME_FRONTEND, &_ret_use_ssl, &default_hostgroup, NULL, NULL, &transaction_persistent, NULL, NULL, &sha1_pass); + ch_dup_account_details_t ch_dup_details { false, true }; + ch_dup_details.sha1_pass = true; + + ch_account_details_t ch_account_details { + GloClickHouseAuth->lookup((char*)user, USERNAME_FRONTEND, ch_dup_details) + }; + + ch_account_to_my(account_details, ch_account_details); #endif /* PROXYSQLCLICKHOUSE */ } else { - password=GloMyAuth->lookup((char *)user, USERNAME_FRONTEND, &_ret_use_ssl, &default_hostgroup, NULL, NULL, &transaction_persistent, NULL, NULL, &sha1_pass, &user_attributes); + account_details = GloMyAuth->lookup((char *)user, USERNAME_FRONTEND, dup_details); } // FIXME: add support for default schema and fast forward, see issue #255 and #256 - (*myds)->sess->default_hostgroup=default_hostgroup; - (*myds)->sess->transaction_persistent=transaction_persistent; + (*myds)->sess->default_hostgroup=account_details.default_hostgroup; + (*myds)->sess->transaction_persistent=account_details.transaction_persistent; // Could be reached several times before auth completion; allocating attributes should be reset if ((*myds)->sess->user_attributes) { free((*myds)->sess->user_attributes); (*myds)->sess->user_attributes = nullptr; } - (*myds)->sess->user_attributes=user_attributes; + (*myds)->sess->user_attributes=account_details.attributes; + account_details.attributes = nullptr; + char* password = get_password(account_details, PASSWORD_TYPE::PRIMARY); + if (password==NULL) { ret=false; } else { @@ -1350,15 +1450,10 @@ bool MySQL_Protocol::process_pkt_COM_CHANGE_USER(unsigned char *pkt, unsigned in // it to authenticate the user. See #3504 for more context. ret = verify_user_pass( session_type, password, reinterpret_cast(user), reinterpret_cast(pass), - pass_len, static_cast(sha1_pass), client_auth_plugin + pass_len, static_cast(account_details.sha1_pass), client_auth_plugin ); } } - //if (_ret_use_ssl==true) { - // if we reached here, use_ssl is false , but _ret_use_ssl is true - // it means that a client is required to use SSL , but it is not - // ret=false; - //} } if (userinfo->username) free(userinfo->username); if (userinfo->password) free(userinfo->password); @@ -1371,16 +1466,13 @@ bool MySQL_Protocol::process_pkt_COM_CHANGE_USER(unsigned char *pkt, unsigned in } else { // we always duplicate username and password, or crashes happen userinfo->username=strdup((const char *)user); - /*if (pass_len) */ userinfo->password=strdup((const char *)""); + userinfo->password=strdup((const char *)""); } if (password) { free(password); password=NULL; } - if (sha1_pass) { - free(sha1_pass); - sha1_pass=NULL; - } + free_account_details(account_details); userinfo->set(NULL,NULL,NULL,NULL); // just to call compute_hash() if (ret) { // we need to process charset if present in CHANGE_USER @@ -1652,7 +1744,6 @@ bool MySQL_Protocol::PPHR_4auth0(unsigned char *pkt, unsigned int len, bool& ret #endif /* PROXYSQLCLICKHOUSE */ user_exists = GloMyAuth->exists((char *)vars1.user); proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user_exists=%d , user='%s'\n", (*myds), (*myds)->sess, user_exists, vars1.user); - //password=GloMyAuth->lookup((char *)user, USERNAME_FRONTEND, &_ret_use_ssl, &default_hostgroup, &default_schema, &schema_locked, &transaction_persistent, &fast_forward, &max_connections, &sha1_pass); #ifdef PROXYSQLCLICKHOUSE } #endif /* PROXYSQLCLICKHOUSE */ @@ -1685,7 +1776,6 @@ bool MySQL_Protocol::PPHR_4auth1(unsigned char *pkt, unsigned int len, bool& ret } else { #endif /* PROXYSQLCLICKHOUSE */ user_exists = GloMyAuth->exists((char *)vars1.user); - //password=GloMyAuth->lookup((char *)user, USERNAME_FRONTEND, &_ret_use_ssl, &default_hostgroup, &default_schema, &schema_locked, &transaction_persistent, &fast_forward, &max_connections, &sha1_pass); #ifdef PROXYSQLCLICKHOUSE } #endif /* PROXYSQLCLICKHOUSE */ @@ -1705,33 +1795,35 @@ bool MySQL_Protocol::PPHR_4auth1(unsigned char *pkt, unsigned int len, bool& ret } void MySQL_Protocol::PPHR_5passwordTrue( - unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, - MyProt_tmp_auth_attrs& attr1) { - //int& default_hostgroup, char *& default_schema, char *& attributes, bool& schema_locked, bool& transaction_persistent, bool& fast_forward, int& max_connections) { + account_details_t& attr1 +) { #ifdef DEBUG - char *tmp_pass=strdup(vars1.password); - int lpass = strlen(tmp_pass); - for (int i=2; isess, vars1.user, tmp_pass); - free(tmp_pass); + proxy_debug( + PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , password='%s'\n", + (*myds), (*myds)->sess, vars1.user, mf_unique_ptr(get_masked_pass(vars1.password)).get() + ); #endif // debug // Could be reached several times before auth completion; allocating attributes should be reset (*myds)->sess->default_hostgroup = attr1.default_hostgroup; - if ((*myds)->sess->default_schema) { + // Protect against multiple calls; only replace on property change + if ((*myds)->sess->default_schema && ((*myds)->sess->default_schema != attr1.default_schema)) { free((*myds)->sess->default_schema); (*myds)->sess->default_schema = nullptr; } - (*myds)->sess->default_schema = attr1.default_schema; // just the pointer is passed - if ((*myds)->sess->user_attributes) { + // TODO: Not ideal, but the flow is currently too complex. Simplifying resource management so + // we can reduce extra alloctions should be part of the next rework. + (*myds)->sess->default_schema = attr1.default_schema ? strdup(attr1.default_schema) : nullptr; + // Protect against multiple calls; only replace on property change + if ((*myds)->sess->user_attributes && (*myds)->sess->user_attributes != attr1.attributes) { free((*myds)->sess->user_attributes); (*myds)->sess->user_attributes = nullptr; } - (*myds)->sess->user_attributes = attr1.attributes; // just the pointer is passed + // TODO: Not ideal, but the flow is currently too complex. Simplifying resource management so + // we can reduce extra alloctions should be part of the next rework. + (*myds)->sess->user_attributes = attr1.attributes ? strdup(attr1.attributes) : nullptr; #ifdef DEBUG debug_spiffe_id(vars1.user,attr1.attributes, __LINE__, __func__); #endif @@ -1747,11 +1839,10 @@ void MySQL_Protocol::PPHR_5passwordTrue( void MySQL_Protocol::PPHR_5passwordFalse_0( // FIXME: does this work only for mysql_native_password ? - unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, - MyProt_tmp_auth_attrs& attr1) { + account_details_t& attr1) { if (strcmp((const char *)vars1.user,mysql_thread___monitor_username)==0) { proxy_scramble(reply, (*myds)->myconn->scramble_buff, mysql_thread___monitor_password); if (memcmp(reply, vars1.pass, SHA_DIGEST_LENGTH)==0) { @@ -1770,12 +1861,11 @@ void MySQL_Protocol::PPHR_5passwordFalse_0( } void MySQL_Protocol::PPHR_5passwordFalse_auth2( - unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, - MyProt_tmp_auth_attrs& attr1, - void *& sha1_pass) { + account_details_t& attr1 +) { if (GloMyLdapAuth) { #ifdef DEBUG { @@ -1791,8 +1881,8 @@ void MySQL_Protocol::PPHR_5passwordFalse_auth2( char *backend_username = NULL; (*myds)->sess->use_ldap_auth = true; vars1.password = GloMyLdapAuth->lookup((char *) vars1.user, (char *) vars1.pass, USERNAME_FRONTEND, - &attr1._ret_use_ssl, &attr1.default_hostgroup, &attr1.default_schema, &attr1.schema_locked, - &attr1.transaction_persistent, &attr1.fast_forward, &attr1.max_connections, &sha1_pass, &attr1.attributes, &backend_username); + &attr1.use_ssl, &attr1.default_hostgroup, &attr1.default_schema, &attr1.schema_locked, + &attr1.transaction_persistent, &attr1.fast_forward, &attr1.max_connections, &attr1.sha1_pass, &attr1.attributes, &backend_username); if (vars1.password) { #ifdef DEBUG char *tmp_pass=strdup(vars1.password); @@ -1815,10 +1905,11 @@ void MySQL_Protocol::PPHR_5passwordFalse_auth2( (*myds)->sess->user_max_connections=attr1.max_connections; if (strcmp(vars1.password, (char *) vars1.pass) == 0) { if (backend_username) { - free(vars1.password); - vars1.password=NULL; - vars1.password=GloMyAuth->lookup(backend_username, USERNAME_BACKEND, &attr1._ret_use_ssl, &attr1.default_hostgroup, &attr1.default_schema, &attr1.schema_locked, &attr1.transaction_persistent, &attr1.fast_forward, &attr1.max_connections, &sha1_pass, &attr1.attributes); - if (vars1.password) { + account_details_t acct { + GloMyAuth->lookup(backend_username, USERNAME_BACKEND, { true, true, true }) + }; + + if (acct.password) { (*myds)->sess->default_hostgroup=attr1.default_hostgroup; // Free the previously set 'default_schema' by 'GloMyLdapAuth' if ((*myds)->sess->default_schema) { @@ -1831,18 +1922,19 @@ void MySQL_Protocol::PPHR_5passwordFalse_auth2( } (*myds)->sess->user_attributes = attr1.attributes; // just the pointer is passed #ifdef DEBUG - proxy_info("Attributes for user %s: %s\n" , vars1.user, attr1.attributes); + proxy_info("Attributes for user %s: %s\n" , acct.username, attr1.attributes); #endif (*myds)->sess->schema_locked=attr1.schema_locked; (*myds)->sess->transaction_persistent=attr1.transaction_persistent; (*myds)->sess->session_fast_forward=attr1.fast_forward; (*myds)->sess->user_max_connections=attr1.max_connections; - char *tmp_user=strdup((const char *)vars1.user); + char *tmp_user=strdup((const char *)acct.username); userinfo->set(backend_username, NULL, NULL, NULL); // 'MySQL_Connection_userinfo::set' duplicates the supplied information, 'free' is required. free(backend_username); - if (sha1_pass==NULL) { + if (attr1.sha1_pass==NULL) { // currently proxysql doesn't know any sha1_pass for that specific user, let's set it! + // TODO: CHECK these usages of 'reply' GloMyAuth->set_SHA1((char *)userinfo->username, USERNAME_FRONTEND,reply); } if (userinfo->sha1_pass) free(userinfo->sha1_pass); @@ -1851,8 +1943,10 @@ void MySQL_Protocol::PPHR_5passwordFalse_auth2( free(tmp_user); ret=true; } else { - proxy_error("Unable to load credentials for backend user %s , associated to LDAP user %s\n", backend_username, vars1.user); + proxy_error("Unable to load credentials for backend user %s , associated to LDAP user %s\n", backend_username, acct.username); } + + free_account_details(acct); } else { proxy_error("Unable to find backend user associated to LDAP user '%s'\n", vars1.user); ret=false; @@ -1888,17 +1982,16 @@ void MySQL_Protocol::PPHR_6auth2( } void MySQL_Protocol::PPHR_7auth1( - unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, - MyProt_tmp_auth_attrs& attr1, - void *& sha1_pass) { + account_details_t& attr1 +) { enum proxysql_session_type session_type = (*myds)->sess->session_type; if (session_type == PROXYSQL_SESSION_MYSQL || session_type == PROXYSQL_SESSION_SQLITE || session_type == PROXYSQL_SESSION_ADMIN || session_type == PROXYSQL_SESSION_STATS) { ret=proxy_scramble_sha1((char *)vars1.pass,(*myds)->myconn->scramble_buff,vars1.password+1, reply); if (ret) { - if (sha1_pass==NULL) { + if (attr1.sha1_pass==NULL) { // currently proxysql doesn't know any sha1_pass for that specific user, let's set it! GloMyAuth->set_SHA1((char *)vars1.user, USERNAME_FRONTEND,reply); } @@ -1911,12 +2004,11 @@ void MySQL_Protocol::PPHR_7auth1( void MySQL_Protocol::PPHR_7auth2( - unsigned char *pkt, unsigned int len, bool& ret, MyProt_tmp_auth_vars& vars1, char * reply, - MyProt_tmp_auth_attrs& attr1, - void *& sha1_pass) { + account_details_t& attr1 +) { enum proxysql_session_type session_type = (*myds)->sess->session_type; if (session_type == PROXYSQL_SESSION_MYSQL || session_type == PROXYSQL_SESSION_SQLITE || session_type == PROXYSQL_SESSION_ADMIN || session_type == PROXYSQL_SESSION_STATS) { proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , session_type=%d\n", (*myds), (*myds)->sess, vars1.user, session_type); @@ -1940,7 +2032,7 @@ void MySQL_Protocol::PPHR_7auth2( if (strcasecmp(double_hashed_password,vars1.password)==0) { ret = true; - if (sha1_pass==NULL) { + if (attr1.sha1_pass==NULL) { // currently proxysql doesn't know any sha1_pass for that specific user, let's set it! GloMyAuth->set_SHA1((char *)vars1.user, USERNAME_FRONTEND,md1_buf); } @@ -1954,11 +2046,55 @@ void MySQL_Protocol::PPHR_7auth2( } } +bool MySQL_Protocol::PPHR_verify_sha2( + MyProt_tmp_auth_vars& vars1, + enum proxysql_auth_plugins passformat, + PASSWORD_TYPE::E passtype +) { + bool ret = false; + + if ((*myds)->switching_auth_stage == 5) { + if (passformat == AUTH_MYSQL_NATIVE_PASSWORD) { + unsigned char md1_buf[SHA_DIGEST_LENGTH]; + unsigned char md2_buf[SHA_DIGEST_LENGTH]; + SHA1(vars1.pass, vars1.pass_len, md1_buf); + SHA1(md1_buf,SHA_DIGEST_LENGTH,md2_buf); + char *double_hashed_password = sha1_pass_hex((char *)md2_buf); // note that sha1_pass_hex() returns a new buffer + if (strcasecmp(double_hashed_password,vars1.password)==0) { + ret = true; + } + free(double_hashed_password); + } else if (passformat == AUTH_MYSQL_CACHING_SHA2_PASSWORD) { + assert(strlen(vars1.password) == 70); + string sp = string(vars1.password); + long rounds = stol(sp.substr(3,3)); + string salt = sp.substr(7,20); + string sha256hash = sp.substr(27,43); + char buf[100]; + salt = "$5$rounds=" + to_string(rounds*1000) + "$" + salt; + sha256_crypt_r((const char*)vars1.pass, salt.c_str(), buf, sizeof(buf)); + string sbuf = string(buf); + std::size_t found = sbuf.find_last_of("$"); + assert(found != string::npos); + sbuf = sbuf.substr(found+1); + if (strcmp(sbuf.c_str(),vars1.password+27)==0) { + ret = true; + } + } else { + // Programatic error; invalid param + assert(0); + } + } + + return ret; +} + void MySQL_Protocol::PPHR_sha2full( bool& ret, MyProt_tmp_auth_vars& vars1, - enum proxysql_auth_plugins passformat - ) { + enum proxysql_auth_plugins passformat, + PASSWORD_TYPE::E passtype +) { if ((*myds)->switching_auth_stage == 0) { const unsigned char perform_full_authentication = '\4'; generate_one_byte_pkt(perform_full_authentication); @@ -2005,7 +2141,7 @@ void MySQL_Protocol::PPHR_sha2full( enum proxysql_session_type session_type = (*myds)->sess->session_type; if (session_type == PROXYSQL_SESSION_MYSQL || session_type == PROXYSQL_SESSION_SQLITE || session_type == PROXYSQL_SESSION_ADMIN || session_type == PROXYSQL_SESSION_STATS) { // currently proxysql doesn't know the clear text password for that specific user, let's set it! - GloMyAuth->set_clear_text_password((char *)vars1.user, USERNAME_FRONTEND, (const char *)vars1.pass); + GloMyAuth->set_clear_text_password((char *)vars1.user, USERNAME_FRONTEND, (const char *)vars1.pass, passtype); // Update 'vars1' password with 'clear text' one, so session can be later updated with it if (vars1.password) { free(vars1.password); } vars1.password = strdup(reinterpret_cast(vars1.pass)); @@ -2016,7 +2152,7 @@ void MySQL_Protocol::PPHR_sha2full( } } -void MySQL_Protocol::PPHR_SetConnAttrs(MyProt_tmp_auth_vars& vars1, MyProt_tmp_auth_attrs& attr1) { +void MySQL_Protocol::PPHR_SetConnAttrs(MyProt_tmp_auth_vars& vars1, account_details_t& attr1) { MySQL_Connection *myconn = NULL; myconn=sess->client_myds->myconn; assert(myconn); @@ -2040,11 +2176,166 @@ void MySQL_Protocol::PPHR_SetConnAttrs(MyProt_tmp_auth_vars& vars1, MyProt_tmp_a //myconn->set_status_compression(true); // don't enable this here. It needs to be enabled after the OK is sent } } - if (attr1._ret_use_ssl==true) { + if (attr1.use_ssl==true) { (*myds)->sess->use_ssl = true; } } +// PPHR_proc_auth_stage :: bool -> MySQL_Protocol::MySQL_Data_Stream -> auth_plugin_id -> OSC +// PPHR_cont_auth :: bool -> MySQL_Protocol::MySQL_Data_Stream -> auth_plugin_id -> OSC +// MySQL_Protocol::verify_password :: vars1 -> account_details_t -> bool +// Template idea for auth in stages; not used at the moment +/* +void MySQL_Protocol::PPHR_next_auth_stage(MyProt_tmp_auth_vars& vars1, PASSWORD_TYPE::E passtype) { + if ( + auth_plugin_id == AUTH_MYSQL_CACHING_SHA2_PASSWORD + && + strlen(vars1.password) == 70 + && + strncasecmp(vars1.password, "$A$0", 4) == 0 + ) { + if ((*myds)->switching_auth_stage == 0) { + const unsigned char perform_full_authentication = '\4'; + generate_one_byte_pkt(perform_full_authentication); + (*myds)->switching_auth_type = auth_plugin_id; + (*myds)->switching_auth_stage = 4; + (*myds)->auth_in_progress = 1; + } else if ((*myds)->switching_auth_stage == 5) { + enum proxysql_session_type session_type = (*myds)->sess->session_type; + if ( + session_type == PROXYSQL_SESSION_MYSQL || + session_type == PROXYSQL_SESSION_SQLITE || + session_type == PROXYSQL_SESSION_ADMIN || + session_type == PROXYSQL_SESSION_STATS + ) { + // Clear text password currently unknown for that specific user; let's set it! + GloMyAuth->set_clear_text_password( + (char *)vars1.user, USERNAME_FRONTEND, (const char *)vars1.pass, passtype + ); + // Update 'vars1' password with 'clear text' one, so session can be later updated with it + if (vars1.password) { free(vars1.password); } + vars1.password = strdup(reinterpret_cast(vars1.pass)); + } + } + } else if (vars1.password[0] != '*') { + if (auth_plugin_id == AUTH_MYSQL_CACHING_SHA2_PASSWORD && (*myds)->switching_auth_stage == 0) { + } + } else { + } +} +*/ + +// +// Update global state if pass verified: +// - If auth finished: +// + Save password: sha1 || clear_text_password +// - If not (caching_sha2_password): +// + Continue auth; trigger contitue of full auth +bool MySQL_Protocol::PPHR_verify_password(MyProt_tmp_auth_vars& vars1, account_details_t& account_details) { + bool ret = false; + char reply[SHA_DIGEST_LENGTH + 1] = { 0 }; + + if (vars1.password == NULL) { + // this is a workaround for bug #603 + if ( + ((*myds)->sess->session_type == PROXYSQL_SESSION_ADMIN) + || + ((*myds)->sess->session_type == PROXYSQL_SESSION_STATS) + || + ((*myds)->sess->session_type == PROXYSQL_SESSION_SQLITE) + ) { + PPHR_5passwordFalse_0(ret, vars1, reply, account_details); + } else { + // assume failure + ret=false; + // try LDAP + if (auth_plugin_id == AUTH_MYSQL_CLEAR_PASSWORD) { + PPHR_5passwordFalse_auth2(ret, vars1, reply, account_details); + } + } + } else { + // update 'MySQL_Session' info using 'account_details'; transfers ownership of: + // - 'ad::default_schema', 'ad::attributes' + PPHR_5passwordTrue(ret, vars1, reply, account_details); + + if (vars1.pass_len==0 && strlen(vars1.password)==0) { + ret=true; + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , password=''\n", (*myds), (*myds)->sess, vars1.user); + } + // For empty passwords client expects either 'OK' or 'ERR' + else if (vars1.pass_len == 0 && strlen(vars1.password) != 0) { + ret=false; + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , password=''\n", (*myds), (*myds)->sess, vars1.user); + } + else { +#ifdef DEBUG + proxy_debug( + PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , password='%s' , auth_plugin_id=%d\n", + (*myds), (*myds)->sess, vars1.user, get_masked_pass(vars1.password).get(), auth_plugin_id + ); +#endif // debug + if ( + auth_plugin_id == AUTH_MYSQL_CACHING_SHA2_PASSWORD + && + strlen(vars1.password) == 70 + && + strncasecmp(vars1.password,"$A$0",4)==0 + ) { + // We have a hashed caching_sha2_password + PPHR_sha2full(ret, vars1, AUTH_MYSQL_CACHING_SHA2_PASSWORD, vars1.passtype); + } else if (vars1.password[0]!='*') { // clear text password + if (auth_plugin_id == AUTH_MYSQL_NATIVE_PASSWORD) { // mysql_native_password + proxy_scramble(reply, (*myds)->myconn->scramble_buff, vars1.password); + if (vars1.pass_len != 0 && memcmp(reply, vars1.pass, SHA_DIGEST_LENGTH)==0) { + ret=true; + } + } else if (auth_plugin_id == AUTH_MYSQL_CLEAR_PASSWORD) { // mysql_clear_password + if (strcmp(vars1.password, (char *) vars1.pass) == 0) { + ret = true; + } + } else if (auth_plugin_id == AUTH_MYSQL_CACHING_SHA2_PASSWORD) { // caching_sha2_password + // Checking 'switching_auth_stage' is required case due to a potential concurrent update + // of pass in 'GloMyAuth'. When the pass found is clear-text it's assumed that full-auth + // is never required and that we are in the first auth stage, because of this, the pass + // received by the client is assumed to be hashed (first auth data received). Yet, during + // during a 'full-auth' the pass stored in 'GloMyAuth' could have been updated either by + // user action or by another concurrent connection that called 'set_clear_text_pass' on + // completion. In this case, we would have received a 'clear-text' pass form 'GloMyAuth' + // but we since we would be in the final auth stage, the pass sent by client should also + // be 'clear-text' (encrypt-pass). + if ((*myds)->switching_auth_stage == 5) { + if (strcmp(vars1.password, reinterpret_cast(vars1.pass)) == 0) { + ret = true; + } + } else { + PPHR_6auth2(ret, vars1); + if (ret == true) { + if ((*myds)->switching_auth_stage == 0) { + const unsigned char fast_auth_success = '\3'; + generate_one_byte_pkt(fast_auth_success); + } + } + } + } else { + assert(0); + } + } else { // password hashed with SHA1 , mysql_native_password format + if (auth_plugin_id == AUTH_MYSQL_NATIVE_PASSWORD) { // mysql_native_password + PPHR_7auth1(ret, vars1, reply, account_details); + } else if (auth_plugin_id == AUTH_MYSQL_CLEAR_PASSWORD) { // mysql_clear_password + PPHR_7auth2(ret, vars1, reply, account_details); + } else if (auth_plugin_id == AUTH_MYSQL_CACHING_SHA2_PASSWORD) { // caching_sha2_password + PPHR_sha2full(ret, vars1, AUTH_MYSQL_NATIVE_PASSWORD, vars1.passtype); + } else { + assert(0); + } + } + } + } + + return ret; +} + /** * @brief Process handshake response from the client, and it needs to be called until * the authentication is completed (successfully or failed) @@ -2060,12 +2351,11 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned bool ret = false; auth_plugin_id = AUTH_UNKNOWN_PLUGIN; - char reply[SHA_DIGEST_LENGTH+1] = { 0 }; enum proxysql_session_type session_type = (*myds)->sess->session_type; - - void *sha1_pass=NULL; MyProt_tmp_auth_vars vars1; - MyProt_tmp_auth_attrs attr1; + account_details_t account_details {}; + dup_account_details_t dup_details { true, true, true }; + vars1._ptr = pkt; mysql_hdr hdr; bool bool_rc = false; @@ -2173,113 +2463,58 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned } if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { #ifdef PROXYSQLCLICKHOUSE - vars1.password=GloClickHouseAuth->lookup((char *)vars1.user, USERNAME_FRONTEND, &attr1._ret_use_ssl, &attr1.default_hostgroup, &attr1.default_schema, &attr1.schema_locked, &attr1.transaction_persistent, &attr1.fast_forward, &attr1.max_connections, &sha1_pass); + ch_dup_account_details_t ch_dup_details { true, true }; + + ch_account_details_t ch_account { + GloClickHouseAuth->lookup((char*)vars1.user, USERNAME_FRONTEND, ch_dup_details) + }; + + ch_account_to_my(account_details, ch_account); #endif /* PROXYSQLCLICKHOUSE */ } else { - vars1.password=GloMyAuth->lookup((char *)vars1.user, USERNAME_FRONTEND, &attr1._ret_use_ssl, &attr1.default_hostgroup, &attr1.default_schema, &attr1.schema_locked, &attr1.transaction_persistent, &attr1.fast_forward, &attr1.max_connections, &sha1_pass, &attr1.attributes); + account_details = GloMyAuth->lookup((char*)vars1.user, USERNAME_FRONTEND, dup_details); } - //assert(default_hostgroup>=0); - //if (vars1.password) { - //} - if (vars1.password == NULL) { - // this is a workaround for bug #603 - if ( - ((*myds)->sess->session_type == PROXYSQL_SESSION_ADMIN) - || - ((*myds)->sess->session_type == PROXYSQL_SESSION_STATS) -//#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) - || - ((*myds)->sess->session_type == PROXYSQL_SESSION_SQLITE) -//#endif // TEST_AURORA || TEST_GALERA - ) { - PPHR_5passwordFalse_0(pkt, len, ret, vars1, reply, attr1); - } else { - ret=false; // by default, assume this will fail - // try LDAP - if (auth_plugin_id == AUTH_MYSQL_CLEAR_PASSWORD) { - PPHR_5passwordFalse_auth2(pkt, len, ret, vars1, reply, attr1, sha1_pass); - } - } - } else { - PPHR_5passwordTrue(pkt, len, ret, vars1, reply, attr1); - if (vars1.pass_len==0 && strlen(vars1.password)==0) { - ret=true; - proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , password=''\n", (*myds), (*myds)->sess, vars1.user); - } - // For empty passwords client expects either 'OK' or 'ERR' - else if (vars1.pass_len == 0 && strlen(vars1.password) != 0) { - ret=false; - proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , password=''\n", (*myds), (*myds)->sess, vars1.user); - } - else { -#ifdef DEBUG - char *tmp_pass=strdup(vars1.password); - int lpass = strlen(tmp_pass); - for (int i=2; isess, vars1.user, tmp_pass, auth_plugin_id); - free(tmp_pass); -#endif // debug - if ( - auth_plugin_id == AUTH_MYSQL_CACHING_SHA2_PASSWORD - && - strlen(vars1.password) == 70 - && - strncasecmp(vars1.password,"$A$0",4)==0 - ) { - // we have a hashed caching_sha2_password - PPHR_sha2full(ret, vars1, AUTH_MYSQL_CACHING_SHA2_PASSWORD); - } else if (vars1.password[0]!='*') { // clear text password - if (auth_plugin_id == AUTH_MYSQL_NATIVE_PASSWORD) { // mysql_native_password - proxy_scramble(reply, (*myds)->myconn->scramble_buff, vars1.password); - if (vars1.pass_len != 0 && memcmp(reply, vars1.pass, SHA_DIGEST_LENGTH)==0) { - ret=true; - } - } else if (auth_plugin_id == AUTH_MYSQL_CLEAR_PASSWORD) { // mysql_clear_password - if (strcmp(vars1.password, (char *) vars1.pass) == 0) { - ret = true; - } - } else if (auth_plugin_id == AUTH_MYSQL_CACHING_SHA2_PASSWORD) { // caching_sha2_password - // Checking 'switching_auth_stage' is required case due to a potential concurrent update - // of pass in 'GloMyAuth'. When the pass found is clear-text it's assumed that full-auth - // is never required and that we are in the first auth stage, because of this, the pass - // received by the client is assumed to be hashed (first auth data received). Yet, during - // during a 'full-auth' the pass stored in 'GloMyAuth' could have been updated either by - // user action or by another concurrent connection that called 'set_clear_text_pass' on - // completion. In this case, we would have received a 'clear-text' pass form 'GloMyAuth' - // but we since we would be in the final auth stage, the pass sent by client should also - // be 'clear-text' (encrypt-pass). - if ((*myds)->switching_auth_stage == 5) { - if (strcmp(vars1.password, reinterpret_cast(vars1.pass)) == 0) { - ret = true; - } - } else { - PPHR_6auth2(ret, vars1); - if (ret == true) { - if ((*myds)->switching_auth_stage == 0) { - const unsigned char fast_auth_success = '\3'; - generate_one_byte_pkt(fast_auth_success); - } - } - } - } else { - assert(0); - } - } else { // password hashed with SHA1 , mysql_native_password format - if (auth_plugin_id == AUTH_MYSQL_NATIVE_PASSWORD) { // mysql_native_password - PPHR_7auth1(pkt, len, ret, vars1, reply, attr1, sha1_pass); - } else if (auth_plugin_id == AUTH_MYSQL_CLEAR_PASSWORD) { // mysql_clear_password - PPHR_7auth2(pkt, len, ret, vars1, reply, attr1, sha1_pass); - } else if (auth_plugin_id == AUTH_MYSQL_CACHING_SHA2_PASSWORD) { // caching_sha2_password - PPHR_sha2full(ret, vars1, AUTH_MYSQL_NATIVE_PASSWORD); - } else { - assert(0); - } + + vars1.password = get_password(account_details, PASSWORD_TYPE::PRIMARY); + vars1.passtype = PASSWORD_TYPE::PRIMARY; + // the async state machine needs to change; we are creating overhead in auth for old-passwords + ret = PPHR_verify_password(vars1, account_details); + + // full_auth have been performed an taken place, we 'may' already have clear_text of addl_pass + if (!ret && (*myds)->auth_in_progress == 0) { + proxy_debug( + PROXY_DEBUG_MYSQL_AUTH, 5, + "Attempting to use additional user password ret='%d', switching_auht_stage='%d'\n", + ret, (*myds)->switching_auth_stage + ); + char* addl_pass = get_password(account_details, PASSWORD_TYPE::ADDITIONAL); + + if (addl_pass) { + if (strlen(addl_pass) > 0) { + if (vars1.password) { free(vars1.password); } + vars1.password = addl_pass; + vars1.passtype = PASSWORD_TYPE::ADDITIONAL; + ret = PPHR_verify_password(vars1, account_details); + } else { + free(addl_pass); } } } + if ( + ret && + (*myds)->auth_in_progress == 0 && + (*myds)->sess->session_type == PROXYSQL_SESSION_MYSQL && + (*myds)->sess->connections_handler == false && + (*myds)->sess->mirror == false + ) { + __sync_add_and_fetch( + vars1.passtype == PASSWORD_TYPE::PRIMARY ? + &MyHGM->status.client_connections_prim_pass : &MyHGM->status.client_connections_addl_pass, + 1 + ); + } + __exit_do_auth: @@ -2302,7 +2537,7 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned assert(sess->client_myds); // set connection attributes (charsets, compression, encryption) - PPHR_SetConnAttrs(vars1, attr1); + PPHR_SetConnAttrs(vars1, account_details); #ifdef DEBUG if (dump_pkt) { __dump_pkt(__func__,vars1._ptr,len); } @@ -2325,6 +2560,7 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned } userinfo->password=strdup((const char *)vars1.password); if (vars1.db) userinfo->set_schemaname(vars1.db,strlen(vars1.db)); + userinfo->passtype = vars1.passtype; } else { // we always duplicate username and password, or crashes happen if (!userinfo->username) // if set already, ignore @@ -2333,6 +2569,7 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned if (userinfo->password) { free(userinfo->password); } userinfo->password=strdup((const char *)""); }; + userinfo->passtype = vars1.passtype; } userinfo->set(NULL,NULL,NULL,NULL); // just to call compute_hash() @@ -2342,10 +2579,6 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned free(vars1.password); vars1.password=NULL; } - if (sha1_pass) { - free(sha1_pass); - sha1_pass=NULL; - } if (vars1.db_tmp) { free(vars1.db_tmp); vars1.db_tmp=NULL; @@ -2353,6 +2586,7 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned if (ret == true) { ret = verify_user_attributes(__LINE__, __func__, vars1.user); } + free_account_details(account_details); return ret; } diff --git a/lib/MySQL_Query_Processor.cpp b/lib/MySQL_Query_Processor.cpp new file mode 100644 index 0000000000..3266f74068 --- /dev/null +++ b/lib/MySQL_Query_Processor.cpp @@ -0,0 +1,992 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include // std::cout +#include // std::sort +#include // std::vector +#include "proxysql.h" +#include "cpp.h" + +#include "Command_Counter.h" +#include "MySQL_PreparedStatement.h" +#include "MySQL_Query_Processor.h" + +extern MySQL_Threads_Handler *GloMTH; +extern ProxySQL_Admin *GloAdmin; + +static __thread Command_Counter* _thr_commands_counters[MYSQL_COM_QUERY___NONE]; + +static char* commands_counters_desc[MYSQL_COM_QUERY___NONE] = { + [MYSQL_COM_QUERY_ALTER_TABLE] = (char*)"ALTER_TABLE", + [MYSQL_COM_QUERY_ALTER_VIEW] = (char*)"ALTER_VIEW", + [MYSQL_COM_QUERY_ANALYZE_TABLE] = (char*)"ANALYZE_TABLE", + [MYSQL_COM_QUERY_BEGIN] = (char*)"BEGIN", + [MYSQL_COM_QUERY_CALL] = (char*)"CALL", + [MYSQL_COM_QUERY_CHANGE_MASTER] = (char*)"CHANGE_MASTER", + [MYSQL_COM_QUERY_COMMIT] = (char*)"COMMIT", + [MYSQL_COM_QUERY_CREATE_DATABASE] = (char*)"CREATE_DATABASE", + [MYSQL_COM_QUERY_CREATE_INDEX] = (char*)"CREATE_INDEX", + [MYSQL_COM_QUERY_CREATE_TABLE] = (char*)"CREATE_TABLE", + [MYSQL_COM_QUERY_CREATE_TEMPORARY] = (char*)"CREATE_TEMPORARY", + [MYSQL_COM_QUERY_CREATE_TRIGGER] = (char*)"CREATE_TRIGGER", + [MYSQL_COM_QUERY_CREATE_USER] = (char*)"CREATE_USER", + [MYSQL_COM_QUERY_CREATE_VIEW] = (char*)"CREATE_VIEW", + [MYSQL_COM_QUERY_DEALLOCATE] = (char*)"DEALLOCATE", + [MYSQL_COM_QUERY_DELETE] = (char*)"DELETE", + [MYSQL_COM_QUERY_DESCRIBE] = (char*)"DESCRIBE", + [MYSQL_COM_QUERY_DROP_DATABASE] = (char*)"DROP_DATABASE", + [MYSQL_COM_QUERY_DROP_INDEX] = (char*)"DROP_INDEX", + [MYSQL_COM_QUERY_DROP_TABLE] = (char*)"DROP_TABLE", + [MYSQL_COM_QUERY_DROP_TRIGGER] = (char*)"DROP_TRIGGER", + [MYSQL_COM_QUERY_DROP_USER] = (char*)"DROP_USER", + [MYSQL_COM_QUERY_DROP_VIEW] = (char*)"DROP_VIEW", + [MYSQL_COM_QUERY_GRANT] = (char*)"GRANT", + [MYSQL_COM_QUERY_EXECUTE] = (char*)"EXECUTE", + [MYSQL_COM_QUERY_EXPLAIN] = (char*)"EXPLAIN", + [MYSQL_COM_QUERY_FLUSH] = (char*)"FLUSH", + [MYSQL_COM_QUERY_INSERT] = (char*)"INSERT", + [MYSQL_COM_QUERY_KILL] = (char*)"KILL", + [MYSQL_COM_QUERY_LOAD] = (char*)"LOAD", + [MYSQL_COM_QUERY_LOCK_TABLE] = (char*)"LOCK_TABLE", + [MYSQL_COM_QUERY_OPTIMIZE] = (char*)"OPTIMIZE", + [MYSQL_COM_QUERY_PREPARE] = (char*)"PREPARE", + [MYSQL_COM_QUERY_PURGE] = (char*)"PURGE", + [MYSQL_COM_QUERY_RELEASE_SAVEPOINT] = (char*)"RELEASE_SAVEPOINT", + [MYSQL_COM_QUERY_RENAME_TABLE] = (char*)"RENAME_TABLE", + [MYSQL_COM_QUERY_RESET_MASTER] = (char*)"RESET_MASTER", + [MYSQL_COM_QUERY_RESET_SLAVE] = (char*)"RESET_SLAVE", + [MYSQL_COM_QUERY_REPLACE] = (char*)"REPLACE", + [MYSQL_COM_QUERY_REVOKE] = (char*)"REVOKE", + [MYSQL_COM_QUERY_ROLLBACK] = (char*)"ROLLBACK", + [MYSQL_COM_QUERY_ROLLBACK_SAVEPOINT] = (char*)"ROLLBACK_SAVEPOINT", + [MYSQL_COM_QUERY_SAVEPOINT] = (char*)"SAVEPOINT", + [MYSQL_COM_QUERY_SELECT] = (char*)"SELECT", + [MYSQL_COM_QUERY_SELECT_FOR_UPDATE] = (char*)"SELECT_FOR_UPDATE", + [MYSQL_COM_QUERY_SET] = (char*)"SET", + [MYSQL_COM_QUERY_SHOW_TABLE_STATUS] = (char*)"SHOW_TABLE_STATUS", + [MYSQL_COM_QUERY_START_TRANSACTION] = (char*)"START_TRANSACTION", + [MYSQL_COM_QUERY_TRUNCATE_TABLE] = (char*)"TRUNCATE_TABLE", + [MYSQL_COM_QUERY_UNLOCK_TABLES] = (char*)"UNLOCK_TABLES", + [MYSQL_COM_QUERY_UPDATE] = (char*)"UPDATE", + [MYSQL_COM_QUERY_USE] = (char*)"USE", + [MYSQL_COM_QUERY_SHOW] = (char*)"SHOW", + [MYSQL_COM_QUERY_UNKNOWN] = (char*)"UNKNOWN" +}; + +MySQL_Rule_Text::MySQL_Rule_Text(const MySQL_Query_Processor_Rule_t* mqr) { + num_fields = 36; // this count the number of fields + pta = NULL; + pta = (char**)malloc(sizeof(char*) * num_fields); + itostr(pta[0], (long long)mqr->rule_id); + itostr(pta[1], (long long)mqr->active); + pta[2] = strdup_null(mqr->username); + pta[3] = strdup_null(mqr->schemaname); + itostr(pta[4], (long long)mqr->flagIN); + + pta[5] = strdup_null(mqr->client_addr); + pta[6] = strdup_null(mqr->proxy_addr); + itostr(pta[7], (long long)mqr->proxy_port); + + char buf[20]; + if (mqr->digest) { + sprintf(buf, "0x%016llX", (long long unsigned int)mqr->digest); + pta[8] = strdup(buf); + } + else { + pta[8] = NULL; + } + + pta[9] = strdup_null(mqr->match_digest); + pta[10] = strdup_null(mqr->match_pattern); + itostr(pta[11], (long long)mqr->negate_match_pattern); + std::string re_mod; + re_mod = ""; + if ((mqr->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) re_mod = "CASELESS"; + if ((mqr->re_modifiers & QP_RE_MOD_GLOBAL) == QP_RE_MOD_GLOBAL) { + if (re_mod.length()) { + re_mod = re_mod + ","; + } + re_mod = re_mod + "GLOBAL"; + } + pta[12] = strdup_null((char*)re_mod.c_str()); // re_modifiers + itostr(pta[13], (long long)mqr->flagOUT); + pta[14] = strdup_null(mqr->replace_pattern); + itostr(pta[15], (long long)mqr->destination_hostgroup); + itostr(pta[16], (long long)mqr->cache_ttl); + itostr(pta[17], (long long)mqr->cache_empty_result); + itostr(pta[18], (long long)mqr->cache_timeout); + itostr(pta[19], (long long)mqr->reconnect); + itostr(pta[20], (long long)mqr->timeout); + itostr(pta[21], (long long)mqr->retries); + itostr(pta[22], (long long)mqr->delay); + itostr(pta[23], (long long)mqr->next_query_flagIN); + itostr(pta[24], (long long)mqr->mirror_flagOUT); + itostr(pta[25], (long long)mqr->mirror_hostgroup); + pta[26] = strdup_null(mqr->error_msg); + pta[27] = strdup_null(mqr->OK_msg); + itostr(pta[28], (long long)mqr->sticky_conn); + itostr(pta[29], (long long)mqr->multiplex); + itostr(pta[30], (long long)mqr->gtid_from_hostgroup); + itostr(pta[31], (long long)mqr->log); + itostr(pta[32], (long long)mqr->apply); + pta[33] = strdup_null(mqr->attributes); + pta[34] = strdup_null(mqr->comment); // issue #643 + itostr(pta[35], (long long)mqr->hits); +} + +MySQL_Query_Processor::MySQL_Query_Processor() : + Query_Processor(GloMTH->get_variable_int("query_rules_fast_routing_algorithm")) { + + for (int i = 0; i < MYSQL_COM_QUERY___NONE; i++) commands_counters[i] = new Command_Counter(i,15,commands_counters_desc); + + //if (GloMTH) { + // query_rules_fast_routing_algorithm = GloMTH->get_variable_int("query_rules_fast_routing_algorithm"); + //} +} + +MySQL_Query_Processor::~MySQL_Query_Processor() { + for (int i = 0; i < MYSQL_COM_QUERY___NONE; i++) delete commands_counters[i]; +} + +enum MYSQL_COM_QUERY_command MySQL_Query_Processor::query_parser_command_type(SQP_par_t* qp) { + char* text = NULL; // this new variable is a pointer to either qp->digest_text , or to the query + if (qp->digest_text) { + text = qp->digest_text; + } else { + text = qp->query_prefix; + } + + enum MYSQL_COM_QUERY_command ret = MYSQL_COM_QUERY_UNKNOWN; + char c1; + + tokenizer_t tok; + tokenizer(&tok, text, " ", TOKENIZER_NO_EMPTIES); + char* token = NULL; +__get_token: + token = (char*)tokenize(&tok); + if (token == NULL) { + goto __exit__query_parser_command_type; + } +__remove_paranthesis: + if (token[0] == '(') { + if (strlen(token) > 1) { + token++; + goto __remove_paranthesis; + } + else { + goto __get_token; + } + } + c1 = token[0]; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Command:%s Prefix:%c\n", token, c1); + switch (c1) { + case 'a': + case 'A': + if (!mystrcasecmp("ALTER", token)) { // ALTER [ONLINE | OFFLINE] [IGNORE] TABLE + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!mystrcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_ALTER_TABLE; + break; + } + else { + if (!mystrcasecmp("OFFLINE", token) || !mystrcasecmp("ONLINE", token)) { + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!mystrcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_ALTER_TABLE; + break; + } + else { + if (!mystrcasecmp("IGNORE", token)) { + if (token == NULL) break; + token = (char*)tokenize(&tok); + if (!mystrcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_ALTER_TABLE; + break; + } + } + } + } + else { + if (!mystrcasecmp("IGNORE", token)) { + if (token == NULL) break; + token = (char*)tokenize(&tok); + if (!mystrcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_ALTER_TABLE; + break; + } + } + } + } + if (!mystrcasecmp("VIEW", token)) { + ret = MYSQL_COM_QUERY_ALTER_VIEW; + break; + } + break; + } + if (!mystrcasecmp("ANALYZE", token)) { // ANALYZE [NO_WRITE_TO_BINLOG | LOCAL] TABLE + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_ANALYZE_TABLE; + } + else { + if (!strcasecmp("NO_WRITE_TO_BINLOG", token) || !strcasecmp("LOCAL", token)) { + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_ANALYZE_TABLE; + } + } + } + break; + } + break; + case 'b': + case 'B': + if (!strcasecmp("BEGIN", token)) { // BEGIN + ret = MYSQL_COM_QUERY_BEGIN; + } + break; + case 'c': + case 'C': + if (!strcasecmp("CALL", token)) { // CALL + ret = MYSQL_COM_QUERY_CALL; + break; + } + if (!strcasecmp("CHANGE", token)) { // CHANGE + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("MASTER", token)) { + ret = MYSQL_COM_QUERY_CHANGE_MASTER; + break; + } + break; + } + if (!strcasecmp("COMMIT", token)) { // COMMIT + ret = MYSQL_COM_QUERY_COMMIT; + break; + } + if (!strcasecmp("CREATE", token)) { // CREATE + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("DATABASE", token)) { + ret = MYSQL_COM_QUERY_CREATE_DATABASE; + break; + } + if (!strcasecmp("INDEX", token)) { + ret = MYSQL_COM_QUERY_CREATE_INDEX; + break; + } + if (!strcasecmp("SCHEMA", token)) { + ret = MYSQL_COM_QUERY_CREATE_DATABASE; + break; + } + if (!strcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_CREATE_TABLE; + break; + } + if (!strcasecmp("TEMPORARY", token)) { + ret = MYSQL_COM_QUERY_CREATE_TEMPORARY; + break; + } + if (!strcasecmp("TRIGGER", token)) { + ret = MYSQL_COM_QUERY_CREATE_TRIGGER; + break; + } + if (!strcasecmp("USER", token)) { + ret = MYSQL_COM_QUERY_CREATE_USER; + break; + } + if (!strcasecmp("VIEW", token)) { + ret = MYSQL_COM_QUERY_CREATE_VIEW; + break; + } + break; + } + break; + case 'd': + case 'D': + if (!strcasecmp("DEALLOCATE", token)) { // DEALLOCATE PREPARE + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("PREPARE", token)) { + ret = MYSQL_COM_QUERY_DEALLOCATE; + break; + } + } + if (!strcasecmp("DELETE", token)) { // DELETE + ret = MYSQL_COM_QUERY_DELETE; + break; + } + if (!strcasecmp("DESCRIBE", token)) { // DESCRIBE + ret = MYSQL_COM_QUERY_DESCRIBE; + break; + } + if (!strcasecmp("DROP", token)) { // DROP + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_DROP_TABLE; + break; + } + if (!strcasecmp("TRIGGER", token)) { + ret = MYSQL_COM_QUERY_DROP_TRIGGER; + break; + } + if (!strcasecmp("USER", token)) { + ret = MYSQL_COM_QUERY_DROP_USER; + break; + } + if (!strcasecmp("VIEW", token)) { + ret = MYSQL_COM_QUERY_DROP_VIEW; + break; + } + } + break; + case 'e': + case 'E': + if (!strcasecmp("EXECUTE", token)) { // EXECUTE + ret = MYSQL_COM_QUERY_EXECUTE; + } + break; + case 'f': + case 'F': + if (!strcasecmp("FLUSH", token)) { // FLUSH + ret = MYSQL_COM_QUERY_FLUSH; + break; + } + break; + case 'g': + case 'G': + if (!strcasecmp("GRANT", token)) { // GRANT + ret = MYSQL_COM_QUERY_GRANT; + break; + } + break; + case 'i': + case 'I': + if (!strcasecmp("INSERT", token)) { // INSERT + ret = MYSQL_COM_QUERY_INSERT; + break; + } + break; + case 'k': + case 'K': + if (!strcasecmp("KILL", token)) { // KILL + ret = MYSQL_COM_QUERY_KILL; + break; + } + break; + case 'l': + case 'L': + if (!strcasecmp("LOCK", token)) { // LOCK + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_LOCK_TABLE; + break; + } + } + if (!strcasecmp("LOAD", token)) { // LOAD + ret = MYSQL_COM_QUERY_LOAD; + break; + } + break; + case 'o': + case 'O': + if (!strcasecmp("OPTIMIZE", token)) { // OPTIMIZE + ret = MYSQL_COM_QUERY_OPTIMIZE; + break; + } + break; + case 'p': + case 'P': + if (!strcasecmp("PREPARE", token)) { // PREPARE + ret = MYSQL_COM_QUERY_PREPARE; + break; + } + if (!strcasecmp("PURGE", token)) { // PURGE + ret = MYSQL_COM_QUERY_PURGE; + break; + } + break; + case 'r': + case 'R': + if (!strcasecmp("RELEASE", token)) { // RELEASE + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("SAVEPOINT", token)) { + ret = MYSQL_COM_QUERY_RELEASE_SAVEPOINT; + break; + } + } + if (!strcasecmp("RENAME", token)) { // RENAME + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_RENAME_TABLE; + break; + } + } + if (!strcasecmp("REPLACE", token)) { // REPLACE + ret = MYSQL_COM_QUERY_REPLACE; + break; + } + if (!strcasecmp("RESET", token)) { // RESET + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("MASTER", token)) { + ret = MYSQL_COM_QUERY_RESET_MASTER; + break; + } + if (!strcasecmp("SLAVE", token)) { + ret = MYSQL_COM_QUERY_RESET_SLAVE; + break; + } + break; + } + if (!strcasecmp("REVOKE", token)) { // REVOKE + ret = MYSQL_COM_QUERY_REVOKE; + break; + } + if (!strcasecmp("ROLLBACK", token)) { // ROLLBACK + token = (char*)tokenize(&tok); + if (token == NULL) { + ret = MYSQL_COM_QUERY_ROLLBACK; + break; + } + else { + if (!strcasecmp("TO", token)) { + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("SAVEPOINT", token)) { + ret = MYSQL_COM_QUERY_ROLLBACK_SAVEPOINT; + break; + } + } + } + break; + } + break; + case 's': + case 'S': + if (!mystrcasecmp("SAVEPOINT", token)) { // SAVEPOINT + ret = MYSQL_COM_QUERY_SAVEPOINT; + break; + } + if (!mystrcasecmp("SELECT", token)) { // SELECT + ret = MYSQL_COM_QUERY_SELECT; + break; + // FIXME: SELECT FOR UPDATE is not implemented + } + if (!mystrcasecmp("SET", token)) { // SET + ret = MYSQL_COM_QUERY_SET; + break; + } + if (!mystrcasecmp("SHOW", token)) { // SHOW + ret = MYSQL_COM_QUERY_SHOW; + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) { + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("STATUS", token)) { + ret = MYSQL_COM_QUERY_SHOW_TABLE_STATUS; + } + } + break; + } + if (!mystrcasecmp("START", token)) { // START + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TRANSACTION", token)) { + ret = MYSQL_COM_QUERY_START_TRANSACTION; + } + break; + } + break; + case 't': + case 'T': + if (!strcasecmp("TRUNCATE", token)) { // TRUNCATE + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) { + ret = MYSQL_COM_QUERY_TRUNCATE_TABLE; + break; + } + } + break; + case 'u': + case 'U': + if (!strcasecmp("UNLOCK", token)) { // UNLOCK + ret = MYSQL_COM_QUERY_UNLOCK_TABLES; + break; + } + if (!strcasecmp("UPDATE", token)) { // UPDATE + ret = MYSQL_COM_QUERY_UPDATE; + break; + } + break; + default: + break; + } + +__exit__query_parser_command_type: + free_tokenizer(&tok); + if (qp->query_prefix) { + free(qp->query_prefix); + qp->query_prefix = NULL; + } + return ret; +} + +bool MySQL_Query_Processor::_is_valid_gtid(char* gtid, size_t gtid_len) { + if (gtid_len < 3) { + return false; + } + char* sep_pos = index(gtid, ':'); + if (sep_pos == NULL) { + return false; + } + size_t uuid_len = sep_pos - gtid; + if (uuid_len < 1) { + return false; + } + if (gtid_len < uuid_len + 2) { + return false; + } + return true; +} + +void MySQL_Query_Processor::update_query_processor_stats() { + Query_Processor::update_query_processor_stats(); + + for (int i = 0; i < MYSQL_COM_QUERY___NONE; i++) { + commands_counters[i]->add_and_reset(_thr_commands_counters[i]); + } +} + +void MySQL_Query_Processor::init_thread() { + Query_Processor::init_thread(); + for (int i = 0; i < MYSQL_COM_QUERY___NONE; i++) _thr_commands_counters[i] = new Command_Counter(i,15,commands_counters_desc); +} + +void MySQL_Query_Processor::end_thread() { + Query_Processor::end_thread(); + for (int i = 0; i < MYSQL_COM_QUERY___NONE; i++) delete _thr_commands_counters[i]; +}; + +unsigned long long MySQL_Query_Processor::query_parser_update_counters(MySQL_Session* sess, enum MYSQL_COM_QUERY_command c, SQP_par_t* qp, unsigned long long t) { + if (c >= MYSQL_COM_QUERY___NONE) return 0; + unsigned long long ret = _thr_commands_counters[c]->add_time(t); + uint64_t digest = 0; + char* digest_text = NULL; + if (sess->CurrentQuery.stmt_info == NULL && qp->digest_text) { + digest = qp->digest; + digest_text = qp->digest_text; + } else if (sess->CurrentQuery.stmt_info && sess->CurrentQuery.stmt_info->digest_text) { + MySQL_STMT_Global_info* stmt_info = sess->CurrentQuery.stmt_info; + digest = stmt_info->digest; + digest_text = stmt_info->digest_text; + } + if (digest_text) + Query_Processor::query_parser_update_counters(sess, qp->digest_total, digest, digest_text, t); + return ret; +} + +SQLite3_result* MySQL_Query_Processor::get_stats_commands_counters() { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping commands counters\n"); + SQLite3_result* result = new SQLite3_result(15); + result->add_column_definition(SQLITE_TEXT, "Command"); + result->add_column_definition(SQLITE_TEXT, "Total_Cnt"); + result->add_column_definition(SQLITE_TEXT, "Total_Time_us"); + result->add_column_definition(SQLITE_TEXT, "cnt_100us"); + result->add_column_definition(SQLITE_TEXT, "cnt_500us"); + result->add_column_definition(SQLITE_TEXT, "cnt_1ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_5ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_10ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_50ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_100ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_500ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_1s"); + result->add_column_definition(SQLITE_TEXT, "cnt_5s"); + result->add_column_definition(SQLITE_TEXT, "cnt_10s"); + result->add_column_definition(SQLITE_TEXT, "cnt_INFs"); + for (int i = 0; i < MYSQL_COM_QUERY__UNINITIALIZED; i++) { + char** pta = commands_counters[i]->get_row(); + result->add_row(pta); + commands_counters[i]->free_row(pta); + } + return result; +} + +MySQL_Query_Processor_Output* MySQL_Query_Processor::process_query(MySQL_Session* sess, void* ptr, unsigned int size, Query_Info* qi) { + // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE + // to avoid unnecssary deallocation/allocation, we initialize qpo witout new allocation + MySQL_Query_Processor_Output* ret = sess->qpo; + ret->init(); + + SQP_par_t stmt_exec_qp; + SQP_par_t* qp = NULL; + if (qi) { + // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE + if (ptr) { + qp = (SQP_par_t*)&qi->QueryParserArgs; + } + else { + qp = &stmt_exec_qp; + qp->digest = qi->stmt_info->digest; + qp->digest_text = qi->stmt_info->digest_text; + qp->first_comment = qi->stmt_info->first_comment; + } + } +#define stackbuffer_size 128 + char stackbuffer[stackbuffer_size]; + unsigned int len = 0; + char* query = NULL; + // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE + if (ptr) { + len = size - sizeof(mysql_hdr) - 1; + if (len < stackbuffer_size) { + query = stackbuffer; + } + else { + query = (char*)l_alloc(len + 1); + } + memcpy(query, (char*)ptr + sizeof(mysql_hdr) + 1, len); + query[len] = 0; + } + else { + query = qi->stmt_info->query; + len = qi->stmt_info->query_length; + } + + Query_Processor::process_query(sess, ptr == NULL, query, len, ret, qp); + + // FIXME : there is too much data being copied around + if (len < stackbuffer_size) { + // query is in the stack + } else { + if (ptr) { + l_free(len + 1, query); + } + } + + return ret; +} + +MySQL_Query_Processor_Rule_t* MySQL_Query_Processor::new_query_rule(int rule_id, bool active, const char* username, const char* schemaname, int flagIN, const char* client_addr, + const char* proxy_addr, int proxy_port, const char* digest, const char* match_digest, const char* match_pattern, bool negate_match_pattern, + const char* re_modifiers, int flagOUT, const char* replace_pattern, int destination_hostgroup, int cache_ttl, int cache_empty_result, + int cache_timeout, int reconnect, int timeout, int retries, int delay, int next_query_flagIN, int mirror_hostgroup, + int mirror_flagOUT, const char* error_msg, const char* OK_msg, int sticky_conn, int multiplex, int gtid_from_hostgroup, int log, + bool apply, const char* attributes, const char* comment) { + + MySQL_Query_Processor_Rule_t* newQR = (MySQL_Query_Processor_Rule_t*)malloc(sizeof(MySQL_Query_Processor_Rule_t)); + newQR->rule_id = rule_id; + newQR->active = active; + newQR->username = (username ? strdup(username) : NULL); + newQR->schemaname = (schemaname ? strdup(schemaname) : NULL); + newQR->flagIN = flagIN; + newQR->match_digest = (match_digest ? strdup(match_digest) : NULL); + newQR->match_pattern = (match_pattern ? strdup(match_pattern) : NULL); + newQR->negate_match_pattern = negate_match_pattern; + newQR->re_modifiers = 0; + { + tokenizer_t tok; + tokenizer(&tok, re_modifiers, ",", TOKENIZER_NO_EMPTIES); + const char* token; + for (token = tokenize(&tok); token; token = tokenize(&tok)) { + if (strncasecmp(token, (char*)"CASELESS", strlen((char*)"CASELESS")) == 0) { + newQR->re_modifiers |= QP_RE_MOD_CASELESS; + } + if (strncasecmp(token, (char*)"GLOBAL", strlen((char*)"GLOBAL")) == 0) { + newQR->re_modifiers |= QP_RE_MOD_GLOBAL; + } + } + free_tokenizer(&tok); + } + newQR->flagOUT = flagOUT; + newQR->replace_pattern = (replace_pattern ? strdup(replace_pattern) : NULL); + newQR->destination_hostgroup = destination_hostgroup; + newQR->cache_ttl = cache_ttl; + newQR->cache_empty_result = cache_empty_result; + newQR->cache_timeout = cache_timeout; + newQR->reconnect = reconnect; + newQR->timeout = timeout; + newQR->retries = retries; + newQR->delay = delay; + newQR->next_query_flagIN = next_query_flagIN; + newQR->mirror_flagOUT = mirror_flagOUT; + newQR->mirror_hostgroup = mirror_hostgroup; + newQR->error_msg = (error_msg ? strdup(error_msg) : NULL); + newQR->OK_msg = (OK_msg ? strdup(OK_msg) : NULL); + newQR->sticky_conn = sticky_conn; + newQR->multiplex = multiplex; + newQR->gtid_from_hostgroup = gtid_from_hostgroup; + newQR->apply = apply; + newQR->attributes = (attributes ? strdup(attributes) : NULL); + newQR->comment = (comment ? strdup(comment) : NULL); // see issue #643 + newQR->regex_engine1 = NULL; + newQR->regex_engine2 = NULL; + newQR->hits = 0; + + newQR->client_addr_wildcard_position = -1; // not existing by default + newQR->client_addr = (client_addr ? strdup(client_addr) : NULL); + if (newQR->client_addr) { + char* pct = strchr(newQR->client_addr, '%'); + if (pct) { // there is a wildcard . We assume Admin did already all the input validation + if (pct == newQR->client_addr) { + // client_addr == '%' + // % is at the end of the string, but also at the beginning + // becoming a catch all + newQR->client_addr_wildcard_position = 0; + } + else { + // this math is valid also if (pct == newQR->client_addr) + // but we separate it to clarify that client_addr_wildcard_position is a match all + newQR->client_addr_wildcard_position = strlen(newQR->client_addr) - strlen(pct); + } + } + } + newQR->proxy_addr = (proxy_addr ? strdup(proxy_addr) : NULL); + newQR->proxy_port = proxy_port; + newQR->log = log; + newQR->digest = 0; + if (digest) { + unsigned long long num = strtoull(digest, NULL, 0); + if (num != ULLONG_MAX && num != 0) { + newQR->digest = num; + } + else { + proxy_error("Incorrect digest for rule_id %d : %s\n", rule_id, digest); + } + } + newQR->flagOUT_weights_total = 0; + newQR->flagOUT_ids = NULL; + newQR->flagOUT_weights = NULL; + if (newQR->attributes != NULL) { + if (strlen(newQR->attributes)) { + nlohmann::json j_attributes = nlohmann::json::parse(newQR->attributes); + if (j_attributes.find("flagOUTs") != j_attributes.end()) { + newQR->flagOUT_ids = new vector; + newQR->flagOUT_weights = new vector; + const nlohmann::json& flagOUTs = j_attributes["flagOUTs"]; + if (flagOUTs.type() == nlohmann::json::value_t::array) { + for (auto it = flagOUTs.begin(); it != flagOUTs.end(); it++) { + bool parsed = false; + const nlohmann::json& j = *it; + if (j.find("id") != j.end() && j.find("weight") != j.end()) { + if (j["id"].type() == nlohmann::json::value_t::number_unsigned && j["weight"].type() == nlohmann::json::value_t::number_unsigned) { + int id = j["id"]; + int weight = j["weight"]; + newQR->flagOUT_ids->push_back(id); + newQR->flagOUT_weights->push_back(weight); + newQR->flagOUT_weights_total += weight; + parsed = true; + } + } + if (parsed == false) { + proxy_error("Failed to parse flagOUTs in JSON on attributes for rule_id %d : %s\n", newQR->rule_id, j.dump().c_str()); + } + } + } + else { + proxy_error("Failed to parse flagOUTs attributes for rule_id %d : %s\n", newQR->rule_id, flagOUTs.dump().c_str()); + } + } + } + } + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Creating new rule in %p : rule_id:%d, active:%d, username=%s, schemaname=%s, flagIN:%d, %smatch_digest=\"%s\", %smatch_pattern=\"%s\", flagOUT:%d replace_pattern=\"%s\", destination_hostgroup:%d, apply:%d\n", newQR, newQR->rule_id, newQR->active, newQR->username, newQR->schemaname, newQR->flagIN, (newQR->negate_match_pattern ? "(!)" : ""), newQR->match_digest, (newQR->negate_match_pattern ? "(!)" : ""), newQR->match_pattern, newQR->flagOUT, newQR->replace_pattern, newQR->destination_hostgroup, newQR->apply); + return newQR; +} + +MySQL_Query_Processor_Rule_t* MySQL_Query_Processor::new_query_rule(const MySQL_Query_Processor_Rule_t* mqr) { + + char buf[20]; + if (mqr->digest) { // not 0 + sprintf(buf, "0x%016llX", (long long unsigned int)mqr->digest); + } + + std::string re_mod; + re_mod = ""; + if ((mqr->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) re_mod = "CASELESS"; + if ((mqr->re_modifiers & QP_RE_MOD_GLOBAL) == QP_RE_MOD_GLOBAL) { + if (re_mod.length()) { + re_mod = re_mod + ","; + } + re_mod = re_mod + "GLOBAL"; + } + + MySQL_Query_Processor_Rule_t* newQR = (MySQL_Query_Processor_Rule_t*)malloc(sizeof(MySQL_Query_Processor_Rule_t)); + newQR->rule_id = mqr->rule_id; + newQR->active = mqr->active; + newQR->username = (mqr->username ? strdup(mqr->username) : NULL); + newQR->schemaname = (mqr->schemaname ? strdup(mqr->schemaname) : NULL); + newQR->flagIN = mqr->flagIN; + newQR->match_digest = (mqr->match_digest ? strdup(mqr->match_digest) : NULL); + newQR->match_pattern = (mqr->match_pattern ? strdup(mqr->match_pattern) : NULL); + newQR->negate_match_pattern = mqr->negate_match_pattern; + newQR->re_modifiers = 0; + { + tokenizer_t tok; + tokenizer(&tok, re_mod.c_str(), ",", TOKENIZER_NO_EMPTIES); + const char* token; + for (token = tokenize(&tok); token; token = tokenize(&tok)) { + if (strncasecmp(token, (char*)"CASELESS", strlen((char*)"CASELESS")) == 0) { + newQR->re_modifiers |= QP_RE_MOD_CASELESS; + } + if (strncasecmp(token, (char*)"GLOBAL", strlen((char*)"GLOBAL")) == 0) { + newQR->re_modifiers |= QP_RE_MOD_GLOBAL; + } + } + free_tokenizer(&tok); + } + newQR->flagOUT = mqr->flagOUT; + newQR->replace_pattern = (mqr->replace_pattern ? strdup(mqr->replace_pattern) : NULL); + newQR->destination_hostgroup = mqr->destination_hostgroup; + newQR->cache_ttl = mqr->cache_ttl; + newQR->cache_empty_result = mqr->cache_empty_result; + newQR->cache_timeout = mqr->cache_timeout; + newQR->reconnect = mqr->reconnect; + newQR->timeout = mqr->timeout; + newQR->retries = mqr->retries; + newQR->delay = mqr->delay; + newQR->next_query_flagIN = mqr->next_query_flagIN; + newQR->mirror_flagOUT = mqr->mirror_flagOUT; + newQR->mirror_hostgroup = mqr->mirror_hostgroup; + newQR->error_msg = (mqr->error_msg ? strdup(mqr->error_msg) : NULL); + newQR->OK_msg = (mqr->OK_msg ? strdup(mqr->OK_msg) : NULL); + newQR->sticky_conn = mqr->sticky_conn; + newQR->multiplex = mqr->multiplex; + newQR->gtid_from_hostgroup = mqr->gtid_from_hostgroup; + newQR->apply = mqr->apply; + newQR->attributes = (mqr->attributes ? strdup(mqr->attributes) : NULL); + newQR->comment = (mqr->comment ? strdup(mqr->comment) : NULL); // see issue #643 + newQR->regex_engine1 = NULL; + newQR->regex_engine2 = NULL; + newQR->hits = 0; + + newQR->client_addr_wildcard_position = -1; // not existing by default + newQR->client_addr = (mqr->client_addr ? strdup(mqr->client_addr) : NULL); + if (newQR->client_addr) { + char* pct = strchr(newQR->client_addr, '%'); + if (pct) { // there is a wildcard . We assume Admin did already all the input validation + if (pct == newQR->client_addr) { + // client_addr == '%' + // % is at the end of the string, but also at the beginning + // becoming a catch all + newQR->client_addr_wildcard_position = 0; + } + else { + // this math is valid also if (pct == newQR->client_addr) + // but we separate it to clarify that client_addr_wildcard_position is a match all + newQR->client_addr_wildcard_position = strlen(newQR->client_addr) - strlen(pct); + } + } + } + newQR->proxy_addr = (mqr->proxy_addr ? strdup(mqr->proxy_addr) : NULL); + newQR->proxy_port = mqr->proxy_port; + newQR->log = mqr->log; + newQR->digest = 0; + if (mqr->digest) { + unsigned long long num = strtoull(buf, NULL, 0); + if (num != ULLONG_MAX && num != 0) { + newQR->digest = num; + } + else { + proxy_error("Incorrect digest for rule_id %d : %s\n", mqr->rule_id, buf); + } + } + newQR->flagOUT_weights_total = 0; + newQR->flagOUT_ids = NULL; + newQR->flagOUT_weights = NULL; + if (newQR->attributes != NULL) { + if (strlen(newQR->attributes)) { + nlohmann::json j_attributes = nlohmann::json::parse(newQR->attributes); + if (j_attributes.find("flagOUTs") != j_attributes.end()) { + newQR->flagOUT_ids = new vector; + newQR->flagOUT_weights = new vector; + const nlohmann::json& flagOUTs = j_attributes["flagOUTs"]; + if (flagOUTs.type() == nlohmann::json::value_t::array) { + for (auto it = flagOUTs.begin(); it != flagOUTs.end(); it++) { + bool parsed = false; + const nlohmann::json& j = *it; + if (j.find("id") != j.end() && j.find("weight") != j.end()) { + if (j["id"].type() == nlohmann::json::value_t::number_unsigned && j["weight"].type() == nlohmann::json::value_t::number_unsigned) { + int id = j["id"]; + int weight = j["weight"]; + newQR->flagOUT_ids->push_back(id); + newQR->flagOUT_weights->push_back(weight); + newQR->flagOUT_weights_total += weight; + parsed = true; + } + } + if (parsed == false) { + proxy_error("Failed to parse flagOUTs in JSON on attributes for rule_id %d : %s\n", newQR->rule_id, j.dump().c_str()); + } + } + } + else { + proxy_error("Failed to parse flagOUTs attributes for rule_id %d : %s\n", newQR->rule_id, flagOUTs.dump().c_str()); + } + } + } + } + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Creating new rule in %p : rule_id:%d, active:%d, username=%s, schemaname=%s, flagIN:%d, %smatch_digest=\"%s\", %smatch_pattern=\"%s\", flagOUT:%d replace_pattern=\"%s\", destination_hostgroup:%d, apply:%d\n", newQR, newQR->rule_id, newQR->active, newQR->username, newQR->schemaname, newQR->flagIN, (newQR->negate_match_pattern ? "(!)" : ""), newQR->match_digest, (newQR->negate_match_pattern ? "(!)" : ""), newQR->match_pattern, newQR->flagOUT, newQR->replace_pattern, newQR->destination_hostgroup, newQR->apply); + return newQR; +} + +SQLite3_result* MySQL_Query_Processor::get_current_query_rules() { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query rules, using Global version %d\n", version); + SQLite3_result* result = new SQLite3_result(35); + MySQL_Query_Processor_Rule_t* qr1; + rdlock(); + result->add_column_definition(SQLITE_TEXT, "rule_id"); + result->add_column_definition(SQLITE_TEXT, "active"); + result->add_column_definition(SQLITE_TEXT, "username"); + result->add_column_definition(SQLITE_TEXT, "schemaname"); + result->add_column_definition(SQLITE_TEXT, "flagIN"); + result->add_column_definition(SQLITE_TEXT, "client_addr"); + result->add_column_definition(SQLITE_TEXT, "proxy_addr"); + result->add_column_definition(SQLITE_TEXT, "proxy_port"); + result->add_column_definition(SQLITE_TEXT, "digest"); + result->add_column_definition(SQLITE_TEXT, "match_digest"); + result->add_column_definition(SQLITE_TEXT, "match_pattern"); + result->add_column_definition(SQLITE_TEXT, "negate_match_pattern"); + result->add_column_definition(SQLITE_TEXT, "re_modifiers"); + result->add_column_definition(SQLITE_TEXT, "flagOUT"); + result->add_column_definition(SQLITE_TEXT, "replace_pattern"); + result->add_column_definition(SQLITE_TEXT, "destination_hostgroup"); + result->add_column_definition(SQLITE_TEXT, "cache_ttl"); + result->add_column_definition(SQLITE_TEXT, "cache_empty_result"); + result->add_column_definition(SQLITE_TEXT, "cache_timeout"); + result->add_column_definition(SQLITE_TEXT, "reconnect"); + result->add_column_definition(SQLITE_TEXT, "timeout"); + result->add_column_definition(SQLITE_TEXT, "retries"); + result->add_column_definition(SQLITE_TEXT, "delay"); + result->add_column_definition(SQLITE_TEXT, "next_query_flagIN"); + result->add_column_definition(SQLITE_TEXT, "mirror_flagOUT"); + result->add_column_definition(SQLITE_TEXT, "mirror_hostgroup"); + result->add_column_definition(SQLITE_TEXT, "error_msg"); + result->add_column_definition(SQLITE_TEXT, "OK_msg"); + result->add_column_definition(SQLITE_TEXT, "sticky_conn"); + result->add_column_definition(SQLITE_TEXT, "multiplex"); + result->add_column_definition(SQLITE_TEXT, "gtid_from_hostgroup"); + result->add_column_definition(SQLITE_TEXT, "log"); + result->add_column_definition(SQLITE_TEXT, "apply"); + result->add_column_definition(SQLITE_TEXT, "attributes"); + result->add_column_definition(SQLITE_TEXT, "comment"); // issue #643 + result->add_column_definition(SQLITE_TEXT, "hits"); + for (std::vector::iterator it = rules.begin(); it != rules.end(); ++it) { + qr1 = static_cast(*it); + MySQL_Rule_Text* qt = new MySQL_Rule_Text(qr1); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping Query Rule id: %d\n", qr1->rule_id); + result->add_row(qt->pta); + delete qt; + } + wrunlock(); + return result; +} diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index d7c81a273b..4907463061 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include "MySQL_HostGroups_Manager.h" #include "MySQL_Thread.h" #include "proxysql.h" @@ -8,7 +12,7 @@ #include "mysqld_error.h" #include "MySQL_Data_Stream.h" -#include "query_processor.h" +#include "MySQL_Query_Processor.h" #include "MySQL_PreparedStatement.h" #include "MySQL_Logger.hpp" #include "StatCounters.h" @@ -25,9 +29,9 @@ #define SELECT_VERSION_COMMENT "select @@version_comment limit 1" #define SELECT_VERSION_COMMENT_LEN 32 -#define SELECT_DB_USER "select DATABASE(), USER() limit 1" +//#define SELECT_DB_USER "select DATABASE(), USER() limit 1" #define SELECT_DB_USER_LEN 33 -#define SELECT_CHARSET_STATUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" +//#define SELECT_CHARSET_STATUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" #define SELECT_CHARSET_STATUS_LEN 115 #define PROXYSQL_VERSION_COMMENT "\x01\x00\x00\x01\x01\x27\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x11\x40\x40\x76\x65\x72\x73\x69\x6f\x6e\x5f\x63\x6f\x6d\x6d\x65\x6e\x74\x00\x0c\x21\x00\x18\x00\x00\x00\xfd\x00\x00\x1f\x00\x00\x05\x00\x00\x03\xfe\x00\x00\x02\x00\x0b\x00\x00\x04\x0a(ProxySQL)\x05\x00\x00\x05\xfe\x00\x00\x02\x00" #define PROXYSQL_VERSION_COMMENT_LEN 81 @@ -139,12 +143,12 @@ extern ClickHouse_Authentication *GloClickHouseAuth; extern ClickHouse_Server *GloClickHouseServer; #endif /* PROXYSQLCLICKHOUSE */ - /** * @brief Converts session type to a human-readable string. * @param session_type The session type to convert. * @return A string representing the session type. */ +/* std::string proxysql_session_type_str(enum proxysql_session_type session_type) { if (session_type == PROXYSQL_SESSION_MYSQL) { return "PROXYSQL_SESSION_MYSQL"; @@ -161,55 +165,21 @@ std::string proxysql_session_type_str(enum proxysql_session_type session_type) { } else { return "PROXYSQL_SESSION_NONE"; } -}; +};*/ -/** - * @brief Constructs a Session_Regex object with the specified pattern. - * - * This constructor initializes a Session_Regex object with the provided pattern. - * It sets up the regular expression engine with case insensitivity. - * - * @param[in] p The regular expression pattern. - */ -Session_Regex::Session_Regex(char *p) { - s=strdup(p); - re2::RE2::Options *opt2=new re2::RE2::Options(RE2::Quiet); - opt2->set_case_sensitive(false); - opt=(void *)opt2; - re=(RE2 *)new RE2(s, *opt2); -} -/** - * @brief Destroys the Session_Regex object. - * - * This destructor releases the memory allocated for the regular expression pattern, - * the regular expression object, and its associated options. - */ -Session_Regex::~Session_Regex() { - free(s); - delete (RE2 *)re; - delete (re2::RE2::Options *)opt; -} - -/** - * @brief Matches the given input against the regular expression pattern. - * - * This function attempts to match the input string against the regular expression pattern. - * - * @param[in] m The input string to match. - * @return true if the input matches the pattern, false otherwise. - */ -bool Session_Regex::match(char *m) { - bool rc=false; - rc=RE2::PartialMatch(m,*(RE2 *)re); - return rc; -} KillArgs::KillArgs(char* u, char* p, char* h, unsigned int P, unsigned int _hid, unsigned long i, int kt, int _use_ssl, MySQL_Thread* _mt) : KillArgs(u, p, h, P, _hid, i, kt, _use_ssl, _mt, NULL) { // resolving DNS if available in Cache - resolve_hostname(); + if (h && P) { + const std::string& ip = MySQL_Monitor::dns_lookup(h, false); + + if (ip.empty() == false) { + ip_addr = strdup(ip.c_str()); + } + } } KillArgs::KillArgs(char* u, char* p, char* h, unsigned int P, unsigned int _hid, unsigned long i, int kt, int _use_ssl, MySQL_Thread *_mt, char *ip) { @@ -244,25 +214,6 @@ const char* KillArgs::get_host_address() const { return host_address; } -void KillArgs::resolve_hostname() { - if (ip_addr) { - free(ip_addr); - ip_addr = NULL; - } - if (hostname && port) { - const std::string& ip = MySQL_Monitor::dns_lookup(hostname, false); - - if (ip.empty() == false) { - ip_addr = strdup(ip.c_str()); - } - } -} - -void KillArgs::remove_dns_record() { - if (hostname && port) { - MySQL_Monitor::remove_dns_record_from_dns_cache(hostname); - } -} /** * @brief Thread function to kill a query or connection on a MySQL server. @@ -338,8 +289,6 @@ void* kill_query_thread(void *arg) { ret=mysql_real_connect(mysql,"localhost",ka->username,ka->password,NULL,0,ka->hostname,0); } if (!ret) { - ka->remove_dns_record(); - //ka->resolve_hostname(); int myerr = mysql_errno(mysql); if (ssl_params != NULL && myerr == 2026) { proxy_error("Failed to connect to server %s:%d to run KILL %s %lu. SSL Params: %s , %s , %s , %s , %s , %s , %s , %s\n", @@ -383,7 +332,7 @@ void* kill_query_thread(void *arg) { return NULL; } -extern Query_Processor *GloQPro; +extern MySQL_Query_Processor* GloMyQPro; extern Query_Cache *GloQC; extern ProxySQL_Admin *GloAdmin; extern MySQL_Threads_Handler *GloMTH; @@ -417,7 +366,7 @@ Query_Info::Query_Info() { * Frees resources associated with QueryParserArgs and stmt_info. */ Query_Info::~Query_Info() { - GloQPro->query_parser_free(&QueryParserArgs); + GloMyQPro->query_parser_free(&QueryParserArgs); if (stmt_info) { stmt_info=NULL; } @@ -508,7 +457,7 @@ void Query_Info::init(unsigned char *_p, int len, bool mysql_header) { * @brief Initializes the query parser. */ void Query_Info::query_parser_init() { - GloQPro->query_parser_init(&QueryParserArgs,(char *)QueryPointer,QueryLength,0); + GloMyQPro->query_parser_init(&QueryParserArgs,(char *)QueryPointer,QueryLength,0); } /** @@ -516,7 +465,7 @@ void Query_Info::query_parser_init() { * @return The command type of the query. */ enum MYSQL_COM_QUERY_command Query_Info::query_parser_command_type() { - MyComQueryCmd=GloQPro->query_parser_command_type(&QueryParserArgs); + MyComQueryCmd= GloMyQPro->query_parser_command_type(&QueryParserArgs); return MyComQueryCmd; } @@ -524,7 +473,7 @@ enum MYSQL_COM_QUERY_command Query_Info::query_parser_command_type() { * @brief Frees resources associated with the query parser. */ void Query_Info::query_parser_free() { - GloQPro->query_parser_free(&QueryParserArgs); + GloMyQPro->query_parser_free(&QueryParserArgs); } /** @@ -537,7 +486,7 @@ unsigned long long Query_Info::query_parser_update_counters() { } if (MyComQueryCmd==MYSQL_COM_QUERY___NONE) return 0; // this means that it was never initialized if (MyComQueryCmd == MYSQL_COM_QUERY__UNINITIALIZED) return 0; // this means that it was never initialized - unsigned long long ret=GloQPro->query_parser_update_counters(sess, MyComQueryCmd, &QueryParserArgs, end_time-start_time); + unsigned long long ret= GloMyQPro->query_parser_update_counters(sess, MyComQueryCmd, &QueryParserArgs, end_time-start_time); MyComQueryCmd=MYSQL_COM_QUERY___NONE; QueryPointer=NULL; QueryLength=0; @@ -549,7 +498,7 @@ unsigned long long Query_Info::query_parser_update_counters() { * @return The digest text of the query. */ char * Query_Info::get_digest_text() { - return GloQPro->get_digest_text(&QueryParserArgs); + return GloMyQPro->get_digest_text(&QueryParserArgs); } /** @@ -607,16 +556,6 @@ bool Query_Info::is_select_NOT_for_update() { // let simplify. If NOWAIT is used, we assume FOR UPDATE|SHARE is used __sync_fetch_and_add(&MyHGM->status.select_for_update_or_equivalent, 1); return false; -/* - if (strcasestr(QP," FOR UPDATE ")==NULL) { - __sync_fetch_and_add(&MyHGM->status.select_for_update_or_equivalent, 1); - return false; - } - if (strcasestr(QP," FOR SHARE ")==NULL) { - __sync_fetch_and_add(&MyHGM->status.select_for_update_or_equivalent, 1); - return false; - } -*/ } p=QP; p+=ql-12; @@ -624,16 +563,6 @@ bool Query_Info::is_select_NOT_for_update() { // let simplify. If SKIP LOCKED is used, we assume FOR UPDATE|SHARE is used __sync_fetch_and_add(&MyHGM->status.select_for_update_or_equivalent, 1); return false; -/* - if (strcasestr(QP," FOR UPDATE ")) { - __sync_fetch_and_add(&MyHGM->status.select_for_update_or_equivalent, 1); - return false; - } - if (strcasestr(QP," FOR SHARE ")) { - __sync_fetch_and_add(&MyHGM->status.select_for_update_or_equivalent, 1); - return false; - } -*/ } p=QP; char buf[129]; @@ -661,14 +590,6 @@ bool Query_Info::is_select_NOT_for_update() { return true; } -void * MySQL_Session::operator new(size_t size) { - return l_alloc(size); -} - -void MySQL_Session::operator delete(void *ptr) { - l_free(sizeof(MySQL_Session),ptr); -} - void MySQL_Session::set_status(enum session_status e) { if (e==session_status___NONE) { @@ -691,7 +612,8 @@ MySQL_Session::MySQL_Session() { thread_session_id=0; //handler_ret = 0; pause_until=0; - qpo=new Query_Processor_Output(); + qpo=new MySQL_Query_Processor_Output(); + qpo->init(); start_time=0; command_counters=new StatCounters(15,10); healthy=1; @@ -759,17 +681,6 @@ MySQL_Session::MySQL_Session() { use_ldap_auth = false; } -/** - * @brief Initializes the MySQL session. - */ -void MySQL_Session::init() { - transaction_persistent_hostgroup=-1; - transaction_persistent=false; - mybes= new PtrArray(4); - sess_STMTs_meta=new MySQL_STMTs_meta(); - SLDH=new StmtLongDataHandler(); -} - /** * @brief Resets the MySQL session to its initial state. */ @@ -840,13 +751,26 @@ MySQL_Session::~MySQL_Session() { #endif /* PROXYSQLCLICKHOUSE */ default: if (use_ldap_auth == false) { - GloMyAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->username); + GloMyAuth->decrease_frontend_user_connections( + client_myds->myconn->userinfo->username, + client_myds->myconn->userinfo->passtype + ); } else { GloMyLdapAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->fe_username); } break; } + + if (client_myds->myconn) { + __sync_fetch_and_sub( + client_myds->myconn->userinfo->passtype == PASSWORD_TYPE::PRIMARY ? + &MyHGM->status.client_connections_prim_pass : + &MyHGM->status.client_connections_addl_pass, + 1 + ); + } } + delete client_myds; } if (default_schema) { @@ -874,243 +798,6 @@ MySQL_Session::~MySQL_Session() { } } - -/** - * @brief Find a backend associated with the specified hostgroup ID. - * - * @param hostgroup_id The ID of the hostgroup to search for. - * @return A pointer to the MySQL backend associated with the specified hostgroup ID, or nullptr if not found. - */ -MySQL_Backend * MySQL_Session::find_backend(int hostgroup_id) { - MySQL_Backend *_mybe; - unsigned int i; - for (i=0; i < mybes->len; i++) { - _mybe=(MySQL_Backend *)mybes->index(i); - if (_mybe->hostgroup_id==hostgroup_id) { - return _mybe; - } - } - return NULL; // NULL = backend not found -}; - -/** - * @brief Update expired connections based on specified checks. - * - * This function iterates through the list of backends and their connections - * to determine if any connections have expired based on the provided checks. - * If a connection is found to be expired, its hostgroup ID is added to the - * list of expired connections for further processing. - * - * @param checks A vector of function objects representing checks to determine if a connection has expired. - */ -void MySQL_Session::update_expired_conns(const vector>& checks) { - for (uint32_t i = 0; i < mybes->len; i++) { // iterate through the list of backends - MySQL_Backend* mybe = static_cast(mybes->index(i)); - MySQL_Data_Stream* myds = mybe != nullptr ? mybe->server_myds : nullptr; - MySQL_Connection* myconn = myds != nullptr ? myds->myconn : nullptr; - - //! it performs a series of checks to determine if it has expired - if (myconn != nullptr) { - const bool is_active_transaction = myconn->IsActiveTransaction(); - const bool multiplex_disabled = myconn->MultiplexDisabled(false); - const bool is_idle = myconn->async_state_machine == ASYNC_IDLE; - - // Make sure the connection is reusable before performing any check - if (myconn->reusable==true && is_active_transaction==false && multiplex_disabled==false && is_idle) { - for (const function& check : checks) { - if (check(myconn)) { - // If a connection is found to be expired based on the provided checks, - // its hostgroup ID is added to the list of expired connections (hgs_expired_conns) - // for further processing. - this->hgs_expired_conns.push_back(mybe->hostgroup_id); - break; - } - } - } - } - } -} - -/** - * @brief Create a new MySQL backend associated with the specified hostgroup ID and data stream. - * - * This function creates a new MySQL backend object and associates it with the provided hostgroup ID - * and data stream. If the data stream is not provided (_myds is nullptr), a new MySQL_Data_Stream - * object is created and initialized. - * - * @param hostgroup_id The ID of the hostgroup to which the backend belongs. - * @param _myds The MySQL data stream associated with the backend. - * @return A pointer to the newly created MySQL_Backend object. - */ -MySQL_Backend * MySQL_Session::create_backend(int hostgroup_id, MySQL_Data_Stream *_myds) { - MySQL_Backend *_mybe=new MySQL_Backend(); - proxy_debug(PROXY_DEBUG_NET,4,"HID=%d, _myds=%p, _mybe=%p\n" , hostgroup_id, _myds, _mybe); - _mybe->hostgroup_id=hostgroup_id; - if (_myds) { - _mybe->server_myds=_myds; - } else { - _mybe->server_myds = new MySQL_Data_Stream(); - _mybe->server_myds->DSS=STATE_NOT_INITIALIZED; - _mybe->server_myds->init(MYDS_BACKEND_NOT_CONNECTED, this, 0); - } - // the newly created backend is added to the session's list of backends (mybes) and a pointer to it is returned. - mybes->add(_mybe); - return _mybe; -}; - -/** - * @brief Find or create a MySQL backend associated with the specified hostgroup ID and data stream. - * - * This function first attempts to find an existing MySQL backend associated with the provided - * hostgroup ID. If a backend is found, its pointer is returned. Otherwise, a new MySQL backend - * is created and associated with the hostgroup ID and data stream. If the data stream is not provided - * (_myds is nullptr), a new MySQL_Data_Stream object is created and initialized for the new backend. - * - * @param hostgroup_id The ID of the hostgroup to which the backend belongs. - * @param _myds The MySQL data stream associated with the backend. - * @return A pointer to the MySQL_Backend object found or created. - */ -MySQL_Backend * MySQL_Session::find_or_create_backend(int hostgroup_id, MySQL_Data_Stream *_myds) { - MySQL_Backend *_mybe=find_backend(hostgroup_id); - proxy_debug(PROXY_DEBUG_NET,4,"HID=%d, _myds=%p, _mybe=%p\n" , hostgroup_id, _myds, _mybe); - // The pointer to the found or newly created backend is returned. - return ( _mybe ? _mybe : create_backend(hostgroup_id, _myds) ); -}; - -/** - * @brief Reset all MySQL backends associated with this session. - * - * This function resets all MySQL backends associated with the current session. - * It iterates over all backends stored in the session, resets each backend, and then deletes it. - * - */ -void MySQL_Session::reset_all_backends() { - MySQL_Backend *mybe; - while(mybes->len) { - mybe=(MySQL_Backend *)mybes->remove_index_fast(0); - mybe->reset(); - delete mybe; - } -}; - -/** - * @brief Writes data from the session to the network with optional throttling and flow control. - * - * The writeout() function in the MySQL_Session class is responsible for writing data from the session to the network. - * It supports throttling, which limits the rate at which data is sent to the client. Throttling is controlled by the - * mysql_thread___throttle_max_bytes_per_second_to_client configuration parameter. If throttling is disabled (the parameter - * is set to 0), the function bypasses throttling. - * - * This function first ensures that any pending data in the session's data stream (client_myds) is written to the network. - * This ensures that the network buffers are emptied, allowing new data to be sent. - * - * After writing data to the network, the function checks if flow control is necessary. If the total amount of data written - * exceeds the maximum allowed per call (mwpl), or if the data is sent too quickly, the function pauses writing for a brief - * period to control the flow of data. - * - * If throttling is enabled, the function adjusts the throttle based on the amount of data written and the configured maximum - * bytes per second. If the current throughput exceeds the configured limit, the function increases the pause duration to - * regulate the flow of data. - * - * Finally, if the session has a backend associated with it (mybe), and the backend has a server data stream (server_myds), - * the function also writes data from the server data stream to the network. - * - * @note This function assumes that necessary session and network structures are properly initialized. - * - * @see mysql_thread___throttle_max_bytes_per_second_to_client - * @see MySQL_Session::client_myds - * @see MySQL_Session::mybe - * @see MySQL_Backend::server_myds - */ -void MySQL_Session::writeout() { - int tps = 10; // throttling per second , by default every 100ms - int total_written = 0; - unsigned long long last_sent_=0; - bool disable_throttle = mysql_thread___throttle_max_bytes_per_second_to_client == 0; - int mwpl = mysql_thread___throttle_max_bytes_per_second_to_client; // max writes per call - mwpl = mwpl/tps; - if (session_type!=PROXYSQL_SESSION_MYSQL) { - disable_throttle = true; - } - if (client_myds) client_myds->array2buffer_full(); - if (mybe && mybe->server_myds && mybe->server_myds->myds_type==MYDS_BACKEND) { - if (session_type==PROXYSQL_SESSION_MYSQL) { - if (mybe->server_myds->net_failure==false) { - if (mybe->server_myds->poll_fds_idx>-1) { // NOTE: attempt to force writes - mybe->server_myds->array2buffer_full(); - } - } - } else { - mybe->server_myds->array2buffer_full(); - } - } - if (client_myds && thread->curtime >= client_myds->pause_until) { - if (mirror==false) { - bool runloop=false; - if (client_myds->mypolls) { - last_sent_ = client_myds->mypolls->last_sent[client_myds->poll_fds_idx]; - } - int retbytes=client_myds->write_to_net_poll(); - total_written+=retbytes; - if (retbytes==QUEUE_T_DEFAULT_SIZE) { // optimization to solve memory bloat - runloop=true; - } - while (runloop && (disable_throttle || total_written < mwpl)) { - runloop=false; // the default - client_myds->array2buffer_full(); - struct pollfd fds; - fds.fd=client_myds->fd; - fds.events=POLLOUT; - fds.revents=0; - int retpoll=poll(&fds, 1, 0); - if (retpoll>0) { - if (fds.revents==POLLOUT) { - retbytes=client_myds->write_to_net_poll(); - total_written+=retbytes; - if (retbytes==QUEUE_T_DEFAULT_SIZE) { // optimization to solve memory bloat - runloop=true; - } - } - } - } - } - } - - // flow control - if (!disable_throttle && total_written > 0) { - if (total_written > mwpl) { - unsigned long long add_ = 1000000/tps + 1000000/tps*((unsigned long long)total_written - (unsigned long long)mwpl)/mwpl; - pause_until = thread->curtime + add_; - client_myds->remove_pollout(); - client_myds->pause_until = thread->curtime + add_; - } else { - if (total_written >= QUEUE_T_DEFAULT_SIZE) { - unsigned long long time_diff = thread->curtime - last_sent_; - if (time_diff == 0) { // sending data really too fast! - unsigned long long add_ = 1000000/tps + 1000000/tps*((unsigned long long)total_written - (unsigned long long)mwpl)/mwpl; - pause_until = thread->curtime + add_; - client_myds->remove_pollout(); - client_myds->pause_until = thread->curtime + add_; - } else { - float current_Bps = (float)total_written*1000*1000/time_diff; - if (current_Bps > mysql_thread___throttle_max_bytes_per_second_to_client) { - unsigned long long add_ = 1000000/tps; - pause_until = thread->curtime + add_; - assert(pause_until > thread->curtime); - client_myds->remove_pollout(); - client_myds->pause_until = thread->curtime + add_; - } - } - } - } - } - - if (mybe) { - if (mybe->server_myds) mybe->server_myds->write_to_net_poll(); - } - proxy_debug(PROXY_DEBUG_NET,1,"Thread=%p, Session=%p -- Writeout Session %p\n" , this->thread, this, this); -} - /** * @brief Handles COMMIT or ROLLBACK commands received from the client. * @@ -1432,102 +1119,6 @@ void MySQL_Session::generate_proxysql_internal_session_json(json &j) { } } -void MySQL_Session::return_proxysql_internal(PtrSize_t *pkt) { - unsigned int l = 0; - l = strlen((char *)"PROXYSQL INTERNAL SESSION"); - if (pkt->size==(5+l) && strncasecmp((char *)"PROXYSQL INTERNAL SESSION", (char *)pkt->ptr+5, l)==0) { - json j; - generate_proxysql_internal_session_json(j); - std::string s = j.dump(4, ' ', false, json::error_handler_t::replace); - SQLite3_result *resultset = new SQLite3_result(1); - resultset->add_column_definition(SQLITE_TEXT,"session_info"); - char *pta[1]; - pta[0] = (char *)s.c_str(); - resultset->add_row(pta); - bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; - SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); - delete resultset; - l_free(pkt->size,pkt->ptr); - return; - } - // default - client_myds->DSS=STATE_QUERY_SENT_NET; - client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,1064,(char *)"42000",(char *)"Unknown PROXYSQL INTERNAL command",true); - if (mirror==false) { - RequestEnd(NULL); - } else { - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; - } - l_free(pkt->size,pkt->ptr); -} - -/** - * @brief Handles special queries executed by the STATUS command in mysql cli . - * Specifically: - * "select DATABASE(), USER() limit 1" - * "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" - * See Github issues 4396 and 4426 - * - * @param PtrSize_t The packet from the client - * - * @return True if the queries are handled - */ -bool MySQL_Session::handler_special_queries_STATUS(PtrSize_t *pkt) { - if (pkt->size == (SELECT_DB_USER_LEN+5)) { - if (strncasecmp(SELECT_DB_USER,(char *)pkt->ptr+5, SELECT_DB_USER_LEN)==0) { - SQLite3_result *resultset = new SQLite3_result(2); - resultset->add_column_definition(SQLITE_TEXT,"DATABASE()"); - resultset->add_column_definition(SQLITE_TEXT,"USER()"); - char *pta[2]; - pta[0] = client_myds->myconn->userinfo->username; - pta[1] = client_myds->myconn->userinfo->schemaname; - resultset->add_row(pta); - bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; - SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); - delete resultset; - l_free(pkt->size,pkt->ptr); - return true; - } - } - if (pkt->size == (SELECT_CHARSET_STATUS_LEN+5)) { - if (strncasecmp(SELECT_CHARSET_STATUS,(char *)pkt->ptr+5, SELECT_CHARSET_STATUS_LEN)==0) { - SQLite3_result *resultset = new SQLite3_result(4); - resultset->add_column_definition(SQLITE_TEXT,"@@character_set_client"); - resultset->add_column_definition(SQLITE_TEXT,"@@character_set_connection"); - resultset->add_column_definition(SQLITE_TEXT,"@@character_set_server"); - resultset->add_column_definition(SQLITE_TEXT,"@@character_set_database"); - - string vals[4]; - json j = {}; - json& jc = j["conn"]; - MySQL_Connection *conn = client_myds->myconn; - // here we do a bit back and forth to and from JSON to reuse existing code instead of writing new code. - // This is not great for performance, but this query is rarely executed. - conn->variables[SQL_CHARACTER_SET_CLIENT].fill_client_internal_session(jc, SQL_CHARACTER_SET_CLIENT); - conn->variables[SQL_CHARACTER_SET_CONNECTION].fill_client_internal_session(jc, SQL_CHARACTER_SET_CONNECTION); - conn->variables[SQL_CHARACTER_SET_DATABASE].fill_client_internal_session(jc, SQL_CHARACTER_SET_DATABASE); - - vals[0] = jc[mysql_tracked_variables[SQL_CHARACTER_SET_CLIENT].internal_variable_name]; - vals[1] = jc[mysql_tracked_variables[SQL_CHARACTER_SET_CONNECTION].internal_variable_name]; - vals[2] = string(mysql_thread___default_variables[SQL_CHARACTER_SET]); - vals[3] = jc[mysql_tracked_variables[SQL_CHARACTER_SET_DATABASE].internal_variable_name]; - - const char *pta[4]; - for (int i=0; i<4; i++) { - pta[i] = vals[i].c_str(); - } - resultset->add_row(pta); - bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; - SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); - delete resultset; - l_free(pkt->size,pkt->ptr); - return true; - } - } - return false; -} - bool MySQL_Session::handler_special_queries(PtrSize_t *pkt) { bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; @@ -1893,7 +1484,7 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C thread->mirror_queue_mysql_sessions->add(newsess); } else { GloMTH->status_variables.p_gauge_array[p_th_gauge::mirror_concurrency]->Increment(); - thread->register_session(newsess); + thread->register_session(thread,newsess); newsess->handler(); // execute immediately //newsess->to_process=0; if (newsess->status==WAITING_CLIENT_DATA) { // the mirror session has completed @@ -3594,14 +3185,14 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C if (thread->variables.stats_time_query_processor) { clock_gettime(CLOCK_THREAD_CPUTIME_ID,&begint); } - qpo=GloQPro->process_mysql_query(this,pkt.ptr,pkt.size,&CurrentQuery); + qpo= GloMyQPro->process_query(this,pkt.ptr,pkt.size,&CurrentQuery); if (thread->variables.stats_time_query_processor) { clock_gettime(CLOCK_THREAD_CPUTIME_ID,&endt); thread->status_variables.stvar[st_var_query_processor_time] = thread->status_variables.stvar[st_var_query_processor_time] + (endt.tv_sec*1000000000+endt.tv_nsec) - (begint.tv_sec*1000000000+begint.tv_nsec); } - assert(qpo); // GloQPro->process_mysql_query() should always return a qpo + assert(qpo); // GloMyQPro->process_mysql_query() should always return a qpo // setting 'prepared' to prevent fetching results from the cache if the digest matches rc_break=handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(&pkt, &lock_hostgroup, ps_type_prepare_stmt); if (rc_break==true) { @@ -3738,7 +3329,7 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C if (thread->variables.stats_time_query_processor) { clock_gettime(CLOCK_THREAD_CPUTIME_ID,&begint); } - qpo=GloQPro->process_mysql_query(this,NULL,0,&CurrentQuery); + qpo= GloMyQPro->process_query(this,NULL,0,&CurrentQuery); if (qpo->max_lag_ms >= 0) { thread->status_variables.stvar[st_var_queries_with_max_lag_ms]++; } @@ -3748,7 +3339,7 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C (endt.tv_sec*1000000000+endt.tv_nsec) - (begint.tv_sec*1000000000+begint.tv_nsec); } - assert(qpo); // GloQPro->process_mysql_query() should always return a qpo + assert(qpo); // GloMyQPro->process_mysql_query() should always return a qpo // we now take the metadata associated with STMT_EXECUTE from MySQL_STMTs_meta bool stmt_meta_found=true; // let's be optimistic and we assume we will found it stmt_execute_metadata_t *stmt_meta=sess_STMTs_meta->find(stmt_global_id); @@ -3889,9 +3480,9 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C issqli = libinjection_is_sqli(&state); if (issqli) { bool allow_sqli = false; - allow_sqli = GloQPro->whitelisted_sqli_fingerprint(state.fingerprint); + allow_sqli = GloMyQPro->whitelisted_sqli_fingerprint(state.fingerprint); if (allow_sqli) { - thread->status_variables.stvar[st_var_whitelisted_sqli_fingerprint]++; + thread->status_variables.stvar[st_var_mysql_whitelisted_sqli_fingerprint]++; } else { thread->status_variables.stvar[st_var_automatic_detected_sqli]++; char * username = client_myds->myconn->userinfo->username; @@ -4362,6 +3953,14 @@ int MySQL_Session::get_pkts_from_client(bool& wrong_pass, PtrSize_t& pkt) { if (mirror==false) { client_myds->PSarrayIN->remove_index(0,&pkt); } + + if (pkt.size <= sizeof(mysql_hdr)) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Malformed packet received\n"); + l_free(pkt.size, pkt.ptr); + handler_ret = -1; + return handler_ret; + } + switch (status) { case WAITING_CLIENT_DATA: if (pkt.size==(0xFFFFFF+sizeof(mysql_hdr))) { // we are handling a multi-packet @@ -4399,6 +3998,33 @@ int MySQL_Session::get_pkts_from_client(bool& wrong_pass, PtrSize_t& pkt) { } c=*((unsigned char *)pkt.ptr+sizeof(mysql_hdr)); + + /*if (client_myds != NULL && client_myds->is_postgres == true) { + if (session_type == PROXYSQL_SESSION_ADMIN || session_type == PROXYSQL_SESSION_STATS) { + c = *((unsigned char*)pkt.ptr + 0); + if (c == 'Q') { + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___not_mysql(pkt); + } + else if (c == 'X') { + //proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_QUIT packet\n"); + //if (GloMyLogger) { GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_QUIT, this, NULL); } + l_free(pkt.size, pkt.ptr); + handler_ret = -1; + return handler_ret; + } + else { + proxy_error("Experimental feature"); + //assert(0); + } + } + else { + proxy_error("Experimental feature"); + assert(0); + } + break; + }*/ + + if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { if ((enum_mysql_command)c == _MYSQL_COM_INIT_DB) { handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_INIT_DB_replace_CLICKHOUSE(pkt); @@ -4448,14 +4074,14 @@ int MySQL_Session::get_pkts_from_client(bool& wrong_pass, PtrSize_t& pkt) { if (thread->variables.stats_time_query_processor) { clock_gettime(CLOCK_THREAD_CPUTIME_ID,&begint); } - qpo=GloQPro->process_mysql_query(this,pkt.ptr,pkt.size,&CurrentQuery); + qpo= GloMyQPro->process_query(this,pkt.ptr,pkt.size,&CurrentQuery); if (thread->variables.stats_time_query_processor) { clock_gettime(CLOCK_THREAD_CPUTIME_ID,&endt); thread->status_variables.stvar[st_var_query_processor_time]=thread->status_variables.stvar[st_var_query_processor_time] + (endt.tv_sec*1000000000+endt.tv_nsec) - (begint.tv_sec*1000000000+begint.tv_nsec); } - assert(qpo); // GloQPro->process_mysql_query() should always return a qpo + assert(qpo); // GloMyQPro->process_mysql_query() should always return a qpo { bool need_break = GPFC_QueryUSE(pkt, handler_ret); @@ -5067,71 +4693,8 @@ int MySQL_Session::RunQuery(MySQL_Data_Stream *myds, MySQL_Connection *myconn) { // this function was inline void MySQL_Session::handler___status_WAITING_CLIENT_DATA() { // NOTE: Maintenance of 'multiplex_delayed' has been moved to 'housekeeping_before_pkts'. The previous impl -// is left below as an example of how to perform a more passive maintenance over session connections. -/* - if (mybes) { - MySQL_Backend *_mybe; - unsigned int i; - for (i=0; i < mybes->len; i++) { - _mybe=(MySQL_Backend *)mybes->index(i); - if (_mybe->server_myds) { - MySQL_Data_Stream *_myds=_mybe->server_myds; - if (_myds->myconn) { - if (_myds->myconn->multiplex_delayed) { - if (_myds->wait_until <= thread->curtime) { - _myds->wait_until=0; - _myds->myconn->multiplex_delayed=false; - _myds->DSS=STATE_NOT_INITIALIZED; - _myds->return_MySQL_Connection_To_Pool(); - } - } - } - } - } - } -*/ } -/** - * @brief Perform housekeeping tasks before processing packets. - * - * This function is responsible for performing necessary housekeeping tasks - * before processing packets. These tasks include handling expired connections - * for multiplexing scenarios. If multiplexing is enabled, it iterates over - * the list of expired backend connections and either returns them to the connection pool - * or destroys them based on certain conditions. - * - * @note This function assumes that the `hgs_expired_conns` vector contains the IDs - * of the backend connections that have expired. - * - * @return None. - */ -void MySQL_Session::housekeeping_before_pkts() { - if (mysql_thread___multiplexing) { - for (const int hg_id : hgs_expired_conns) { - MySQL_Backend* mybe = find_backend(hg_id); - - if (mybe != nullptr) { - MySQL_Data_Stream* myds = mybe->server_myds; - - if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit()==false) { - if (mysql_thread___reset_connection_algorithm == 2) { - create_new_session_and_reset_connection(myds); - } else { - myds->destroy_MySQL_Connection_From_Pool(true); - } - } else { - myds->return_MySQL_Connection_To_Pool(); - } - } - } - // We are required to perform a cleanup after consuming the elements, thus preventing any subsequent - // 'handler' call to perform recomputing of the already processed elements. - if (hgs_expired_conns.empty() == false) { - hgs_expired_conns.clear(); - } - } -} // this function was inline /** @@ -5898,7 +5461,11 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE( case PROXYSQL_SESSION_MYSQL: proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION,8,"Session=%p , DS=%p , session_type=PROXYSQL_SESSION_MYSQL\n", this, client_myds); if (use_ldap_auth == false) { - free_users = GloMyAuth->increase_frontend_user_connections(client_myds->myconn->userinfo->username, &used_users); + free_users = GloMyAuth->increase_frontend_user_connections( + client_myds->myconn->userinfo->username, + client_myds->myconn->userinfo->passtype, + &used_users + ); } else { free_users = GloMyLdapAuth->increase_frontend_user_connections(client_myds->myconn->userinfo->fe_username, &used_users); } @@ -6285,84 +5852,6 @@ void MySQL_Session::handler_WCD_SS_MCQ_qpo_LargePacket(PtrSize_t *pkt) { l_free(pkt->size,pkt->ptr); } -/* -// this function as inline in handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo -// returned values: -// 0 : no action -// 1 : return false -// 2 : return true -int MySQL_Session::handler_WCD_SS_MCQ_qpo_Parse_SQL_LOG_BIN(PtrSize_t *pkt, bool *lock_hostgroup, unsigned int nTrx, string& nq) { - re2::RE2::Options *opt2=new re2::RE2::Options(RE2::Quiet); - opt2->set_case_sensitive(false); - char *pattern=(char *)"(?: *)SET *(?:|SESSION +|@@|@@session.)SQL_LOG_BIN *(?:|:)= *(\\d+) *(?:(|;|-- .*|#.*))$"; - re2::RE2 *re=new RE2(pattern, *opt2); - int i; - int rc=RE2::PartialMatch(nq, *re, &i); - delete re; - delete opt2; - if (rc && ( i==0 || i==1) ) { - //fprintf(stderr,"sql_log_bin=%d\n", i); - if (i == 1) { - if (!mysql_variables.client_set_value(this, SQL_SQL_LOG_BIN, "1")) - return 1; - } - else if (i == 0) { - if (!mysql_variables.client_set_value(this, SQL_SQL_LOG_BIN, "0")) - return 1; - } - -#ifdef DEBUG - proxy_info("Setting SQL_LOG_BIN to %d\n", i); -#endif -#ifdef DEBUG - { - string nqn = string((char *)CurrentQuery.QueryPointer,CurrentQuery.QueryLength); - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Setting SQL_LOG_BIN to %d for query: %s\n", i, nqn.c_str()); - } -#endif - // we recompute command_type instead of taking it from the calling function - unsigned char command_type=*((unsigned char *)pkt->ptr+sizeof(mysql_hdr)); - if (command_type == _MYSQL_COM_QUERY) { - client_myds->DSS=STATE_QUERY_SENT_NET; - uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); - if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; - client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; - RequestEnd(NULL); - l_free(pkt->size,pkt->ptr); - return 2; - } - } else { - int kq = 0; - kq = strncmp((const char *)CurrentQuery.QueryPointer, (const char *)"SET @@SESSION.SQL_LOG_BIN = @MYSQLDUMP_TEMP_LOG_BIN;" , CurrentQuery.QueryLength); -#ifdef DEBUG - { - string nqn = string((char *)CurrentQuery.QueryPointer,CurrentQuery.QueryLength); - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Setting SQL_LOG_BIN to %d for query: %s\n", i, nqn.c_str()); - } -#endif - if (kq == 0) { - client_myds->DSS=STATE_QUERY_SENT_NET; - uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); - if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; - client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - client_myds->DSS=STATE_SLEEP; - status=WAITING_CLIENT_DATA; - RequestEnd(NULL); - l_free(pkt->size,pkt->ptr); - return 2; - } else { - string nqn = string((char *)CurrentQuery.QueryPointer,CurrentQuery.QueryLength); - proxy_error("Unable to parse query. If correct, report it as a bug: %s\n", nqn.c_str()); - unable_to_parse_set_statement(lock_hostgroup); - return 1; - } - } - return 0; -} -*/ - bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(PtrSize_t *pkt, bool *lock_hostgroup, ps_type prepare_stmt_type) { /* lock_hostgroup: @@ -6499,15 +5988,6 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C if (pos != nq.npos) { nq.erase(pos + 1); // remove trailing spaces and semicolumns } -/* - // we do not threat SET SQL_LOG_BIN as a special case - if (match_regexes && match_regexes[0]->match(dig)) { - int rc = handler_WCD_SS_MCQ_qpo_Parse_SQL_LOG_BIN(pkt, lock_hostgroup, nTrx, nq); - if (rc == 1) return false; - if (rc == 2) return true; - // if rc == 0 , continue as normal - } -*/ if ( ( match_regexes && (match_regexes[1]->match(dig)) @@ -7399,7 +6879,10 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C init(); if (client_authenticated) { if (use_ldap_auth == false) { - GloMyAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->username); + GloMyAuth->decrease_frontend_user_connections( + client_myds->myconn->userinfo->username, + client_myds->myconn->userinfo->passtype + ); } else { GloMyLdapAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->fe_username); } @@ -7414,7 +6897,11 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C client_authenticated=true; //int free_users=0; int used_users=0; - /*free_users */GloMyAuth->increase_frontend_user_connections(client_myds->myconn->userinfo->username, &used_users); + GloMyAuth->increase_frontend_user_connections( + client_myds->myconn->userinfo->username, + client_myds->myconn->userinfo->passtype, + &used_users + ); // FIXME: max_connections is not handled for CHANGE_USER } else { l_free(pkt->size,pkt->ptr); @@ -7459,6 +6946,7 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C proxy_error("ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)\n", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,2,1045,(char *)"28000", _s, true); free(_s); + if (client_addr) { free(client_addr); } __sync_fetch_and_add(&MyHGM->status.access_denied_wrong_password, 1); } } else { @@ -7908,102 +7396,6 @@ void MySQL_Session::SQLite3_to_MySQL(SQLite3_result *result, char *error, int af } } -void MySQL_Session::set_unhealthy() { - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess:%p\n", this); - healthy=0; -} - - -unsigned int MySQL_Session::NumActiveTransactions(bool check_savepoint) { - unsigned int ret=0; - if (mybes==0) return ret; - MySQL_Backend *_mybe; - unsigned int i; - for (i=0; i < mybes->len; i++) { - _mybe=(MySQL_Backend *)mybes->index(i); - if (_mybe->server_myds) { - if (_mybe->server_myds->myconn) { - if (_mybe->server_myds->myconn->IsActiveTransaction()) { - ret++; - } else { - // we use check_savepoint to check if we shouldn't ignore COMMIT or ROLLBACK due - // to MySQL bug https://bugs.mysql.com/bug.php?id=107875 related to - // SAVEPOINT and autocommit=0 - if (check_savepoint) { - if (_mybe->server_myds->myconn->AutocommitFalse_AndSavepoint() == true) { - ret++; - } - } - } - } - } - } - return ret; -} - -bool MySQL_Session::HasOfflineBackends() { - bool ret=false; - if (mybes==0) return ret; - MySQL_Backend *_mybe; - unsigned int i; - for (i=0; i < mybes->len; i++) { - _mybe=(MySQL_Backend *)mybes->index(i); - if (_mybe->server_myds) - if (_mybe->server_myds->myconn) - if (_mybe->server_myds->myconn->IsServerOffline()) { - ret=true; - return ret; - } - } - return ret; -} - -bool MySQL_Session::SetEventInOfflineBackends() { - bool ret=false; - if (mybes==0) return ret; - MySQL_Backend *_mybe; - unsigned int i; - for (i=0; i < mybes->len; i++) { - _mybe=(MySQL_Backend *)mybes->index(i); - if (_mybe->server_myds) - if (_mybe->server_myds->myconn) - if (_mybe->server_myds->myconn->IsServerOffline()) { - _mybe->server_myds->revents|=POLLIN; - ret = true; - } - } - return ret; -} - -int MySQL_Session::FindOneActiveTransaction(bool check_savepoint) { - int ret=-1; - if (mybes==0) return ret; - MySQL_Backend *_mybe; - unsigned int i; - for (i=0; i < mybes->len; i++) { - _mybe=(MySQL_Backend *)mybes->index(i); - if (_mybe->server_myds) { - if (_mybe->server_myds->myconn) { - if (_mybe->server_myds->myconn->IsKnownActiveTransaction()) { - return (int)_mybe->server_myds->myconn->parent->myhgc->hid; - } else if (_mybe->server_myds->myconn->IsActiveTransaction()) { - ret = (int)_mybe->server_myds->myconn->parent->myhgc->hid; - } else { - // we use check_savepoint to check if we shouldn't ignore COMMIT or ROLLBACK due - // to MySQL bug https://bugs.mysql.com/bug.php?id=107875 related to - // SAVEPOINT and autocommit=0 - if (check_savepoint) { - if (_mybe->server_myds->myconn->AutocommitFalse_AndSavepoint() == true) { - return (int)_mybe->server_myds->myconn->parent->myhgc->hid; - } - } - } - } - } - } - return ret; -} - unsigned long long MySQL_Session::IdleTime() { unsigned long long ret = 0; if (client_myds==0) return 0; @@ -8019,7 +7411,6 @@ unsigned long long MySQL_Session::IdleTime() { } - // this is called either from RequestEnd(), or at the end of executing // prepared statements void MySQL_Session::LogQuery(MySQL_Data_Stream *myds) { @@ -8065,7 +7456,7 @@ void MySQL_Session::RequestEnd(MySQL_Data_Stream *myds) { break; } - GloQPro->delete_QP_out(qpo); + GloMyQPro->delete_QP_out(qpo); // if there is an associated myds, clean its status if (myds) { // if there is a mysql connection, clean its status @@ -8194,7 +7585,8 @@ bool MySQL_Session::handle_command_query_kill(PtrSize_t *pkt) { MySQL_Connection *mc = client_myds->myconn; if (mc->userinfo && mc->userinfo->username) { if (CurrentQuery.MyComQueryCmd == MYSQL_COM_QUERY_KILL) { - char *qu = mysql_query_strip_comments((char *)pkt->ptr+1+sizeof(mysql_hdr), pkt->size-1-sizeof(mysql_hdr)); + char* qu = query_strip_comments((char *)pkt->ptr+1+sizeof(mysql_hdr), pkt->size-1-sizeof(mysql_hdr), + mysql_thread___query_digests_lowercase); string nq=string(qu,strlen(qu)); re2::RE2::Options *opt2=new re2::RE2::Options(RE2::Quiet); opt2->set_case_sensitive(false); @@ -8435,25 +7827,6 @@ void MySQL_Session::unable_to_parse_set_statement(bool *lock_hostgroup) { } } -/** - * @brief Check if any backend has an active MySQL connection. - * - * This function iterates through all backends associated with the session and checks if any backend has an - * active MySQL connection. If any backend has an active connection, it returns true; otherwise, it returns false. - * - * @return true if any backend has an active MySQL connection, otherwise false. - */ -bool MySQL_Session::has_any_backend() { - for (unsigned int j=0;j < mybes->len;j++) { - MySQL_Backend *tmp_mybe=(MySQL_Backend *)mybes->index(j); - MySQL_Data_Stream *__myds=tmp_mybe->server_myds; - if (__myds->myconn) { - return true; - } - } - return false; -} - /** * @brief Handler for MYSQL_COM_STMT_RESET command in WAITING_CLIENT_DATA state with STATE_SLEEP. * diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index eaf60184b2..5fca5cf23b 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + //#define __CLASS_STANDARD_MYSQL_THREAD_H #include @@ -15,7 +19,7 @@ #include "re2/regexp.h" #include "MySQL_Data_Stream.h" -#include "query_processor.h" +#include "MySQL_Query_Processor.h" #include "StatCounters.h" #include "MySQL_PreparedStatement.h" #include "MySQL_Logger.hpp" @@ -26,7 +30,7 @@ using std::vector; using std::function; #ifdef DEBUG -MySQL_Session *sess_stopat; +static MySQL_Session *sess_stopat; #endif #ifdef epoll_create1 @@ -102,7 +106,7 @@ MySQL_Session *sess_stopat; mysql_thread___ ## name = GloMTH->get_variable_string((char *)STRINGIFY(name)); \ } while (0) -extern Query_Processor *GloQPro; +extern MySQL_Query_Processor* GloMyQPro; extern MySQL_Authentication *GloMyAuth; extern MySQL_Threads_Handler *GloMTH; extern MySQL_Monitor *GloMyMon; @@ -157,7 +161,7 @@ mythr_st_vars_t MySQL_Thread_status_variables_counter_array[] { { st_var_backend_offline_during_query,p_th_counter::backend_offline_during_query, (char *)"backend_offline_during_query" }, { st_var_aws_aurora_replicas_skipped_during_query , p_th_counter::aws_aurora_replicas_skipped_during_query, (char *)"get_aws_aurora_replicas_skipped_during_query" }, { st_var_automatic_detected_sqli, p_th_counter::automatic_detected_sql_injection, (char *)"automatic_detected_sql_injection" }, - { st_var_whitelisted_sqli_fingerprint,p_th_counter::whitelisted_sqli_fingerprint, (char *)"whitelisted_sqli_fingerprint" }, + { st_var_mysql_whitelisted_sqli_fingerprint,p_th_counter::mysql_whitelisted_sqli_fingerprint, (char *)"mysql_whitelisted_sqli_fingerprint" }, { st_var_max_connect_timeout_err, p_th_counter::max_connect_timeouts, (char *)"max_connect_timeouts" }, { st_var_generated_pkt_err, p_th_counter::generated_error_packets, (char *)"generated_error_packets" }, { st_var_client_host_error_killed_connections, p_th_counter::client_host_error_killed_connections, (char *)"client_host_error_killed_connections" }, @@ -773,8 +777,8 @@ th_metrics_map = std::make_tuple( metric_tags {} ), std::make_tuple ( - p_th_counter::whitelisted_sqli_fingerprint, - "proxysql_whitelisted_sqli_fingerprint_total", + p_th_counter::mysql_whitelisted_sqli_fingerprint, + "proxysql_mysql_whitelisted_sqli_fingerprint_total", "Detected a whitelisted 'sql injection' fingerprint.", metric_tags {} ), @@ -1619,12 +1623,12 @@ bool MySQL_Threads_Handler::set_variable(char *name, const char *value) { // thi } } if (nameS == "query_rules_fast_routing_algorithm") { - if (GloQPro) { + if (GloMyQPro) { int intv = atoi(value); if (intv >= std::get<1>(it->second) && intv <= std::get<2>(it->second)) { - GloQPro->wrlock(); - GloQPro->query_rules_fast_routing_algorithm = intv; - GloQPro->wrunlock(); + GloMyQPro->wrlock(); + GloMyQPro->query_rules_fast_routing_algorithm = intv; + GloMyQPro->wrunlock(); } } } @@ -1983,8 +1987,7 @@ bool MySQL_Threads_Handler::set_variable(char *name, const char *value) { // thi variables.auditlog_filename = strdup(value); free(full_path); return true; - } - else { + } else { proxy_error("%s is an invalid value for auditlog_filename path, the directory cannot be accessed\n", eval_dirname); free(full_path); return false; @@ -2009,8 +2012,7 @@ bool MySQL_Threads_Handler::set_variable(char *name, const char *value) { // thi variables.eventslog_filename = strdup(value); free(full_path); return true; - } - else { + } else { proxy_error("%s is an invalid value for eventslog_filename path, the directory cannot be accessed\n", eval_dirname); free(full_path); return false; @@ -2517,7 +2519,7 @@ void MySQL_Threads_Handler::stop_listeners() { * @return Either an string holding the string representation of internal * member 'client_addr', or empty string if this member is NULL. */ -std::string get_client_addr(struct sockaddr* client_addr) { +static std::string get_client_addr(struct sockaddr* client_addr) { char buf[INET6_ADDRSTRLEN]; std::string str_client_addr {}; @@ -2617,7 +2619,7 @@ char** client_host_cache_entry_row( * * @param row The pointer array holding the row values to be freed. */ -void free_client_host_cache_row(char** row) { +static void free_client_host_cache_row(char** row) { for (int i = 0; i < CLIENT_HOST_CACHE_COLUMNS; i++) { free(row[i]); } @@ -2781,7 +2783,7 @@ MySQL_Thread::~MySQL_Thread() { } delete mysql_sessions; mysql_sessions=NULL; - GloQPro->end_thread(); // only for real threads + GloMyQPro->end_thread(); // only for real threads } if (mirror_queue_mysql_sessions) { @@ -2910,78 +2912,6 @@ MySQL_Thread::~MySQL_Thread() { } -MySQL_Session * MySQL_Thread::create_new_session_and_client_data_stream(int _fd) { - int arg_on=1; - MySQL_Session *sess=new MySQL_Session; - register_session(sess); // register session - sess->client_myds = new MySQL_Data_Stream(); - sess->client_myds->fd=_fd; - - // set not blocking for client connections too! - { - // PMC-10004 - // While implementing SSL and fast_forward it was noticed that all frontend connections - // are in blocking, although this was never a problem because we call poll() before reading. - // Although it became a problem with fast_forward, SSL and large packets because SSL handled - // data in chunks of 16KB and there may be data inside SSL even when there is no data - // received from the network. - // The only modules that seems to be affected by this issue are Admin, SQLite3 Server - // and Clickhouse Server - int prevflags = fcntl(_fd, F_GETFL, 0); - if (prevflags == -1) { - proxy_error("For FD %d fcntl() returned -1 errno %d\n", _fd, errno); - if (shutdown == 0) - assert (prevflags != -1); - } - int nb = fcntl(_fd, F_SETFL, prevflags | O_NONBLOCK); - if (nb == -1) { - proxy_error("For FD %d fcntl() returned -1 , previous flags %d , errno %d\n", _fd, prevflags, errno); - // previously we were asserting here. But it is possible that this->shutdown is still 0 during the - // shutdown itself: - // - the current thread is processing connections - // - the signal handler thread is still setting shutdown = 0 - //if (shutdown == 0) - // assert (nb != -1); - } - } - setsockopt(sess->client_myds->fd, IPPROTO_TCP, TCP_NODELAY, (char *) &arg_on, sizeof(arg_on)); - - if (mysql_thread___use_tcp_keepalive) { - setsockopt(sess->client_myds->fd, SOL_SOCKET, SO_KEEPALIVE, (char *) &arg_on, sizeof(arg_on)); -#ifdef TCP_KEEPIDLE - if (mysql_thread___tcp_keepalive_time > 0) { - int keepalive_time = mysql_thread___tcp_keepalive_time; - setsockopt(sess->client_myds->fd, IPPROTO_TCP, TCP_KEEPIDLE, (char *) &keepalive_time, sizeof(keepalive_time)); - } -#endif - } - -#ifdef __APPLE__ - setsockopt(sess->client_myds->fd, SOL_SOCKET, SO_NOSIGPIPE, (char *) &arg_on, sizeof(int)); -#endif - sess->client_myds->init(MYDS_FRONTEND, sess, sess->client_myds->fd); - proxy_debug(PROXY_DEBUG_NET,1,"Thread=%p, Session=%p, DataStream=%p -- Created new client Data Stream\n", sess->thread, sess, sess->client_myds); -#ifdef DEBUG - sess->client_myds->myprot.dump_pkt=true; -#endif - MySQL_Connection *myconn=new MySQL_Connection(); - sess->client_myds->attach_connection(myconn); - myconn->set_is_client(); // this is used for prepared statements - myconn->last_time_used=curtime; - myconn->myds=sess->client_myds; // 20141011 - myconn->fd=sess->client_myds->fd; // 20141011 - - sess->client_myds->myprot.init(&sess->client_myds, sess->client_myds->myconn->userinfo, sess); - uint32_t session_track_gtids_int=SpookyHash::Hash32(mysql_thread___default_session_track_gtids,strlen(mysql_thread___default_session_track_gtids),10); - sess->client_myds->myconn->options.session_track_gtids_int = session_track_gtids_int; - if (sess->client_myds->myconn->options.session_track_gtids) { - free(sess->client_myds->myconn->options.session_track_gtids); - } - sess->client_myds->myconn->options.session_track_gtids=strdup(mysql_thread___default_session_track_gtids); - - return sess; -} - bool MySQL_Thread::init() { int i; mysql_sessions = new PtrArray(); @@ -3009,7 +2939,7 @@ bool MySQL_Thread::init() { shutdown=0; my_idle_conns=(MySQL_Connection **)malloc(sizeof(MySQL_Connection *)*SESSIONS_FOR_CONNECTIONS_HANDLER); memset(my_idle_conns,0,sizeof(MySQL_Connection *)*SESSIONS_FOR_CONNECTIONS_HANDLER); - GloQPro->init_thread(); + GloMyQPro->init_thread(); refresh_variables(); i=pipe(pipefd); ioctl_FIONBIO(pipefd[0],1); @@ -3059,18 +2989,6 @@ void MySQL_Thread::poll_listener_del(int sock) { } } -void MySQL_Thread::register_session(MySQL_Session *_sess, bool up_start) { - if (mysql_sessions==NULL) { - mysql_sessions = new PtrArray(); - } - mysql_sessions->add(_sess); - _sess->thread=this; - _sess->match_regexes=match_regexes; - if (up_start) - _sess->start_time=curtime; - proxy_debug(PROXY_DEBUG_NET,1,"Thread=%p, Session=%p -- Registered new session\n", _sess->thread, _sess); -} - void MySQL_Thread::unregister_session(int idx) { if (mysql_sessions==NULL) return; proxy_debug(PROXY_DEBUG_NET,1,"Thread=%p, Session=%p -- Unregistered session\n", this, mysql_sessions->index(idx)); @@ -3123,100 +3041,6 @@ void MySQL_Thread::run___get_multiple_idle_connections(int& num_idles) { last_processing_idles=curtime; } -// this function was inline in MySQL_Thread::run() -void MySQL_Thread::ProcessAllMyDS_BeforePoll() { - bool check_if_move_to_idle_thread = false; -#ifdef IDLE_THREADS - if (GloVars.global.idle_threads) { - if (curtime > last_move_to_idle_thread_time + (unsigned long long)mysql_thread___session_idle_ms * 1000) { - last_move_to_idle_thread_time=curtime; - check_if_move_to_idle_thread=true; - } - } -#endif - for (unsigned int n = 0; n < mypolls.len; n++) { - MySQL_Data_Stream *myds=NULL; - myds=mypolls.myds[n]; - mypolls.fds[n].revents=0; - if (myds) { -#ifdef IDLE_THREADS - if (check_if_move_to_idle_thread == true) { - // here we try to move it to the maintenance thread - if (myds->myds_type==MYDS_FRONTEND && myds->sess) { - if (myds->DSS==STATE_SLEEP && myds->sess->status==WAITING_CLIENT_DATA) { - if (move_session_to_idle_mysql_sessions(myds, n)) { - n--; // compensate mypolls.remove_index_fast(n) and n++ of loop - continue; - } - } - } - } -#endif // IDLE_THREADS - if (unlikely(myds->wait_until)) { - tune_timeout_for_myds_needs_pause(myds); - } - if (myds->sess) { - if (unlikely(myds->sess->pause_until > 0)) { - tune_timeout_for_session_needs_pause(myds); - } - } - myds->revents=0; - if (myds->myds_type!=MYDS_LISTENER) { - configure_pollout(myds, n); - } - } - proxy_debug(PROXY_DEBUG_NET,1,"Poll for DataStream=%p will be called with FD=%d and events=%d\n", mypolls.myds[n], mypolls.fds[n].fd, mypolls.fds[n].events); - } -} - - -// this function was inline in MySQL_Thread::run() -/** - * @brief Processes all MySQL Data Streams after polling. - * - * This function iterates through all MySQL polls and processes the associated data streams. - * For each poll, it prints debug information about the file descriptor and its events. - * If a MySQL Data Stream is associated with the poll, it checks for events on the file descriptor. - * If there are no events and a poll timeout is enabled, it checks for sessions timing out. - * If there are events, it checks for invalid file descriptors and handles new connections - * for listener type data streams. For other types of data streams, it processes data and - * handles any potential errors. - */ -void MySQL_Thread::ProcessAllMyDS_AfterPoll() { - for (unsigned int n = 0; n < mypolls.len; n++) { - proxy_debug(PROXY_DEBUG_NET,3, "poll for fd %d events %d revents %d\n", mypolls.fds[n].fd , mypolls.fds[n].events, mypolls.fds[n].revents); - - MySQL_Data_Stream *myds=mypolls.myds[n]; - if (myds==NULL) { - read_one_byte_from_pipe(n); - continue; - } - if (mypolls.fds[n].revents==0) { - if (poll_timeout_bool) { - check_timing_out_session(n); - } - } else { - check_for_invalid_fd(n); // this is designed to assert in case of failure - switch(myds->myds_type) { - // Note: this logic that was here was removed completely because we added mariadb client library. - case MYDS_LISTENER: - // we got a new connection! - listener_handle_new_connection(myds,n); - continue; - break; - default: - break; - } - // data on exiting connection - bool rc=process_data_on_data_stream(myds, n); - if (rc==false) { - n--; - } - } - } -} - - // this function was inline in MySQL_Thread::run() /** * @brief Cleans up the mirror queue by removing excess sessions. @@ -3348,24 +3172,6 @@ void MySQL_Thread::run_StopListener() { } } - -void MySQL_Thread::run_SetAllSession_ToProcess0() { - unsigned int n; -#ifdef IDLE_THREADS - // @note: in MySQL_Thread::run we have: bool idle_maintenance_thread=epoll_thread; - // Thus idle_maintenance_thread and epoll_thread are equivalent. - if (epoll_thread==false) { -#endif // IDLE_THREADS - for (n=0; nlen; n++) { - MySQL_Session *_sess=(MySQL_Session *)mysql_sessions->index(n); - _sess->to_process=0; - } -#ifdef IDLE_THREADS - } -#endif // IDLE_THREADS -} - - // main loop /** * @brief Main loop for the MySQL thread. @@ -3424,7 +3230,7 @@ void MySQL_Thread::run() { handle_mirror_queue_mysql_sessions(); - ProcessAllMyDS_BeforePoll(); + ProcessAllMyDS_BeforePoll(); #ifdef IDLE_THREADS run_MoveSessionsBetweenThreads(); @@ -3511,7 +3317,7 @@ void MySQL_Thread::run() { ) { // house keeping run___cleanup_mirror_queue(); - GloQPro->update_query_processor_stats(); + GloMyQPro->update_query_processor_stats(); } if (rc == -1 && errno == EINTR) @@ -3529,7 +3335,7 @@ void MySQL_Thread::run() { refresh_variables(); } - run_SetAllSession_ToProcess0(); + run_SetAllSession_ToProcess0(); #ifdef IDLE_THREADS // here we handle epoll_wait() @@ -3544,7 +3350,7 @@ void MySQL_Thread::run() { } } else { #endif // IDLE_THREADS - ProcessAllMyDS_AfterPoll(); + ProcessAllMyDS_AfterPoll(); // iterate through all sessions and process the session logic process_all_sessions(); return_local_connections(); @@ -3556,17 +3362,6 @@ void MySQL_Thread::run() { } // end of ::run() -unsigned int MySQL_Thread::find_session_idx_in_mysql_sessions(MySQL_Session *sess) { - unsigned int i=0; - for (i=0;ilen;i++) { - MySQL_Session *mysess=(MySQL_Session *)mysql_sessions->index(i); - if (mysess==sess) { - return i; - } - } - return i; -} - #ifdef IDLE_THREADS /** @@ -3747,7 +3542,7 @@ void MySQL_Thread::worker_thread_gets_sessions_from_idle_thread() { //unsigned int maxsess=GloMTH->resume_mysql_sessions->len; while (myexchange.resume_mysql_sessions->len) { MySQL_Session *mysess=(MySQL_Session *)myexchange.resume_mysql_sessions->remove_index_fast(0); - register_session(mysess, false); + register_session(this, mysess, false); MySQL_Data_Stream *myds=mysess->client_myds; mypolls.add(POLLIN, myds->fd, myds, monotonic_time()); } @@ -3915,38 +3710,6 @@ bool MySQL_Thread::process_data_on_data_stream(MySQL_Data_Stream *myds, unsigned } - -// this function was inline in MySQL_Thread::process_all_sessions() -/** - * @brief Sort all sessions based on maximum connection time. - * - * This function iterates through all MySQL sessions and sorts them based on their maximum connection time. - * Sessions with a valid maximum connection time are compared, and if one session has a greater maximum connection - * time than another, their positions in the session list are swapped. The sorting is performed in-place. - * - * @note This function assumes that MySQL sessions and their associated data structures have been initialized - * and are accessible within the MySQL Thread. - */ -void MySQL_Thread::ProcessAllSessions_SortingSessions() { - unsigned int a=0; - for (unsigned int n=0; nlen; n++) { - MySQL_Session *sess=(MySQL_Session *)mysql_sessions->index(n); - if (sess->mybe && sess->mybe->server_myds) { - if (sess->mybe->server_myds->max_connect_time) { - MySQL_Session *sess2=(MySQL_Session *)mysql_sessions->index(a); - if (sess2->mybe && sess2->mybe->server_myds && sess2->mybe->server_myds->max_connect_time && sess2->mybe->server_myds->max_connect_time <= sess->mybe->server_myds->max_connect_time) { - // do nothing - } else { - void *p=mysql_sessions->pdata[a]; - mysql_sessions->pdata[a]=mysql_sessions->pdata[n]; - mysql_sessions->pdata[n]=p; - a++; - } - } - } - } -} - // this function was inline in MySQL_Thread::process_all_sessions() void MySQL_Thread::ProcessAllSessions_CompletedMirrorSession(unsigned int& n, MySQL_Session *sess) { unregister_session(n); @@ -4170,7 +3933,7 @@ void MySQL_Thread::process_all_sessions() { } #endif // IDLE_THREADS if (sess_sort && mysql_sessions->len > 3) { - ProcessAllSessions_SortingSessions(); + ProcessAllSessions_SortingSessions(); } for (n=0; nlen; n++) { MySQL_Session *sess=(MySQL_Session *)mysql_sessions->index(n); @@ -4511,7 +4274,7 @@ MySQL_Thread::MySQL_Thread() { status_variables.active_transactions=0; - for (unsigned int i = 0; i < st_var_END ; i++) { + for (unsigned int i = 0; i < MY_st_var_END ; i++) { status_variables.stvar[i] = 0; } match_regexes=NULL; @@ -4607,7 +4370,7 @@ void MySQL_Thread::listener_handle_new_connection(MySQL_Data_Stream *myds, unsig // create a new client connection mypolls.fds[n].revents=0; - MySQL_Session *sess=create_new_session_and_client_data_stream(c); + MySQL_Session *sess=create_new_session_and_client_data_stream(c); __sync_add_and_fetch(&MyHGM->status.client_connections_created,1); if (__sync_add_and_fetch(&MyHGM->status.client_connections,1) > mysql_thread___max_connections) { sess->max_connections_reached=true; @@ -4697,12 +4460,30 @@ SQLite3_result * MySQL_Threads_Handler::SQL3_GlobalStatus(bool _memory) { pta[1]=buf; result->add_row(pta); } + { // Connections + pta[0]=(char *)"Client_Connections_connected_prim_pass"; + sprintf(buf,"%d",MyHGM->status.client_connections_prim_pass); + pta[1]=buf; + result->add_row(pta); + } + { // Connections + pta[0]=(char *)"Client_Connections_connected_addl_pass"; + sprintf(buf,"%d",MyHGM->status.client_connections_addl_pass); + pta[1]=buf; + result->add_row(pta); + } { // Connections created pta[0]=(char *)"Client_Connections_created"; sprintf(buf,"%lu",MyHGM->status.client_connections_created); pta[1]=buf; result->add_row(pta); } + { // Connections created using cached 'clear_text_passwords' + pta[0]=(char *)"Client_Connections_sha2cached"; + sprintf(buf,"%lu",MyHGM->status.client_connections_sha2cached); + pta[1]=buf; + result->add_row(pta); + } { // Connections pta[0]=(char *)"Server_Connections_aborted"; @@ -5951,58 +5732,6 @@ void MySQL_Thread::Scan_Sessions_to_Kill(PtrArray *mysess) { } } -#ifdef IDLE_THREADS -/** - * @brief Moves a session to the idle session array if it meets the idle criteria. - * - * This function checks if a session should be moved to the idle session array based on its idle time - * and other conditions. If the session meets the idle criteria, it is moved to the idle session array. - * - * @param myds Pointer to the MySQL data stream associated with the session. - * @param n The index of the session in the poll array. - * @return True if the session is moved to the idle session array, false otherwise. - */ -bool MySQL_Thread::move_session_to_idle_mysql_sessions(MySQL_Data_Stream *myds, unsigned int n) { - unsigned long long _tmp_idle = mypolls.last_recv[n] > mypolls.last_sent[n] ? mypolls.last_recv[n] : mypolls.last_sent[n] ; - if (_tmp_idle < ( (curtime > (unsigned int)mysql_thread___session_idle_ms * 1000) ? (curtime - mysql_thread___session_idle_ms * 1000) : 0)) { - // make sure data stream has no pending data out and session is not throttled (#1939) - // because epoll thread does not handle data stream with data out - if (myds->sess->client_myds == myds && !myds->available_data_out() && myds->sess->pause_until <= curtime) { - //unsigned int j; - bool has_backends = myds->sess->has_any_backend(); - if (has_backends==false) { - unsigned long long idle_since = curtime - myds->sess->IdleTime(); - mypolls.remove_index_fast(n); - myds->mypolls=NULL; - unsigned int i = find_session_idx_in_mysql_sessions(myds->sess); - myds->sess->thread=NULL; - unregister_session(i); - myds->sess->idle_since = idle_since; - idle_mysql_sessions->add(myds->sess); - return true; - } - } - } - return false; -} -#endif // IDLE_THREADS - -bool MySQL_Thread::set_backend_to_be_skipped_if_frontend_is_slow(MySQL_Data_Stream *myds, unsigned int n) { - if (myds->sess && myds->sess->client_myds && myds->sess->mirror==false) { - unsigned int buffered_data=0; - buffered_data = myds->sess->client_myds->PSarrayOUT->len * RESULTSET_BUFLEN; - buffered_data += myds->sess->client_myds->resultset->len * RESULTSET_BUFLEN; - // we pause receiving from backend at mysql_thread___threshold_resultset_size * 8 - // but assuming that client isn't completely blocked, we will stop checking for data - // only at mysql_thread___threshold_resultset_size * 4 - if (buffered_data > (unsigned int)mysql_thread___threshold_resultset_size*4) { - mypolls.fds[n].events = 0; - return true; - } - } - return false; -} - #ifdef IDLE_THREADS /** * @brief Moves sessions from the idle thread's session array to the worker thread's session array. @@ -6017,7 +5746,7 @@ void MySQL_Thread::idle_thread_gets_sessions_from_worker_thread() { pthread_mutex_lock(&myexchange.mutex_idles); while (myexchange.idle_mysql_sessions->len) { MySQL_Session *mysess=(MySQL_Session *)myexchange.idle_mysql_sessions->remove_index_fast(0); - register_session(mysess, false); + register_session(this, mysess, false); MySQL_Data_Stream *myds=mysess->client_myds; mypolls.add(POLLIN, myds->fd, myds, monotonic_time()); // add in epoll() @@ -6044,7 +5773,7 @@ void MySQL_Thread::handle_mirror_queue_mysql_sessions() { int idx; idx=fastrand()%(mirror_queue_mysql_sessions->len); MySQL_Session *newsess=(MySQL_Session *)mirror_queue_mysql_sessions->remove_index_fast(idx); - register_session(newsess); + register_session(this, newsess); newsess->handler(); // execute immediately if (newsess->status==WAITING_CLIENT_DATA) { // the mirror session has completed unregister_session(mysql_sessions->len-1); @@ -6088,120 +5817,3 @@ void MySQL_Thread::handle_kill_queues() { } pthread_mutex_unlock(&kq.m); } - - -/** - * @brief Checks for timing out session and marks them for processing. - * - * This function checks for timing out sessions and marks them for processing. Although the logic for managing connection timeout - * was removed due to the addition of the MariaDB client library, this function remains as a placeholder. It checks if the session - * has reached its wait_until or pause_until time, and if so, marks the session for processing. - * - * @param n The index of the session in the MySQL_Data_Stream array. - */ -void MySQL_Thread::check_timing_out_session(unsigned int n) { - // FIXME: this logic was removed completely because we added mariadb client library. Yet, we need to implement a way to manage connection timeout - // check for timeout - // no events. This section is copied from process_data_on_data_stream() - MySQL_Data_Stream *_myds=mypolls.myds[n]; - if (_myds && _myds->sess) { - if (_myds->wait_until && curtime > _myds->wait_until) { - // timeout - _myds->sess->to_process=1; - } else { - if (_myds->sess->pause_until && curtime > _myds->sess->pause_until) { - // timeout - _myds->sess->to_process=1; - } - } - } -} - - -/** - * @brief Checks for an invalid file descriptor (FD) and raises an error if found. - * - * This function checks if the file descriptor (FD) at the specified index in the `mypolls.fds` array is invalid (`POLLNVAL`). - * If an invalid FD is found, it raises an error and asserts to ensure that the program does not proceed with an invalid FD. - * - * @param n The index of the file descriptor in the `mypolls.fds` array. - */ -void MySQL_Thread::check_for_invalid_fd(unsigned int n) { - // check if the FD is valid - if (mypolls.fds[n].revents==POLLNVAL) { - // debugging output before assert - MySQL_Data_Stream *_myds=mypolls.myds[n]; - if (_myds) { - if (_myds->myconn) { - proxy_error("revents==POLLNVAL for FD=%d, events=%d, MyDSFD=%d, MyConnFD=%d\n", mypolls.fds[n].fd, mypolls.fds[n].events, _myds->fd, _myds->myconn->fd); - assert(mypolls.fds[n].revents!=POLLNVAL); - } - } - // if we reached her, we didn't assert() yet - proxy_error("revents==POLLNVAL for FD=%d, events=%d, MyDSFD=%d\n", mypolls.fds[n].fd, mypolls.fds[n].events, _myds->fd); - assert(mypolls.fds[n].revents!=POLLNVAL); - } -} - -void MySQL_Thread::read_one_byte_from_pipe(unsigned int n) { - if (mypolls.fds[n].revents) { - unsigned char c; - if (read(mypolls.fds[n].fd, &c, 1)==-1) {// read just one byte - proxy_error("Error during read from signal_all_threads()\n"); - } - proxy_debug(PROXY_DEBUG_GENERIC,3, "Got signal from admin , done nothing\n"); - //fprintf(stderr,"Got signal from admin , done nothing\n"); // FIXME: this is just the skeleton for issue #253 - if (c) { - // we are being signaled to sleep for some ms. Before going to sleep we also release the mutex - pthread_mutex_unlock(&thread_mutex); - usleep(c*1000); - pthread_mutex_lock(&thread_mutex); - // we enter in maintenance loop only if c is set - // when threads are signaling each other, there is no need to set maintenance_loop - maintenance_loop=true; - } - } -} - -void MySQL_Thread::tune_timeout_for_myds_needs_pause(MySQL_Data_Stream *myds) { - if (myds->wait_until > curtime) { - if (mypolls.poll_timeout==0 || (myds->wait_until - curtime < mypolls.poll_timeout) ) { - mypolls.poll_timeout= myds->wait_until - curtime; - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Session=%p , poll_timeout=%u , wait_until=%llu , curtime=%llu\n", myds->sess, mypolls.poll_timeout, myds->wait_until, curtime); - } - } -} - -void MySQL_Thread::tune_timeout_for_session_needs_pause(MySQL_Data_Stream *myds) { - if (mypolls.poll_timeout==0 || (myds->sess->pause_until - curtime < mypolls.poll_timeout) ) { - mypolls.poll_timeout= myds->sess->pause_until - curtime; - proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Session=%p , poll_timeout=%u , pause_until=%llu , curtime=%llu\n", myds->sess, mypolls.poll_timeout, myds->sess->pause_until, curtime); - } -} - -void MySQL_Thread::configure_pollout(MySQL_Data_Stream *myds, unsigned int n) { - if (myds->myds_type==MYDS_FRONTEND && myds->DSS==STATE_SLEEP && myds->sess && myds->sess->status==WAITING_CLIENT_DATA) { - myds->set_pollout(); - } else { - if (myds->DSS > STATE_MARIADB_BEGIN && myds->DSS < STATE_MARIADB_END) { - mypolls.fds[n].events = POLLIN; - if (mypolls.myds[n]->myconn->async_exit_status & MYSQL_WAIT_WRITE) - mypolls.fds[n].events |= POLLOUT; - } else { - myds->set_pollout(); - } - } - if (unlikely(myds->sess->pause_until > curtime)) { - if (myds->myds_type==MYDS_FRONTEND) { - myds->remove_pollout(); - } - if (myds->myds_type==MYDS_BACKEND) { - if (mysql_thread___throttle_ratio_server_to_client) { - mypolls.fds[n].events = 0; - } - } - } - if (myds->myds_type==MYDS_BACKEND) { - set_backend_to_be_skipped_if_frontend_is_slow(myds, n); - } -} diff --git a/lib/MySrvList.cpp b/lib/MySrvList.cpp deleted file mode 100644 index ab39ef22e7..0000000000 --- a/lib/MySrvList.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include "MySQL_HostGroups_Manager.h" - -class MySrvConnList; -class MySrvC; -class MySrvList; -class MyHGC; - -MySrvList::MySrvList(MyHGC *_myhgc) { - myhgc=_myhgc; - servers=new PtrArray(); -} - -void MySrvList::add(MySrvC *s) { - if (s->myhgc==NULL) { - s->myhgc=myhgc; - } - servers->add(s); - myhgc->refresh_online_server_count(); -} - - -int MySrvList::find_idx(MySrvC *s) { - for (unsigned int i=0; ilen; i++) { - MySrvC *mysrv=(MySrvC *)servers->index(i); - if (mysrv==s) { - return (unsigned int)i; - } - } - return -1; -} - -void MySrvList::remove(MySrvC *s) { - int i=find_idx(s); - assert(i>=0); - servers->remove_index_fast((unsigned int)i); - myhgc->refresh_online_server_count(); -} - -MySrvList::~MySrvList() { - myhgc=NULL; - while (servers->len) { - MySrvC *mysrvc=(MySrvC *)servers->remove_index_fast(0); - delete mysrvc; - } - delete servers; -} diff --git a/lib/PgSQL_Authentication.cpp b/lib/PgSQL_Authentication.cpp new file mode 100644 index 0000000000..102bc3f65d --- /dev/null +++ b/lib/PgSQL_Authentication.cpp @@ -0,0 +1,729 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +//#include "btree_map.h" +#include "proxysql.h" +#include "cpp.h" +#include "proxysql_atomic.h" + +#include "PgSQL_Authentication.h" + +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif + +PgSQL_Authentication::PgSQL_Authentication() { +#ifdef DEBUG + if (glovars.has_debug==false) { +#else + if (glovars.has_debug==true) { +#endif /* DEBUG */ + perror("Incompatible debugging version"); + exit(EXIT_FAILURE); + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_init(&creds_backends.lock, NULL); + pthread_rwlock_init(&creds_frontends.lock, NULL); +#else + spinlock_rwlock_init(&creds_backends.lock); + spinlock_rwlock_init(&creds_frontends.lock); +#endif + creds_backends.cred_array = new PtrArray(); + creds_frontends.cred_array = new PtrArray(); +}; + +PgSQL_Authentication::~PgSQL_Authentication() { + reset(); + delete creds_backends.cred_array; + delete creds_frontends.cred_array; +}; + +void PgSQL_Authentication::print_version() { + fprintf(stderr,"Standard PgSQL Authentication rev. %s -- %s -- %s\n", PGSQL_AUTHENTICATION_VERSION, __FILE__, __TIMESTAMP__); + }; + +void PgSQL_Authentication::set_all_inactive(enum cred_username_type usertype) { + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_wrlock(&cg.lock); +#else + spin_wrlock(&cg.lock); +#endif + unsigned int i; + for (i=0; ilen; i++) { + pgsql_account_details_t *ado=(pgsql_account_details_t *)cg.cred_array->index(i); + ado->__active=false; + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_wrunlock(&cg.lock); +#endif +} + +void PgSQL_Authentication::remove_inactives(enum cred_username_type usertype) { + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_wrlock(&cg.lock); +#else + spin_wrlock(&cg.lock); +#endif + unsigned int i; +__loop_remove_inactives: + for (i=0; ilen; i++) { + pgsql_account_details_t *ado=(pgsql_account_details_t *)cg.cred_array->index(i); + if (ado->__active==false) { + del(ado->username,usertype,false); + goto __loop_remove_inactives; // we aren't sure how the underlying structure changes, so we jump back to 0 + } + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_wrunlock(&cg.lock); +#endif +} + +bool PgSQL_Authentication::add(char * username, char * password, enum cred_username_type usertype, bool use_ssl, int default_hostgroup, bool transaction_persistent, bool fast_forward, int max_connections, char* attributes, char *comment) { + uint64_t hash1, hash2; + SpookyHash myhash; + myhash.Init(1,2); + myhash.Update(username,strlen(username)); + myhash.Final(&hash1,&hash2); + + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); + +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_wrlock(&cg.lock); +#else + spin_wrlock(&cg.lock); +#endif + std::map::iterator lookup; + lookup = cg.bt_map.find(hash1); + // few changes will follow, due to issue #802 + pgsql_account_details_t *ad=NULL; + bool new_ad=false; + if (lookup != cg.bt_map.end()) { + ad=lookup->second; + if (strcmp(ad->password,password)) { + free(ad->password); + ad->password=strdup(password); + if (ad->sha1_pass) { + free(ad->sha1_pass); + ad->sha1_pass=NULL; + } + } + if (strcmp(ad->comment,comment)) { + free(ad->comment); + ad->comment=strdup(comment); + } + if (strcasecmp(ad->attributes, attributes)) { + free(ad->attributes); + if (strlen(attributes)) { + // NOTE: add() is only place where we do input validation + try { + nlohmann::json valid=nlohmann::json::parse(attributes); + // we do further input validation here, and possibly transforming the JSON itself + bool json_rewritten = false; + auto default_transaction_isolation = valid.find("default-transaction_isolation"); + if (default_transaction_isolation != valid.end()) { + std::string dti = valid["default-transaction_isolation"].get(); + for (unsigned int i = 0; i < dti.length(); ++i) { + if (dti[i] == '-') { + dti[i] = ' '; + json_rewritten = true; // the json needs to be rewritten + } + } + // input validation + if ( + (strcasecmp(dti.c_str(), "READ UNCOMMITTED")==0 ) + || (strcasecmp(dti.c_str(), "READ COMMITTED")==0 ) + || (strcasecmp(dti.c_str(), "REPEATABLE READ")==0 ) + || (strcasecmp(dti.c_str(), "SERIALIZABLE")==0 ) + ) { + if (json_rewritten) { + valid["default-transaction_isolation"]=dti; + } + } else { + std::string dti_orig = valid["default-transaction_isolation"].get(); + proxy_error("Invalid default-transaction_isolation for user %s : %s . Removing it from runtime\n", username, dti_orig.c_str()); + valid.erase("default-transaction_isolation"); + json_rewritten = true; // the json was rewritten + } + } + if (json_rewritten) { + std::string d = valid.dump(); + if (d.length()==2) { // empty json + ad->attributes=strdup(""); // empty string + } else { + ad->attributes=strdup(d.c_str()); + } + } else { + // the JSON wasn't rewritten for the purpose of input validation, therefore we copy the original value + ad->attributes=strdup(attributes); + } + } + catch(nlohmann::json::exception& e) { + ad->attributes=strdup(""); + proxy_error("Invalid attributes for user %s: %s\n", username, attributes); + } + } else { + ad->attributes=strdup(attributes); // default, empty string + } + } + } else { + ad=(pgsql_account_details_t *)malloc(sizeof(pgsql_account_details_t )); + ad->username=strdup(username); + ad->comment=strdup(comment); + ad->password=strdup(password); + if (strlen(attributes)) { + // NOTE: add() is only place where we do input validation + try { + nlohmann::json valid=nlohmann::json::parse(attributes); + ad->attributes=strdup(attributes); + } + catch(nlohmann::json::exception& e) { + ad->attributes=strdup(""); + proxy_error("Invalid attributes for user %s: %s\n", username, attributes); + } + } else { + ad->attributes=strdup(attributes); // default, empty string + } + new_ad=true; + ad->sha1_pass=NULL; + ad->num_connections_used=0; + } + + ad->use_ssl=use_ssl; + ad->default_hostgroup=default_hostgroup; + ad->transaction_persistent=transaction_persistent; + ad->fast_forward=fast_forward; + ad->max_connections=max_connections; + ad->__active=true; + if (new_ad) { + cg.bt_map.insert(std::make_pair(hash1,ad)); + cg.cred_array->add(ad); + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_wrunlock(&cg.lock); +#endif + return true; +}; + + +unsigned int PgSQL_Authentication::memory_usage() { + unsigned int ret=0; +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_rdlock(&creds_frontends.lock); + pthread_rwlock_rdlock(&creds_backends.lock); +#else + spin_rdlock(&creds_frontends.lock); + spin_rdlock(&creds_backends.lock); +#endif + unsigned i=0; + for (i=0; ilen; i++) { + pgsql_account_details_t *ado=(pgsql_account_details_t *)creds_frontends.cred_array->index(i); + ret += sizeof(pgsql_account_details_t ); + if (ado->username) ret += strlen(ado->username) + 1; + if (ado->password) ret += strlen(ado->password) + 1; + if (ado->sha1_pass) ret += SHA_DIGEST_LENGTH; + if (ado->comment) ret += strlen(ado->comment) + 1; + if (ado->attributes) ret += strlen(ado->attributes) + 1; + } + ret += sizeof(creds_group_t); + ret += sizeof(PtrArray); + ret += (creds_frontends.cred_array->size * sizeof(void *)); + for (i=0; ilen; i++) { + pgsql_account_details_t *ado=(pgsql_account_details_t *)creds_backends.cred_array->index(i); + ret += sizeof(pgsql_account_details_t); + if (ado->username) ret += strlen(ado->username) + 1; + if (ado->password) ret += strlen(ado->password) + 1; + if (ado->sha1_pass) ret += SHA_DIGEST_LENGTH; + if (ado->comment) ret += strlen(ado->comment) + 1; + if (ado->attributes) ret += strlen(ado->attributes) + 1; + } + ret += sizeof(creds_group_t); + ret += sizeof(PtrArray); + ret += (creds_backends.cred_array->size * sizeof(void *)); +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&creds_frontends.lock); + pthread_rwlock_unlock(&creds_backends.lock); +#else + spin_rdunlock(&creds_frontends.lock); + spin_rdunlock(&creds_backends.lock); +#endif + return ret; +} + +int PgSQL_Authentication::dump_all_users(pgsql_account_details_t***ads, bool _complete) { +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_rdlock(&creds_frontends.lock); + pthread_rwlock_rdlock(&creds_backends.lock); +#else + spin_rdlock(&creds_frontends.lock); + spin_rdlock(&creds_backends.lock); +#endif + int total_size; + int idx_=0; + unsigned i=0; + pgsql_account_details_t** _ads; + total_size=creds_frontends.cred_array->len; + if (_complete) { + total_size+=creds_backends.cred_array->len; + } + if (!total_size) goto __exit_dump_all_users; + _ads=(pgsql_account_details_t**)malloc(sizeof(pgsql_account_details_t*)*total_size); + for (i=0; ilen; i++) { + pgsql_account_details_t *ad=(pgsql_account_details_t*)malloc(sizeof(pgsql_account_details_t)); + pgsql_account_details_t *ado=(pgsql_account_details_t*)creds_frontends.cred_array->index(i); + ad->username=strdup(ado->username); + ad->max_connections=ado->max_connections; + ad->default_hostgroup=ado->default_hostgroup; + if (_complete==false) { + ad->password=NULL; + ad->attributes=NULL; + ad->comment=NULL; + ad->num_connections_used=ado->num_connections_used; + } else { + ad->num_connections_used=ado->num_connections_used; + ad->password=strdup(ado->password); + ad->sha1_pass=NULL; + ad->use_ssl=ado->use_ssl; + ad->attributes=strdup(ado->attributes); + ad->comment=strdup(ado->comment); + ad->transaction_persistent=ado->transaction_persistent; + ad->fast_forward=ado->fast_forward; + ad->__frontend=1; + ad->__backend=0; + } + _ads[idx_]=ad; + idx_++; + } + if (_complete==true) { + for (i=0; ilen; i++) { + pgsql_account_details_t *ad=(pgsql_account_details_t *)malloc(sizeof(pgsql_account_details_t)); + pgsql_account_details_t *ado=(pgsql_account_details_t *)creds_backends.cred_array->index(i); + ad->num_connections_used=0; + ad->username=strdup(ado->username); + ad->password=strdup(ado->password); + ad->sha1_pass=NULL; + ad->use_ssl=ado->use_ssl; + ad->default_hostgroup=ado->default_hostgroup; + ad->attributes=strdup(ado->attributes); + ad->comment=strdup(ado->comment); + ad->transaction_persistent=ado->transaction_persistent; + ad->fast_forward=ado->fast_forward; + ad->max_connections=ado->max_connections; + ad->__frontend=0; + ad->__backend=1; + _ads[idx_]=ad; + idx_++; + } + } + *ads=_ads; +__exit_dump_all_users: +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&creds_frontends.lock); + pthread_rwlock_unlock(&creds_backends.lock); +#else + spin_rdunlock(&creds_frontends.lock); + spin_rdunlock(&creds_backends.lock); +#endif + return total_size; +} + + +int PgSQL_Authentication::increase_frontend_user_connections(char *username, int *mc) { + uint64_t hash1, hash2; + SpookyHash *myhash=new SpookyHash(); + myhash->Init(1,2); + myhash->Update(username,strlen(username)); + myhash->Final(&hash1,&hash2); + delete myhash; + creds_group_t &cg=creds_frontends; + int ret=0; +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_wrlock(&cg.lock); +#else + spin_wrlock(&cg.lock); +#endif + std::map::iterator it; + it = cg.bt_map.find(hash1); + if (it != cg.bt_map.end()) { + pgsql_account_details_t *ad=it->second; + if (ad->max_connections > ad->num_connections_used) { + ret=ad->max_connections-ad->num_connections_used; + ad->num_connections_used++; + } + if (mc) { + *mc=ad->max_connections; + } + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_wrunlock(&cg.lock); +#endif + return ret; +} + +void PgSQL_Authentication::decrease_frontend_user_connections(char *username) { + uint64_t hash1, hash2; + SpookyHash *myhash=new SpookyHash(); + myhash->Init(1,2); + myhash->Update(username,strlen(username)); + myhash->Final(&hash1,&hash2); + delete myhash; + creds_group_t &cg=creds_frontends; +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_wrlock(&cg.lock); +#else + spin_wrlock(&cg.lock); +#endif + std::map::iterator it; + it = cg.bt_map.find(hash1); + if (it != cg.bt_map.end()) { + pgsql_account_details_t *ad=it->second; + if (ad->num_connections_used > 0) { + ad->num_connections_used--; + } + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_wrunlock(&cg.lock); +#endif +} + +bool PgSQL_Authentication::del(char * username, enum cred_username_type usertype, bool set_lock) { + bool ret=false; + uint64_t hash1, hash2; + SpookyHash *myhash=new SpookyHash(); + myhash->Init(1,2); + myhash->Update(username,strlen(username)); + myhash->Final(&hash1,&hash2); + delete myhash; + + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); + + if (set_lock) +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_wrlock(&cg.lock); +#else + spin_wrlock(&cg.lock); +#endif + std::map::iterator lookup; + lookup = cg.bt_map.find(hash1); + if (lookup != cg.bt_map.end()) { + pgsql_account_details_t *ad=lookup->second; + cg.cred_array->remove_fast(ad); + cg.bt_map.erase(lookup); + free(ad->username); + free(ad->password); + if (ad->sha1_pass) { free(ad->sha1_pass); ad->sha1_pass=NULL; } + free(ad->attributes); + free(ad->comment); + free(ad); + ret=true; + } + if (set_lock) +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_wrunlock(&cg.lock); +#endif + return ret; +}; + +bool PgSQL_Authentication::set_SHA1(char * username, enum cred_username_type usertype, void *sha_pass) { + bool ret=false; + uint64_t hash1, hash2; + SpookyHash *myhash=new SpookyHash(); + myhash->Init(1,2); + myhash->Update(username,strlen(username)); + myhash->Final(&hash1,&hash2); + delete myhash; + + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); + +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_wrlock(&cg.lock); +#else + spin_wrlock(&cg.lock); +#endif + std::map::iterator lookup; + lookup = cg.bt_map.find(hash1); + if (lookup != cg.bt_map.end()) { + pgsql_account_details_t *ad=lookup->second; + if (ad->sha1_pass) { free(ad->sha1_pass); ad->sha1_pass=NULL; } + if (sha_pass) { + ad->sha1_pass=malloc(SHA_DIGEST_LENGTH); + memcpy(ad->sha1_pass,sha_pass,SHA_DIGEST_LENGTH); + } + ret=true; + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_wrunlock(&cg.lock); +#endif + return ret; +}; + +bool PgSQL_Authentication::exists(char * username) { + bool ret = false; + uint64_t hash1, hash2; + SpookyHash myhash; + myhash.Init(1,2); + myhash.Update(username,strlen(username)); + myhash.Final(&hash1,&hash2); + + creds_group_t &cg = creds_frontends ; + pthread_rwlock_rdlock(&cg.lock); + std::map::iterator lookup; + lookup = cg.bt_map.find(hash1); + if (lookup != cg.bt_map.end()) { + ret = true; + } + pthread_rwlock_unlock(&cg.lock); + return ret; +} + +char * PgSQL_Authentication::lookup(char * username, enum cred_username_type usertype, bool *use_ssl, int *default_hostgroup, bool *transaction_persistent, bool *fast_forward, int *max_connections, void **sha1_pass, char **attributes) { + char *ret=NULL; + uint64_t hash1, hash2; + SpookyHash myhash; + myhash.Init(1,2); + myhash.Update(username,strlen(username)); + myhash.Final(&hash1,&hash2); + + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); + +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_rdlock(&cg.lock); +#else + spin_rdlock(&cg.lock); +#endif + std::map::iterator lookup; + lookup = cg.bt_map.find(hash1); + if (lookup != cg.bt_map.end()) { + pgsql_account_details_t *ad=lookup->second; + ret=l_strdup(ad->password); + if (use_ssl) *use_ssl=ad->use_ssl; + if (default_hostgroup) *default_hostgroup=ad->default_hostgroup; + if (transaction_persistent) *transaction_persistent=ad->transaction_persistent; + if (fast_forward) *fast_forward=ad->fast_forward; + if (max_connections) *max_connections=ad->max_connections; + if (sha1_pass) { + if (ad->sha1_pass) { + *sha1_pass=malloc(SHA_DIGEST_LENGTH); + memcpy(*sha1_pass,ad->sha1_pass,SHA_DIGEST_LENGTH); + } + } + if (attributes) *attributes=l_strdup(ad->attributes); + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_rdunlock(&cg.lock); +#endif + return ret; + +} + +bool PgSQL_Authentication::_reset(enum cred_username_type usertype) { + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); + +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_wrlock(&cg.lock); +#else + spin_wrlock(&cg.lock); +#endif + std::map::iterator lookup; + + while (cg.bt_map.size()) { + lookup = cg.bt_map.begin(); + if ( lookup != cg.bt_map.end() ) { + pgsql_account_details_t *ad=lookup->second; + cg.bt_map.erase(lookup); + free(ad->username); + free(ad->password); + if (ad->sha1_pass) { free(ad->sha1_pass); ad->sha1_pass=NULL; } + free(ad->comment); + free(ad->attributes); + //free(ad->scram_keys); + free(ad); + } + } + while (cg.cred_array->len) { + cg.cred_array->remove_index_fast(0); + } +#ifdef PROXYSQL_AUTH_PTHREAD_MUTEX + pthread_rwlock_unlock(&cg.lock); +#else + spin_wrunlock(&cg.lock); +#endif + return true; +}; + +bool PgSQL_Authentication::reset() { + _reset(USERNAME_BACKEND); + _reset(USERNAME_FRONTEND); + return true; +} + +using std::map; + +static uint64_t compute_accounts_hash(const umap_pgauth& accs_map) { + if (accs_map.size() == 0) { + return 0; + } + bool foundany = false; + SpookyHash acc_map_hash; + acc_map_hash.Init(13,4); + + for (const pair& map_entry : accs_map) { + const pgsql_account_details_t* ad = map_entry.second; + + if (ad->default_hostgroup >= 0) { + foundany = true; + acc_map_hash.Update(&ad->use_ssl,sizeof(ad->use_ssl)); + acc_map_hash.Update(&ad->default_hostgroup,sizeof(ad->default_hostgroup)); + acc_map_hash.Update(&ad->transaction_persistent,sizeof(ad->transaction_persistent)); + acc_map_hash.Update(&ad->fast_forward,sizeof(ad->fast_forward)); + acc_map_hash.Update(&ad->max_connections,sizeof(ad->max_connections)); + acc_map_hash.Update(ad->username,strlen(ad->username)); + acc_map_hash.Update(ad->password,strlen(ad->password)); + if (ad->comment) + acc_map_hash.Update(ad->comment,strlen(ad->comment)); + if (ad->attributes) { + acc_map_hash.Update(ad->attributes,strlen(ad->attributes)); + } + } + } + + if (foundany == false) { + return 0; + } else { + uint64_t hash1 = 0, hash2 = 0; + acc_map_hash.Final(&hash1, &hash2); + + return hash1; + } +} + +uint64_t PgSQL_Authentication::_get_runtime_checksum(enum cred_username_type usertype) { + creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); + uint64_t accs_hash = compute_accounts_hash(cg.bt_map); + + return accs_hash; +} + +uint64_t PgSQL_Authentication::get_runtime_checksum() { + uint64_t hashB = _get_runtime_checksum(USERNAME_BACKEND); + uint64_t hashF = _get_runtime_checksum(USERNAME_FRONTEND); + return hashB+hashF; +} + +static pair extract_accounts_details(MYSQL_RES* resultset, unique_ptr& all_users) { + if (resultset == nullptr) { return { umap_pgauth {}, umap_pgauth {} }; } + + // The following order is assumed for the resulset received fields: + // - username, password, active, use_ssl, default_hostgroup, + // transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment. + umap_pgauth f_accs_map {}; + umap_pgauth b_accs_map {}; + + // Create the SQLite3 resultsets for 'frontend' and 'backend' users + uint32_t num_fields = mysql_num_fields(resultset); + MYSQL_FIELD* fields = mysql_fetch_fields(resultset); + + SQLite3_result* _all_users { new SQLite3_result(num_fields) }; + + for (uint32_t i = 0; i < num_fields; i++) { + _all_users->add_column_definition(SQLITE_TEXT, fields[i].name); + } + + const auto create_account_details = [] (MYSQL_ROW row) -> pgsql_account_details_t* { + pgsql_account_details_t* acc_details { new pgsql_account_details_t {} }; + + acc_details->username = row[0]; + acc_details->password = row[1] ? row[1] : const_cast(""); + acc_details->__active = true; + acc_details->use_ssl = strcmp(row[2], "1") == 0 ? true : false; + acc_details->default_hostgroup = atoi(row[3]); + acc_details->transaction_persistent = strcmp(row[4], "1") == 0 ? true : false; + acc_details->fast_forward = strcmp(row[5], "1") == 0 ? true : false; + acc_details->__backend = strcmp(row[6], "1") == 0 ? true : false; + acc_details->__frontend = strcmp(row[7], "1") == 0 ? true : false; + acc_details->max_connections = atoi(row[8]); + acc_details->attributes = row[9] ? row[9] : const_cast(""); + acc_details->comment = row[10] ? row[10] : const_cast(""); + + return acc_details; + }; + + vector pta(static_cast(num_fields)); + while (MYSQL_ROW row = mysql_fetch_row(resultset)) { + // compute the 'username' hash for the map + uint64_t u_hash = 0, _u_hash2 = 0; + SpookyHash myhash {}; + myhash.Init(1,2); + myhash.Update(row[0], strlen(row[0])); + myhash.Final(&u_hash, &_u_hash2); + + // is backend + if (strcmp(row[6], "1") == 0) { + pgsql_account_details_t* acc_details = create_account_details(row); + b_accs_map.insert({u_hash, acc_details}); + } + // is frontend + if (strcmp(row[7], "1") == 0) { + pgsql_account_details_t* acc_details = create_account_details(row); + f_accs_map.insert({u_hash, acc_details}); + } + + // Update the contents of the row for the SQLite3 resultset + for (uint32_t i = 0; i < num_fields; i++) { + pta[i] = row[i]; + } + _all_users->add_row(&pta[0]); + } + + mysql_data_seek(resultset, 0); + + // Update the supplied 'unique_ptr' with the target resultsets + all_users.reset(_all_users); + + return { b_accs_map, f_accs_map }; +} + +uint64_t PgSQL_Authentication::get_runtime_checksum(MYSQL_RES* resultset, unique_ptr& all_users) { + if (resultset == NULL) { return 0; } + + pair acc_maps { extract_accounts_details(resultset, all_users) }; + + uint64_t b_acc_hash = compute_accounts_hash(acc_maps.first); + uint64_t f_acc_hash = compute_accounts_hash(acc_maps.second); + + for (pair& map_entry : acc_maps.first) { + delete map_entry.second; + } + for (pair& map_entry : acc_maps.second) { + delete map_entry.second; + } + + return b_acc_hash + f_acc_hash; +} + +void PgSQL_Authentication::save_pgsql_users(unique_ptr&& users) { + this->pgsql_users_resultset = std::move(users); +} + +SQLite3_result* PgSQL_Authentication::get_current_pgsql_users() { + return this->pgsql_users_resultset.get(); +} diff --git a/lib/PgSQL_Backend.cpp b/lib/PgSQL_Backend.cpp new file mode 100644 index 0000000000..0b926d0c06 --- /dev/null +++ b/lib/PgSQL_Backend.cpp @@ -0,0 +1,31 @@ +#include "proxysql.h" +#include "cpp.h" +#include "PgSQL_Data_Stream.h" + +void * PgSQL_Backend::operator new(size_t size) { + return l_alloc(size); +} + +void PgSQL_Backend::operator delete(void *ptr) { + l_free(sizeof(PgSQL_Backend),ptr); +} + +PgSQL_Backend::PgSQL_Backend() { + hostgroup_id=-1; + server_myds=NULL; + server_bytes_at_cmd.bytes_recv=0; + server_bytes_at_cmd.bytes_sent=0; + memset(gtid_uuid,0,sizeof(gtid_uuid)); + gtid_trxid=0; +} + +PgSQL_Backend::~PgSQL_Backend() { +} + +void PgSQL_Backend::reset() { + + if (server_myds) { + server_myds->reset_connection(); + delete server_myds; + } +} diff --git a/lib/PgSQL_Connection.cpp b/lib/PgSQL_Connection.cpp new file mode 100644 index 0000000000..a51a11b600 --- /dev/null +++ b/lib/PgSQL_Connection.cpp @@ -0,0 +1,2905 @@ + +#include +#include +#include + +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON +#include "PgSQL_HostGroups_Manager.h" +#include "proxysql.h" +#include "cpp.h" +#include "MySQL_PreparedStatement.h" +#include "PgSQL_Data_Stream.h" +#include "PgSQL_Query_Processor.h" +#include "MySQL_Variables.h" + + +#if 0 +// some of the code that follows is from mariadb client library memory allocator +typedef int myf; // Type of MyFlags in my_funcs +#define MYF(v) (myf) (v) +#define MY_KEEP_PREALLOC 1 +#define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1)) +#define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double)) +static void ma_free_root(MA_MEM_ROOT *root, myf MyFLAGS); +static void *ma_alloc_root(MA_MEM_ROOT *mem_root, size_t Size); +#define MAX(a,b) (((a) > (b)) ? (a) : (b)) + + +static void * ma_alloc_root(MA_MEM_ROOT *mem_root, size_t Size) +{ + size_t get_size; + void * point; + MA_USED_MEM *next= 0; + MA_USED_MEM **prev; + + Size= ALIGN_SIZE(Size); + + if ((*(prev= &mem_root->free))) + { + if ((*prev)->left < Size && + mem_root->first_block_usage++ >= 16 && + (*prev)->left < 4096) + { + next= *prev; + *prev= next->next; + next->next= mem_root->used; + mem_root->used= next; + mem_root->first_block_usage= 0; + } + for (next= *prev; next && next->left < Size; next= next->next) + prev= &next->next; + } + if (! next) + { /* Time to alloc new block */ + get_size= MAX(Size+ALIGN_SIZE(sizeof(MA_USED_MEM)), + (mem_root->block_size & ~1) * ( (mem_root->block_num >> 2) < 4 ? 4 : (mem_root->block_num >> 2) ) ); + + if (!(next = (MA_USED_MEM*) malloc(get_size))) + { + if (mem_root->error_handler) + (*mem_root->error_handler)(); + return((void *) 0); /* purecov: inspected */ + } + mem_root->block_num++; + next->next= *prev; + next->size= get_size; + next->left= get_size-ALIGN_SIZE(sizeof(MA_USED_MEM)); + *prev=next; + } + point= (void *) ((char*) next+ (next->size-next->left)); + if ((next->left-= Size) < mem_root->min_malloc) + { /* Full block */ + *prev=next->next; /* Remove block from list */ + next->next=mem_root->used; + mem_root->used=next; + mem_root->first_block_usage= 0; + } + return(point); +} + + +static void ma_free_root(MA_MEM_ROOT *root, myf MyFlags) +{ + MA_USED_MEM *next,*old; + + if (!root) + return; /* purecov: inspected */ + if (!(MyFlags & MY_KEEP_PREALLOC)) + root->pre_alloc=0; + + for ( next=root->used; next ;) + { + old=next; next= next->next ; + if (old != root->pre_alloc) + free(old); + } + for (next= root->free ; next ; ) + { + old=next; next= next->next ; + if (old != root->pre_alloc) + free(old); + } + root->used=root->free=0; + if (root->pre_alloc) + { + root->free=root->pre_alloc; + root->free->left=root->pre_alloc->size-ALIGN_SIZE(sizeof(MA_USED_MEM)); + root->free->next=0; + } +} +#endif // 0 + +extern char * binary_sha1; + +#include "proxysql_find_charset.h" + +void PgSQL_Variable::fill_server_internal_session(json &j, int conn_num, int idx) { + if (idx == SQL_CHARACTER_SET_RESULTS || idx == SQL_CHARACTER_SET_CLIENT || idx == SQL_CHARACTER_SET_DATABASE) { + const MARIADB_CHARSET_INFO *ci = NULL; + if (!value) { + ci = proxysql_find_charset_name(mysql_tracked_variables[idx].default_value); + } else if (strcasecmp("NULL", value) && strcasecmp("binary", value)) { + ci = proxysql_find_charset_nr(atoi(value)); + } + if (!ci) { + if (idx == SQL_CHARACTER_SET_RESULTS && (!strcasecmp("NULL", value) || !strcasecmp("binary", value))) { + if (!strcasecmp("NULL", value)) { + j["backends"][conn_num]["conn"][mysql_tracked_variables[idx].internal_variable_name] = ""; + } else { + j["backends"][conn_num]["conn"][mysql_tracked_variables[idx].internal_variable_name] = value; + } + } else { + // LCOV_EXCL_START + proxy_error("Cannot find charset [%s] for variables %d\n", value, idx); + assert(0); + // LCOV_EXCL_STOP + } + } else { + j["backends"][conn_num]["conn"][mysql_tracked_variables[idx].internal_variable_name] = std::string((ci && ci->csname)?ci->csname:""); + } + } else if (idx == SQL_CHARACTER_SET_CONNECTION) { + const MARIADB_CHARSET_INFO *ci = NULL; + if (!value) + ci = proxysql_find_charset_name(mysql_tracked_variables[idx].default_value); + else + ci = proxysql_find_charset_nr(atoi(value)); + + j["backends"][conn_num]["conn"][mysql_tracked_variables[idx].internal_variable_name] = std::string((ci && ci->csname)?ci->csname:""); + } else if (idx == SQL_COLLATION_CONNECTION) { + const MARIADB_CHARSET_INFO *ci = NULL; + if (!value) + ci = proxysql_find_charset_collate(mysql_tracked_variables[idx].default_value); + else + ci = proxysql_find_charset_nr(atoi(value)); + + j["backends"][conn_num]["conn"][mysql_tracked_variables[idx].internal_variable_name] = std::string((ci && ci->name)?ci->name:""); +/* +// NOTE: it seems we treat SQL_LOG_BIN in a special way +// it doesn't seem necessary + } else if (idx == SQL_SQL_LOG_BIN) { + if (!value) + j["backends"][conn_num]["conn"][mysql_tracked_variables[idx].internal_variable_name] = mysql_tracked_variables[idx].default_value; + else + j["backends"][conn_num]["conn"][mysql_tracked_variables[idx].internal_variable_name] = std::string(!strcmp("1",value)?"ON":"OFF"); +*/ + } else { + j["backends"][conn_num]["conn"][mysql_tracked_variables[idx].internal_variable_name] = std::string(value?value:""); + } +} + +void PgSQL_Variable::fill_client_internal_session(json &j, int idx) { + if (idx == SQL_CHARACTER_SET_RESULTS || idx == SQL_CHARACTER_SET_CLIENT || idx == SQL_CHARACTER_SET_DATABASE) { + const MARIADB_CHARSET_INFO *ci = NULL; + if (!value) { + ci = proxysql_find_charset_name(mysql_tracked_variables[idx].default_value); + } else if (strcasecmp("NULL", value) && strcasecmp("binary", value)) { + ci = proxysql_find_charset_nr(atoi(value)); + } + if (!ci) { + if (idx == SQL_CHARACTER_SET_RESULTS && (!strcasecmp("NULL", value) || !strcasecmp("binary", value))) { + if (!strcasecmp("NULL", value)) { + j["conn"][mysql_tracked_variables[idx].internal_variable_name] = ""; + } else { + j["conn"][mysql_tracked_variables[idx].internal_variable_name] = value; + } + } else { + // LCOV_EXCL_START + proxy_error("Cannot find charset [%s] for variables %d\n", value, idx); + assert(0); + // LCOV_EXCL_STOP + } + } else { + j["conn"][mysql_tracked_variables[idx].internal_variable_name] = (ci && ci->csname)?ci->csname:""; + } + } else if (idx == SQL_CHARACTER_SET_CONNECTION) { + const MARIADB_CHARSET_INFO *ci = NULL; + if (!value) + ci = proxysql_find_charset_collate(mysql_tracked_variables[idx].default_value); + else + ci = proxysql_find_charset_nr(atoi(value)); + j["conn"][mysql_tracked_variables[idx].internal_variable_name] = (ci && ci->csname)?ci->csname:""; + } else if (idx == SQL_COLLATION_CONNECTION) { + const MARIADB_CHARSET_INFO *ci = NULL; + if (!value) + ci = proxysql_find_charset_collate(mysql_tracked_variables[idx].default_value); + else + ci = proxysql_find_charset_nr(atoi(value)); + j["conn"][mysql_tracked_variables[idx].internal_variable_name] = (ci && ci->name)?ci->name:""; +/* +// NOTE: it seems we treat SQL_LOG_BIN in a special way +// it doesn't seem necessary + } else if (idx == SQL_LOG_BIN) { + if (!value) + j["conn"][mysql_tracked_variables[idx].internal_variable_name] = mysql_tracked_variables[idx].default_value; + else + j["conn"][mysql_tracked_variables[idx].internal_variable_name] = !strcmp("1", value)?"ON":"OFF"; +*/ + } else { + j["conn"][mysql_tracked_variables[idx].internal_variable_name] = value?value:""; + } +} + +static int +mysql_status(short event, short cont) { + int status= 0; + if (event & POLLIN) + status|= MYSQL_WAIT_READ; + if (event & POLLOUT) + status|= MYSQL_WAIT_WRITE; +// if (event==0 && cont==true) { +// status |= MYSQL_WAIT_TIMEOUT; +// } +// FIXME: handle timeout +// if (event & PROXY_TIMEOUT) +// status|= MYSQL_WAIT_TIMEOUT; + return status; +} + +/* deprecating session_vars[] because we are introducing a better algorithm +// Defining list of session variables for comparison with query digest to disable multiplexing for "SET " commands +static char * session_vars[]= { + // For issue #555 , multiplexing is disabled if --safe-updates is used + //(char *)"SQL_SAFE_UPDATES=?,SQL_SELECT_LIMIT=?,MAX_JOIN_SIZE=?", + // for issue #1832 , we are splitting the above into 3 variables +// (char *)"SQL_SAFE_UPDATES", +// (char *)"SQL_SELECT_LIMIT", +// (char *)"MAX_JOIN_SIZE", + (char *)"FOREIGN_KEY_CHECKS", + (char *)"UNIQUE_CHECKS", + (char *)"AUTO_INCREMENT_INCREMENT", + (char *)"AUTO_INCREMENT_OFFSET", + (char *)"TIMESTAMP", + (char *)"GROUP_CONCAT_MAX_LEN" +}; +*/ + +PgSQL_Connection_userinfo::PgSQL_Connection_userinfo() { + username=NULL; + password=NULL; + sha1_pass=NULL; + dbname=NULL; + fe_username=NULL; + hash=0; +} + +PgSQL_Connection_userinfo::~PgSQL_Connection_userinfo() { + if (username) free(username); + if (fe_username) free(fe_username); + if (password) free(password); + if (sha1_pass) free(sha1_pass); + if (dbname) free(dbname); +} + +void PgSQL_Connection_Placeholder::compute_unknown_transaction_status() { + if (pgsql) { + int _myerrno=mysql_errno(pgsql); + if (_myerrno == 0) { + unknown_transaction_status = false; // no error + return; + } + if (_myerrno >= 2000 && _myerrno < 3000) { // client error + // do not change it + return; + } + if (_myerrno >= 1000 && _myerrno < 2000) { // server error + unknown_transaction_status = true; + return; + } + if (_myerrno >= 3000 && _myerrno < 4000) { // server error + unknown_transaction_status = true; + return; + } + // all other cases, server error + } +} + +uint64_t PgSQL_Connection_userinfo::compute_hash() { + int l=0; + if (username) + l+=strlen(username); + if (password) + l+=strlen(password); + if (dbname) + l+=strlen(dbname); +// two random seperator +#define _COMPUTE_HASH_DEL1_ "-ujhtgf76y576574fhYTRDF345wdt-" +#define _COMPUTE_HASH_DEL2_ "-8k7jrhtrgJHRgrefgreyhtRFewg6-" + l+=strlen(_COMPUTE_HASH_DEL1_); + l+=strlen(_COMPUTE_HASH_DEL2_); + char *buf=(char *)malloc(l+1); + l=0; + if (username) { + strcpy(buf+l,username); + l+=strlen(username); + } + strcpy(buf+l,_COMPUTE_HASH_DEL1_); + l+=strlen(_COMPUTE_HASH_DEL1_); + if (password) { + strcpy(buf+l,password); + l+=strlen(password); + } + if (dbname) { + strcpy(buf+l, dbname); + l+=strlen(dbname); + } + strcpy(buf+l,_COMPUTE_HASH_DEL2_); + l+=strlen(_COMPUTE_HASH_DEL2_); + hash=SpookyHash::Hash64(buf,l,0); + free(buf); + return hash; +} + +void PgSQL_Connection_userinfo::set(char *user, char *pass, char *db, char *sh1) { + if (user) { + if (username) { + if (strcmp(user,username)) { + free(username); + username=strdup(user); + } + } else { + username=strdup(user); + } + } + if (pass) { + if (password) { + if (strcmp(pass,password)) { + free(password); + password=strdup(pass); + } + } else { + password=strdup(pass); + } + } + if (db) { + if (dbname) { + if (strcmp(db,dbname)) { + free(dbname); + dbname=strdup(db); + } + } else { + dbname=strdup(db); + } + } + if (sh1) { + if (sha1_pass) { + free(sha1_pass); + } + sha1_pass=strdup(sh1); + } + compute_hash(); +} + +void PgSQL_Connection_userinfo::set(PgSQL_Connection_userinfo *ui) { + set(ui->username, ui->password, ui->dbname, ui->sha1_pass); +} + +bool PgSQL_Connection_userinfo::set_dbname(const char* db) { + assert(db); + const int new_db_len = db ? strlen(db) : 0; + const int old_db_len = dbname ? strlen(dbname) : 0; + + if (old_db_len == 0 || + old_db_len != new_db_len || + strncmp(db, dbname, new_db_len)) { + if (dbname) { + free(dbname); + } + dbname = (char*)malloc(new_db_len + 1); + memcpy(dbname, db, new_db_len); + dbname[new_db_len] = 0; + compute_hash(); + return true; + } + return false; +} + +PgSQL_Connection_Placeholder::PgSQL_Connection_Placeholder() { + pgsql=NULL; + async_state_machine=ASYNC_CONNECT_START; + ret_mysql=NULL; + send_quit=true; + myds=NULL; + inserted_into_pool=0; + reusable=false; + parent=NULL; + userinfo=new PgSQL_Connection_userinfo(); + fd=-1; + status_flags=0; + last_time_used=0; + + for (auto i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + variables[i].value = NULL; + var_hash[i] = 0; + } + + options.client_flag = 0; + options.server_capabilities = 0; + options.compression_min_length=0; + options.server_version=NULL; + options.last_set_autocommit=-1; // -1 = never set + options.autocommit=true; + options.no_backslash_escapes=false; + options.init_connect=NULL; + options.init_connect_sent=false; + options.session_track_gtids = NULL; + options.session_track_gtids_sent = false; + options.ldap_user_variable=NULL; + options.ldap_user_variable_value=NULL; + options.ldap_user_variable_sent=false; + options.session_track_gtids_int=0; + compression_pkt_id=0; + mysql_result=NULL; + query.ptr=NULL; + query.length=0; + query.stmt=NULL; + query.stmt_meta=NULL; + query.stmt_result=NULL; + largest_query_length=0; + warning_count=0; + multiplex_delayed=false; + MyRS=NULL; + MyRS_reuse=NULL; + unknown_transaction_status = false; + creation_time=0; + auto_increment_delay_token = 0; + processing_multi_statement=false; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "Creating new PgSQL_Connection %p\n", this); + local_stmts=new MySQL_STMTs_local_v14(false); // false by default, it is a backend + bytes_info.bytes_recv = 0; + bytes_info.bytes_sent = 0; + statuses.questions = 0; + statuses.pgconnpoll_get = 0; + statuses.pgconnpoll_put = 0; + memset(gtid_uuid,0,sizeof(gtid_uuid)); + memset(&connected_host_details, 0, sizeof(connected_host_details)); +}; + +PgSQL_Connection_Placeholder::~PgSQL_Connection_Placeholder() { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "Destroying PgSQL_Connection %p\n", this); + if (options.server_version) free(options.server_version); + if (options.init_connect) free(options.init_connect); + if (options.ldap_user_variable) free(options.ldap_user_variable); + if (options.ldap_user_variable_value) free(options.ldap_user_variable_value); + if (userinfo) { + delete userinfo; + userinfo=NULL; + } + if (local_stmts) { + delete local_stmts; + } + if (pgsql) { + // always decrease the counter + if (ret_mysql) { + __sync_fetch_and_sub(&PgHGM->status.server_connections_connected,1); + if (query.stmt_result) { + if (query.stmt_result->handle) { + query.stmt_result->handle->status = MYSQL_STATUS_READY; // avoid calling mthd_my_skip_result() + } + } + if (mysql_result) { + if (mysql_result->handle) { + mysql_result->handle->status = MYSQL_STATUS_READY; // avoid calling mthd_my_skip_result() + } + } + async_free_result(); + } + close_mysql(); // this take care of closing pgsql connection + pgsql=NULL; + } + if (MyRS) { + delete MyRS; + MyRS = NULL; + } + if (MyRS_reuse) { + delete MyRS_reuse; + MyRS_reuse = NULL; + } + if (query.stmt) { + query.stmt=NULL; + } + + if (options.session_track_gtids) { + free(options.session_track_gtids); + options.session_track_gtids=NULL; + } + + for (auto i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + if (variables[i].value) { + free(variables[i].value); + variables[i].value = NULL; + var_hash[i] = 0; + } + } + + if (connected_host_details.hostname) { + free(connected_host_details.hostname); + connected_host_details.hostname = NULL; + } + if (connected_host_details.ip) { + free(connected_host_details.ip); + connected_host_details.hostname = NULL; + } +}; + +bool PgSQL_Connection_Placeholder::set_autocommit(bool _ac) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "Setting autocommit %d\n", _ac); + options.autocommit=_ac; + return _ac; +} + +bool PgSQL_Connection_Placeholder::set_no_backslash_escapes(bool _ac) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "Setting no_backslash_escapes %d\n", _ac); + options.no_backslash_escapes=_ac; + return _ac; +} + +void print_backtrace(void); + +unsigned int PgSQL_Connection_Placeholder::set_charset(unsigned int _c, enum pgsql_charset_action action) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "Setting charset %d\n", _c); + + // SQL_CHARACTER_SET should be set befor setting SQL_CHRACTER_ACTION + std::stringstream ss; + ss << _c; + pgsql_variables.client_set_value(myds->sess, SQL_CHARACTER_SET, ss.str()); + + // When SQL_CHARACTER_ACTION is set character set variables are set according to + // SQL_CHRACTER_SET value + ss.str(std::string()); + ss.clear(); + ss << action; + pgsql_variables.client_set_value(myds->sess, SQL_CHARACTER_ACTION, ss.str()); + + return _c; +} + +void PgSQL_Connection_Placeholder::update_warning_count_from_connection() { + // if a prepared statement was cached while 'mysql_thread_query_digest' was true, and subsequently, + // 'mysql_thread_query_digest' is set to false, fetching that statement from the cache may still contain the digest text. + // To prevent this, we will check the digest text in conjunction with 'mysql_thread_query_digest' to verify whether it + // is enabled or disabled. + if (myds && myds->sess && myds->sess->CurrentQuery.QueryParserArgs.digest_text) { + const char* dig_text = myds->sess->CurrentQuery.QueryParserArgs.digest_text; + const size_t dig_len = strlen(dig_text); + // SHOW WARNINGS doesn't have any impact warning count, + // so we are replication same behaviour here + if (parent->myhgc->handle_warnings_enabled() && + (dig_len != 13 || strncasecmp(dig_text, "SHOW WARNINGS", 13) != 0)) { + warning_count = mysql_warning_count(pgsql); + } + } +} + +void PgSQL_Connection_Placeholder::update_warning_count_from_statement() { + // if a prepared statement was cached while 'mysql_thread_query_digest' was true, and subsequently, + // 'mysql_thread_query_digest' is set to false, fetching that statement from the cache may still contain the digest text. + // To prevent this, we will check the digest text in conjunction with 'mysql_thread_query_digest' to verify whether it + // is enabled or disabled. + if (myds && myds->sess && myds->sess->CurrentQuery.stmt_info && myds->sess->CurrentQuery.stmt_info->digest_text && + pgsql_thread___query_digests == true) { + if (parent->myhgc->handle_warnings_enabled()) { + warning_count = mysql_stmt_warning_count(query.stmt); + } + } +} + +bool PgSQL_Connection_Placeholder::is_expired(unsigned long long timeout) { +// FIXME: here the check should be a sanity check +// FIXME: for now this is just a temporary (and stupid) check + return false; +} + +void PgSQL_Connection_Placeholder::set_status(bool set, uint32_t status_flag) { + if (set) { + this->status_flags |= status_flag; + } else { + this->status_flags &= ~status_flag; + } +} + +bool PgSQL_Connection_Placeholder::get_status(uint32_t status_flag) { + return this->status_flags & status_flag; +} + +#if 0 +void PgSQL_Connection_Placeholder::set_status_sql_log_bin0(bool v) { + if (v) { + status_flags |= STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0; + } else { + status_flags &= ~STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0; + } +} + +bool PgSQL_Connection_Placeholder::get_status_sql_log_bin0() { + return status_flags & STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0; +} +#endif // 0 + +unsigned int PgSQL_Connection_Placeholder::reorder_dynamic_variables_idx() { + dynamic_variables_idx.clear(); + // note that we are inserting the index already ordered + for (auto i = SQL_NAME_LAST_LOW_WM + 1 ; i < SQL_NAME_LAST_HIGH_WM ; i++) { + if (var_hash[i] != 0) { + dynamic_variables_idx.push_back(i); + } + } + unsigned int r = dynamic_variables_idx.size(); + return r; +} + +unsigned int PgSQL_Connection_Placeholder::number_of_matching_session_variables(const PgSQL_Connection *client_conn, unsigned int& not_matching) { + unsigned int ret=0; + for (auto i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (client_conn->var_hash[i] && i != SQL_CHARACTER_ACTION) { // client has a variable set + if (var_hash[i] == client_conn->var_hash[i]) { // server conection has the variable set to the same value + ret++; + } else { + not_matching++; + } + } + } + // increse not_matching y the sum of client and server variables + // when a match is found the counter will be reduced by 2 + not_matching += client_conn->dynamic_variables_idx.size(); + not_matching += dynamic_variables_idx.size(); + std::vector::const_iterator it_c = client_conn->dynamic_variables_idx.begin(); // client connection iterator + std::vector::const_iterator it_s = dynamic_variables_idx.begin(); // server connection iterator + for ( ; it_c != client_conn->dynamic_variables_idx.end() && it_s != dynamic_variables_idx.end() ; it_c++) { + while (it_s != dynamic_variables_idx.end() && *it_s < *it_c) { + it_s++; + } + if (it_s != dynamic_variables_idx.end()) { + if (*it_s == *it_c) { + if (var_hash[*it_s] == client_conn->var_hash[*it_c]) { // server conection has the variable set to the same value + // when a match is found the counter is reduced by 2 + not_matching-=2; + ret++; + } + } + } + } + return ret; +} + +#if 0 +void PgSQL_Connection_Placeholder::initdb_start() { + PROXY_TRACE(); + PgSQL_Connection_userinfo *client_ui=myds->sess->client_myds->myconn->userinfo; + async_exit_status = mysql_select_db_start(&interr,pgsql,client_ui->dbname); +} + +void PgSQL_Connection_Placeholder::initdb_cont(short event) { + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); + async_exit_status = mysql_select_db_cont(&interr,pgsql, mysql_status(event, true)); +} + +void PgSQL_Connection_Placeholder::set_option_start() { + PROXY_TRACE(); + + enum_mysql_set_option set_option; + set_option=((options.client_flag & CLIENT_MULTI_STATEMENTS) ? MYSQL_OPTION_MULTI_STATEMENTS_ON : MYSQL_OPTION_MULTI_STATEMENTS_OFF); + async_exit_status = mysql_set_server_option_start(&interr,pgsql,set_option); +} + +void PgSQL_Connection_Placeholder::set_option_cont(short event) { + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); + async_exit_status = mysql_set_server_option_cont(&interr,pgsql, mysql_status(event, true)); +} + +void PgSQL_Connection_Placeholder::set_autocommit_start() { + PROXY_TRACE(); + async_exit_status = mysql_autocommit_start(&ret_bool, pgsql, options.autocommit); +} + +void PgSQL_Connection_Placeholder::set_autocommit_cont(short event) { + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); + async_exit_status = mysql_autocommit_cont(&ret_bool, pgsql, mysql_status(event, true)); +} +#endif // 0 + +void PgSQL_Connection_Placeholder::set_names_start() { + PROXY_TRACE(); + const MARIADB_CHARSET_INFO * c = proxysql_find_charset_nr(atoi(pgsql_variables.client_get_value(myds->sess, SQL_CHARACTER_SET))); + if (!c) { + // LCOV_EXCL_START + proxy_error("Not existing charset number %u\n", atoi(pgsql_variables.client_get_value(myds->sess, SQL_CHARACTER_SET))); + assert(0); + // LCOV_EXCL_STOP + } + async_exit_status = mysql_set_character_set_start(&interr,pgsql, NULL, atoi(pgsql_variables.client_get_value(myds->sess, SQL_CHARACTER_SET))); +} + +void PgSQL_Connection_Placeholder::set_names_cont(short event) { + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); + async_exit_status = mysql_set_character_set_cont(&interr,pgsql, mysql_status(event, true)); +} + +void PgSQL_Connection_Placeholder::set_query(char *stmt, unsigned long length) { + query.length=length; + query.ptr=stmt; + if (length > largest_query_length) { + largest_query_length=length; + } + if (query.stmt) { + query.stmt=NULL; + } +} + +void PgSQL_Connection_Placeholder::stmt_prepare_start() { + PROXY_TRACE(); + query.stmt=mysql_stmt_init(pgsql); + my_bool my_arg=true; + mysql_stmt_attr_set(query.stmt, STMT_ATTR_UPDATE_MAX_LENGTH, &my_arg); + async_exit_status = mysql_stmt_prepare_start(&interr , query.stmt, query.ptr, query.length); +} + +void PgSQL_Connection_Placeholder::stmt_prepare_cont(short event) { + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); + async_exit_status = mysql_stmt_prepare_cont(&interr , query.stmt , mysql_status(event, true)); +} + +void PgSQL_Connection_Placeholder::stmt_execute_start() { + PROXY_TRACE(); + int _rc=0; + assert(query.stmt->mysql); // if we reached here, we hit bug #740 + _rc=mysql_stmt_bind_param(query.stmt, query.stmt_meta->binds); // FIXME : add error handling + if (_rc) { + proxy_error("mysql_stmt_bind_param() failed: %s", mysql_stmt_error(query.stmt)); + } + // if for whatever reason the previous execution failed, state is left to an inconsistent value + // see bug #3547 + // here we force the state to be MYSQL_STMT_PREPARED + // it is a nasty hack because we shouldn't change states that should belong to the library + // I am not sure if this is a bug in the backend library or not + query.stmt->state= MYSQL_STMT_PREPARED; + async_exit_status = mysql_stmt_execute_start(&interr , query.stmt); +} + +void PgSQL_Connection_Placeholder::stmt_execute_cont(short event) { + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); + async_exit_status = mysql_stmt_execute_cont(&interr , query.stmt , mysql_status(event, true)); +} + +void PgSQL_Connection_Placeholder::stmt_execute_store_result_start() { + PROXY_TRACE(); + async_exit_status = mysql_stmt_store_result_start(&interr, query.stmt); +} + +void PgSQL_Connection_Placeholder::stmt_execute_store_result_cont(short event) { + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); + async_exit_status = mysql_stmt_store_result_cont(&interr , query.stmt , mysql_status(event, true)); +} + +#ifndef PROXYSQL_USE_RESULT +void PgSQL_Connection_Placeholder::store_result_start() { + PROXY_TRACE(); + async_exit_status = mysql_store_result_start(&mysql_result, pgsql); +} + +void PgSQL_Connection_Placeholder::store_result_cont(short event) { + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); + async_exit_status = mysql_store_result_cont(&mysql_result , pgsql , mysql_status(event, true)); +} +#endif // PROXYSQL_USE_RESULT + +void PgSQL_Connection_Placeholder::set_is_client() { + //-- local_stmts->set_is_client(myds->sess); +} + +#define NEXT_IMMEDIATE(new_st) do { async_state_machine = new_st; goto handler_again; } while (0) + +#if 0 +void PgSQL_Connection_Placeholder::process_rows_in_ASYNC_STMT_EXECUTE_STORE_RESULT_CONT(unsigned long long& processed_bytes) { + PROXY_TRACE2(); + // there is more than 1 row + unsigned long long total_size=0; + long long unsigned int irs = 0; + MYSQL_ROWS *ir = query.stmt->result.data; + for (irs = 0; irs < query.stmt->result.rows -1 ; irs++) { + // while iterating the rows we also count the bytes + total_size+=ir->length; + if (ir->length > 0xFFFFFF) { + total_size+=(ir->length / 0xFFFFFF) * sizeof(mysql_hdr); + } + total_size+=sizeof(mysql_hdr); + // add the row to the resulset + unsigned int br=MyRS->add_row(ir); + // increment counters for the bytes processed + __sync_fetch_and_add(&parent->bytes_recv,br); + myds->sess->thread->status_variables.stvar[st_var_queries_backends_bytes_recv]+=br; + myds->bytes_info.bytes_recv += br; + bytes_info.bytes_recv += br; + processed_bytes+=br; // issue #527 : this variable will store the amount of bytes processed during this event + + // we stop when we 'ir->next' will be pointing to the last row + if (irs <= query.stmt->result.rows - 2) { + ir = ir->next; + } + } + // at this point, ir points to the last row + // next, we create a new MYSQL_ROWS that is a copy of the last row + MYSQL_ROWS *lcopy = (MYSQL_ROWS *)malloc(sizeof(MYSQL_ROWS) + ir->length); + lcopy->length = ir->length; + lcopy->data= (MYSQL_ROW)(lcopy + 1); + memcpy((char *)lcopy->data, (char *)ir->data, ir->length); + // next we proceed to reset all the buffer + + // this invalidates the local variables inside the coroutines + // pointing to the previous allocated memory for 'stmt->result'. + // For more context see: #3324 + ma_free_root(&query.stmt->result.alloc, MYF(MY_KEEP_PREALLOC)); + query.stmt->result.data= NULL; + query.stmt->result_cursor= NULL; + query.stmt->result.rows = 0; + + // we will now copy back the last row and make it the only row available + MYSQL_ROWS *current = (MYSQL_ROWS *)ma_alloc_root(&query.stmt->result.alloc, sizeof(MYSQL_ROWS) + lcopy->length); + current->data= (MYSQL_ROW)(current + 1); + // update 'stmt->result.data' to the new allocated memory and copy the backed last row + query.stmt->result.data = current; + memcpy((char *)current->data, (char *)lcopy->data, lcopy->length); + // update the 'current->length' with the length of the copied row + current->length = lcopy->length; + + // we free the copy + free(lcopy); + // change the rows count to 1 + query.stmt->result.rows = 1; + // we should also configure the cursor, but because we scan it using our own + // algorithm, this is not needed + + // now we update bytes counter + __sync_fetch_and_add(&parent->bytes_recv,total_size); + myds->sess->thread->status_variables.stvar[st_var_queries_backends_bytes_recv]+=total_size; + myds->bytes_info.bytes_recv += total_size; + bytes_info.bytes_recv += total_size; +} + +int PgSQL_Connection_Placeholder::async_set_autocommit(short event, bool ac) { + PROXY_TRACE(); + assert(pgsql); + assert(ret_mysql); + server_status=parent->status; // we copy it here to avoid race condition. The caller will see this + if (IsServerOffline()) + return -1; + + switch (async_state_machine) { + case ASYNC_SET_AUTOCOMMIT_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine=ASYNC_IDLE; + return 0; + break; + case ASYNC_SET_AUTOCOMMIT_FAILED: + return -1; + break; + case ASYNC_QUERY_END: + case ASYNC_IDLE: + set_autocommit(ac); + async_state_machine=ASYNC_SET_AUTOCOMMIT_START; + default: + handler(event); + break; + } + + // check again + switch (async_state_machine) { + case ASYNC_SET_AUTOCOMMIT_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine=ASYNC_IDLE; + return 0; + break; + case ASYNC_SET_AUTOCOMMIT_FAILED: + return -1; + break; + default: + return 1; + break; + } + return 1; +} +#endif // 0 + +int PgSQL_Connection_Placeholder::async_set_names(short event, unsigned int c) { + PROXY_TRACE(); + assert(pgsql); + assert(ret_mysql); + server_status=parent->status; // we copy it here to avoid race condition. The caller will see this + if (IsServerOffline()) + return -1; + + switch (async_state_machine) { + case ASYNC_SET_NAMES_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine=ASYNC_IDLE; + return 0; + break; + case ASYNC_SET_NAMES_FAILED: + return -1; + break; + case ASYNC_IDLE: + /* useless statement. should be removed after thorough testing */ + //set_charset(c, CONNECT_START); + async_state_machine=ASYNC_SET_NAMES_START; + default: + handler(event); + break; + } + + // check again + switch (async_state_machine) { + case ASYNC_SET_NAMES_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine=ASYNC_IDLE; + return 0; + break; + case ASYNC_SET_NAMES_FAILED: + return -1; + break; + default: + return 1; + break; + } + return 1; +} + +int PgSQL_Connection_Placeholder::async_set_option(short event, bool mask) { + PROXY_TRACE(); + assert(pgsql); + assert(ret_mysql); + server_status=parent->status; // we copy it here to avoid race condition. The caller will see this + if (IsServerOffline()) + return -1; + + switch (async_state_machine) { + case ASYNC_SET_OPTION_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine=ASYNC_IDLE; + return 0; + break; + case ASYNC_SET_OPTION_FAILED: + return -1; + break; + case ASYNC_IDLE: + if (mask) + options.client_flag |= CLIENT_MULTI_STATEMENTS; + else + options.client_flag &= ~CLIENT_MULTI_STATEMENTS; + async_state_machine=ASYNC_SET_OPTION_START; + default: + handler(event); + break; + } + + // check again + switch (async_state_machine) { + case ASYNC_SET_OPTION_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine=ASYNC_IDLE; + return 0; + break; + case ASYNC_SET_OPTION_FAILED: + return -1; + break; + default: + return 1; + break; + } + return 1; +} + +void PgSQL_Connection_Placeholder::async_free_result() { + PROXY_TRACE(); + assert(pgsql); + //assert(ret_mysql); + //assert(async_state_machine==ASYNC_QUERY_END); + if (query.ptr) { + query.ptr=NULL; + query.length=0; + } + if (query.stmt_result) { + mysql_free_result(query.stmt_result); + query.stmt_result=NULL; + } + if (userinfo) { + // if userinfo is NULL , the connection is being destroyed + // because it is reset on destructor ( ~PgSQL_Connection() ) + // therefore this section is skipped completely + // this should prevent bug #1046 + if (query.stmt) { + if (query.stmt->mysql) { + if (query.stmt->mysql == pgsql) { // extra check + mysql_stmt_free_result(query.stmt); + } + } + // If we reached here from 'ASYNC_STMT_PREPARE_FAILED', the + // prepared statement was never added to 'local_stmts', thus + // it will never be freed when 'local_stmts' are purged. If + // initialized, it must be freed. For more context see #3525. + if (this->async_state_machine == ASYNC_STMT_PREPARE_FAILED) { + if (query.stmt != NULL) { + proxy_mysql_stmt_close(query.stmt); + } + } + query.stmt=NULL; + } + if (mysql_result) { + mysql_free_result(mysql_result); + mysql_result=NULL; + } + } + compute_unknown_transaction_status(); + async_state_machine=ASYNC_IDLE; + if (MyRS) { + if (MyRS_reuse) { + delete (MyRS_reuse); + } + MyRS_reuse = MyRS; + MyRS=NULL; + } +} + +// This function check if autocommit=0 and if there are any savepoint. +// this is an attempt to mitigate MySQL bug https://bugs.pgsql.com/bug.php?id=107875 +bool PgSQL_Connection_Placeholder::AutocommitFalse_AndSavepoint() { + bool ret=false; + if (IsAutoCommit() == false) { + if (get_status(STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT) == true) { + ret = true; + } + } + return ret; +} + +bool PgSQL_Connection_Placeholder::IsAutoCommit() { + bool ret=false; + if (pgsql) { + ret = (pgsql->server_status & SERVER_STATUS_AUTOCOMMIT); + if (ret) { + if (options.last_set_autocommit==0) { + // it seems we hit bug http://bugs.pgsql.com/bug.php?id=66884 + // we last sent SET AUTOCOMMIT = 0 , but the server says it is 1 + // we assume that what we sent last is correct . #873 + ret = false; + } + } else { + if (options.last_set_autocommit==-1) { + // if a connection was reset (thus last_set_autocommit==-1) + // the information related to SERVER_STATUS_AUTOCOMMIT is lost + // therefore we fall back on the safe assumption that autocommit==1 + ret = true; + } + } + } + return ret; +} + +bool PgSQL_Connection_Placeholder::MultiplexDisabled(bool check_delay_token) { +// status_flags stores information about the status of the connection +// can be used to determine if multiplexing can be enabled or not + bool ret=false; + if (status_flags & (STATUS_MYSQL_CONNECTION_USER_VARIABLE | STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT | + STATUS_MYSQL_CONNECTION_LOCK_TABLES | STATUS_MYSQL_CONNECTION_TEMPORARY_TABLE | STATUS_MYSQL_CONNECTION_GET_LOCK | STATUS_MYSQL_CONNECTION_NO_MULTIPLEX | + STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0 | STATUS_MYSQL_CONNECTION_FOUND_ROWS | STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG | + STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT | STATUS_MYSQL_CONNECTION_HAS_WARNINGS) ) { + ret=true; + } + if (check_delay_token && auto_increment_delay_token) return true; + return ret; +} + +bool PgSQL_Connection_Placeholder::IsKeepMultiplexEnabledVariables(char *query_digest_text) { + if (query_digest_text==NULL) return true; + + char *query_digest_text_filter_select = NULL; + unsigned long query_digest_text_len=strlen(query_digest_text); + if (strncasecmp(query_digest_text,"SELECT ",strlen("SELECT "))==0){ + query_digest_text_filter_select=(char*)malloc(query_digest_text_len-7+1); + memcpy(query_digest_text_filter_select,&query_digest_text[7],query_digest_text_len-7); + query_digest_text_filter_select[query_digest_text_len-7]='\0'; + } else { + return false; + } + //filter @@session., @@local. and @@ + char *match=NULL; + char* last_pos=NULL; + const int at_session_offset = strlen("@@session."); + const int at_local_offset = strlen("@@local."); // Alias of session + const int double_at_offset = strlen("@@"); + while (query_digest_text_filter_select && (match = strcasestr(query_digest_text_filter_select,"@@session."))) { + memmove(match, match + at_session_offset, strlen(match) - at_session_offset); + last_pos = match + strlen(match) - at_session_offset; + *last_pos = '\0'; + } + while (query_digest_text_filter_select && (match = strcasestr(query_digest_text_filter_select, "@@local."))) { + memmove(match, match + at_local_offset, strlen(match) - at_local_offset); + last_pos = match + strlen(match) - at_local_offset; + *last_pos = '\0'; + } + while (query_digest_text_filter_select && (match = strcasestr(query_digest_text_filter_select,"@@"))) { + memmove(match, match + double_at_offset, strlen(match) - double_at_offset); + last_pos = match + strlen(match) - double_at_offset; + *last_pos = '\0'; + } + + std::vectorquery_digest_text_filter_select_v; + char* query_digest_text_filter_select_tok = NULL; + char* save_query_digest_text_ptr = NULL; + if (query_digest_text_filter_select) { + query_digest_text_filter_select_tok = strtok_r(query_digest_text_filter_select, ",", &save_query_digest_text_ptr); + } + while(query_digest_text_filter_select_tok){ + //filter "as"/space/alias,such as select @@version as a, @@version b + while (1){ + char c = *query_digest_text_filter_select_tok; + if (!isspace(c)){ + break; + } + query_digest_text_filter_select_tok++; + } + char* match_as; + match_as=strcasestr(query_digest_text_filter_select_tok," "); + if(match_as){ + query_digest_text_filter_select_tok[match_as-query_digest_text_filter_select_tok]='\0'; + query_digest_text_filter_select_v.push_back(query_digest_text_filter_select_tok); + }else{ + query_digest_text_filter_select_v.push_back(query_digest_text_filter_select_tok); + } + query_digest_text_filter_select_tok=strtok_r(NULL, ",", &save_query_digest_text_ptr); + } + + std::vectorkeep_multiplexing_variables_v; + char* keep_multiplexing_variables_tmp; + char* save_keep_multiplexing_variables_ptr = NULL; + unsigned long keep_multiplexing_variables_len=strlen(pgsql_thread___keep_multiplexing_variables); + keep_multiplexing_variables_tmp=(char*)malloc(keep_multiplexing_variables_len+1); + memcpy(keep_multiplexing_variables_tmp, pgsql_thread___keep_multiplexing_variables, keep_multiplexing_variables_len); + keep_multiplexing_variables_tmp[keep_multiplexing_variables_len]='\0'; + char* keep_multiplexing_variables_tok=strtok_r(keep_multiplexing_variables_tmp, " ,", &save_keep_multiplexing_variables_ptr); + while (keep_multiplexing_variables_tok){ + keep_multiplexing_variables_v.push_back(keep_multiplexing_variables_tok); + keep_multiplexing_variables_tok=strtok_r(NULL, " ,", &save_keep_multiplexing_variables_ptr); + } + + for (std::vector::iterator it=query_digest_text_filter_select_v.begin();it!=query_digest_text_filter_select_v.end();it++){ + bool is_match=false; + for (std::vector::iterator it1=keep_multiplexing_variables_v.begin();it1!=keep_multiplexing_variables_v.end();it1++){ + //printf("%s,%s\n",*it,*it1); + if (strncasecmp(*it,*it1,strlen(*it1))==0){ + is_match=true; + break; + } + } + if(is_match){ + is_match=false; + continue; + }else{ + free(query_digest_text_filter_select); + free(keep_multiplexing_variables_tmp); + return false; + } + } + free(query_digest_text_filter_select); + free(keep_multiplexing_variables_tmp); + return true; +} + +void PgSQL_Connection_Placeholder::ProcessQueryAndSetStatusFlags(char *query_digest_text) { + if (query_digest_text==NULL) return; + // unknown what to do with multiplex + int mul=-1; + if (myds) { + if (myds->sess) { + if (myds->sess->qpo) { + mul=myds->sess->qpo->multiplex; + if (mul==0) { + set_status(true, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX); + } else { + if (mul==1) { + set_status(false, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX); + } + } + } + } + } + // checking warnings and disabling multiplexing will be effective only when the pgsql-query_digests is enabled + if (get_status(STATUS_MYSQL_CONNECTION_HAS_WARNINGS) == false) { + if (warning_count > 0) { + // 'warning_in_hg' will be used if the next query is 'SHOW WARNINGS' or + // 'SHOW COUNT(*) WARNINGS' + if (myds && myds->sess) + myds->sess->warning_in_hg = myds->sess->current_hostgroup; + // enabling multiplexing + set_status(true, STATUS_MYSQL_CONNECTION_HAS_WARNINGS); + } + } else { // reset warning_in_hg + const char* dig = query_digest_text; + const size_t dig_len = strlen(dig); + // disable multiplexing and reset the 'warning_in_hg' flag only when the current executed query is not + // 'SHOW WARNINGS' or 'SHOW COUNT(*) WARNINGS', as these queries do not clear the warning message list + // on backend. + if (!((dig_len == 22 && strncasecmp(dig, "SHOW COUNT(*) WARNINGS", 22) == 0) || + (dig_len == 13 && strncasecmp(dig, "SHOW WARNINGS", 13) == 0))) { + if (myds && myds->sess) + myds->sess->warning_in_hg = -1; + warning_count = 0; + // disabling multiplexing + set_status(false, STATUS_MYSQL_CONNECTION_HAS_WARNINGS); + } + } + + if (get_status(STATUS_MYSQL_CONNECTION_USER_VARIABLE)==false) { // we search for variables only if not already set +// if ( +// strncasecmp(query_digest_text,"SELECT @@tx_isolation", strlen("SELECT @@tx_isolation")) +// && +// strncasecmp(query_digest_text,"SELECT @@version", strlen("SELECT @@version")) + if (strncasecmp(query_digest_text,"SET ",4)==0) { + // For issue #555 , multiplexing is disabled if --safe-updates is used (see session_vars definition) + int sqloh = pgsql_thread___set_query_lock_on_hostgroup; + switch (sqloh) { + case 0: // old algorithm + if (mul!=2) { + if (index(query_digest_text,'@')) { // mul = 2 has a special meaning : do not disable multiplex for variables in THIS QUERY ONLY + if (!IsKeepMultiplexEnabledVariables(query_digest_text)) { + set_status(true, STATUS_MYSQL_CONNECTION_USER_VARIABLE); + } +/* deprecating session_vars[] because we are introducing a better algorithm + } else { + for (unsigned int i = 0; i < sizeof(session_vars)/sizeof(char *); i++) { + if (strcasestr(query_digest_text,session_vars[i])!=NULL) { + set_status(true, STATUS_MYSQL_CONNECTION_USER_VARIABLE); + break; + } + } +*/ + } + } + break; + case 1: // new algorithm + if (myds->sess->locked_on_hostgroup > -1) { + // locked_on_hostgroup was set, so some variable wasn't parsed + set_status(true, STATUS_MYSQL_CONNECTION_USER_VARIABLE); + } + break; + default: + break; + } + } else { + if (mul!=2 && index(query_digest_text,'@')) { // mul = 2 has a special meaning : do not disable multiplex for variables in THIS QUERY ONLY + if (!IsKeepMultiplexEnabledVariables(query_digest_text)) { + set_status(true, STATUS_MYSQL_CONNECTION_USER_VARIABLE); + } + } + } + } + if (get_status(STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT)==false) { // we search if prepared was already executed + if (!strncasecmp(query_digest_text,"PREPARE ", strlen("PREPARE "))) { + set_status(true, STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT); + } + } + if (get_status(STATUS_MYSQL_CONNECTION_TEMPORARY_TABLE)==false) { // we search for temporary if not already set + if (!strncasecmp(query_digest_text,"CREATE TEMPORARY TABLE ", strlen("CREATE TEMPORARY TABLE "))) { + set_status(true, STATUS_MYSQL_CONNECTION_TEMPORARY_TABLE); + } + } + if (get_status(STATUS_MYSQL_CONNECTION_LOCK_TABLES)==false) { // we search for lock tables only if not already set + if (!strncasecmp(query_digest_text,"LOCK TABLE", strlen("LOCK TABLE"))) { + set_status(true, STATUS_MYSQL_CONNECTION_LOCK_TABLES); + } + } + if (get_status(STATUS_MYSQL_CONNECTION_LOCK_TABLES)==false) { // we search for lock tables only if not already set + if (!strncasecmp(query_digest_text,"FLUSH TABLES WITH READ LOCK", strlen("FLUSH TABLES WITH READ LOCK"))) { // issue 613 + set_status(true, STATUS_MYSQL_CONNECTION_LOCK_TABLES); + } + } + if (get_status(STATUS_MYSQL_CONNECTION_LOCK_TABLES)==true) { + if (!strncasecmp(query_digest_text,"UNLOCK TABLES", strlen("UNLOCK TABLES"))) { + set_status(false, STATUS_MYSQL_CONNECTION_LOCK_TABLES); + } + } + if (get_status(STATUS_MYSQL_CONNECTION_GET_LOCK)==false) { // we search for get_lock if not already set + if (strcasestr(query_digest_text,"GET_LOCK(")) { + set_status(true, STATUS_MYSQL_CONNECTION_GET_LOCK); + } + } + if (get_status(STATUS_MYSQL_CONNECTION_FOUND_ROWS)==false) { // we search for SQL_CALC_FOUND_ROWS if not already set + if (strcasestr(query_digest_text,"SQL_CALC_FOUND_ROWS")) { + set_status(true, STATUS_MYSQL_CONNECTION_FOUND_ROWS); + } + } + if (get_status(STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT)==false) { + if (pgsql) { + if ( + (pgsql->server_status & SERVER_STATUS_IN_TRANS) + || + ((pgsql->server_status & SERVER_STATUS_AUTOCOMMIT) == 0) + ) { + if (!strncasecmp(query_digest_text,"SAVEPOINT ", strlen("SAVEPOINT "))) { + set_status(true, STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT); + } + } + } + } else { + if ( // get_status(STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT) == true + ( + // make sure we don't have a transaction running + // checking just for COMMIT and ROLLBACK is not enough, because `SET autocommit=1` can commit too + (pgsql->server_status & SERVER_STATUS_AUTOCOMMIT) + && + ( (pgsql->server_status & SERVER_STATUS_IN_TRANS) == 0 ) + ) + || + (strcasecmp(query_digest_text,"COMMIT") == 0) + || + (strcasecmp(query_digest_text,"ROLLBACK") == 0) + ) { + set_status(false, STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT); + } + } + if (pgsql) { + if (myds && myds->sess) { + if (myds->sess->client_myds && myds->sess->client_myds->myconn) { + // if SERVER_STATUS_NO_BACKSLASH_ESCAPES is changed it is likely + // because of sql_mode was changed + // we set the same on the client connection + unsigned int ss = pgsql->server_status & SERVER_STATUS_NO_BACKSLASH_ESCAPES; + myds->sess->client_myds->myconn->set_no_backslash_escapes(ss); + } + } + } +} + +void PgSQL_Connection_Placeholder::optimize() { + if (pgsql->net.max_packet > 65536) { // FIXME: temporary, maybe for very long time . This needs to become a global variable + if ( ( pgsql->net.buff == pgsql->net.read_pos ) && ( pgsql->net.read_pos == pgsql->net.write_pos ) ) { + free(pgsql->net.buff); + pgsql->net.max_packet=8192; + pgsql->net.buff=(unsigned char *)malloc(pgsql->net.max_packet); + memset(pgsql->net.buff,0,pgsql->net.max_packet); + pgsql->net.read_pos=pgsql->net.buff; + pgsql->net.write_pos=pgsql->net.buff; + pgsql->net.buff_end=pgsql->net.buff+pgsql->net.max_packet; + } + } +} + +// close_mysql() is a replacement for mysql_close() +// if avoids that a QUIT command stops forever +// FIXME: currently doesn't support encryption and compression +void PgSQL_Connection_Placeholder::close_mysql() { + if ((send_quit) && (pgsql->net.pvio) && ret_mysql) { + char buff[5]; + mysql_hdr myhdr; + myhdr.pkt_id=0; + myhdr.pkt_length=1; + memcpy(buff, &myhdr, sizeof(mysql_hdr)); + buff[4]=0x01; + int fd=pgsql->net.fd; +#ifdef __APPLE__ + int arg_on=1; + setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, (char *) &arg_on, sizeof(int)); + send(fd, buff, 5, 0); +#else + send(fd, buff, 5, MSG_NOSIGNAL); +#endif + } +// int rc=0; + mysql_close_no_command(pgsql); +} + + +// this function is identical to async_query() , with the only exception that MyRS should never be set +int PgSQL_Connection_Placeholder::async_send_simple_command(short event, char *stmt, unsigned long length) { + PROXY_TRACE(); + assert(pgsql); + assert(ret_mysql); + server_status=parent->status; // we copy it here to avoid race condition. The caller will see this + if ( + (parent->status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user + || + (parent->status==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic==true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + ) { + return -1; + } + switch (async_state_machine) { + case ASYNC_QUERY_END: + processing_multi_statement=false; // no matter if we are processing a multi statement or not, we reached the end + //return 0; <= bug. Do not return here, because we need to reach the if (async_state_machine==ASYNC_QUERY_END) few lines below + break; + case ASYNC_IDLE: + set_query(stmt,length); + async_state_machine=ASYNC_QUERY_START; + default: + handler(event); + break; + } + if (MyRS) { + // this is a severe mistake, we shouldn't have reach here + // for now we do not assert but report the error + // PMC-10003: Retrieved a resultset while running a simple command using async_send_simple_command() . + // async_send_simple_command() is used by ProxySQL to configure the connection, thus it + // shouldn't retrieve any resultset. + // A common issue for triggering this error is to have configure pgsql-init_connect to + // run a statement that returns a resultset. + proxy_error2(10003, "PMC-10003: Retrieved a resultset while running a simple command. This is an error!! Simple command: %s\n", stmt); + return -2; + } + if (async_state_machine==ASYNC_QUERY_END) { + compute_unknown_transaction_status(); + if (mysql_errno(pgsql)) { + return -1; + } else { + async_state_machine=ASYNC_IDLE; + return 0; + } + } + if (async_state_machine==ASYNC_NEXT_RESULT_START) { + // if we reached this point it measn we are processing a multi-statement + // and we need to exit to give control to MySQL_Session + processing_multi_statement=true; + return 2; + } + if (processing_multi_statement==true) { + // we are in the middle of processing a multi-statement + return 3; + } + return 1; +} + +void PgSQL_Connection_Placeholder::reset() { + bool old_no_multiplex_hg = get_status(STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + bool old_compress = get_status(STATUS_MYSQL_CONNECTION_COMPRESSION); + status_flags=0; + // reconfigure STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG + set_status(old_no_multiplex_hg,STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + // reconfigure STATUS_MYSQL_CONNECTION_COMPRESSION + set_status(old_compress,STATUS_MYSQL_CONNECTION_COMPRESSION); + reusable=true; + options.last_set_autocommit=-1; // never sent + warning_count=0; + delete local_stmts; + local_stmts=new MySQL_STMTs_local_v14(false); + creation_time = monotonic_time(); + + for (auto i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + var_hash[i] = 0; + if (variables[i].value) { + free(variables[i].value); + variables[i].value = NULL; + var_hash[i] = 0; + } + } + dynamic_variables_idx.clear(); + + if (options.init_connect) { + free(options.init_connect); + options.init_connect = NULL; + options.init_connect_sent = false; + } + auto_increment_delay_token = 0; + if (options.ldap_user_variable) { + if (options.ldap_user_variable_value) { + free(options.ldap_user_variable_value); + options.ldap_user_variable_value = NULL; + } + options.ldap_user_variable = NULL; + options.ldap_user_variable_sent = false; + } + options.session_track_gtids_int = 0; + if (options.session_track_gtids) { + free (options.session_track_gtids); + options.session_track_gtids = NULL; + options.session_track_gtids_sent = false; + } +} + +bool PgSQL_Connection_Placeholder::get_gtid(char *buff, uint64_t *trx_id) { + // note: current implementation for for OWN GTID only! + bool ret = false; + if (buff==NULL || trx_id == NULL) { + return ret; + } + if (pgsql) { + if (pgsql->net.last_errno==0) { // only if there is no error + if (pgsql->server_status & SERVER_SESSION_STATE_CHANGED) { // only if status changed + const char *data; + size_t length; + if (mysql_session_track_get_first(pgsql, SESSION_TRACK_GTIDS, &data, &length) == 0) { + if (length >= (sizeof(gtid_uuid) - 1)) { + length = sizeof(gtid_uuid) - 1; + } + if (memcmp(gtid_uuid,data,length)) { + // copy to local buffer in PgSQL_Connection + memcpy(gtid_uuid,data,length); + gtid_uuid[length]=0; + // copy to external buffer in MySQL_Backend + memcpy(buff,data,length); + buff[length]=0; + __sync_fetch_and_add(&myds->sess->thread->status_variables.stvar[st_var_gtid_session_collected],1); + ret = true; + } + } + } + } + } + return ret; +} + + + +PgSQL_Connection::PgSQL_Connection() { + pgsql_conn = NULL; + result_type = 0; + pgsql_result = NULL; + query_result = NULL; + query_result_reuse = NULL; + new_result = true; + reset_error(); +} + +PgSQL_Connection::~PgSQL_Connection() { + + if (userinfo) { + delete userinfo; + userinfo = NULL; + } + if (pgsql_result) { + PQclear(pgsql_result); + pgsql_result = NULL; + } + if (pgsql_conn) { + PQfinish(pgsql_conn); + pgsql_conn = NULL; + } + if (query_result) { + delete query_result; + query_result = NULL; + } + if (query_result_reuse) { + delete query_result_reuse; + query_result_reuse = NULL; + } + for (auto i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + if (variables[i].value) { + free(variables[i].value); + variables[i].value = NULL; + var_hash[i] = 0; + } + } + + if (connected_host_details.hostname) { + free(connected_host_details.hostname); + connected_host_details.hostname = NULL; + } + if (connected_host_details.ip) { + free(connected_host_details.ip); + connected_host_details.hostname = NULL; + } + +} + +void PgSQL_Connection::next_event(PG_ASYNC_ST new_st) { +#ifdef DEBUG + int fd; +#endif /* DEBUG */ + wait_events = 0; + + if (async_exit_status & PG_EVENT_READ) + wait_events |= POLLIN; + if (async_exit_status & PG_EVENT_WRITE) + wait_events |= POLLOUT; + if (wait_events) +#ifdef DEBUG + fd = PQsocket(pgsql_conn); +#else + PQsocket(pgsql_conn); +#endif /* DEBUG */ + else +#ifdef DEBUG + fd = -1; +#endif /* DEBUG */ + + proxy_debug(PROXY_DEBUG_NET, 8, "fd=%d, wait_events=%d , old_ST=%d, new_ST=%d\n", fd, wait_events, async_state_machine, new_st); + async_state_machine = new_st; +}; + + +PG_ASYNC_ST PgSQL_Connection::handler(short event) { +#if ENABLE_TIMER + Timer timer(myds->sess->thread->Timers.Connections_Handlers); +#endif // ENABLE_TIMER + unsigned long long processed_bytes = 0; // issue #527 : this variable will store the amount of bytes processed during this event + if (pgsql_conn == NULL) { + // it is the first time handler() is being called + async_state_machine = ASYNC_CONNECT_START; + myds->wait_until = myds->sess->thread->curtime + pgsql_thread___connect_timeout_server * 1000; + if (myds->max_connect_time) { + if (myds->wait_until > myds->max_connect_time) { + myds->wait_until = myds->max_connect_time; + } + } + } +handler_again: + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6, "async_state_machine=%d\n", async_state_machine); + switch (async_state_machine) { + case ASYNC_CONNECT_START: + connect_start(); + if (async_exit_status) { + next_event(ASYNC_CONNECT_CONT); + } + else { + NEXT_IMMEDIATE(ASYNC_CONNECT_END); + } + break; + case ASYNC_CONNECT_CONT: + if (event) { + connect_cont(event); + } + if (async_exit_status) { + if (myds->sess->thread->curtime >= myds->wait_until) { + NEXT_IMMEDIATE(ASYNC_CONNECT_TIMEOUT); + } + next_event(ASYNC_CONNECT_CONT); + } else { + NEXT_IMMEDIATE(ASYNC_CONNECT_END); + } + break; + case ASYNC_CONNECT_END: + if (myds) { + if (myds->sess) { + if (myds->sess->thread) { + unsigned long long curtime = monotonic_time(); + myds->sess->thread->atomic_curtime = curtime; + } + } + } + if (is_error_present()) { + // always increase the counter + proxy_error("Failed to PQconnectStart() on %u:%s:%d , FD (Conn:%d , MyDS:%d) , %s.\n", parent->myhgc->hid, parent->address, parent->port, PQsocket(pgsql_conn), myds->fd, get_error_code_with_message().c_str()); + NEXT_IMMEDIATE(ASYNC_CONNECT_FAILED); + } else { + if (PQisnonblocking(pgsql_conn) == false) { + // Set non-blocking mode + if (PQsetnonblocking(pgsql_conn, 1) != 0) { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + set_error_from_result(result); + proxy_error("Failed to set non-blocking mode: %s\n", get_error_code_with_message().c_str()); + NEXT_IMMEDIATE(ASYNC_CONNECT_FAILED); + } + } + NEXT_IMMEDIATE(ASYNC_CONNECT_SUCCESSFUL); + } + break; + case ASYNC_CONNECT_SUCCESSFUL: + if (!is_connected()) + assert(0); // shouldn't ever reach here, we have messed up the state machine + + __sync_fetch_and_add(&PgHGM->status.server_connections_connected, 1); + __sync_fetch_and_add(&parent->connect_OK, 1); + //MySQL_Monitor::update_dns_cache_from_mysql_conn(pgsql); + break; + case ASYNC_CONNECT_FAILED: + //PQfinish(pgsql_conn);//release connection even on error + //pgsql_conn = NULL; + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::pgsql, parent->myhgc->hid, parent->address, parent->port, mysql_errno(pgsql)); + parent->connect_error(mysql_errno(pgsql)); + break; + case ASYNC_CONNECT_TIMEOUT: + // to fix + //PQfinish(pgsql_conn);//release connection + //pgsql_conn = NULL; + proxy_error("Connect timeout on %s:%d : exceeded by %lluus\n", parent->address, parent->port, myds->sess->thread->curtime - myds->wait_until); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::pgsql, parent->myhgc->hid, parent->address, parent->port, mysql_errno(pgsql)); + parent->connect_error(mysql_errno(pgsql)); + break; + case ASYNC_QUERY_START: + query_start(); + __sync_fetch_and_add(&parent->queries_sent, 1); + update_bytes_sent(query.length + 5); + statuses.questions++; + if (async_exit_status) { + next_event(ASYNC_QUERY_CONT); + } else { + if (is_error_present()) { + NEXT_IMMEDIATE(ASYNC_QUERY_END); + } + NEXT_IMMEDIATE(ASYNC_USE_RESULT_START); + } + break; + case ASYNC_QUERY_CONT: + if (event) { + query_cont(event); + } + if (async_exit_status) { + next_event(ASYNC_QUERY_CONT); + } else { + if (is_error_present() || + !set_single_row_mode()) { + NEXT_IMMEDIATE(ASYNC_QUERY_END); + } + NEXT_IMMEDIATE(ASYNC_USE_RESULT_START); + } + break; + case ASYNC_USE_RESULT_START: + fetch_result_start(); + if (async_exit_status == PG_EVENT_NONE) { + if (is_error_present()) { + NEXT_IMMEDIATE(ASYNC_QUERY_END); + } + new_result = true; + if (myds->sess->mirror == false) { + if (query_result_reuse == NULL) { + query_result = new PgSQL_Query_Result(); + query_result->init(&myds->sess->client_myds->myprot, myds, this); + } else { + query_result = query_result_reuse; + query_result_reuse = NULL; + query_result->init(&myds->sess->client_myds->myprot, myds, this); + } + } else { + if (query_result_reuse == NULL) { + query_result = new PgSQL_Query_Result(); + query_result->init(NULL, myds, this); + } else { + query_result = query_result_reuse; + query_result_reuse = NULL; + query_result->init(NULL, myds, this); + } + } + NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); + } else { + assert(0); // shouldn't ever reach here + } + break; + case ASYNC_USE_RESULT_CONT: + { + if (myds->sess && myds->sess->client_myds && myds->sess->mirror == false /* && + myds->sess->status != SHOW_WARNINGS*/) { // see issue#4072 + unsigned int buffered_data = 0; + buffered_data = myds->sess->client_myds->PSarrayOUT->len * PGSQL_RESULTSET_BUFLEN; + buffered_data += myds->sess->client_myds->resultset->len * PGSQL_RESULTSET_BUFLEN; + if (buffered_data > (unsigned int)pgsql_thread___threshold_resultset_size * 8) { + next_event(ASYNC_USE_RESULT_CONT); // we temporarily pause . See #1232 + break; + } + } + + fetch_result_cont(event); + if (async_exit_status) { + next_event(ASYNC_USE_RESULT_CONT); + break; + } + + if (result_type == 1) { + std::unique_ptr result(get_result(), PQclear); + + if (result) { + + const ExecStatusType exec_status_type = PQresultStatus(result.get()); + + if ((query_result->get_result_packet_type() & (PGSQL_QUERY_RESULT_COMMAND | PGSQL_QUERY_RESULT_EMPTY | PGSQL_QUERY_RESULT_ERROR))) { + next_multi_statement_result(result.release()); + next_event(ASYNC_USE_RESULT_START); + break; + } + + switch (exec_status_type) { + case PGRES_COMMAND_OK: + { + const unsigned int bytes_recv = query_result->add_command_completion(result.get()); + update_bytes_recv(bytes_recv); + } + NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); + break; + case PGRES_EMPTY_QUERY: + { + const unsigned int bytes_recv = query_result->add_empty_query_response(result.get()); + update_bytes_recv(bytes_recv); + } + NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); + break; + case PGRES_TUPLES_OK: + case PGRES_SINGLE_TUPLE: + break; + case PGRES_COPY_OUT: + case PGRES_COPY_IN: + case PGRES_COPY_BOTH: + // NOT IMPLEMENTED + assert(0); + break; + case PGRES_BAD_RESPONSE: + case PGRES_NONFATAL_ERROR: + case PGRES_FATAL_ERROR: + default: + // if on previous call we encountered a FATAL error, we will not process the result, as it will contain residual protocol messages + // from the broken connection + if (is_error_present() == true && get_error_severity() == PGSQL_ERROR_SEVERITY::ERRSEVERITY_FATAL) { + NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); + } + + // we don't have a command completion, empty query responseor error packet in the result. This check is here to + // handle internal cleanup of libpq that might return residual protocol messages from the broken connection and + // may add multiple final packets. + //if ((query_result->get_result_packet_type() & (PGSQL_QUERY_RESULT_COMMAND | PGSQL_QUERY_RESULT_EMPTY | PGSQL_QUERY_RESULT_ERROR)) == 0) { + set_error_from_result(result.get(), PGSQL_ERROR_FIELD_ALL); + assert(is_error_present()); + + // we will not send FATAL error messages to the client + const PGSQL_ERROR_SEVERITY severity = get_error_severity(); + if (severity == PGSQL_ERROR_SEVERITY::ERRSEVERITY_ERROR || + severity == PGSQL_ERROR_SEVERITY::ERRSEVERITY_WARNING || + severity == PGSQL_ERROR_SEVERITY::ERRSEVERITY_NOTICE) { + + const unsigned int bytes_recv = query_result->add_error(result.get()); + update_bytes_recv(bytes_recv); + } + + const PGSQL_ERROR_CATEGORY error_category = get_error_category(); + if (error_category != PGSQL_ERROR_CATEGORY::ERRCATEGORY_SYNTAX_ERROR && + error_category != PGSQL_ERROR_CATEGORY::ERRCATEGORY_STATUS && + error_category != PGSQL_ERROR_CATEGORY::ERRCATEGORY_DATA_ERROR) { + proxy_error("Error: %s, Multi-Statement: %d\n", get_error_code_with_message().c_str(), processing_multi_statement); + } + //} + NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); + } + + if (new_result == true) { + const unsigned int bytes_recv = query_result->add_row_description(result.get()); + update_bytes_recv(bytes_recv); + new_result = false; + } + + if (PQntuples(result.get()) > 0) { + const unsigned int bytes_recv = query_result->add_row(result.get()); + update_bytes_recv(bytes_recv); + processed_bytes += bytes_recv; // issue #527 : this variable will store the amount of bytes processed during this event + if ( + (processed_bytes > (unsigned int)pgsql_thread___threshold_resultset_size * 8) + || + (pgsql_thread___throttle_ratio_server_to_client && pgsql_thread___throttle_max_bytes_per_second_to_client && (processed_bytes > (unsigned long long)pgsql_thread___throttle_max_bytes_per_second_to_client / 10 * (unsigned long long)pgsql_thread___throttle_ratio_server_to_client)) + ) { + next_event(ASYNC_USE_RESULT_CONT); // we temporarily pause + break; + } else { + NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); // we continue looping + } + } else { + const unsigned int bytes_recv=query_result->add_command_completion(result.get(), false); + update_bytes_recv(bytes_recv); + NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); + } + } + } else if (result_type == 2) { + if (ps_result.id == 'D') { + unsigned int bytes_recv=query_result->add_row(&ps_result); + update_bytes_recv(bytes_recv); + processed_bytes += bytes_recv; // issue #527 : this variable will store the amount of bytes processed during this event + + if ( + (processed_bytes > (unsigned int)pgsql_thread___threshold_resultset_size * 8) + || + (pgsql_thread___throttle_ratio_server_to_client && pgsql_thread___throttle_max_bytes_per_second_to_client && (processed_bytes > (unsigned long long)pgsql_thread___throttle_max_bytes_per_second_to_client / 10 * (unsigned long long)pgsql_thread___throttle_ratio_server_to_client)) + ) { + next_event(ASYNC_USE_RESULT_CONT); // we temporarily pause + break; + } else { + NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); // we continue looping + } + } else { + assert(0); + } + } else { + assert(0); + } + + if ((query_result->get_result_packet_type() & (PGSQL_QUERY_RESULT_COMMAND | PGSQL_QUERY_RESULT_EMPTY | PGSQL_QUERY_RESULT_ERROR)) == 0) { + // if we reach here we assume that error_info is already set in previous call + if (!is_error_present()) + assert(0); // we might have missed setting error_info in previous call + + query_result->add_error(NULL); + } + + // finally add ready for query packet + query_result->add_ready_status(PQtransactionStatus(pgsql_conn)); + update_bytes_recv(6); + //processing_multi_statement = false; + NEXT_IMMEDIATE(ASYNC_QUERY_END); + } + break; + case ASYNC_QUERY_END: + PROXY_TRACE2(); + if (is_error_present()) { + compute_unknown_transaction_status(); + } else { + unknown_transaction_status = false; + } + // should be NULL + assert(!pgsql_result); + break; + case ASYNC_RESET_SESSION_START: + reset_session_start(); + update_bytes_sent((reset_session_in_txn == false ? (sizeof("DISCARD ALL") + 5) : (sizeof("ROLLBACK") + 5))); + if (async_exit_status) { + next_event(ASYNC_RESET_SESSION_CONT); + } else { + if (is_error_present()) { + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_END); + } + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_CONT); + } + break; + case ASYNC_RESET_SESSION_CONT: + { + reset_session_cont(event); + if (async_exit_status) { + if (myds->wait_until != 0 && myds->sess->thread->curtime >= myds->wait_until) { + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_TIMEOUT); + } + next_event(ASYNC_RESET_SESSION_CONT); + break; + } + if (is_error_present()) { + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_END); + } + PGresult* result = get_result(); + if (result) { + if (PQresultStatus(result) != PGRES_COMMAND_OK) { + set_error_from_result(result, PGSQL_ERROR_FIELD_ALL); + assert(is_error_present()); + } + PQclear(result); + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_CONT); + } + if (reset_session_in_txn) { + //assert(IsKnownActiveTransaction() == false); + reset_session_in_txn = false; + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_START); + } + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_END); + } + break; + case ASYNC_RESET_SESSION_END: + if (is_error_present()) { + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_FAILED); + } + NEXT_IMMEDIATE(ASYNC_RESET_SESSION_SUCCESSFUL); + break; + case ASYNC_RESET_SESSION_FAILED: + case ASYNC_RESET_SESSION_SUCCESSFUL: + case ASYNC_RESET_SESSION_TIMEOUT: + break; + default: + // not implemented yet + assert(0); + } + return async_state_machine; +} + +void PgSQL_Connection::connect_start() { + PROXY_TRACE(); + assert(pgsql_conn == NULL); // already there is a connection + reset_error(); + async_exit_status = PG_EVENT_NONE; + + std::ostringstream conninfo; + conninfo << "user=" << userinfo->username << " "; // username + conninfo << "password=" << userinfo->password << " "; // password + conninfo << "host=" << parent->address << " "; // backend address + conninfo << "port=" << parent->port << " "; // backend port + conninfo << "dbname=" << userinfo->dbname << " "; + conninfo << "application_name=proxysql "; // application name + //conninfo << "require_auth=" << AUTHENTICATION_METHOD_STR[pgsql_thread___authentication_method]; // authentication method + if (parent->use_ssl) { + conninfo << "sslmode=require "; // SSL required + if (pgsql_thread___ssl_p2s_key) + conninfo << "sslkey=" << pgsql_thread___ssl_p2s_key << " "; + if (pgsql_thread___ssl_p2s_cert) + conninfo << "sslcert=" << pgsql_thread___ssl_p2s_cert << " "; + if (pgsql_thread___ssl_p2s_ca) + conninfo << "sslrootcert=" << pgsql_thread___ssl_p2s_ca << " "; + if (pgsql_thread___ssl_p2s_crl) + conninfo << "sslcrl=" << pgsql_thread___ssl_p2s_crl << " "; + if (pgsql_thread___ssl_p2s_crlpath) + conninfo << "sslcrldir=" << pgsql_thread___ssl_p2s_crlpath << " "; + // Only supported in PostgreSQL Server + // if (pgsql_thread___ssl_p2s_cipher) + // conninfo << "sslcipher=" << pgsql_thread___ssl_p2s_cipher << " "; + } else { + conninfo << "sslmode=disable "; // not supporting SSL + } + + /*conninfo << "postgres://"; + conninfo << userinfo->username << ":" << userinfo->password; // username and password + conninfo << "@"; + conninfo << parent->address << ":" << parent->port; // backend address and port + conninfo << "/"; + conninfo << userinfo->schemaname; // currently schemaname consists of datasename (have to improve this in future). In PostgreSQL database and schema are NOT the same. + conninfo << "?"; + //conninfo << "require_auth=" << AUTHENTICATION_METHOD_STR[pgsql_thread___authentication_method]; // authentication method + conninfo << "application_name=proxysql"; + */ + + const std::string& conninfo_str = conninfo.str(); + pgsql_conn = PQconnectStart(conninfo_str.c_str()); + //pgsql_conn = PQconnectdb(conninfo_str.c_str()); + + //PQsetErrorVerbosity(pgsql_conn, PQERRORS_VERBOSE); + //PQsetErrorContextVisibility(pgsql_conn, PQSHOW_CONTEXT_ERRORS); + + if (pgsql_conn == NULL || PQstatus(pgsql_conn) == CONNECTION_BAD) { + if (pgsql_conn) { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + set_error_from_result(result); + } else { + set_error(PGSQL_GET_ERROR_CODE_STR(ERRCODE_OUT_OF_MEMORY), "Out of memory", false); + } + proxy_error("Connect failed. %s\n", get_error_code_with_message().c_str()); + return; + } + if (PQsetnonblocking(pgsql_conn, 1) != 0) { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + set_error_from_result(result); + proxy_error("Failed to set non-blocking mode: %s\n", get_error_code_with_message().c_str()); + return; + } + fd = PQsocket(pgsql_conn); + async_exit_status = PG_EVENT_WRITE; +} + +void PgSQL_Connection::connect_cont(short event) { + PROXY_TRACE(); + assert(pgsql_conn); + reset_error(); + async_exit_status = PG_EVENT_NONE; + +// For troubleshooting connection issue +#if 0 + const char* message = nullptr; + switch (PQstatus(pgsql_conn)) + { + case CONNECTION_STARTED: + message = "Connecting..."; + break; + + case CONNECTION_MADE: + message = "Connected to server (waiting to send) ..."; + break; + + case CONNECTION_AWAITING_RESPONSE: + message = "Waiting for a response from the server..."; + break; + + case CONNECTION_AUTH_OK: + message = "Received authentication; waiting for backend start - up to finish..."; + break; + + case CONNECTION_SSL_STARTUP: + message = "Negotiating SSL encryption..."; + break; + + case CONNECTION_SETENV: + message = "Negotiating environment-driven parameter settings..."; + break; + + default: + message = "Connecting..."; + } + + proxy_info("Connection status: %d %s\n", PQsocket(pgsql_conn), message); +#endif + + PostgresPollingStatusType poll_res = PQconnectPoll(pgsql_conn); + switch (poll_res) { + case PGRES_POLLING_WRITING: + async_exit_status = PG_EVENT_WRITE; + break; + case PGRES_POLLING_ACTIVE: + case PGRES_POLLING_READING: + async_exit_status = PG_EVENT_READ; + break; + case PGRES_POLLING_OK: + async_exit_status = PG_EVENT_NONE; + break; + //case PGRES_POLLING_FAILED: + default: + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + set_error_from_result(result); + proxy_error("Connect failed. %s\n", get_error_code_with_message().c_str()); + return; + } +} + +void PgSQL_Connection::query_start() { + PROXY_TRACE(); + reset_error(); + processing_multi_statement = false; + async_exit_status = PG_EVENT_NONE; + if (PQsendQuery(pgsql_conn, query.ptr) == 0) { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + set_error_from_result(result); + proxy_error("Failed to send query. %s\n", get_error_code_with_message().c_str()); + return; + } + flush(); +} + +void PgSQL_Connection::query_cont(short event) { + PROXY_TRACE(); + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6, "event=%d\n", event); + reset_error(); + async_exit_status = PG_EVENT_NONE; + if (event & POLLOUT) { + flush(); + } +} + +void PgSQL_Connection::fetch_result_start() { + PROXY_TRACE(); + reset_error(); + async_exit_status = PG_EVENT_NONE; +} + +void PgSQL_Connection::fetch_result_cont(short event) { + PROXY_TRACE(); + async_exit_status = PG_EVENT_NONE; + + // Avoid fetching a new result if one is already available. + // This situation can happen when a multi-statement query has been executed. + if (pgsql_result) + return; + + switch (PShandleRowData(pgsql_conn, &ps_result)) { + case 0: + result_type = 2; + return; + case 1: + // we already have data available in buffer + if (PQisBusy(pgsql_conn) == 0) { + result_type = 1; + pgsql_result = PQgetResult(pgsql_conn); + return; + } + break; + } + + if (PQconsumeInput(pgsql_conn) == 0) { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + /* We will only set the error if the result is not NULL or we didn't capture error in last call. If the result is NULL, + * it indicates that an error was already captured during a previous PQconsumeInput call, + * and we do not want to overwrite that information. + */ + if (result || is_error_present() == false) { + set_error_from_result(result); + proxy_error("Failed to consume input. %s\n", get_error_code_with_message().c_str()); + } + return; + } + + switch (PShandleRowData(pgsql_conn, &ps_result)) { + case 0: + result_type = 2; + return; + case 1: + if (PQisBusy(pgsql_conn)) { + async_exit_status = PG_EVENT_READ; + return; + } + break; + default: + async_exit_status = PG_EVENT_READ; + return; + } + result_type = 1; + pgsql_result = PQgetResult(pgsql_conn); +} + +void PgSQL_Connection::flush() { + reset_error(); + int res = PQflush(pgsql_conn); + + if (res > 0) { + async_exit_status = PG_EVENT_WRITE; + } + else if (res == 0) { + async_exit_status = PG_EVENT_READ; + } + else { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + set_error_from_result(result); + proxy_error("Failed to flush data to backend. %s\n", get_error_code_with_message().c_str()); + async_exit_status = PG_EVENT_NONE; + } +} + +int PgSQL_Connection::async_connect(short event) { + PROXY_TRACE(); + if (pgsql_conn == NULL && async_state_machine != ASYNC_CONNECT_START) { + // LCOV_EXCL_START + assert(0); + // LCOV_EXCL_STOP + } + if (async_state_machine == ASYNC_IDLE) { + myds->wait_until = 0; + return 0; + } + if (async_state_machine == ASYNC_CONNECT_SUCCESSFUL) { + compute_unknown_transaction_status(); + async_state_machine = ASYNC_IDLE; + myds->wait_until = 0; + creation_time = monotonic_time(); + return 0; + } + handler(event); + switch (async_state_machine) { + case ASYNC_CONNECT_SUCCESSFUL: + compute_unknown_transaction_status(); + async_state_machine = ASYNC_IDLE; + myds->wait_until = 0; + return 0; + break; + case ASYNC_CONNECT_FAILED: + return -1; + break; + case ASYNC_CONNECT_TIMEOUT: + return -2; + break; + default: + return 1; + } + return 1; +} + +bool PgSQL_Connection::is_connected() const { + if (pgsql_conn == nullptr || PQstatus(pgsql_conn) != CONNECTION_OK) { + return false; + } + return true; +} + +void PgSQL_Connection::compute_unknown_transaction_status() { + + if (pgsql_conn) { + // make sure we have not missed even a single error + if (is_error_present() == false) { + unknown_transaction_status = false; + return; + } + + /*if (is_connected() == false) { + unknown_transaction_status = true; + return; + }*/ + + switch (PQtransactionStatus(pgsql_conn)) { + case PQTRANS_INTRANS: + case PQTRANS_INERROR: + case PQTRANS_ACTIVE: + unknown_transaction_status = true; + break; + case PQTRANS_UNKNOWN: + default: + //unknown_transaction_status = false; + break; + } + } +} + +void PgSQL_Connection::async_free_result() { + PROXY_TRACE(); + //assert(pgsql_conn); + + if (query.ptr) { + query.ptr = NULL; + query.length = 0; + } + if (query.stmt_result) { + mysql_free_result(query.stmt_result); + query.stmt_result = NULL; + } + if (userinfo) { + // if userinfo is NULL , the connection is being destroyed + // because it is reset on destructor ( ~PgSQL_Connection() ) + // therefore this section is skipped completely + // this should prevent bug #1046 + //if (query.stmt) { + // if (query.stmt->mysql) { + // if (query.stmt->mysql == pgsql) { // extra check + // mysql_stmt_free_result(query.stmt); + // } + // } + // // If we reached here from 'ASYNC_STMT_PREPARE_FAILED', the + // // prepared statement was never added to 'local_stmts', thus + // // it will never be freed when 'local_stmts' are purged. If + // // initialized, it must be freed. For more context see #3525. + // if (this->async_state_machine == ASYNC_STMT_PREPARE_FAILED) { + // if (query.stmt != NULL) { + // proxy_mysql_stmt_close(query.stmt); + // } + // } + // query.stmt = NULL; + //} + } + if (pgsql_result) { + PQclear(pgsql_result); + pgsql_result = NULL; + } + compute_unknown_transaction_status(); + async_state_machine = ASYNC_IDLE; + if (query_result) { + if (query_result_reuse) { + delete (query_result_reuse); + } + query_result_reuse = query_result; + query_result = NULL; + } + new_result = false; +} + +#if 0 +int PgSQL_Connection::async_set_autocommit(short event, bool ac) { + PROXY_TRACE(); + assert(pgsql_conn); + server_status = parent->status; // we copy it here to avoid race condition. The caller will see this + if (IsServerOffline()) + return -1; + + switch (async_state_machine) { + case ASYNC_SET_AUTOCOMMIT_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine = ASYNC_IDLE; + return 0; + break; + case ASYNC_SET_AUTOCOMMIT_FAILED: + return -1; + break; + case ASYNC_QUERY_END: + case ASYNC_IDLE: + set_autocommit(ac); + async_state_machine = ASYNC_SET_AUTOCOMMIT_START; + default: + handler(event); + break; + } + + // check again + switch (async_state_machine) { + case ASYNC_SET_AUTOCOMMIT_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine = ASYNC_IDLE; + return 0; + break; + case ASYNC_SET_AUTOCOMMIT_FAILED: + return -1; + break; + default: + return 1; + break; + } + return 1; +} +#endif // 0 + +bool PgSQL_Connection::IsAutoCommit() { + bool ret = true; + /*if (pgsql) { + ret = (pgsql->server_status & SERVER_STATUS_AUTOCOMMIT); + if (ret) { + if (options.last_set_autocommit == 0) { + // it seems we hit bug http://bugs.pgsql.com/bug.php?id=66884 + // we last sent SET AUTOCOMMIT = 0 , but the server says it is 1 + // we assume that what we sent last is correct . #873 + ret = false; + } + } + else { + if (options.last_set_autocommit == -1) { + // if a connection was reset (thus last_set_autocommit==-1) + // the information related to SERVER_STATUS_AUTOCOMMIT is lost + // therefore we fall back on the safe assumption that autocommit==1 + ret = true; + } + } + }*/ + return ret; +} + +// Returns: +// 0 when the query is completed +// 1 when the query is not completed +// the calling function should check pgsql error in pgsql struct +int PgSQL_Connection::async_query(short event, char* stmt, unsigned long length, MYSQL_STMT** _stmt, stmt_execute_metadata_t* stmt_meta) { + PROXY_TRACE(); + PROXY_TRACE2(); + assert(pgsql_conn); + + server_status = parent->status; // we copy it here to avoid race condition. The caller will see this + if (IsServerOffline()) + return -1; + + if (myds) { + if (myds->DSS != STATE_MARIADB_QUERY) { + myds->DSS = STATE_MARIADB_QUERY; + } + } + switch (async_state_machine) { + case ASYNC_QUERY_END: + processing_multi_statement = false; // no matter if we are processing a multi statement or not, we reached the end + return 0; + break; + case ASYNC_IDLE: + if (myds && myds->sess) { + if (myds->sess->active_transactions == 0) { + // every time we start a query (no matter if COM_QUERY, STMT_PREPARE or otherwise) + // also a transaction starts, even if in autocommit mode + myds->sess->active_transactions = 1; + myds->sess->transaction_started_at = myds->sess->thread->curtime; + } + } + if (stmt_meta == NULL) + set_query(stmt, length); + async_state_machine = ASYNC_QUERY_START; + if (_stmt) { + query.stmt = *_stmt; + if (stmt_meta == NULL) { + async_state_machine = ASYNC_STMT_PREPARE_START; + } + else { + if (query.stmt_meta == NULL) { + query.stmt_meta = stmt_meta; + } + async_state_machine = ASYNC_STMT_EXECUTE_START; + } + } + default: + handler(event); + break; + } + + if (async_state_machine == ASYNC_QUERY_END) { + PROXY_TRACE2(); + compute_unknown_transaction_status(); + if (is_error_present()) { + return -1; + } + else { + return 0; + } + } + if (async_state_machine == ASYNC_STMT_EXECUTE_END) { + PROXY_TRACE2(); + query.stmt_meta = NULL; + async_state_machine = ASYNC_QUERY_END; + compute_unknown_transaction_status(); + if (mysql_stmt_errno(query.stmt)) { + return -1; + } + else { + return 0; + } + } + if (async_state_machine == ASYNC_STMT_PREPARE_SUCCESSFUL || async_state_machine == ASYNC_STMT_PREPARE_FAILED) { + query.stmt_meta = NULL; + compute_unknown_transaction_status(); + if (async_state_machine == ASYNC_STMT_PREPARE_FAILED) { + return -1; + } + else { + *_stmt = query.stmt; + return 0; + } + } + if (async_state_machine == ASYNC_USE_RESULT_START) { + // if we reached this point it measn we are processing a multi-statement + // and we need to exit to give control to MySQL_Session + processing_multi_statement = true; + return 2; + } + if (processing_multi_statement == true) { + // we are in the middle of processing a multi-statement + return 3; + } + return 1; +} + +// Returns: +// 0 when the query is completed +// 1 when the query is not completed +// the calling function should check pgsql error in pgsql struct +int PgSQL_Connection::async_reset_session(short event) { + PROXY_TRACE(); + PROXY_TRACE2(); + assert(pgsql_conn); + + server_status = parent->status; // we copy it here to avoid race condition. The caller will see this + if (IsServerOffline()) + return -1; + + /*if (myds) { + if (myds->DSS != STATE_MARIADB_QUERY) { + myds->DSS = STATE_MARIADB_QUERY; + } + }*/ + + switch (async_state_machine) { + case ASYNC_RESET_SESSION_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine = ASYNC_IDLE; + return 0; + break; + case ASYNC_RESET_SESSION_FAILED: + return -1; + break; + case ASYNC_RESET_SESSION_TIMEOUT: + return -2; + break; + case ASYNC_IDLE: + if (myds && myds->sess) { + if (myds->sess->active_transactions == 0) { + myds->sess->active_transactions = 1; + myds->sess->transaction_started_at = myds->sess->thread->curtime; + } + } + async_state_machine = ASYNC_RESET_SESSION_START; + default: + handler(event); + break; + } + + switch (async_state_machine) { + case ASYNC_RESET_SESSION_SUCCESSFUL: + if (myds && myds->sess) { + if (myds->sess->active_transactions != 0) { + myds->sess->active_transactions = 0; + myds->sess->transaction_started_at = 0; + } + } + unknown_transaction_status = false; + async_state_machine = ASYNC_IDLE; + return 0; + break; + case ASYNC_RESET_SESSION_FAILED: + if (myds && myds->sess) { + if (myds->sess->active_transactions != 0) { + myds->sess->active_transactions = 0; + myds->sess->transaction_started_at = 0; + } + } + return -1; + break; + case ASYNC_RESET_SESSION_TIMEOUT: + if (myds && myds->sess) { + if (myds->sess->active_transactions != 0) { + myds->sess->active_transactions = 0; + myds->sess->transaction_started_at = 0; + } + } + return -2; + break; + default: + break; + } + return 1; +} + +// Returns: +// 0 when the ping is completed successfully +// -1 when the ping is completed not successfully +// 1 when the ping is not completed +// -2 on timeout +// the calling function should check pgsql error in pgsql struct +int PgSQL_Connection::async_ping(short event) { + PROXY_TRACE(); + assert(pgsql_conn); + switch (async_state_machine) { + case ASYNC_PING_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine = ASYNC_IDLE; + return 0; + break; + case ASYNC_PING_FAILED: + return -1; + break; + case ASYNC_PING_TIMEOUT: + return -2; + break; + case ASYNC_IDLE: + async_state_machine = ASYNC_PING_START; + default: + //handler(event); + async_state_machine = ASYNC_PING_SUCCESSFUL; + break; + } + + // check again + switch (async_state_machine) { + case ASYNC_PING_SUCCESSFUL: + unknown_transaction_status = false; + async_state_machine = ASYNC_IDLE; + return 0; + break; + case ASYNC_PING_FAILED: + return -1; + break; + case ASYNC_PING_TIMEOUT: + return -2; + break; + default: + return 1; + break; + } + return 1; +} + +bool PgSQL_Connection::IsKnownActiveTransaction() { + bool in_txn = false; + if (pgsql_conn) { + // Get the transaction status + PGTransactionStatusType status = PQtransactionStatus(pgsql_conn); + if (status == PQTRANS_INTRANS || status == PQTRANS_INERROR) { + in_txn = true; + } + } + return in_txn; +} + +bool PgSQL_Connection::IsActiveTransaction() { + bool in_txn = false; + if (pgsql_conn) { + + // Get the transaction status + PGTransactionStatusType status = PQtransactionStatus(pgsql_conn); + + switch (status) { + case PQTRANS_INTRANS: + case PQTRANS_INERROR: + in_txn = true; + break; + case PQTRANS_UNKNOWN: + case PQTRANS_IDLE: + case PQTRANS_ACTIVE: + default: + in_txn = false; + } + + if (in_txn == false && is_error_present() && unknown_transaction_status == true) { + in_txn = true; + } + /*if (ret == false) { + //bool r = ( mysql_thread___autocommit_false_is_transaction || mysql_thread___forward_autocommit ); // deprecated , see #3253 + bool r = (mysql_thread___autocommit_false_is_transaction); + if (r && (IsAutoCommit() == false)) { + ret = true; + } + }*/ + } + return in_txn; +} + +bool PgSQL_Connection::IsServerOffline() { + bool ret = false; + if (parent == NULL) + return ret; + server_status = parent->status; // we copy it here to avoid race condition. The caller will see this + if ( + (server_status == MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user + || + (server_status == MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic == true && parent->shunned_and_kill_all_connections == true) // the server is SHUNNED due to a serious issue + || + (server_status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 + ) { + ret = true; + } + return ret; +} + +bool PgSQL_Connection::is_connection_in_reusable_state() const { + const PGTransactionStatusType txn_status = PQtransactionStatus(pgsql_conn); + const bool conn_usable = !(txn_status == PQTRANS_UNKNOWN || txn_status == PQTRANS_ACTIVE); + assert(!(conn_usable == false && is_error_present() == false)); + return conn_usable; +} + +PGresult* PgSQL_Connection::get_result() { + PGresult* result_tmp = pgsql_result; + pgsql_result = nullptr; + return result_tmp; +} + +bool PgSQL_Connection::set_single_row_mode() { + assert(pgsql_conn); + if (PQsetSingleRowMode(pgsql_conn) == 0) { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + set_error_from_result(result); + proxy_error("Failed to set single row mode. %s\n", get_error_code_with_message().c_str()); + return false; + } + return true; +} + +void PgSQL_Connection::next_multi_statement_result(PGresult* result) { + // set unprocessed result to pgsql_result + pgsql_result = result; + // copy buffer to PSarrayOut + query_result->buffer_to_PSarrayOut(); +} + +void PgSQL_Connection::reset_session_start() { + PROXY_TRACE(); + assert(pgsql_conn); + reset_error(); + async_exit_status = PG_EVENT_NONE; + reset_session_in_txn = IsKnownActiveTransaction(); + if (PQsendQuery(pgsql_conn, (reset_session_in_txn == false ? "DISCARD ALL" : "ROLLBACK")) == 0) { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + set_error_from_result(result); + proxy_error("Failed to send query. %s\n", get_error_code_with_message().c_str()); + return; + } + flush(); +} + +void PgSQL_Connection::reset_session_cont(short event) { + PROXY_TRACE(); + proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6, "event=%d\n", event); + reset_error(); + async_exit_status = PG_EVENT_NONE; + if (event & POLLOUT) { + flush(); + return; + } + + if (PQconsumeInput(pgsql_conn) == 0) { + // WARNING: DO NOT RELEASE this PGresult + const PGresult* result = PQgetResultFromPGconn(pgsql_conn); + /* We will only set the error if the result is not NULL or we didn't capture error in last call. If the result is NULL, + * it indicates that an error was already captured during a previous PQconsumeInput call, + * and we do not want to overwrite that information. + */ + if (result || is_error_present() == false) { + set_error_from_result(result); + proxy_error("Failed to consume input. %s\n", get_error_code_with_message().c_str()); + } + return; + } + + if (PQisBusy(pgsql_conn)) { + async_exit_status = PG_EVENT_READ; + return; + } + + pgsql_result = PQgetResult(pgsql_conn); +} + +bool PgSQL_Connection::requires_RESETTING_CONNECTION(const PgSQL_Connection* client_conn) { + for (auto i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (client_conn->var_hash[i] == 0) { + if (var_hash[i]) { + // this connection has a variable set that the + // client connection doesn't have. + // Since connection cannot be unset , this connection + // needs to be reset with CHANGE_USER + return true; + } + } + } + if (client_conn->dynamic_variables_idx.size() < dynamic_variables_idx.size()) { + // the server connection has more variables set than the client + return true; + } + std::vector::const_iterator it_c = client_conn->dynamic_variables_idx.begin(); // client connection iterator + std::vector::const_iterator it_s = dynamic_variables_idx.begin(); // server connection iterator + for (; it_s != dynamic_variables_idx.end(); it_s++) { + while (it_c != client_conn->dynamic_variables_idx.end() && (*it_c < *it_s)) { + it_c++; + } + if (it_c != client_conn->dynamic_variables_idx.end() && *it_c == *it_s) { + // the backend variable idx matches the frontend variable idx + } + else { + // we are processing a backend variable but there are + // no more frontend variables + return true; + } + } + return false; +} + +bool PgSQL_Connection::has_same_connection_options(const PgSQL_Connection* client_conn) { + if (userinfo->hash != client_conn->userinfo->hash) { + if (strcmp(userinfo->username, client_conn->userinfo->username)) { + return false; + } + if (strcmp(userinfo->dbname, client_conn->userinfo->dbname)) { + return false; + } + } + return true; +} + +unsigned int PgSQL_Connection::get_memory_usage() const { + // TODO: need to create new function in libpq + unsigned int memory_bytes = (16 * 1024) * 2; //PSgetMemoryUsage(pgsql_conn); + return /*sizeof(PGconn) +*/ memory_bytes; +} + +void PgSQL_Connection::update_bytes_recv(uint64_t bytes_recv) { + __sync_fetch_and_add(&parent->bytes_recv, bytes_recv); + myds->sess->thread->status_variables.stvar[st_var_queries_backends_bytes_recv] += bytes_recv; + myds->bytes_info.bytes_recv += bytes_recv; + bytes_info.bytes_recv += bytes_recv; +} + +void PgSQL_Connection::update_bytes_sent(uint64_t bytes_sent) { + __sync_fetch_and_add(&parent->bytes_sent, bytes_sent); + myds->sess->thread->status_variables.stvar[st_var_queries_backends_bytes_sent] += bytes_sent; + myds->bytes_info.bytes_sent += bytes_sent; + bytes_info.bytes_sent += bytes_sent; +} + +const char* PgSQL_Connection::get_pg_server_version_str(char* buff, int buff_size) { + const int postgresql_version = get_pg_server_version(); + snprintf(buff, buff_size, "%d.%d.%d", postgresql_version / 10000, (postgresql_version / 100) % 100, postgresql_version % 100); + return buff; +} + +const char* PgSQL_Connection::get_pg_connection_status_str() { + switch (get_pg_connection_status()) { + case CONNECTION_OK: + return "OK"; + case CONNECTION_BAD: + return "BAD"; + case CONNECTION_STARTED: + return "STARTED"; + case CONNECTION_MADE: + return "MADE"; + case CONNECTION_AWAITING_RESPONSE: + return "AWAITING_RESPONSE"; + case CONNECTION_AUTH_OK: + return "AUTH_OK"; + case CONNECTION_SETENV: + return "SETENV"; + case CONNECTION_SSL_STARTUP: + return "SSL_STARTUP"; + case CONNECTION_NEEDED: + return "NEEDED"; + case CONNECTION_CHECK_WRITABLE: + return "CHECK_WRITABLE"; + case CONNECTION_CONSUME: + return "CONSUME"; + case CONNECTION_GSS_STARTUP: + return "GSS_STARTUP"; + case CONNECTION_CHECK_TARGET: + return "CHECK_TARGET"; + case CONNECTION_CHECK_STANDBY: + return "CHECK_STANDBY"; + } + return "UNKNOWN"; +} + +const char* PgSQL_Connection::get_pg_transaction_status_str() { + switch (get_pg_transaction_status()) { + case PQTRANS_IDLE: + return "IDLE"; + case PQTRANS_ACTIVE: + return "ACTIVE"; + case PQTRANS_INTRANS: + return "IN-TRANSACTION"; + case PQTRANS_INERROR: + return "IN-ERROR-TRANSACTION"; + case PQTRANS_UNKNOWN: + return "UNKNOWN"; + } + return "INVALID"; +} diff --git a/lib/PgSQL_Data_Stream.cpp b/lib/PgSQL_Data_Stream.cpp new file mode 100644 index 0000000000..f7fb1c16b0 --- /dev/null +++ b/lib/PgSQL_Data_Stream.cpp @@ -0,0 +1,1417 @@ + +#include "proxysql.h" +#include "cpp.h" +#include +#ifndef UNIX_PATH_MAX +#define UNIX_PATH_MAX 108 +#endif + +#include "MySQL_PreparedStatement.h" +#include "PgSQL_Data_Stream.h" + +#include "openssl/x509v3.h" + +/* + +in libssl 1.1.0 +struct bio_st { + const BIO_METHOD *method; + long (*callback) (struct bio_st *, int, const char *, int, long, long); + char *cb_arg; + int init; + int shutdown; + int flags; + int retry_reason; + int num; + void *ptr; + struct bio_st *next_bio; + struct bio_st *prev_bio; + int references; + uint64_t num_read; + uint64_t num_write; + CRYPTO_EX_DATA ex_data; + CRYPTO_RWLOCK *lock; +}; +*/ + +typedef int CRYPTO_REF_COUNT; + +/** + * @brief This is the 'bio_st' struct definition from libssl 3.0.0. NOTE: This is an internal struct from + * OpenSSL library, currently it's used for performing checks on the reads/writes performed on the BIO objects. + * It's extremely important to keep this struct up to date with each OpenSSL dependency update. + */ +struct bio_st { + OSSL_LIB_CTX* libctx; + const BIO_METHOD* method; + /* bio, mode, argp, argi, argl, ret */ +#ifndef OPENSSL_NO_DEPRECATED_3_0 + BIO_callback_fn callback; +#endif + BIO_callback_fn_ex callback_ex; + char* cb_arg; /* first argument for the callback */ + int init; + int shutdown; + int flags; /* extra storage */ + int retry_reason; + int num; + void* ptr; + struct bio_st* next_bio; /* used by filter BIOs */ + struct bio_st* prev_bio; /* used by filter BIOs */ + CRYPTO_REF_COUNT references; + uint64_t num_read; + uint64_t num_write; + CRYPTO_EX_DATA ex_data; + CRYPTO_RWLOCK* lock; +}; + + +#define RESULTSET_BUFLEN_DS_16K 16000 +#define RESULTSET_BUFLEN_DS_1M 1000*1024 + +extern PgSQL_Threads_Handler* GloPTH; + +#ifdef DEBUG +static void __dump_pkt(const char* func, unsigned char* _ptr, unsigned int len) { + + if (GloVars.global.gdbg == 0) return; + if (GloVars.global.gdbg_lvl[PROXY_DEBUG_PKT_ARRAY].verbosity < 8) return; + unsigned int i; + fprintf(stderr, "DUMP %d bytes FROM %s\n", len, func); + for (i = 0; i < len; i++) { + if (isprint(_ptr[i])) fprintf(stderr, "%c", _ptr[i]); else fprintf(stderr, "."); + if (i > 0 && (i % 16 == 15 || i == len - 1)) { + unsigned int j; + if (i % 16 != 15) { + j = 15 - i % 16; + while (j--) fprintf(stderr, " "); + } + fprintf(stderr, " --- "); + for (j = (i == len - 1 ? ((int)(i / 16)) * 16 : i - 15); j <= i; j++) { + fprintf(stderr, "%02x ", _ptr[j]); + } + fprintf(stderr, "\n"); + } + } + fprintf(stderr, "\n\n"); + + +} +#endif + +#define queue_init(_q,_s) { \ + _q.size=_s; \ + _q.buffer=malloc(_q.size); \ + _q.head=0; \ + _q.tail=0; \ + _q.partial=0; \ + _q.pkt.ptr=NULL; \ + _q.pkt.size=0; \ +} + +#define queue_destroy(_q) { \ + if (_q.buffer) free(_q.buffer); \ + _q.buffer=NULL; \ + if (_q.pkt.ptr) { \ + l_free(_q.pkt.size,_q.pkt.ptr); \ + queueOUT.pkt.ptr=NULL; \ + } \ +} + +#define queue_zero(_q) { \ + if (_q.tail != 0) { \ + memcpy(_q.buffer, (unsigned char *)_q.buffer + _q.tail, _q.head - _q.tail); \ + } \ + _q.head-=_q.tail; \ + _q.tail=0; \ +} + +#define queue_available(_q) (_q.size-_q.head) +#define queue_data(_q) (_q.head-_q.tail) + +#define queue_r(_q, _s) { \ + _q.tail+=_s; \ + if (_q.tail==_q.head) { \ + _q.head=0; \ + _q.tail=0; \ + } \ +} + +#define queue_w(_q,_s) (_q.head+=_s) + +#define queue_r_ptr(_q) ((unsigned char *)_q.buffer+_q.tail) +#define queue_w_ptr(_q) ((unsigned char *)_q.buffer+_q.head) + +#define add_to_data_packet_history(_o,_p,_s) if (unlikely(GloVars.global.data_packets_history_size)) { \ + if (static_cast(_o.get_max_size()) != GloVars.global.data_packets_history_size) { \ + _o.set_max_size(GloVars.global.data_packets_history_size); \ + } \ + _o.push(_p,_s);\ +} + +// memory deallocation responsibility is now transferred to the queue as the buffer is directly assigned to it. +// if the size of data_packet_history is 0, the memory will be released. +#define add_to_data_packet_history_without_alloc(_o,_p,_s) if (unlikely(GloVars.global.data_packets_history_size)) { \ + if (static_cast(_o.get_max_size()) != GloVars.global.data_packets_history_size) { \ + _o.set_max_size(GloVars.global.data_packets_history_size); \ + } \ + _o.push(_p,_s);\ +} else { \ + l_free(_s,_p); \ +} +//enum sslstatus { SSLSTATUS_OK, SSLSTATUS_WANT_IO, SSLSTATUS_FAIL}; + +static enum pgsql_sslstatus get_sslstatus(SSL* ssl, int n) +{ + int err = SSL_get_error(ssl, n); + ERR_clear_error(); + switch (err) { + case SSL_ERROR_NONE: + return PGSQL_SSLSTATUS_OK; + case SSL_ERROR_WANT_WRITE: + case SSL_ERROR_WANT_READ: + return PGSQL_SSLSTATUS_WANT_IO; + case SSL_ERROR_ZERO_RETURN: + case SSL_ERROR_SYSCALL: + default: + return PGSQL_SSLSTATUS_FAIL; + } +} + + +void PgSQL_Data_Stream::queue_encrypted_bytes(const char* buf, size_t len) { + ssl_write_buf = (char*)realloc(ssl_write_buf, ssl_write_len + len); + memcpy(ssl_write_buf + ssl_write_len, buf, len); + ssl_write_len += len; + //proxy_info("New ssl_write_len size: %u\n", ssl_write_len); +} + +enum pgsql_sslstatus PgSQL_Data_Stream::do_ssl_handshake() { + char buf[MY_SSL_BUFFER]; + enum pgsql_sslstatus status; + int n = SSL_do_handshake(ssl); + if (n == 1) { + //proxy_info("SSL handshake completed\n"); + X509* cert; + cert = SSL_get_peer_certificate(ssl); + if (cert) { + GENERAL_NAMES* alt_names = (stack_st_GENERAL_NAME*)X509_get_ext_d2i((X509*)cert, NID_subject_alt_name, 0, 0); + int alt_name_count = sk_GENERAL_NAME_num(alt_names); + + // Iterate all the SAN names, looking for SPIFFE identifier + for (int i = 0; i < alt_name_count; i++) { + GENERAL_NAME* san = sk_GENERAL_NAME_value(alt_names, i); + + // We only care about URI names + if (san->type == GEN_URI) { + if (san->d.uniformResourceIdentifier->data) { + const char* resource_data = + reinterpret_cast(san->d.uniformResourceIdentifier->data); + const char* spiffe_loc = strstr(resource_data, "spiffe"); + + // First name starting with 'spiffe' is considered the match. + if (spiffe_loc == resource_data) { + x509_subject_alt_name = strdup(resource_data); + } + } + } + } + + sk_GENERAL_NAME_pop_free(alt_names, GENERAL_NAME_free); + X509_free(cert); + } + else { + // we currently disable this annoying error + // in future we can configure this as per user level, specifying if the certificate is mandatory or not + // see issue #3424 + //proxy_error("X509 error: no required certificate sent by client\n"); + } + // In case the supplied certificate has a 'SAN'-'URI' identifier + // starting with 'spiffe', client certificate verification is performed. + if (x509_subject_alt_name != NULL) { + long rc = SSL_get_verify_result(ssl); + if (rc != X509_V_OK) { + proxy_error("Disconnecting %s:%d: X509 client SSL certificate verify error: (%ld:%s)\n", addr.addr, addr.port, rc, X509_verify_cert_error_string(rc)); + return PGSQL_SSLSTATUS_FAIL; + } + } + } + status = get_sslstatus(ssl, n); + //proxy_info("SSL status = %d\n", status); + /* Did SSL request to write bytes? */ + if (status == PGSQL_SSLSTATUS_WANT_IO) { + //proxy_info("SSL status is WANT_IO %d\n", status); + do { + n = BIO_read(wbio_ssl, buf, sizeof(buf)); + //proxy_info("BIO read = %d\n", n); + if (n > 0) { + //proxy_info("Queuing %d encrypted bytes\n", n); + queue_encrypted_bytes(buf, n); + } + else if (!BIO_should_retry(wbio_ssl)) { + //proxy_info("BIO_should_retry failed\n"); + return PGSQL_SSLSTATUS_FAIL; + } + } while (n > 0); + } + return status; +} + +void* PgSQL_Data_Stream::operator new(size_t size) { + return l_alloc(size); +} + +void PgSQL_Data_Stream::operator delete(void* ptr) { + l_free(sizeof(PgSQL_Data_Stream), ptr); +} + +// Constructor +PgSQL_Data_Stream::PgSQL_Data_Stream() { + bytes_info.bytes_recv = 0; + bytes_info.bytes_sent = 0; + pkts_recv = 0; + pkts_sent = 0; + client_addr = NULL; + + addr.addr = NULL; + addr.port = 0; + proxy_addr.addr = NULL; + proxy_addr.port = 0; + + sess = NULL; + mysql_real_query.pkt.ptr = NULL; + mysql_real_query.pkt.size = 0; + mysql_real_query.QueryPtr = NULL; + mysql_real_query.QuerySize = 0; + + query_retries_on_failure = 0; + connect_retries_on_failure = 0; + max_connect_time = 0; + wait_until = 0; + pause_until = 0; + kill_type = 0; + connect_tries = 0; + poll_fds_idx = -1; + resultset_length = 0; + + revents = 0; + + PSarrayIN = NULL; + PSarrayOUT = NULL; + resultset = NULL; + queue_init(queueIN, QUEUE_T_DEFAULT_SIZE); + queue_init(queueOUT, QUEUE_T_DEFAULT_SIZE); + mybe = NULL; + active = 1; + mypolls = NULL; + myconn = NULL; // 20141011 + DSS = STATE_NOT_CONNECTED; + encrypted = false; + switching_auth_stage = 0; + switching_auth_type = 0; + x509_subject_alt_name = NULL; + ssl = NULL; + rbio_ssl = NULL; + wbio_ssl = NULL; + ssl_write_len = 0; + ssl_write_buf = NULL; + net_failure = false; + CompPktIN.pkt.ptr = NULL; + CompPktIN.pkt.size = 0; + CompPktIN.partial = 0; + CompPktOUT.pkt.ptr = NULL; + CompPktOUT.pkt.size = 0; + CompPktOUT.partial = 0; + multi_pkt.ptr = NULL; + multi_pkt.size = 0; + + statuses.questions = 0; + statuses.pgconnpoll_get = 0; + statuses.pgconnpoll_put = 0; + + com_field_wild = NULL; + scram_state = nullptr; +} + +// Destructor +PgSQL_Data_Stream::~PgSQL_Data_Stream() { + + queue_destroy(queueIN); + queue_destroy(queueOUT); + if (client_addr) { + free(client_addr); + client_addr = NULL; + } + if (addr.addr) { + free(addr.addr); + addr.addr = NULL; + } + if (proxy_addr.addr) { + free(proxy_addr.addr); + proxy_addr.addr = NULL; + } + + free_mysql_real_query(); + + if (com_field_wild) { + free(com_field_wild); + com_field_wild = NULL; + } + + proxy_debug(PROXY_DEBUG_NET, 1, "Shutdown Data Stream. Session=%p, DataStream=%p\n", sess, this); + PtrSize_t pkt; + if (PSarrayIN) { + while (PSarrayIN->len) { + PSarrayIN->remove_index_fast(0, &pkt); + l_free(pkt.size, pkt.ptr); + } + delete PSarrayIN; + } + if (PSarrayOUT) { + while (PSarrayOUT->len) { + PSarrayOUT->remove_index_fast(0, &pkt); + l_free(pkt.size, pkt.ptr); + } + delete PSarrayOUT; + } + if (resultset) { + while (resultset->len) { + resultset->remove_index_fast(0, &pkt); + l_free(pkt.size, pkt.ptr); + } + delete resultset; + } + if (mypolls) mypolls->remove_index_fast(poll_fds_idx); + + + if (fd > 0) { + // // Changing logic here. The socket should be closed only if it is not a backend + if (myds_type == MYDS_FRONTEND) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess:%p , MYDS:%p , PgSQL_Connection %p %s: shutdown socket\n", sess, this, myconn, (myconn ? "not reusable" : "is empty")); + shut_hard(); + } + } + // Commenting the follow line of code and adding an assert. We should ensure that if a myconn exists it should be removed *before* + if (myds_type == MYDS_BACKEND || myds_type == MYDS_BACKEND_NOT_CONNECTED) { + assert(myconn == NULL); + } + if ((myconn) && (myds_type == MYDS_FRONTEND)) { delete myconn; myconn = NULL; } + if (encrypted) { + if (ssl) { + // NOTE: SSL standard requires a final 'close_notify' alert on socket + // shutdown. But for avoiding any kind of locking IO waiting for the + // other part, we perform a 'quiet' shutdown. For more context see + // MYSQL #29579. + SSL_set_quiet_shutdown(ssl, 1); + if (SSL_shutdown(ssl) < 0) + ERR_clear_error(); + } + if (ssl) SSL_free(ssl); + } + if (multi_pkt.ptr) { + l_free(multi_pkt.size, multi_pkt.ptr); + multi_pkt.ptr = NULL; + multi_pkt.size = 0; + } + if (CompPktIN.pkt.ptr) { + l_free(CompPktIN.pkt.size, CompPktIN.pkt.ptr); + CompPktIN.pkt.ptr = NULL; + CompPktIN.pkt.size = 0; + } + if (CompPktOUT.pkt.ptr) { + l_free(CompPktOUT.pkt.size, CompPktOUT.pkt.ptr); + CompPktOUT.pkt.ptr = NULL; + CompPktOUT.pkt.size = 0; + } + if (x509_subject_alt_name) { + free(x509_subject_alt_name); + x509_subject_alt_name = NULL; + } + + free_scram_state(scram_state); +} + +// this function initializes a PgSQL_Data_Stream +void PgSQL_Data_Stream::init() { + if (myds_type != MYDS_LISTENER) { + proxy_debug(PROXY_DEBUG_NET, 1, "Init Data Stream. Session=%p, DataStream=%p -- type %d\n", sess, this, myds_type); + if (PSarrayIN == NULL) PSarrayIN = new PtrSizeArray(); + if (PSarrayOUT == NULL) PSarrayOUT = new PtrSizeArray(); + // if (PSarrayOUTpending==NULL) PSarrayOUTpending= new PtrSizeArray(); + if (resultset == NULL) resultset = new PtrSizeArray(); + + if (unlikely(GloVars.global.data_packets_history_size)) { + data_packets_history_IN.set_max_size(GloVars.global.data_packets_history_size); + data_packets_history_OUT.set_max_size(GloVars.global.data_packets_history_size); + } + } + if (myds_type != MYDS_FRONTEND) { + queue_destroy(queueIN); + queue_destroy(queueOUT); + } +} + +void PgSQL_Data_Stream::reinit_queues() { + if (queueIN.buffer == NULL) + queue_init(queueIN, QUEUE_T_DEFAULT_SIZE); + if (queueOUT.buffer == NULL) + queue_init(queueOUT, QUEUE_T_DEFAULT_SIZE); +} + +// this function initializes a PgSQL_Data_Stream with arguments +void PgSQL_Data_Stream::init(enum MySQL_DS_type _type, PgSQL_Session* _sess, int _fd) { + myds_type = _type; + sess = _sess; + init(); + fd = _fd; + proxy_debug(PROXY_DEBUG_NET, 1, "Initialized Data Stream. Session=%p, DataStream=%p, type=%d, fd=%d, myconn=%p\n", sess, this, myds_type, fd, myconn); + //if (myconn==NULL) myconn = new PgSQL_Connection(); + if (myconn) myconn->fd = fd; +} + +// Soft shutdown of socket : it only deactivate the data stream +// TODO: should check the status of the data stream, and identify if it is safe to reconnect or if the session should be destroyed +void PgSQL_Data_Stream::shut_soft() { + proxy_debug(PROXY_DEBUG_NET, 4, "Shutdown soft fd=%d. Session=%p, DataStream=%p\n", fd, sess, this); + active = 0; + set_net_failure(); + //if (sess) sess->net_failure=1; +} + +// Hard shutdown of socket +void PgSQL_Data_Stream::shut_hard() { + proxy_debug(PROXY_DEBUG_NET, 4, "Shutdown hard fd=%d. Session=%p, DataStream=%p\n", fd, sess, this); + set_net_failure(); + if (encrypted) { + // NOTE: SSL standard requires a final 'close_notify' alert on socket + // shutdown. But for avoiding any kind of locking IO waiting for the + // other part, we perform a 'quiet' shutdown. For more context see + // MYSQL #29579. + SSL_set_quiet_shutdown(ssl, 1); + if (SSL_shutdown(ssl) < 0) + ERR_clear_error(); + } + if (fd >= 0) { + shutdown(fd, SHUT_RDWR); + close(fd); + fd = -1; + } +} + +void PgSQL_Data_Stream::check_data_flow() { + if ((PSarrayIN->len || queue_data(queueIN)) && (PSarrayOUT->len || queue_data(queueOUT))) { + // there is data at both sides of the data stream: this is considered a fatal error + proxy_error("Session=%p, DataStream=%p -- Data at both ends of a MySQL data stream: IN <%d bytes %d packets> , OUT <%d bytes %d packets>\n", sess, this, PSarrayIN->len, queue_data(queueIN), PSarrayOUT->len, queue_data(queueOUT)); + shut_soft(); + generate_coredump(); + } + if ((myds_type == MYDS_BACKEND) && myconn && (myconn->fd == 0) && (revents & POLLOUT)) { + int rc; + int error; + socklen_t len = sizeof(error); + rc = getsockopt(fd, SOL_SOCKET, SO_ERROR, &error, &len); + assert(rc == 0); + if (error == 0) { + myconn->fd = fd; // connect succeeded + } + else { + errno = error; + perror("check_data_flow"); + shut_soft(); + } + } +} + +int PgSQL_Data_Stream::read_from_net() { + if (encrypted) { + //proxy_info("Entering\n"); + } + if ((revents & POLLHUP) && ((revents & POLLIN) == 0)) { + // Previously this was (revents & POLLHUP) , but now + // we call shut_soft() only if POLLIN is not set . + // + // This means that if we receive data (POLLIN) we process it + // temporarily ignoring POLLHUP . + // In this way we can intercept a COM_QUIT executed by the client + // before closing the socket + shut_soft(); + return -1; + } + // this check was moved after the previous one about POLLHUP, + // otherwise the previous check was never true + if ((revents & POLLIN) == 0) return 0; + + int r = 0; + int s = queue_available(queueIN); + if (encrypted) { + // proxy_info("Queue available of %d bytes\n", s); + } + if (encrypted == false) { + if (pkts_recv) { + r = recv(fd, queue_w_ptr(queueIN), s, 0); + } + else { + if (queueIN.partial == 0) { + // we are reading the very first packet + // to avoid issue with SSL, we will only read the header and eventually the first packet + r = recv(fd, queue_w_ptr(queueIN), 5, 0); + if (r == 5) { + // let's try to read a whole packet + unsigned int read_pos = 0; + unsigned char* buff = (unsigned char*)queueIN.buffer; + const uint8_t type8 = buff[0]; + if (type8 != 0) + read_pos++; + + uint32_t length = 0; + unsigned a, b, c, d; + a = buff[read_pos++]; + b = buff[read_pos++]; + c = buff[read_pos++]; + d = buff[read_pos++]; + length = (a << 24) | (b << 16) | (c << 8) | d; + + r += recv(fd, queue_w_ptr(queueIN) + 5, length, 0); + } + } + else { + r = recv(fd, queue_w_ptr(queueIN), s, 0); + } + } + } + else { // encrypted == true + /* + if (!SSL_is_init_finished(ssl)) { + int ret = SSL_do_handshake(ssl); + int ret2; + if (ret != 1) { + //ERR_print_errors_fp(stderr); + ret2 = SSL_get_error(ssl, ret); + fprintf(stderr,"%d\n",ret2); + } + return 0; + } else { + r = SSL_read (ssl, queue_w_ptr(queueIN), s); + } + */ + PROXY_TRACE(); + if (s < MY_SSL_BUFFER) { + return 0; // no enough space for reads + } + char buf[MY_SSL_BUFFER]; + //ssize_t n = read(fd, buf, sizeof(buf)); + int n = recv(fd, buf, sizeof(buf), 0); + //proxy_info("SSL recv of %d bytes\n", n); + proxy_debug(PROXY_DEBUG_NET, 7, "Session=%p: recv() read %d bytes. num_write: %lu , num_read: %lu\n", sess, n, rbio_ssl->num_write, rbio_ssl->num_read); + if (n > 0 || rbio_ssl->num_write > rbio_ssl->num_read) { + //on_read_cb(buf, (size_t)n); + + char buf2[MY_SSL_BUFFER]; + int n2; + //enum pgsql_sslstatus pgsql_status; + char* src = buf; + int len = n; + while (len > 0) { + n2 = BIO_write(rbio_ssl, src, len); + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p: write %d bytes into BIO %p, len=%d\n", sess, n2, rbio_ssl, len); + //proxy_info("BIO_write with len = %d and %d bytes\n", len , n2); + if (n2 <= 0) { + shut_soft(); + return -1; + } + src += n2; + len -= n2; + if (!SSL_is_init_finished(ssl)) { + //proxy_info("SSL_is_init_finished NOT completed\n"); + if (do_ssl_handshake() == PGSQL_SSLSTATUS_FAIL) { + //proxy_info("SSL_is_init_finished failed!!\n"); + shut_soft(); + return -1; + } + if (!SSL_is_init_finished(ssl)) { + //proxy_info("SSL_is_init_finished yet NOT completed\n"); + return 0; + } + } + else { + //proxy_info("SSL_is_init_finished completed\n"); + } + } + n2 = SSL_read(ssl, queue_w_ptr(queueIN), s); + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p: read %d bytes from BIO %p into a buffer with %d bytes free\n", sess, n2, rbio_ssl, s); + r = n2; + //proxy_info("Read %d bytes from SSL\n", r); + if (n2 > 0) { + } + /* + do { + n2 = SSL_read(ssl, buf2, sizeof(buf2)); + if (n2 > 0) { + + } + } while (n > 0); + */ + status = get_sslstatus(ssl, n2); + //proxy_info("SSL status = %d\n", status); + if (status == PGSQL_SSLSTATUS_WANT_IO) { + do { + n2 = BIO_read(wbio_ssl, buf2, sizeof(buf2)); + //proxy_info("BIO_read with %d bytes\n", n2); + if (n2 > 0) { + queue_encrypted_bytes(buf2, n2); + } + else if (!BIO_should_retry(wbio_ssl)) { + shut_soft(); + return -1; + } + } while (n2 > 0); + } + if (status == PGSQL_SSLSTATUS_FAIL) { + shut_soft(); + return -1; + } + } + else { + r = n; + //r += SSL_read (ssl, queue_w_ptr(queueIN), s); + //proxy_info("Read %d bytes from SSL\n", r); + } + } + //__exit_read_from_next: + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p: read %d bytes from fd %d into a buffer of %d bytes free\n", sess, r, fd, s); + //proxy_error("read %d bytes from fd %d into a buffer of %d bytes free\n", r, fd, s); + if (r < 1) { + if (encrypted == false) { + int myds_errno = errno; + if (r == 0 || (r == -1 && myds_errno != EINTR && myds_errno != EAGAIN)) { + shut_soft(); + } + } + else { + int ssl_ret = SSL_get_error(ssl, r); + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p, Datastream=%p -- session_id: %u , SSL_get_error(): %d , errno: %d\n", sess, this, sess->thread_session_id, ssl_ret, errno); + if (ssl_ret == SSL_ERROR_SYSCALL && (errno == EINTR || errno == EAGAIN)) { + // the read was interrupted, do nothing + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p, Datastream=%p -- SSL_get_error() is SSL_ERROR_SYSCALL, errno: %d\n", sess, this, errno); + } + else { + if (r == 0) { // we couldn't read any data + if (revents & POLLIN) { + // If revents is holding either POLLIN, or POLLIN and POLLHUP, but 'recv()' returns 0, + // reading no data, the socket has been already closed by the peer. Due to this we can + // ignore POLLHUP in this check, since we should reach here ONLY if POLLIN was set. + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p, Datastream=%p -- shutdown soft\n", sess, this); + shut_soft(); + } + } + if (ssl_ret != SSL_ERROR_WANT_READ && ssl_ret != SSL_ERROR_WANT_WRITE) shut_soft(); + // it seems we end in shut_soft() anyway + } + } + } + else { + queue_w(queueIN, r); + bytes_info.bytes_recv += r; + if (mypolls) mypolls->last_recv[poll_fds_idx] = sess->thread->curtime; + } + return r; +} + +int PgSQL_Data_Stream::write_to_net() { + int bytes_io = 0; + int s = queue_data(queueOUT); + int n; + if (encrypted) { + //proxy_info("Data in write buffer: %d bytes\n", s); + } + if (s == 0) { + if (encrypted == false) { + return 0; + } + if (ssl_write_len == 0 && wbio_ssl->num_write == wbio_ssl->num_read) { + return 0; + } + } + //VALGRIND_DISABLE_ERROR_REPORTING; + // splitting the ternary operation in IF condition for better readability + if (encrypted) { + bytes_io = SSL_write(ssl, queue_r_ptr(queueOUT), s); + //proxy_info("Used SSL_write to write %d bytes\n", bytes_io); + proxy_debug(PROXY_DEBUG_NET, 7, "Session=%p, Datastream=%p: SSL_write() wrote %d bytes . queueOUT before: %u\n", sess, this, bytes_io, queue_data(queueOUT)); + if (ssl_write_len || wbio_ssl->num_write > wbio_ssl->num_read) { + //proxy_info("ssl_write_len = %d , num_write = %d , num_read = %d\n", ssl_write_len , wbio_ssl->num_write , wbio_ssl->num_read); + char buf[MY_SSL_BUFFER]; + do { + n = BIO_read(wbio_ssl, buf, sizeof(buf)); + proxy_debug(PROXY_DEBUG_NET, 7, "Session=%p, Datastream=%p: BIO_read() read %d bytes\n", sess, this, n); + //proxy_info("BIO read = %d\n", n); + if (n > 0) { + //proxy_info("Setting %d byte in queue encrypted\n", n); + queue_encrypted_bytes(buf, n); + } + else if (!BIO_should_retry(wbio_ssl)) { + //proxy_info("BIO_should_retry failed\n"); + shut_soft(); + return -1; + } + } while (n > 0); + } + proxy_debug(PROXY_DEBUG_NET, 7, "Session=%p, Datastream=%p: current ssl_write_len is %lu bytes\n", sess, this, ssl_write_len); + if (ssl_write_len) { + n = write(fd, ssl_write_buf, ssl_write_len); + proxy_debug(PROXY_DEBUG_NET, 7, "Session=%p, Datastream=%p: write() wrote %d bytes in FD %d\n", sess, this, n, fd); + //proxy_info("Calling write() on SSL: %d\n", n); + if (n > 0) { + if ((size_t)n < ssl_write_len) { + memmove(ssl_write_buf, ssl_write_buf + n, ssl_write_len - n); + } + ssl_write_len -= n; + if (ssl_write_len) { + ssl_write_buf = (char*)realloc(ssl_write_buf, ssl_write_len); + } + else { + free(ssl_write_buf); + ssl_write_buf = NULL; + } + //proxy_info("new ssl_write_len: %u\n", ssl_write_len); + //if (ssl_write_len) { + // return n; // stop here + //} else { + // rc = n; // and continue + //} + //bytes_io += n; + } + else { + int myds_errno = errno; + if (n == 0 || (n == -1 && myds_errno != EINTR && myds_errno != EAGAIN)) { + shut_soft(); + return 0; + } + else { + return -1; + } + } + } + } + else { +#ifdef __APPLE__ + bytes_io = send(fd, queue_r_ptr(queueOUT), s, 0); +#else + bytes_io = send(fd, queue_r_ptr(queueOUT), s, MSG_NOSIGNAL); +#endif + proxy_debug(PROXY_DEBUG_NET, 7, "Session=%p, Datastream=%p: send() wrote %d bytes in FD %d\n", sess, this, bytes_io, fd); + } + if (encrypted) { + //proxy_info("bytes_io: %d\n", bytes_io); + } + //VALGRIND_ENABLE_ERROR_REPORTING; + if (bytes_io < 0) { + if (encrypted == false) { + if ((poll_fds_idx < 0) || (mypolls->fds[poll_fds_idx].revents & POLLOUT)) { // in write_to_net_poll() we has remove this safety + // so we enforce it here + shut_soft(); + } + } + else { + int ssl_ret = SSL_get_error(ssl, bytes_io); + if (ssl_ret != SSL_ERROR_WANT_READ && ssl_ret != SSL_ERROR_WANT_WRITE) shut_soft(); + } + } + else { + queue_r(queueOUT, bytes_io); + if (mypolls) mypolls->last_sent[poll_fds_idx] = sess->thread->curtime; + bytes_info.bytes_sent += bytes_io; + } + if (bytes_io > 0) { + if (myds_type == MYDS_FRONTEND) { + if (sess) { + if (sess->thread) { + sess->thread->status_variables.stvar[st_var_queries_frontends_bytes_sent] += bytes_io; + } + } + } + } + return bytes_io; +} + +bool PgSQL_Data_Stream::available_data_out() { + int buflen = queue_data(queueOUT); + if (buflen || PSarrayOUT->len) { + return true; + } + return false; +} + +void PgSQL_Data_Stream::remove_pollout() { + struct pollfd* _pollfd; + _pollfd = &mypolls->fds[poll_fds_idx]; + _pollfd->events = 0; +} + +void PgSQL_Data_Stream::set_pollout() { + struct pollfd* _pollfd; + _pollfd = &mypolls->fds[poll_fds_idx]; + if (DSS > STATE_MARIADB_BEGIN && DSS < STATE_MARIADB_END) { + _pollfd->events = myconn->wait_events; + } + else { + _pollfd->events = POLLIN; + //if (PSarrayOUT->len || available_data_out() || queueOUT.partial || (encrypted && !SSL_is_init_finished(ssl))) { + if (PSarrayOUT->len || available_data_out() || queueOUT.partial) { + _pollfd->events |= POLLOUT; + } + if (encrypted) { + if (ssl_write_len || wbio_ssl->num_write > wbio_ssl->num_read) { + _pollfd->events |= POLLOUT; + } + else { + if (!SSL_is_init_finished(ssl)) { + //proxy_info("SSL_is_init_finished NOT completed\n"); + if (do_ssl_handshake() == PGSQL_SSLSTATUS_FAIL) { + //proxy_info("SSL_is_init_finished failed!!\n"); + shut_soft(); + return; + } + if (!SSL_is_init_finished(ssl)) { + //proxy_info("SSL_is_init_finished yet NOT completed\n"); + return; + } + _pollfd->events |= POLLOUT; + } + else { + //proxy_info("SSL_is_init_finished completed\n"); + } + } + } + } + proxy_debug(PROXY_DEBUG_NET, 1, "Session=%p, DataStream=%p -- Setting poll events %d for FD %d , DSS=%d , myconn=%p\n", sess, this, _pollfd->events, fd, DSS, myconn); +} + +int PgSQL_Data_Stream::write_to_net_poll() { + int rc = 0; + if (active == 0) return rc; + /* + if (encrypted && !SSL_is_init_finished(ssl)) { + int ret = SSL_do_handshake(ssl); + int ret2; + if (ret != 1) { + //ERR_print_errors_fp(stderr); + ret2 = SSL_get_error(ssl, ret); + fprintf(stderr,"%d\n",ret2); + } + return 0; + } + */ + if (encrypted) { + if (!SSL_is_init_finished(ssl)) { + //proxy_info("SSL_is_init_finished completed: NO!\n"); + if (do_ssl_handshake() == PGSQL_SSLSTATUS_FAIL) { + //proxy_info("SSL_is_init_finished failed!!\n"); + shut_soft(); + return -1; + } + } + else { + //proxy_info("SSL_is_init_finished completed: YES\n"); + } + /* + if (!SSL_is_init_finished(ssl)) { + proxy_info("SSL_is_init_finished completed: NO!\n"); + if (fd>0 && sess->session_type == PROXYSQL_SESSION_PGSQL) { + set_pollout(); + return 0; + } + } + */ + //proxy_info("ssl_write_len: %u\n", ssl_write_len); + if (ssl_write_len) { + int n = write(fd, ssl_write_buf, ssl_write_len); + //proxy_info("Calling write() on SSL: %d\n", n); + if (n > 0) { + if ((size_t)n < ssl_write_len) { + memmove(ssl_write_buf, ssl_write_buf + n, ssl_write_len - n); + } + ssl_write_len -= n; + if (ssl_write_len) { + ssl_write_buf = (char*)realloc(ssl_write_buf, ssl_write_len); + } + else { + free(ssl_write_buf); + ssl_write_buf = NULL; + } + //proxy_info("new ssl_write_len: %u\n", ssl_write_len); + if (ssl_write_len) { + return n; // stop here + } + else { + rc = n; // and continue + } + } + else { + int myds_errno = errno; + if (n == 0 || (n == -1 && myds_errno != EINTR && myds_errno != EAGAIN)) { + shut_soft(); + return 0; + } + else { + return -1; + } + } + } + } + proxy_debug(PROXY_DEBUG_NET, 1, "Session=%p, DataStream=%p --\n", sess, this); + bool call_write_to_net = false; + if (queue_data(queueOUT)) { + call_write_to_net = true; + } + if (call_write_to_net == false) { + if (encrypted) { + if (ssl_write_len || wbio_ssl->num_write > wbio_ssl->num_read) { + call_write_to_net = true; + } + } + } + if (call_write_to_net) { + if (sess->session_type == PROXYSQL_SESSION_PGSQL) { + if (poll_fds_idx > -1) { // NOTE: attempt to force writes + if (net_failure == false) + rc += write_to_net(); + } + } + else { + rc += write_to_net(); + } + } + if (fd > 0 && sess->session_type == PROXYSQL_SESSION_PGSQL) { + // PROXYSQL_SESSION_PGSQL is a requirement, because it uses threads pool + // the other session types do not + set_pollout(); + } + return rc; +} + +int PgSQL_Data_Stream::read_pkts() { + int rc = 0; + int r = 0; + while ((r = buffer2array())) rc += r; + return rc; +} + +void PgSQL_Data_Stream::generate_compressed_packet() { +#define MAX_COMPRESSED_PACKET_SIZE 10*1024*1024 + unsigned int total_size = 0; + unsigned int i = 0; + PtrSize_t* p = NULL; + while (i < PSarrayOUT->len && total_size < MAX_COMPRESSED_PACKET_SIZE) { + p = PSarrayOUT->index(i); + total_size += p->size; + i++; + } + if (i >= 2) { + // we successfully read at least 2 packets + if (total_size > MAX_COMPRESSED_PACKET_SIZE) { + // total_size is too big, we remove the last packet read + total_size -= p->size; + } + } + if (total_size <= MAX_COMPRESSED_PACKET_SIZE) { + // this worked in the past . it applies for small packets + uLong sourceLen = total_size; + Bytef* source = (Bytef*)l_alloc(total_size); + uLongf destLen = total_size * 120 / 100 + 12; + Bytef* dest = (Bytef*)malloc(destLen); + i = 0; + total_size = 0; + while (total_size < sourceLen) { + PtrSize_t p2; + PSarrayOUT->remove_index(0, &p2); + memcpy(source + total_size, p2.ptr, p2.size); + total_size += p2.size; + l_free(p2.size, p2.ptr); + } + int rc = compress(dest, &destLen, source, sourceLen); + assert(rc == Z_OK); + l_free(total_size, source); + queueOUT.pkt.size = destLen + 7; + queueOUT.pkt.ptr = l_alloc(queueOUT.pkt.size); + mysql_hdr hdr; + hdr.pkt_length = destLen; + hdr.pkt_id = ++myconn->compression_pkt_id; + memcpy((unsigned char*)queueOUT.pkt.ptr, &hdr, sizeof(mysql_hdr)); + hdr.pkt_length = total_size; + memcpy((unsigned char*)queueOUT.pkt.ptr + 4, &hdr, 3); + memcpy((unsigned char*)queueOUT.pkt.ptr + 7, dest, destLen); + free(dest); + } + else { + // if we reach here, it means we have one single packet larger than MAX_COMPRESSED_PACKET_SIZE + PtrSize_t p2; + PSarrayOUT->remove_index(0, &p2); + + unsigned int len1 = MAX_COMPRESSED_PACKET_SIZE / 2; + unsigned int len2 = p2.size - len1; + uLongf destLen1; + uLongf destLen2; + Bytef* dest1; + Bytef* dest2; + int rc; + + mysql_hdr hdr; + + destLen1 = len1 * 120 / 100 + 12; + dest1 = (Bytef*)malloc(destLen1 + 7); + destLen2 = len2 * 120 / 100 + 12; + dest2 = (Bytef*)malloc(destLen2 + 7); + rc = compress(dest1 + 7, &destLen1, (const unsigned char*)p2.ptr, len1); + assert(rc == Z_OK); + rc = compress(dest2 + 7, &destLen2, (const unsigned char*)p2.ptr + len1, len2); + assert(rc == Z_OK); + + hdr.pkt_length = destLen1; + hdr.pkt_id = ++myconn->compression_pkt_id; + memcpy(dest1, &hdr, sizeof(mysql_hdr)); + hdr.pkt_length = len1; + memcpy((char*)dest1 + sizeof(mysql_hdr), &hdr, 3); + + hdr.pkt_length = destLen2; + hdr.pkt_id = ++myconn->compression_pkt_id; + memcpy(dest2, &hdr, sizeof(mysql_hdr)); + hdr.pkt_length = len2; + memcpy((char*)dest2 + sizeof(mysql_hdr), &hdr, 3); + + queueOUT.pkt.size = destLen1 + destLen2 + 7 + 7; + queueOUT.pkt.ptr = l_alloc(queueOUT.pkt.size); + memcpy((char*)queueOUT.pkt.ptr, dest1, destLen1 + 7); + memcpy((char*)queueOUT.pkt.ptr + destLen1 + 7, dest2, destLen2 + 7); + free(dest1); + free(dest2); + l_free(p2.size, p2.ptr); + } +} + + +int PgSQL_Data_Stream::array2buffer() { + int ret = 0; + unsigned int idx = 0; + bool cont = true; + if (sess) { + if (sess->mirror == true) { // if this is a mirror session, just empty it + idx = PSarrayOUT->len; + goto __exit_array2buffer; + } + } + while (cont) { + //VALGRIND_DISABLE_ERROR_REPORTING; + if (queue_available(queueOUT) == 0) { + goto __exit_array2buffer; + } + if (queueOUT.partial == 0) { // read a new packet + if (PSarrayOUT->len - idx) { + proxy_debug(PROXY_DEBUG_PKT_ARRAY, 5, "Session=%p . DataStream: %p -- Removing a packet from array\n", sess, this); + if (queueOUT.pkt.ptr) { + //l_free(queueOUT.pkt.size,queueOUT.pkt.ptr); + add_to_data_packet_history_without_alloc(data_packets_history_OUT, queueOUT.pkt.ptr, queueOUT.pkt.size); + queueOUT.pkt.ptr = NULL; + } + //VALGRIND_ENABLE_ERROR_REPORTING; + if (myconn->get_status(STATUS_MYSQL_CONNECTION_COMPRESSION) == true) { + proxy_debug(PROXY_DEBUG_PKT_ARRAY, 5, "Session=%p . DataStream: %p -- Compression enabled\n", sess, this); + generate_compressed_packet(); // it is copied directly into queueOUT.pkt + } + else { + //VALGRIND_DISABLE_ERROR_REPORTING; + memcpy(&queueOUT.pkt, PSarrayOUT->index(idx), sizeof(PtrSize_t)); + idx++; + //VALGRIND_ENABLE_ERROR_REPORTING; + // this is a special case, needed because compression is enabled *after* the first OK + if (DSS == STATE_CLIENT_AUTH_OK) { + DSS = STATE_SLEEP; + // enable compression + if (myconn->options.server_capabilities & CLIENT_COMPRESS) { + if (myconn->options.compression_min_length) { + myconn->set_status(true, STATUS_MYSQL_CONNECTION_COMPRESSION); + } + } + else { + //explicitly disable compression + myconn->options.compression_min_length = 0; + myconn->set_status(false, STATUS_MYSQL_CONNECTION_COMPRESSION); + } + } + } +#ifdef DEBUG + { __dump_pkt(__func__, (unsigned char*)queueOUT.pkt.ptr, queueOUT.pkt.size); } +#endif + } + else { + cont = false; + continue; + } + } + int b = (queue_available(queueOUT) > (queueOUT.pkt.size - queueOUT.partial) ? (queueOUT.pkt.size - queueOUT.partial) : queue_available(queueOUT)); + //VALGRIND_DISABLE_ERROR_REPORTING; + memcpy(queue_w_ptr(queueOUT), (unsigned char*)queueOUT.pkt.ptr + queueOUT.partial, b); + //VALGRIND_ENABLE_ERROR_REPORTING; + queue_w(queueOUT, b); + proxy_debug(PROXY_DEBUG_PKT_ARRAY, 5, "Session=%p . DataStream: %p -- Copied %d bytes into send buffer\n", sess, this, b); + queueOUT.partial += b; + ret = b; + if (queueOUT.partial == queueOUT.pkt.size) { + if (queueOUT.pkt.ptr) { + //l_free(queueOUT.pkt.size,queueOUT.pkt.ptr); + add_to_data_packet_history_without_alloc(data_packets_history_OUT, queueOUT.pkt.ptr, queueOUT.pkt.size); + queueOUT.pkt.ptr = NULL; + } + proxy_debug(PROXY_DEBUG_PKT_ARRAY, 5, "Session=%p . DataStream: %p -- Packet completely written into send buffer\n", sess, this); + queueOUT.partial = 0; + pkts_sent += 1; + } + } +__exit_array2buffer: + if (idx) { + PSarrayOUT->remove_index_range(0, idx); + } + return ret; +} + +unsigned char* PgSQL_Data_Stream::resultset2buffer(bool del) { + unsigned int i; + unsigned int l = 0; + unsigned char* mybuff = (unsigned char*)l_alloc(resultset_length); + PtrSize_t* ps; + for (i = 0; i < resultset->len; i++) { + ps = resultset->index(i); + memcpy(mybuff + l, ps->ptr, ps->size); + if (del) l_free(ps->size, ps->ptr); + l += ps->size; + } + return mybuff; +}; + +void PgSQL_Data_Stream::buffer2resultset(unsigned char* ptr, unsigned int size) { + unsigned char* __ptr = ptr; + mysql_hdr hdr; + unsigned int l; + void* buff = NULL; + unsigned int bl; + unsigned int bf; + while (__ptr < ptr + size) { + memcpy(&hdr, __ptr, sizeof(mysql_hdr)); + l = hdr.pkt_length + sizeof(mysql_hdr); // amount of space we need + if (buff) { + if (bf < l) { + // we ran out of space + resultset->add(buff, bl - bf); + buff = NULL; + } + } + if (buff == NULL) { + if (__ptr + RESULTSET_BUFLEN_DS_1M <= ptr + size) { + bl = RESULTSET_BUFLEN_DS_1M; + } + else { + bl = RESULTSET_BUFLEN_DS_16K; + } + if (l > bl) { + bl = l; // make sure there is the space to copy a packet + } + buff = malloc(bl); + bf = bl; + } + memcpy((char*)buff + (bl - bf), __ptr, l); + bf -= l; + __ptr += l; + /* + l=hdr.pkt_length+sizeof(mysql_hdr); + pkt=l_alloc(l); + memcpy(pkt,__ptr,l); + resultset->add(pkt,l); + __ptr+=l; + */ + } + if (buff) { + // last buffer to add + resultset->add(buff, bl - bf); + } +}; + +int PgSQL_Data_Stream::array2buffer_full() { + int rc = 0; + int r = 0; + while ((r = array2buffer())) rc += r; + return rc; +} + +int PgSQL_Data_Stream::assign_fd_from_mysql_conn() { + assert(myconn); + //proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p, myds=%p, oldFD=%d, newFD=%d\n", this->sess, this, fd, myconn->myconn.net.fd); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p, myds=%p, oldFD=%d, newFD=%d\n", this->sess, this, fd, myconn->fd); + fd = myconn->fd; + return fd; +} + +void PgSQL_Data_Stream::unplug_backend() { + DSS = STATE_NOT_INITIALIZED; + myconn = NULL; + myds_type = MYDS_BACKEND_NOT_CONNECTED; + mypolls->remove_index_fast(poll_fds_idx); + mypolls = NULL; + fd = 0; +} + +void PgSQL_Data_Stream::set_net_failure() { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p, myds=%p , myds_type:%d\n", this->sess, this, myds_type); +#ifdef DEBUG + if (myds_type != MYDS_FRONTEND) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p, myds=%p , myds_type:%d not frontend\n", this->sess, this, myds_type); + } +#endif /* DEBUG */ + net_failure = true; +} + +void PgSQL_Data_Stream::setDSS_STATE_QUERY_SENT_NET() { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p, myds=%p\n", this->sess, this); + DSS = STATE_QUERY_SENT_NET; +} + +void PgSQL_Data_Stream::return_MySQL_Connection_To_Pool() { + PgSQL_Connection* mc = myconn; + mc->last_time_used = sess->thread->curtime; + // before detaching, check if last_HG_affected_rows matches . if yes, set it back to -1 + if (mybe) { + if (mybe->hostgroup_id == sess->last_HG_affected_rows) { + sess->last_HG_affected_rows = -1; + } + } + unsigned long long intv = pgsql_thread___connection_max_age_ms; + intv *= 1000; + if ( + (((intv) && (mc->last_time_used > mc->creation_time + intv)) + || + (mc->local_stmts->get_num_backend_stmts() > (unsigned int)GloPTH->variables.max_stmts_per_connection)) + && + // NOTE: If the current session if in 'PINGING_SERVER' status, there is + // no need to reset the session. The destruction and creation of a new + // session in case this session has exceeded the time specified by + // 'connection_max_age_ms' will be deferred to the next time the session + // is used outside 'PINGING_SERVER' operation. For more context see #3502. + sess->status != PINGING_SERVER + ) { + sess->create_new_session_and_reset_connection(this); + } else { + detach_connection(); + unplug_backend(); +#ifdef STRESSTEST_POOL + PgHGM->push_MyConn_to_pool(mc); // #644 +#else + sess->thread->push_MyConn_local(mc); +#endif + } +} + +void PgSQL_Data_Stream::free_mysql_real_query() { + if (mysql_real_query.QueryPtr) { + mysql_real_query.end(); + } +} + +void PgSQL_Data_Stream::destroy_queues() { + queue_destroy(queueIN); + queue_destroy(queueOUT); +} + +void PgSQL_Data_Stream::destroy_MySQL_Connection_From_Pool(bool sq) { + PgSQL_Connection* mc = myconn; + PgSQL_SrvC* mysrvc = mc->parent; + if (sq && mysrvc->status == MYSQL_SERVER_STATUS_ONLINE && + mc->async_state_machine == ASYNC_IDLE && + mc->is_connection_in_reusable_state() == true) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Trying to reset PgSQL_Connection %p, server %s:%d\n", mc, mysrvc->address, mysrvc->port); + sess->create_new_session_and_reset_connection(this); + } else { + mc->last_time_used = sess->thread->curtime; + mc->send_quit = sq; + detach_connection(); + unplug_backend(); + PgHGM->destroy_MyConn_from_pool(mc); + } +} + +bool PgSQL_Data_Stream::data_in_rbio() { + if (rbio_ssl->num_write > rbio_ssl->num_read) { + return true; + } + return false; +} + +void PgSQL_Data_Stream::reset_connection() { + if (myconn) { + if (pgsql_thread___multiplexing && (DSS == STATE_MARIADB_GENERIC || DSS == STATE_READY) && myconn->reusable == true && + myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false && myconn->async_state_machine == ASYNC_IDLE) { + myconn->last_time_used = sess->thread->curtime; + return_MySQL_Connection_To_Pool(); + } else { + if (sess && sess->session_fast_forward == false) { + destroy_MySQL_Connection_From_Pool(true); + } else { + destroy_MySQL_Connection_From_Pool(false); + } + } + } +} + +int PgSQL_Data_Stream::buffer2array() { + int ret = 0; + { + unsigned long s = queue_data(queueIN); + if (s == 0) return ret; + if ((queueIN.pkt.size == 0) && s < 5) { + queue_zero(queueIN); + } + } + unsigned char header[5]; + if ((queueIN.pkt.size == 0) && queue_data(queueIN) >= sizeof(header)) { + proxy_debug(PROXY_DEBUG_PKT_ARRAY, 5, "Session=%p . Reading the header of a new packet\n", sess); + memcpy(header, queue_r_ptr(queueIN), sizeof(header)); + //pkt_sid=queueIN.hdr.pkt_id; + queue_r(queueIN, sizeof(header)); + uint32_t pkgsize = 0; + + + unsigned int read_pos = 0; + const uint8_t type8 = header[0]; + if (type8 != 0) { + read_pos++; + pkgsize++; + } + + unsigned a, b, c, d; + + a = header[read_pos++]; + b = header[read_pos++]; + c = header[read_pos++]; + d = header[read_pos++]; + pkgsize += (a << 24) | (b << 16) | (c << 8) | d; + + queueIN.pkt.size = pkgsize; + queueIN.pkt.ptr = l_alloc(queueIN.pkt.size); + memcpy(queueIN.pkt.ptr, header, sizeof(header)); // immediately copy the header into the packet + queueIN.partial = sizeof(header); + ret += sizeof(header); + } + if ((queueIN.pkt.size > 0) && queue_data(queueIN)) { + int b = (queue_data(queueIN) > (queueIN.pkt.size - queueIN.partial) ? (queueIN.pkt.size - queueIN.partial) : queue_data(queueIN)); + proxy_debug(PROXY_DEBUG_PKT_ARRAY, 5, "Session=%p . Copied %d bytes into packet\n", sess, b); + memcpy((unsigned char*)queueIN.pkt.ptr + queueIN.partial, queue_r_ptr(queueIN), b); + queue_r(queueIN, b); + queueIN.partial += b; + ret += b; + } + if ((queueIN.pkt.size > 0) && (queueIN.pkt.size == queueIN.partial)) { + PSarrayIN->add(queueIN.pkt.ptr, queueIN.pkt.size); + pkts_recv++; + queueIN.pkt.size = 0; + queueIN.pkt.ptr = NULL; + } + return ret; +} \ No newline at end of file diff --git a/lib/PgSQL_Error_Helper.cpp b/lib/PgSQL_Error_Helper.cpp new file mode 100644 index 0000000000..7b172ebf3c --- /dev/null +++ b/lib/PgSQL_Error_Helper.cpp @@ -0,0 +1,453 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include "PgSQL_Error_Helper.h" +#include "proxysql.h" +#include "cpp.h" +#include "libpq-fe.h" + +constexpr const char* PgSQL_Error_Helper::error_code_str[]; + +PGSQL_ERROR_CODES PgSQL_Error_Helper::identify_error_code(const char* errorCode) { + if (strlen(errorCode) != 5) + return PGSQL_ERROR_CODES::ERRCODE_UNKNOWN; + + for (uint8_t i = 0; i < static_cast(PGSQL_ERROR_CODES::PGSQL_ERROR_CODES_COUNT); i++) { + if (strncmp(errorCode, error_code_str[i], 5) == 0) { + return static_cast(i); + } + } + + return PGSQL_ERROR_CODES::ERRCODE_UNKNOWN; +} + +PGSQL_ERROR_CLASS PgSQL_Error_Helper::identify_error_class(const char* errorCode) { + if (strncmp(errorCode, "00", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_SUCCESS; + } + else if (strncmp(errorCode, "01", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_WARNING; + } + else if (strncmp(errorCode, "02", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_NO_DATA; + } + else if (strncmp(errorCode, "03", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_SQL_STATEMENT_NOT_YET_COMPLETE; + } + else if (strncmp(errorCode, "08", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_CONNECTION_EXCEPTION; + } + else if (strncmp(errorCode, "09", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_TRIGGERED_ACTION_EXCEPTION; + } + else if (strncmp(errorCode, "0A", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_FEATURE_NOT_SUPPORTED; + } + else if (strncmp(errorCode, "0B", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_TRANSACTION_INITIATION; + } + else if (strncmp(errorCode, "0F", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_LOCATOR_EXCEPTION; + } + else if (strncmp(errorCode, "0L", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_GRANTOR; + } + else if (strncmp(errorCode, "0P", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_ROLE_SPECIFICATION; + } + else if (strncmp(errorCode, "0Z", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_DIAGNOSTICS_EXCEPTION; + } + else if (strncmp(errorCode, "20", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_CASE_NOT_FOUND; + } + else if (strncmp(errorCode, "21", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_CARDINALITY_VIOLATION; + } + else if (strncmp(errorCode, "22", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_DATA_EXCEPTION; + } + else if (strncmp(errorCode, "23", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INTEGRITY_CONSTRAINT_VIOLATION; + } + else if (strncmp(errorCode, "24", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_CURSOR_STATE; + } + else if (strncmp(errorCode, "25", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_TRANSACTION_STATE; + } + else if (strncmp(errorCode, "26", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_SQL_STATEMENT_NAME; + } + else if (strncmp(errorCode, "27", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_TRIGGERED_DATA_CHANGE_VIOLATION; + } + else if (strncmp(errorCode, "28", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_AUTHORIZATION_SPECIFICATION; + } + else if (strncmp(errorCode, "2B", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_DEPENDENT_PRIVILEGE_DESCRIPTORS_STILL_EXIST; + } + else if (strncmp(errorCode, "2D", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_TRANSACTION_TERMINATION; + } + else if (strncmp(errorCode, "2F", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_SQL_ROUTINE_EXCEPTION; + } + else if (strncmp(errorCode, "34", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_CURSOR_NAME; + } + else if (strncmp(errorCode, "38", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_EXTERNAL_ROUTINE_EXCEPTION; + } + else if (strncmp(errorCode, "39", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION; + } + else if (strncmp(errorCode, "3B", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_SAVEPOINT_EXCEPTION; + } + else if (strncmp(errorCode, "3D", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_CATALOG_NAME; + } + else if (strncmp(errorCode, "3F", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INVALID_SCHEMA_NAME; + } + else if (strncmp(errorCode, "40", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_TRANSACTION_ROLLBACK; + } + else if (strncmp(errorCode, "42", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION; + } + else if (strncmp(errorCode, "44", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_WITH_CHECK_OPTION_VIOLATION; + } + else if (strncmp(errorCode, "53", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INSUFFICIENT_RESOURCES; + } + else if (strncmp(errorCode, "54", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_PROGRAM_LIMIT_EXCEEDED; + } + else if (strncmp(errorCode, "55", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_OBJECT_NOT_IN_PREREQUISITE_STATE; + } + else if (strncmp(errorCode, "57", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_OPERATOR_INTERVENTION; + } + else if (strncmp(errorCode, "58", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_SYSTEM_ERROR_UNSPECIFIED; + } + else if (strncmp(errorCode, "72", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_CRASH_SHUTDOWN; + } + else if (strncmp(errorCode, "F0", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_CONFIG_FILE_ERROR; + } + else if (strncmp(errorCode, "HV", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_FOREIGN_DATA_WRAPPER_ERROR; + } + else if (strncmp(errorCode, "P0", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_PLPGSQL_ERROR; + } + else if (strncmp(errorCode, "XX", 2) == 0) { + return PGSQL_ERROR_CLASS::ERRCLASS_INTERNAL_ERROR; + } + else { + return PGSQL_ERROR_CLASS::ERRCLASS_UNKNOWN_ERROR; + } +} + +PGSQL_ERROR_CATEGORY PgSQL_Error_Helper::categorize_error_class(PGSQL_ERROR_CLASS err_class) { + switch (err_class) { + case PGSQL_ERROR_CLASS::ERRCLASS_SUCCESS: + case PGSQL_ERROR_CLASS::ERRCLASS_WARNING: + case PGSQL_ERROR_CLASS::ERRCLASS_NO_DATA: + case PGSQL_ERROR_CLASS::ERRCLASS_SQL_STATEMENT_NOT_YET_COMPLETE: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_STATUS; + + case PGSQL_ERROR_CLASS::ERRCLASS_CONNECTION_EXCEPTION: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_CONNECTION_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_INVALID_AUTHORIZATION_SPECIFICATION: + case PGSQL_ERROR_CLASS::ERRCLASS_INVALID_GRANTOR: + case PGSQL_ERROR_CLASS::ERRCLASS_INVALID_ROLE_SPECIFICATION: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_AUTHORIZATION_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_INSUFFICIENT_RESOURCES: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_RESOURCE_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_CONFIG_FILE_ERROR: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_CONFIGURATION_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_SYNTAX_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_FEATURE_NOT_SUPPORTED: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_FEATURE_NOT_SUPPORTED; + + case PGSQL_ERROR_CLASS::ERRCLASS_TRIGGERED_ACTION_EXCEPTION: + case PGSQL_ERROR_CLASS::ERRCLASS_INVALID_TRANSACTION_INITIATION: + case PGSQL_ERROR_CLASS::ERRCLASS_INVALID_TRANSACTION_STATE: + case PGSQL_ERROR_CLASS::ERRCLASS_INVALID_TRANSACTION_TERMINATION: + case PGSQL_ERROR_CLASS::ERRCLASS_TRANSACTION_ROLLBACK: + case PGSQL_ERROR_CLASS::ERRCLASS_SAVEPOINT_EXCEPTION: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_TRANSACTION_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_CASE_NOT_FOUND: + case PGSQL_ERROR_CLASS::ERRCLASS_CARDINALITY_VIOLATION: + case PGSQL_ERROR_CLASS::ERRCLASS_DATA_EXCEPTION: + case PGSQL_ERROR_CLASS::ERRCLASS_INTEGRITY_CONSTRAINT_VIOLATION: + case PGSQL_ERROR_CLASS::ERRCLASS_WITH_CHECK_OPTION_VIOLATION: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_DATA_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_SQL_ROUTINE_EXCEPTION: + case PGSQL_ERROR_CLASS::ERRCLASS_TRIGGERED_DATA_CHANGE_VIOLATION: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_ROUTINE_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_INVALID_CURSOR_STATE: + case PGSQL_ERROR_CLASS::ERRCLASS_INVALID_CURSOR_NAME: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_CURSOR_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_EXTERNAL_ROUTINE_EXCEPTION: + case PGSQL_ERROR_CLASS::ERRCLASS_EXTERNAL_ROUTINE_INVOCATION_EXCEPTION: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_EXTERNAL_ROUTINE_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_PROGRAM_LIMIT_EXCEEDED: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_RESOURCE_LIMIT_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_OBJECT_NOT_IN_PREREQUISITE_STATE: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_OBJECT_STATE_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_OPERATOR_INTERVENTION: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_OPERATOR_INTERVENTION_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_FOREIGN_DATA_WRAPPER_ERROR: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_FDW_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_PLPGSQL_ERROR: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_PLPGSQL_ERROR; + + case PGSQL_ERROR_CLASS::ERRCLASS_INTERNAL_ERROR: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_INTERNAL_ERROR_CATEGORY; + + default: + return PGSQL_ERROR_CATEGORY::ERRCATEGORY_UNKNOWN_CATEGORY; + } +} + +PGSQL_ERROR_SEVERITY PgSQL_Error_Helper::identify_error_severity(const char* severity) { + + PGSQL_ERROR_SEVERITY ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_UNKNOWN_SEVERITY; + + if (strcasecmp(severity, "PANIC") == 0) { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_PANIC; + } else if (strcasecmp(severity, "FATAL") == 0) { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_FATAL; + } else if (strcasecmp(severity, "ERROR") == 0) { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_ERROR; + } else if (strcasecmp(severity, "WARNING") == 0) { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_WARNING; + } else if (strcasecmp(severity, "NOTICE") == 0) { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_NOTICE; + } else if (strcasecmp(severity, "DEBUG") == 0) { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_DEBUG; + } else if (strcasecmp(severity, "LOG") == 0) { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_LOG; + } else if (strcasecmp(severity, "INFO") == 0) { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_INFO; + } else { + ret = PGSQL_ERROR_SEVERITY::ERRSEVERITY_UNKNOWN_SEVERITY; + } + return ret; +} + + +void PgSQL_ErrorInfo_Ext::reset() { + text = PGSQL_ERROR_SEVERITY::ERRSEVERITY_UNKNOWN_SEVERITY; + detail.clear(); + hint.clear(); + position.clear(); + internal_position.clear(); + internal_query.clear(); + context.clear(); + schema_name.clear(); + table_name.clear(); + column_name.clear(); + datatype_name.clear(); + constraint_name.clear(); + source_file.clear(); + source_line.clear(); + source_function.clear(); +} + +void PgSQL_Error_Helper::fill_error_info(PgSQL_ErrorInfo& err_info, const char* code, const char* msg, const char* severity) { + strncpy(err_info.sqlstate, code, 5); + err_info.sqlstate[5] = '\0'; + err_info.severity = PgSQL_Error_Helper::identify_error_severity(severity); + err_info.code = PgSQL_Error_Helper::identify_error_code(code); + err_info.type = PgSQL_Error_Helper::identify_error_class(code); + err_info.category = PgSQL_Error_Helper::categorize_error_class(err_info.type); + err_info.message = msg; +} + +void PgSQL_Error_Helper::fill_error_info(PgSQL_ErrorInfo& err_info, PGSQL_ERROR_CODES code, const char* msg, PGSQL_ERROR_SEVERITY severity) { + fill_error_info(err_info, get_error_code(code), msg, PgSQL_Error_Helper::get_severity(severity)); +} + +/* +void PgSQL_Error_Helper::fill_error_info_from_error_message(PgSQL_ErrorInfo& err_info, const char* error_msg) { + std::string errorMsgStr(error_msg); + std::string sqlState; + std::string primaryErrorMsg; + std::string severity; + + // Initialize positions + size_t startPos = 0; + size_t endPos = 0; + + // Extract severity (assume it's the first word in the primary error message) + size_t severityEndPos = errorMsgStr.find(": "); + if (severityEndPos != std::string::npos) { + severity = errorMsgStr.substr(0, severityEndPos); + startPos = severityEndPos + 2; // Skip the ": " + } else { + severity = get_severity(PGSQL_ERROR_SEVERITY::ERRSEVERITY_UNKNOWN_SEVERITY); + } + + // Extract SQL state in the format [XXXXX] + startPos = errorMsgStr.find('[', startPos); + endPos = (startPos != std::string::npos) ? errorMsgStr.find(']', startPos) : std::string::npos; + + if (startPos != std::string::npos && endPos != std::string::npos && endPos == startPos + 6) { + sqlState = errorMsgStr.substr(startPos + 1, 5); // Extract the SQL state + primaryErrorMsg = errorMsgStr.substr(severityEndPos + 2, startPos - (severityEndPos + 2)); // Extract the primary error message up to the SQL state + } else { + sqlState = get_error_code(PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION); // Default SQL state + primaryErrorMsg = errorMsgStr.substr(severityEndPos + 2); // No SQL state found, remainder is the error message + } + + fill_error_info(err_info, sqlState.c_str(), primaryErrorMsg.c_str(), severity.c_str()); +} +*/ + +void reset_error_info(PgSQL_ErrorInfo& err_info, bool release_extented) { + err_info.sqlstate[0] = '\0'; + err_info.code = PGSQL_ERROR_CODES::ERRCODE_SUCCESSFUL_COMPLETION; + err_info.severity = PGSQL_ERROR_SEVERITY::ERRSEVERITY_UNKNOWN_SEVERITY; + err_info.type = PGSQL_ERROR_CLASS::ERRCLASS_UNKNOWN_ERROR; + err_info.category = PGSQL_ERROR_CATEGORY::ERRCATEGORY_UNKNOWN_CATEGORY; + err_info.message.clear(); + if (err_info.ext_info) { + if (release_extented) { + delete err_info.ext_info; + err_info.ext_info = NULL; + } else { + err_info.ext_info->reset(); + } + } +} + +void PgSQL_Error_Helper::fill_extended_error_info(PgSQL_ErrorInfo& err_info, const PGresult* result, uint16_t ext_fields) { + + if (ext_fields == 0) { + if (err_info.ext_info != NULL) { + delete err_info.ext_info; + err_info.ext_info = NULL; + } + return; + } + + char* val = NULL; + + if (err_info.ext_info == NULL) { + err_info.ext_info = new PgSQL_ErrorInfo_Ext(); + } else { + err_info.ext_info->reset(); + } + + if (ext_fields & PGSQL_ERROR_FIELD_TEXT) { + val = PQresultErrorField(result, PG_DIAG_SEVERITY_NONLOCALIZED); + err_info.ext_info->text = identify_error_severity(val ? val : ""); + } + + if (ext_fields & PGSQL_ERROR_FIELD_DETAIL) { + val = PQresultErrorField(result, PG_DIAG_MESSAGE_DETAIL); + err_info.ext_info->detail = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_HINT) { + val = PQresultErrorField(result, PG_DIAG_MESSAGE_HINT); + err_info.ext_info->hint = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_POSITION) { + val = PQresultErrorField(result, PG_DIAG_STATEMENT_POSITION); + err_info.ext_info->position = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_INTERNAL_POSITION) { + val = PQresultErrorField(result, PG_DIAG_INTERNAL_POSITION); + err_info.ext_info->internal_position = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_INTERNAL_QUERY) { + val = PQresultErrorField(result, PG_DIAG_INTERNAL_QUERY); + err_info.ext_info->internal_query = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_CONTEXT) { + val = PQresultErrorField(result, PG_DIAG_CONTEXT); + err_info.ext_info->context = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_SCHEMA_NAME) { + val = PQresultErrorField(result, PG_DIAG_SCHEMA_NAME); + err_info.ext_info->schema_name = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_TABLE_NAME) { + val = PQresultErrorField(result, PG_DIAG_TABLE_NAME); + err_info.ext_info->table_name = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_COLUMN_NAME) { + val = PQresultErrorField(result, PG_DIAG_COLUMN_NAME); + err_info.ext_info->column_name = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_DATA_TYPE_NAME) { + val = PQresultErrorField(result, PG_DIAG_DATATYPE_NAME); + err_info.ext_info->datatype_name = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_CONSTRAINT_NAME) { + val = PQresultErrorField(result, PG_DIAG_CONSTRAINT_NAME); + err_info.ext_info->constraint_name = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_FILE) { + val = PQresultErrorField(result, PG_DIAG_SOURCE_FILE); + err_info.ext_info->source_file = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_LINE) { + val = PQresultErrorField(result, PG_DIAG_SOURCE_LINE); + err_info.ext_info->source_line = val ? val : ""; + } + + if (ext_fields & PGSQL_ERROR_FIELD_ROUTINE) { + val = PQresultErrorField(result, PG_DIAG_SOURCE_FUNCTION); + err_info.ext_info->source_function = val ? val : ""; + } +} + +void PgSQL_Error_Helper::fill_error_info(PgSQL_ErrorInfo& err_info, const PGresult* result, uint16_t ext_fields) { + if (result == nullptr) { + return; + } + const char* sqlstate = PQresultErrorField(result, PG_DIAG_SQLSTATE); + const char* message = PQresultErrorField(result, PG_DIAG_MESSAGE_PRIMARY); + const char* severity = PQresultErrorField(result, PG_DIAG_SEVERITY); + fill_error_info(err_info, sqlstate ? sqlstate : "00000", message ? message : "", severity ? severity : ""); + fill_extended_error_info(err_info, result, ext_fields); +} diff --git a/lib/PgSQL_HostGroups_Manager.cpp b/lib/PgSQL_HostGroups_Manager.cpp new file mode 100644 index 0000000000..99247a85ab --- /dev/null +++ b/lib/PgSQL_HostGroups_Manager.cpp @@ -0,0 +1,4603 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include "PgSQL_HostGroups_Manager.h" +#include "proxysql.h" +#include "cpp.h" + +#include "MySQL_PreparedStatement.h" +#include "PgSQL_Data_Stream.h" + +#include +#include +#include + +#include "prometheus/counter.h" +#include "prometheus/detail/builder.h" +#include "prometheus/family.h" +#include "prometheus/gauge.h" + +#include "prometheus_helpers.h" +#include "proxysql_utils.h" + +#define char_malloc (char *)malloc +#define itostr(__s, __i) { __s=char_malloc(32); sprintf(__s, "%lld", __i); } + +#include "thread.h" +#include "wqueue.h" + +#include "ev.h" + +#include +#include +#include + +using std::function; + +#ifdef TEST_AURORA +static unsigned long long array_mysrvc_total = 0; +static unsigned long long array_mysrvc_cands = 0; +#endif // TEST_AURORA + +#define SAFE_SQLITE3_STEP(_stmt) do {\ + do {\ + rc=(*proxy_sqlite3_step)(_stmt);\ + if (rc!=SQLITE_DONE) {\ + assert(rc==SQLITE_LOCKED);\ + usleep(100);\ + }\ + } while (rc!=SQLITE_DONE);\ +} while (0) + +extern ProxySQL_Admin *GloAdmin; + +extern MySQL_Threads_Handler *GloMTH; + +extern MySQL_Monitor *GloMyMon; + +class PgSQL_SrvConnList; +class PgSQL_SrvC; +class PgSQL_SrvList; +class PgSQL_HGC; + +//static struct ev_async * gtid_ev_async; + +static pthread_mutex_t ev_loop_mutex; + +//static std::unordered_map gtid_map; + +const int PgSQL_ERRORS_STATS_FIELD_NUM = 11; + +/** + * @brief Helper function used to try to extract a value from the JSON field 'servers_defaults'. + * + * @param j JSON object constructed from 'servers_defaults' field. + * @param hid Hostgroup for which the 'servers_defaults' is defined in 'pgsql_hostgroup_attributes'. Used for + * error logging. + * @param key The key for the value to be extracted. + * @param val_check A validation function, checks if the value is within a expected range. + * + * @return The value extracted from the supplied JSON. In case of error '-1', and error cause is logged. + */ +template ::value, bool>::type = true> +T PgSQL_j_get_srv_default_int_val( + const json& j, uint32_t hid, const string& key, const function& val_check +) { + if (j.find(key) != j.end()) { + const json::value_t val_type = j[key].type(); + const char* type_name = j[key].type_name(); + + if (val_type == json::value_t::number_integer || val_type == json::value_t::number_unsigned) { + T val = j[key].get(); + + if (val_check(val)) { + return val; + } else { + proxy_error( + "Invalid value %ld supplied for 'pgsql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." + " Value NOT UPDATED.\n", + static_cast(val), key.c_str(), hid + ); + } + } else { + proxy_error( + "Invalid type '%s'(%hhu) supplied for 'pgsql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d." + " Value NOT UPDATED.\n", + type_name, static_cast(val_type), key.c_str(), hid + ); + } + } + + return static_cast(-1); +} + +PgSQL_Connection *PgSQL_SrvConnList::index(unsigned int _k) { + return (PgSQL_Connection *)conns->index(_k); +} + +PgSQL_Connection * PgSQL_SrvConnList::remove(int _k) { + return (PgSQL_Connection *)conns->remove_index_fast(_k); +} + +PgSQL_SrvConnList::PgSQL_SrvConnList(PgSQL_SrvC *_mysrvc) { + mysrvc=_mysrvc; + conns=new PtrArray(); +} + +void PgSQL_SrvConnList::add(PgSQL_Connection *c) { + conns->add(c); +} + +PgSQL_SrvConnList::~PgSQL_SrvConnList() { + mysrvc=NULL; + while (conns_length()) { + PgSQL_Connection *conn=(PgSQL_Connection *)conns->remove_index_fast(0); + delete conn; + } + delete conns; +} + +void PgSQL_SrvConnList::drop_all_connections() { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Dropping all connections (%u total) on PgSQL_SrvConnList %p for server %s:%d , hostgroup=%d , status=%d\n", conns_length(), this, mysrvc->address, mysrvc->port, mysrvc->myhgc->hid, mysrvc->status); + while (conns_length()) { + PgSQL_Connection *conn=(PgSQL_Connection *)conns->remove_index_fast(0); + delete conn; + } +} + + +PgSQL_SrvC::PgSQL_SrvC( + char* add, uint16_t p, int64_t _weight, enum MySerStatus _status, unsigned int _compression, + int64_t _max_connections, unsigned int _max_replication_lag, int32_t _use_ssl, unsigned int _max_latency_ms, + char* _comment +) { + address=strdup(add); + port=p; + weight=_weight; + status=_status; + compression=_compression; + max_connections=_max_connections; + max_replication_lag=_max_replication_lag; + use_ssl=_use_ssl; + cur_replication_lag_count=0; + max_latency_us=_max_latency_ms*1000; + current_latency_us=0; + aws_aurora_current_lag_us = 0; + connect_OK=0; + connect_ERR=0; + queries_sent=0; + bytes_sent=0; + bytes_recv=0; + max_connections_used=0; + time_last_detected_error=0; + connect_ERR_at_time_last_detected_error=0; + shunned_automatic=false; + shunned_and_kill_all_connections=false; // false to default + //charset=_charset; + myhgc=NULL; + comment=strdup(_comment); + ConnectionsUsed=new PgSQL_SrvConnList(this); + ConnectionsFree=new PgSQL_SrvConnList(this); +} + +void PgSQL_SrvC::connect_error(int err_num, bool get_mutex) { + // NOTE: this function operates without any mutex + // although, it is not extremely important if any counter is lost + // as a single connection failure won't make a significant difference + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Connect failed with code '%d'\n", err_num); + __sync_fetch_and_add(&connect_ERR,1); + __sync_fetch_and_add(&PgHGM->status.server_connections_aborted,1); + if (err_num >= 1048 && err_num <= 1052) + return; + if (err_num >= 1054 && err_num <= 1075) + return; + if (err_num >= 1099 && err_num <= 1104) + return; + if (err_num >= 1106 && err_num <= 1113) + return; + if (err_num >= 1116 && err_num <= 1118) + return; + if (err_num == 1136 || (err_num >= 1138 && err_num <= 1149)) + return; + switch (err_num) { + case 1007: // Can't create database + case 1008: // Can't drop database + case 1044: // access denied + case 1045: // access denied +/* + case 1048: // Column cannot be null + case 1049: // Unknown database + case 1050: // Table already exists + case 1051: // Unknown table + case 1052: // Column is ambiguous +*/ + case 1120: + case 1203: // User %s already has more than 'max_user_connections' active connections + case 1226: // User '%s' has exceeded the '%s' resource (current value: %ld) + case 3118: // Access denied for user '%s'. Account is locked.. + return; + break; + default: + break; + } + time_t t=time(NULL); + if (t > time_last_detected_error) { + time_last_detected_error=t; + connect_ERR_at_time_last_detected_error=1; + } else { + if (t < time_last_detected_error) { + // time_last_detected_error is in the future + // this means that monitor has a ping interval too big and tuned that in the future + return; + } + // same time + /** + * @brief The expected configured retries set by 'pgsql-connect_retries_on_failure' + '2' extra expected + * connection errors. + * @details This two extra connections errors are expected: + * 1. An initial connection error generated by the datastream and the connection when being created, + * this is, right after the session has requested a connection to the connection pool. This error takes + * places directly in the state machine from 'PgSQL_Connection'. Because of this, we consider this + * additional error to be a consequence of the two states machines, and it's not considered for + * 'connect_retries'. + * 2. A second connection connection error, which is the initial connection error generated by 'PgSQL_Session' + * when already in the 'CONNECTING_SERVER' state. This error is an 'extra error' to always consider, since + * it's not part of the retries specified by 'pgsql_thread___connect_retries_on_failure', thus, we set the + * 'connect_retries' to be 'pgsql_thread___connect_retries_on_failure + 1'. + */ + int connect_retries = pgsql_thread___connect_retries_on_failure + 1; + int max_failures = pgsql_thread___shun_on_failures > connect_retries ? connect_retries : pgsql_thread___shun_on_failures; + + if (__sync_add_and_fetch(&connect_ERR_at_time_last_detected_error,1) >= (unsigned int)max_failures) { + bool _shu=false; + if (get_mutex==true) + PgHGM->wrlock(); // to prevent race conditions, lock here. See #627 + if (status==MYSQL_SERVER_STATUS_ONLINE) { + status=MYSQL_SERVER_STATUS_SHUNNED; + shunned_automatic=true; + _shu=true; + } else { + _shu=false; + } + if (get_mutex==true) + PgHGM->wrunlock(); + if (_shu) { + proxy_error("Shunning server %s:%d with %u errors/sec. Shunning for %u seconds\n", address, port, connect_ERR_at_time_last_detected_error , pgsql_thread___shun_recovery_time_sec); + } + } + } +} + +void PgSQL_SrvC::shun_and_killall() { + status=MYSQL_SERVER_STATUS_SHUNNED; + shunned_automatic=true; + shunned_and_kill_all_connections=true; +} + +PgSQL_SrvC::~PgSQL_SrvC() { + if (address) free(address); + if (comment) free(comment); + delete ConnectionsUsed; + delete ConnectionsFree; +} + +using metric_name = std::string; +using metric_help = std::string; +using metric_tags = std::map; + +using hg_counter_tuple = + std::tuple< + PgSQL_p_hg_counter::metric, + metric_name, + metric_help, + metric_tags + >; + +using hg_gauge_tuple = + std::tuple< + PgSQL_p_hg_gauge::metric, + metric_name, + metric_help, + metric_tags + >; + +using hg_dyn_counter_tuple = + std::tuple< + PgSQL_p_hg_dyn_counter::metric, + metric_name, + metric_help, + metric_tags + >; + +using hg_dyn_gauge_tuple = + std::tuple< + PgSQL_p_hg_dyn_gauge::metric, + metric_name, + metric_help, + metric_tags + >; + +using hg_counter_vector = std::vector; +using hg_gauge_vector = std::vector; +using hg_dyn_counter_vector = std::vector; +using hg_dyn_gauge_vector = std::vector; + +/** + * @brief Metrics map holding the metrics for the 'PgSQL_HostGroups_Manager' module. + * + * @note Many metrics in this map, share a common "id name", because + * they differ only by label, because of this, HELP is shared between + * them. For better visual identification of this groups they are + * sepparated using a line separator comment. + */ +const std::tuple< + hg_counter_vector, + hg_gauge_vector, + hg_dyn_counter_vector, + hg_dyn_gauge_vector +> +hg_metrics_map = std::make_tuple( + hg_counter_vector { + std::make_tuple ( + PgSQL_p_hg_counter::servers_table_version, + "proxysql_servers_table_version_total", + "Number of times the \"servers_table\" have been modified.", + metric_tags {} + ), + + // ==================================================================== + std::make_tuple ( + PgSQL_p_hg_counter::server_connections_created, + "proxysql_server_connections_total", + "Total number of server connections (created|delayed|aborted).", + metric_tags { + { "status", "created" } + } + ), + std::make_tuple ( + PgSQL_p_hg_counter::server_connections_delayed, + "proxysql_server_connections_total", + "Total number of server connections (created|delayed|aborted).", + metric_tags { + { "status", "delayed" } + } + ), + std::make_tuple ( + PgSQL_p_hg_counter::server_connections_aborted, + "proxysql_server_connections_total", + "Total number of server connections (created|delayed|aborted).", + metric_tags { + { "status", "aborted" } + } + ), + // ==================================================================== + + // ==================================================================== + std::make_tuple ( + PgSQL_p_hg_counter::client_connections_created, + "proxysql_client_connections_total", + "Total number of client connections created.", + metric_tags { + { "status", "created" } + } + ), + std::make_tuple ( + PgSQL_p_hg_counter::client_connections_aborted, + "proxysql_client_connections_total", + "Total number of client failed connections (or closed improperly).", + metric_tags { + { "status", "aborted" } + } + ), + // ==================================================================== + + /*std::make_tuple( + PgSQL_p_hg_counter::com_autocommit, + "proxysql_com_autocommit_total", + "Total queries autocommited.", + metric_tags {} + ), + std::make_tuple( + PgSQL_p_hg_counter::com_autocommit_filtered, + "proxysql_com_autocommit_filtered_total", + "Total queries filtered autocommit.", + metric_tags {} + ),*/ + std::make_tuple ( + PgSQL_p_hg_counter::com_rollback, + "proxysql_com_rollback_total", + "Total queries rollbacked.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::com_rollback_filtered, + "proxysql_com_rollback_filtered_total", + "Total queries filtered rollbacked.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::com_backend_reset_connection, + "proxysql_com_backend_reset_connection_total", + "Total backend_reset_connection queries backend.", + metric_tags {} + ), + /*std::make_tuple( + PgSQL_p_hg_counter::com_backend_init_db, + "proxysql_com_backend_init_db_total", + "Total queries backend INIT DB.", + metric_tags {} + ),*/ + std::make_tuple ( + PgSQL_p_hg_counter::com_backend_set_client_encoding, + "proxysql_com_backend_set_client_encoding_total", + "Total queries backend SET client_encoding.", + metric_tags {} + ), + /*std::make_tuple( + PgSQL_p_hg_counter::com_frontend_init_db, + "proxysql_com_frontend_init_db_total", + "Total INIT DB queries frontend.", + metric_tags {} + ),*/ + std::make_tuple ( + PgSQL_p_hg_counter::com_frontend_set_client_encoding, + "proxysql_com_frontend_set_client_encoding_total", + "Total SET client_encoding frontend queries.", + metric_tags {} + ), + /*std::make_tuple( + PgSQL_p_hg_counter::com_frontend_use_db, + "proxysql_com_frontend_use_db_total", + "Total USE DB queries frontend.", + metric_tags {} + ),*/ + std::make_tuple ( + PgSQL_p_hg_counter::com_commit_cnt, + "proxysql_com_commit_cnt_total", + "Total queries commit.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::com_commit_cnt_filtered, + "proxysql_com_commit_cnt_filtered_total", + "Total queries commit filtered.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::selects_for_update__autocommit0, + "proxysql_selects_for_update__autocommit0_total", + "Total queries that are SELECT for update or equivalent.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::access_denied_wrong_password, + "proxysql_access_denied_wrong_password_total", + "Total access denied \"wrong password\".", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::access_denied_max_connections, + "proxysql_access_denied_max_connections_total", + "Total access denied \"max connections\".", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::access_denied_max_user_connections, + "proxysql_access_denied_max_user_connections_total", + "Total access denied \"max user connections\".", + metric_tags {} + ), + + // ==================================================================== + std::make_tuple ( + PgSQL_p_hg_counter::pghgm_pgconnpool_get, + "proxysql_pghgm_pgconnpool_get_total", + "The number of requests made to the connection pool.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::pghgm_pgconnpool_get_ok, + "proxysql_pghgm_pgconnpool_get_ok_total", + "The number of successful requests to the connection pool (i.e. where a connection was available).", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::pghgm_pgconnpool_get_ping, + "proxysql_pghgm_myconnpool_get_ping_total", + "The number of connections that were taken from the pool to run a ping to keep them alive.", + metric_tags {} + ), + // ==================================================================== + + std::make_tuple ( + PgSQL_p_hg_counter::pghgm_pgconnpool_push, + "proxysql_pghgm_pgconnpool_push_total", + "The number of connections returned to the connection pool.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::pghgm_pgconnpool_reset, + "proxysql_pghgm_pgconnpool_reset_total", + "The number of connections that have been reset / re-initialized using \"COM_CHANGE_USER\"", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_counter::pghgm_pgconnpool_destroy, + "proxysql_pghgm_pgconnpool_destroy_total", + "The number of connections considered unhealthy and therefore closed.", + metric_tags {} + ), + + // ==================================================================== + + std::make_tuple ( + PgSQL_p_hg_counter::auto_increment_delay_multiplex, + "proxysql_myhgm_auto_increment_multiplex_total", + "The number of times that 'auto_increment_delay_multiplex' has been triggered.", + metric_tags {} + ), + }, + // prometheus gauges + hg_gauge_vector { + std::make_tuple ( + PgSQL_p_hg_gauge::server_connections_connected, + "proxysql_server_connections_connected", + "Backend connections that are currently connected.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_gauge::client_connections_connected, + "proxysql_client_connections_connected", + "Client connections that are currently connected.", + metric_tags {} + ) + }, + // prometheus dynamic counters + hg_dyn_counter_vector { + // connection_pool + // ==================================================================== + + // ==================================================================== + std::make_tuple ( + PgSQL_p_hg_dyn_counter::conn_pool_bytes_data_recv, + "proxysql_connpool_data_bytes_total", + "Amount of data (sent|recv) from the backend, excluding metadata.", + metric_tags { + { "traffic_flow", "recv" } + } + ), + std::make_tuple ( + PgSQL_p_hg_dyn_counter::conn_pool_bytes_data_sent, + "proxysql_connpool_data_bytes_total", + "Amount of data (sent|recv) from the backend, excluding metadata.", + metric_tags { + { "traffic_flow", "sent" } + } + ), + // ==================================================================== + + // ==================================================================== + std::make_tuple ( + PgSQL_p_hg_dyn_counter::connection_pool_conn_err, + "proxysql_connpool_conns_total", + "How many connections have been tried to be established.", + metric_tags { + { "status", "err" } + } + ), + std::make_tuple ( + PgSQL_p_hg_dyn_counter::connection_pool_conn_ok, + "proxysql_connpool_conns_total", + "How many connections have been tried to be established.", + metric_tags { + { "status", "ok" } + } + ), + // ==================================================================== + + std::make_tuple ( + PgSQL_p_hg_dyn_counter::connection_pool_queries, + "proxysql_connpool_conns_queries_total", + "The number of queries routed towards this particular backend server.", + metric_tags {} + ), + // gtid + std::make_tuple ( + PgSQL_p_hg_dyn_counter::gtid_executed, + "proxysql_gtid_executed_total", + "Tracks the number of executed gtid per host and port.", + metric_tags {} + ), + // pgsql_error + std::make_tuple ( + PgSQL_p_hg_dyn_counter::proxysql_pgsql_error, + "proxysql_pgsql_error_total", + "Tracks the pgsql errors generated by proxysql.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_dyn_counter::pgsql_error, + "pgsql_error_total", + "Tracks the pgsql errors encountered.", + metric_tags {} + ) + }, + // prometheus dynamic gauges + hg_dyn_gauge_vector { + std::make_tuple ( + PgSQL_p_hg_dyn_gauge::connection_pool_conn_free, + "proxysql_connpool_conns", + "How many backend connections are currently (free|used).", + metric_tags { + { "status", "free" } + } + ), + std::make_tuple ( + PgSQL_p_hg_dyn_gauge::connection_pool_conn_used, + "proxysql_connpool_conns", + "How many backend connections are currently (free|used).", + metric_tags { + { "status", "used" } + } + ), + std::make_tuple ( + PgSQL_p_hg_dyn_gauge::connection_pool_latency_us, + "proxysql_connpool_conns_latency_us", + "The currently ping time in microseconds, as reported from Monitor.", + metric_tags {} + ), + std::make_tuple ( + PgSQL_p_hg_dyn_gauge::connection_pool_status, + "proxysql_connpool_conns_status", + "The status of the backend server (1 - ONLINE, 2 - SHUNNED, 3 - OFFLINE_SOFT, 4 - OFFLINE_HARD).", + metric_tags {} + ) + } +); + +PgSQL_HostGroups_Manager::PgSQL_HostGroups_Manager() { + pthread_mutex_init(&ev_loop_mutex, NULL); + status.client_connections=0; + status.client_connections_aborted=0; + status.client_connections_created=0; + status.server_connections_connected=0; + status.server_connections_aborted=0; + status.server_connections_created=0; + status.server_connections_delayed=0; + status.servers_table_version=0; + pthread_mutex_init(&status.servers_table_version_lock, NULL); + pthread_cond_init(&status.servers_table_version_cond, NULL); + status.pgconnpoll_get=0; + status.pgconnpoll_get_ok=0; + status.pgconnpoll_get_ping=0; + status.pgconnpoll_push=0; + status.pgconnpoll_destroy=0; + status.pgconnpoll_reset=0; + status.autocommit_cnt=0; + status.commit_cnt=0; + status.rollback_cnt=0; + status.autocommit_cnt_filtered=0; + status.commit_cnt_filtered=0; + status.rollback_cnt_filtered=0; + status.backend_reset_connection=0; + //status.backend_init_db=0; + status.backend_set_client_encoding=0; + //status.frontend_init_db=0; + status.frontend_set_client_encoding=0; + //status.frontend_use_db=0; + status.access_denied_wrong_password=0; + status.access_denied_max_connections=0; + status.access_denied_max_user_connections=0; + status.select_for_update_or_equivalent=0; + status.auto_increment_delay_multiplex=0; +#if 0 + pthread_mutex_init(&readonly_mutex, NULL); + pthread_mutex_init(&lock, NULL); + admindb=NULL; // initialized only if needed + mydb=new SQLite3DB(); +#endif // 0 +#ifdef DEBUG + mydb->open((char *)"file:mem_mydb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); +#else + mydb->open((char *)"file:mem_mydb?mode=memory", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); +#endif /* DEBUG */ + mydb->execute(MYHGM_PgSQL_SERVERS); + mydb->execute(MYHGM_PgSQL_SERVERS_INCOMING); + mydb->execute(MYHGM_PgSQL_REPLICATION_HOSTGROUPS); + mydb->execute(MYHGM_PgSQL_HOSTGROUP_ATTRIBUTES); + mydb->execute("CREATE INDEX IF NOT EXISTS idx_pgsql_servers_hostname_port ON pgsql_servers (hostname,port)"); + MyHostGroups=new PtrArray(); + runtime_pgsql_servers=NULL; + incoming_replication_hostgroups=NULL; + incoming_hostgroup_attributes = NULL; + incoming_pgsql_servers_v2 = NULL; + pthread_rwlock_init(>id_rwlock, NULL); + gtid_missing_nodes = false; + gtid_ev_loop=NULL; + gtid_ev_timer=NULL; + gtid_ev_async = (struct ev_async *)malloc(sizeof(struct ev_async)); + pgsql_servers_to_monitor = NULL; + + { + static const char alphanum[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + rand_del[0] = '-'; + for (int i = 1; i < 6; i++) { + rand_del[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; + } + rand_del[6] = '-'; + rand_del[7] = 0; + } + pthread_mutex_init(&pgsql_errors_mutex, NULL); + + // Initialize prometheus metrics + init_prometheus_counter_array(hg_metrics_map, this->status.p_counter_array); + init_prometheus_gauge_array(hg_metrics_map, this->status.p_gauge_array); + init_prometheus_dyn_counter_array(hg_metrics_map, this->status.p_dyn_counter_array); + init_prometheus_dyn_gauge_array(hg_metrics_map, this->status.p_dyn_gauge_array); + + pthread_mutex_init(&pgsql_errors_mutex, NULL); +} + +void PgSQL_HostGroups_Manager::init() { + // gtid initialization; + //GTID_syncer_thread = new std::thread(>ID_syncer_run); + GTID_syncer_thread = nullptr; + + //pthread_create(>ID_syncer_thread_id, NULL, GTID_syncer_run , NULL); +} + +void PgSQL_HostGroups_Manager::shutdown() { + HGCU_thread->join(); + delete HGCU_thread; + ev_async_send(gtid_ev_loop, gtid_ev_async); + GTID_syncer_thread->join(); + delete GTID_syncer_thread; +} + +PgSQL_HostGroups_Manager::~PgSQL_HostGroups_Manager() { + while (MyHostGroups->len) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->remove_index_fast(0); + delete myhgc; + } + delete MyHostGroups; + delete mydb; + if (admindb) { + delete admindb; + } + free(gtid_ev_async); + if (gtid_ev_loop) + ev_loop_destroy(gtid_ev_loop); + if (gtid_ev_timer) + free(gtid_ev_timer); + pthread_mutex_destroy(&lock); +} + +void PgSQL_HostGroups_Manager::p_update_pgsql_error_counter(p_pgsql_error_type err_type, unsigned int hid, char* address, uint16_t port, unsigned int code) { + PgSQL_p_hg_dyn_counter::metric metric = PgSQL_p_hg_dyn_counter::pgsql_error; + if (err_type == p_pgsql_error_type::proxysql) { + metric = PgSQL_p_hg_dyn_counter::proxysql_pgsql_error; + } + + std::string s_hostgroup = std::to_string(hid); + std::string s_address = std::string(address); + std::string s_port = std::to_string(port); + // TODO: Create switch here to classify error codes + std::string s_code = std::to_string(code); + std::string metric_id = s_hostgroup + ":" + address + ":" + s_port + ":" + s_code; + std::map metric_labels { + { "hostgroup", s_hostgroup }, + { "address", address }, + { "port", s_port }, + { "code", s_code } + }; + + pthread_mutex_lock(&pgsql_errors_mutex); + + p_inc_map_counter( + status.p_pgsql_errors_map, + status.p_dyn_counter_array[metric], + metric_id, + metric_labels + ); + + pthread_mutex_unlock(&pgsql_errors_mutex); +} + +void PgSQL_HostGroups_Manager::wait_servers_table_version(unsigned v, unsigned w) { + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + //ts.tv_sec += w; + unsigned int i = 0; + int rc = 0; + pthread_mutex_lock(&status.servers_table_version_lock); + while ((rc == 0 || rc == ETIMEDOUT) && (i < w) && (__sync_fetch_and_add(&glovars.shutdown,0)==0) && (__sync_fetch_and_add(&status.servers_table_version,0) < v)) { + i++; + ts.tv_sec += 1; + rc = pthread_cond_timedwait( &status.servers_table_version_cond, &status.servers_table_version_lock, &ts); + } + pthread_mutex_unlock(&status.servers_table_version_lock); +} + +unsigned int PgSQL_HostGroups_Manager::get_servers_table_version() { + return __sync_fetch_and_add(&status.servers_table_version,0); +} + +// we always assume that the calling thread has acquired a rdlock() +int PgSQL_HostGroups_Manager::servers_add(SQLite3_result *resultset) { + if (resultset==NULL) { + return 0; + } + int rc; + mydb->execute("DELETE FROM pgsql_servers_incoming"); + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + //sqlite3 *mydb3=mydb->get_db(); + char *query1=(char *)"INSERT INTO pgsql_servers_incoming VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + std::string query32s = "INSERT INTO pgsql_servers_incoming VALUES " + generate_multi_rows_query(32,11); + char *query32 = (char *)query32s.c_str(); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = mydb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, mydb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = mydb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, mydb); + MySerStatus status1=MYSQL_SERVER_STATUS_ONLINE; + int row_idx=0; + int max_bulk_row_idx=resultset->rows_count/32; + max_bulk_row_idx=max_bulk_row_idx*32; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + status1=MYSQL_SERVER_STATUS_ONLINE; + if (strcasecmp(r1->fields[3],"ONLINE")) { + if (!strcasecmp(r1->fields[3],"SHUNNED")) { + status1=MYSQL_SERVER_STATUS_SHUNNED; + } else { + if (!strcasecmp(r1->fields[3],"OFFLINE_SOFT")) { + status1=MYSQL_SERVER_STATUS_OFFLINE_SOFT; + } else { + if (!strcasecmp(r1->fields[3],"OFFLINE_HARD")) { + status1=MYSQL_SERVER_STATUS_OFFLINE_HARD; + } + } + } + } + int idx=row_idx%32; + if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+4, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+5, status1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+10, atoi(r1->fields[9])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, mydb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, status1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoi(r1->fields[9])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + return 0; +} + +void PgSQL_HostGroups_Manager::CUCFT1( + SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum +) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + string query = "SELECT * FROM " + TableName + " ORDER BY " + ColumnName; + mydb->execute_statement(query.c_str(), &error , &cols , &affected_rows , &resultset); + if (resultset) { + if (resultset->rows_count) { + if (init == false) { + init = true; + myhash.Init(19,3); + } + uint64_t hash1_ = resultset->raw_checksum(); + raw_checksum = hash1_; + myhash.Update(&hash1_, sizeof(hash1_)); + proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), hash1_); + } + delete resultset; + } else { + proxy_info("Checksum for table %s is 0x%lX\n", TableName.c_str(), (long unsigned int)0); + } +} + +void PgSQL_HostGroups_Manager::commit_update_checksums_from_tables(SpookyHash& myhash, bool& init) { + // Always reset the current table values before recomputing + for (size_t i = 0; i < table_resultset_checksum.size(); i++) { + if (i != HGM_TABLES::PgSQL_SERVERS && i != HGM_TABLES::PgSQL_SERVERS_V2) { + table_resultset_checksum[i] = 0; + } + } + + CUCFT1(myhash,init,"pgsql_replication_hostgroups","writer_hostgroup", table_resultset_checksum[HGM_TABLES::PgSQL_REPLICATION_HOSTGROUPS]); + CUCFT1(myhash,init,"pgsql_hostgroup_attributes","hostgroup_id", table_resultset_checksum[HGM_TABLES::PgSQL_HOSTGROUP_ATTRIBUTES]); +} + +/** + * @brief This code updates the 'hostgroup_server_mapping' table with the most recent pgsql_servers and pgsql_replication_hostgroups + * records while utilizing checksums to prevent unnecessary updates. + * + * IMPORTANT: Make sure wrlock() is called before calling this method. + * +*/ +void PgSQL_HostGroups_Manager::update_hostgroup_manager_mappings() { + + if (hgsm_pgsql_servers_checksum != table_resultset_checksum[HGM_TABLES::PgSQL_SERVERS] || + hgsm_pgsql_replication_hostgroups_checksum != table_resultset_checksum[HGM_TABLES::PgSQL_REPLICATION_HOSTGROUPS]) + { + proxy_info("Rebuilding 'Hostgroup_Manager_Mapping' due to checksums change - pgsql_servers { old: 0x%lX, new: 0x%lX }, pgsql_replication_hostgroups { old:0x%lX, new:0x%lX }\n", + hgsm_pgsql_servers_checksum, table_resultset_checksum[HGM_TABLES::PgSQL_SERVERS], + hgsm_pgsql_replication_hostgroups_checksum, table_resultset_checksum[HGM_TABLES::PgSQL_REPLICATION_HOSTGROUPS]); + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + + hostgroup_server_mapping.clear(); + + const char* query = "SELECT DISTINCT hostname, port, '1' is_writer, status, reader_hostgroup, writer_hostgroup, mem_pointer FROM pgsql_replication_hostgroups JOIN pgsql_servers ON hostgroup_id=writer_hostgroup WHERE status<>3 \ + UNION \ + SELECT DISTINCT hostname, port, '0' is_writer, status, reader_hostgroup, writer_hostgroup, mem_pointer FROM pgsql_replication_hostgroups JOIN pgsql_servers ON hostgroup_id=reader_hostgroup WHERE status<>3 \ + ORDER BY hostname, port"; + + mydb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + + if (resultset && resultset->rows_count) { + std::string fetched_server_id; + HostGroup_Server_Mapping* fetched_server_mapping = NULL; + + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + + const std::string& server_id = std::string(r->fields[0]) + ":::" + r->fields[1]; + + if (fetched_server_mapping == NULL || server_id != fetched_server_id) { + + auto itr = hostgroup_server_mapping.find(server_id); + + if (itr == hostgroup_server_mapping.end()) { + std::unique_ptr server_mapping(new HostGroup_Server_Mapping(this)); + fetched_server_mapping = server_mapping.get(); + hostgroup_server_mapping.insert( std::pair> { + server_id, std::move(server_mapping) + } ); + } else { + fetched_server_mapping = itr->second.get(); + } + + fetched_server_id = server_id; + } + + HostGroup_Server_Mapping::Node node; + //node.server_status = static_cast(atoi(r->fields[3])); + node.reader_hostgroup_id = atoi(r->fields[4]); + node.writer_hostgroup_id = atoi(r->fields[5]); + node.srv = reinterpret_cast(atoll(r->fields[6])); + + HostGroup_Server_Mapping::Type type = (r->fields[2] && r->fields[2][0] == '1') ? HostGroup_Server_Mapping::Type::WRITER : HostGroup_Server_Mapping::Type::READER; + fetched_server_mapping->add(type, node); + } + } + delete resultset; + + hgsm_pgsql_servers_checksum = table_resultset_checksum[HGM_TABLES::PgSQL_SERVERS]; + hgsm_pgsql_replication_hostgroups_checksum = table_resultset_checksum[HGM_TABLES::PgSQL_REPLICATION_HOSTGROUPS]; + } +} + +/** + * @brief Generates a resultset holding the current Admin 'runtime_pgsql_servers' as reported by Admin. + * @details Requires caller to hold the mutex 'PgSQL_HostGroups_Manager::wrlock'. + * @param mydb The db in which to perform the query, typically 'PgSQL_HostGroups_Manager::mydb'. + * @return An SQLite3 resultset for the query 'MYHGM_GEN_ADMIN_RUNTIME_SERVERS'. + */ +unique_ptr get_admin_runtime_pgsql_servers(SQLite3DB* mydb) { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = nullptr; + + mydb->execute_statement(PGHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + + if (error) { + proxy_error("SQLite3 query generating 'runtime_pgsql_servers' resultset failed with error '%s'\n", error); + assert(0); + } + + return unique_ptr(resultset); +} + +/** + * @brief Generates a resultset with holding the current 'pgsql_servers_v2' table. + * @details Requires caller to hold the mutex 'ProxySQL_Admin::mysql_servers_wrlock'. + * @return A resulset holding 'pgsql_servers_v2'. + */ +unique_ptr get_pgsql_servers_v2() { + char* error = nullptr; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = nullptr; + + if (GloAdmin && GloAdmin->admindb) { + GloAdmin->admindb->execute_statement( + PGHGM_GEN_CLUSTER_ADMIN_PGSQL_SERVERS, &error, &cols, &affected_rows, &resultset + ); + } + + return unique_ptr(resultset); +} + +static void update_glovars_checksum_with_peers( + ProxySQL_Checksum_Value& module_checksum, + const string& new_checksum, + const string& peer_checksum_value, + time_t new_epoch, + time_t peer_checksum_epoch, + bool update_version +) { + module_checksum.set_checksum(const_cast(new_checksum.c_str())); + + if (update_version) + module_checksum.version++; + + bool computed_checksum_matches = + peer_checksum_value != "" && module_checksum.checksum == peer_checksum_value; + + if (peer_checksum_epoch != 0 && computed_checksum_matches) { + module_checksum.epoch = peer_checksum_epoch; + } else { + module_checksum.epoch = new_epoch; + } +} + +/** + * @brief Updates the global 'pgsql_servers' module checksum. + * @details If the new computed checksum matches the supplied 'cluster_checksum', the epoch used for the + * checksum is the supplied epoch instead of current time. This way we ensure the preservation of the + * checksum and epoch fetched from the ProxySQL cluster peer node. + * + * IMPORTANT: This function also generates a new 'global_checksum'. This is because everytime + * 'runtime_pgsql_servers' change, updating the global checksum is unconditional. + * @param new_checksum The new computed checksum for 'runtime_pgsql_servers'. + * @param peer_checksum A checksum fetched from another ProxySQL cluster node, holds the checksum value + * and its epoch. Should be empty if no remote checksum is being considered. + * @param epoch The epoch to be preserved in case the supplied 'peer_checksum' matches the new computed + * checksum. + */ +static void update_glovars_pgsql_servers_checksum( + const string& new_checksum, + const runtime_pgsql_servers_checksum_t& peer_checksum = {}, + bool update_version = false +) { + time_t new_epoch = time(NULL); + + update_glovars_checksum_with_peers( + GloVars.checksums_values.pgsql_servers, + new_checksum, + peer_checksum.value, + new_epoch, + peer_checksum.epoch, + update_version + ); + + GloVars.checksums_values.updates_cnt++; + GloVars.generate_global_checksum(); + GloVars.epoch_version = new_epoch; +} + +/** + * @brief Updates the global 'pgsql_servers_v2' module checksum. + * @details Unlike 'update_glovars_pgsql_servers_checksum' this function doesn't generate a new + * 'global_checksum'. It's caller responsibility to ensure that 'global_checksum' is updated. + * @param new_checksum The new computed checksum for 'pgsql_servers_v2'. + * @param peer_checksum A checksum fetched from another ProxySQL cluster node, holds the checksum value + * and its epoch. Should be empty if no remote checksum is being considered. + * @param epoch The epoch to be preserved in case the supplied 'peer_checksum' matches the new computed + * checksum. + */ +static void update_glovars_pgsql_servers_v2_checksum( + const string& new_checksum, + const pgsql_servers_v2_checksum_t& peer_checksum = {}, + bool update_version = false +) { + time_t new_epoch = time(NULL); + + update_glovars_checksum_with_peers( + GloVars.checksums_values.pgsql_servers_v2, + new_checksum, + peer_checksum.value, + new_epoch, + peer_checksum.epoch, + update_version + ); +} + +uint64_t PgSQL_HostGroups_Manager::commit_update_checksum_from_pgsql_servers(SQLite3_result* runtime_pgsql_servers) { + mydb->execute("DELETE FROM pgsql_servers"); + generate_pgsql_servers_table(); + + if (runtime_pgsql_servers == nullptr) { + unique_ptr resultset { get_admin_runtime_pgsql_servers(mydb) }; + save_runtime_pgsql_servers(resultset.release()); + } else { + save_runtime_pgsql_servers(runtime_pgsql_servers); + } + + uint64_t raw_checksum = this->runtime_pgsql_servers ? this->runtime_pgsql_servers->raw_checksum() : 0; + table_resultset_checksum[HGM_TABLES::PgSQL_SERVERS] = raw_checksum; + + return raw_checksum; +} + +uint64_t PgSQL_HostGroups_Manager::commit_update_checksum_from_pgsql_servers_v2(SQLite3_result* pgsql_servers_v2) { + if (pgsql_servers_v2 == nullptr) { + unique_ptr resultset { get_pgsql_servers_v2() }; + save_pgsql_servers_v2(resultset.release()); + } else { + save_pgsql_servers_v2(pgsql_servers_v2); + } + + uint64_t raw_checksum = this->incoming_pgsql_servers_v2 ? this->incoming_pgsql_servers_v2->raw_checksum() : 0; + table_resultset_checksum[HGM_TABLES::PgSQL_SERVERS_V2] = raw_checksum; + + return raw_checksum; +} + +std::string PgSQL_HostGroups_Manager::gen_global_pgsql_servers_v2_checksum(uint64_t servers_v2_hash) { + bool init = false; + SpookyHash global_hash {}; + + if (servers_v2_hash != 0) { + if (init == false) { + init = true; + global_hash.Init(19, 3); + } + + global_hash.Update(&servers_v2_hash, sizeof(servers_v2_hash)); + } + + commit_update_checksums_from_tables(global_hash, init); + + uint64_t hash_1 = 0, hash_2 = 0; + if (init) { + global_hash.Final(&hash_1,&hash_2); + } + + string mysrvs_checksum { get_checksum_from_hash(hash_1) }; + return mysrvs_checksum; +} + +bool PgSQL_HostGroups_Manager::commit( + const peer_runtime_pgsql_servers_t& peer_runtime_pgsql_servers, + const peer_pgsql_servers_v2_t& peer_pgsql_servers_v2, + bool only_commit_runtime_pgsql_servers, + bool update_version +) { + // if only_commit_runtime_pgsql_servers is true, pgsql_servers_v2 resultset will not be entertained and will cause memory leak. + if (only_commit_runtime_pgsql_servers) { + proxy_info("Generating runtime pgsql servers records only.\n"); + } else { + proxy_info("Generating runtime pgsql servers and pgsql servers v2 records.\n"); + } + + unsigned long long curtime1=monotonic_time(); + wrlock(); + // purge table + purge_pgsql_servers_table(); + + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM pgsql_servers\n"); + mydb->execute("DELETE FROM pgsql_servers"); + generate_pgsql_servers_table(); + + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (GloMTH->variables.hostgroup_manager_verbose) { + mydb->execute_statement((char *)"SELECT * FROM pgsql_servers_incoming", &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on read from pgsql_servers_incoming : %s\n", error); + } else { + if (resultset) { + proxy_info("Dumping pgsql_servers_incoming\n"); + resultset->dump_to_stderr(); + } + } + if (resultset) { delete resultset; resultset=NULL; } + } + char *query=NULL; + query=(char *)"SELECT mem_pointer, t1.hostgroup_id, t1.hostname, t1.port FROM pgsql_servers t1 LEFT OUTER JOIN pgsql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE t2.hostgroup_id IS NULL"; + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Dumping pgsql_servers LEFT JOIN pgsql_servers_incoming\n"); + resultset->dump_to_stderr(); + } + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + long long ptr=atoll(r->fields[0]); + proxy_warning("Removed server at address %lld, hostgroup %s, address %s port %s. Setting status OFFLINE HARD and immediately dropping all free connections. Used connections will be dropped when trying to use them\n", ptr, r->fields[1], r->fields[2], r->fields[3]); + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)ptr; + mysrvc->status=MYSQL_SERVER_STATUS_OFFLINE_HARD; + mysrvc->ConnectionsFree->drop_all_connections(); + char *q1=(char *)"DELETE FROM pgsql_servers WHERE mem_pointer=%lld"; + char *q2=(char *)malloc(strlen(q1)+32); + sprintf(q2,q1,ptr); + mydb->execute(q2); + free(q2); + } + } + if (resultset) { delete resultset; resultset=NULL; } + + // This seems unnecessary. Removed as part of issue #829 + //mydb->execute("DELETE FROM pgsql_servers"); + //generate_pgsql_servers_table(); + + mydb->execute("INSERT OR IGNORE INTO pgsql_servers(hostgroup_id, hostname, port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM pgsql_servers_incoming"); + + // SELECT FROM pgsql_servers whatever is not identical in pgsql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) + query=(char *)"SELECT t1.*, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms, t2.comment FROM pgsql_servers t1 JOIN pgsql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms or t1.comment<>t2.comment"; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Dumping pgsql_servers JOIN pgsql_servers_incoming\n"); + resultset->dump_to_stderr(); + } + // optimization #829 + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement2=NULL; + //sqlite3 *mydb3=mydb->get_db(); + char *query1=(char *)"UPDATE pgsql_servers SET mem_pointer = ?1 WHERE hostgroup_id = ?2 AND hostname = ?3 AND port = ?4"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = mydb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, mydb); + char *query2=(char *)"UPDATE pgsql_servers SET weight = ?1 , status = ?2 , compression = ?3 , max_connections = ?4 , max_replication_lag = ?5 , use_ssl = ?6 , max_latency_ms = ?7 , comment = ?8 WHERE hostgroup_id = ?9 AND hostname = ?10 AND port = ?11"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query2, -1, &statement2, 0); + rc = mydb->prepare_v2(query2, &statement2); + ASSERT_SQLITE_OK(rc, mydb); + + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + long long ptr=atoll(r->fields[11]); // increase this index every time a new column is added + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d , weight=%d, status=%d, mem_pointer=%llu, hostgroup=%d, compression=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), ptr, atoi(r->fields[0]), atoi(r->fields[5])); + //fprintf(stderr,"%lld\n", ptr); + if (ptr==0) { + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Creating new server in HG %d : %s:%d , weight=%d, status=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4])); + } + PgSQL_SrvC *mysrvc=new PgSQL_SrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), r->fields[10]); // add new fields here if adding more columns in pgsql_servers + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), mysrvc, atoi(r->fields[0])); + add(mysrvc,atoi(r->fields[0])); + ptr=(uintptr_t)mysrvc; + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, ptr); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r->fields[0])); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r->fields[2])); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); + } else { + bool run_update=false; + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)ptr; + // carefully increase the 2nd index by 1 for every new column added + + if (atoi(r->fields[3])!=atoi(r->fields[12])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %d:%s:%d (%s:%d) from %d (%ld) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]) , mysrvc->weight , atoi(r->fields[12])); + mysrvc->weight=atoi(r->fields[12]); + } + if (atoi(r->fields[4])!=atoi(r->fields[13])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing status for server %d:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]) , mysrvc->status , atoi(r->fields[13])); + mysrvc->status=(MySerStatus)atoi(r->fields[13]); + if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED) { + mysrvc->shunned_automatic=false; + } + } + if (atoi(r->fields[5])!=atoi(r->fields[14])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing compression for server %d:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[5]) , mysrvc->compression , atoi(r->fields[14])); + mysrvc->compression=atoi(r->fields[14]); + } + if (atoi(r->fields[6])!=atoi(r->fields[15])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_connections for server %d:%s:%d (%s:%d) from %d (%ld) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[6]) , mysrvc->max_connections , atoi(r->fields[15])); + mysrvc->max_connections=atoi(r->fields[15]); + } + if (atoi(r->fields[7])!=atoi(r->fields[16])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_replication_lag for server %u:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[7]) , mysrvc->max_replication_lag , atoi(r->fields[16])); + mysrvc->max_replication_lag=atoi(r->fields[16]); + if (mysrvc->max_replication_lag == 0) { // we just changed it to 0 + if (mysrvc->status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { + // the server is currently shunned due to replication lag + // but we reset max_replication_lag to 0 + // therefore we immediately reset the status too + mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; + } + } + } + if (atoi(r->fields[8])!=atoi(r->fields[17])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing use_ssl for server %d:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[8]) , mysrvc->use_ssl , atoi(r->fields[17])); + mysrvc->use_ssl=atoi(r->fields[17]); + } + if (atoi(r->fields[9])!=atoi(r->fields[18])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing max_latency_ms for server %d:%s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), atoi(r->fields[9]) , mysrvc->max_latency_us/1000 , atoi(r->fields[18])); + mysrvc->max_latency_us=1000*atoi(r->fields[18]); + } + if (strcmp(r->fields[10],r->fields[19])) { + if (GloMTH->variables.hostgroup_manager_verbose) + proxy_info("Changing comment for server %d:%s:%d (%s:%d) from '%s' to '%s'\n" , mysrvc->myhgc->hid , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[10], r->fields[19]); + free(mysrvc->comment); + mysrvc->comment=strdup(r->fields[19]); + } + if (run_update) { + rc=(*proxy_sqlite3_bind_int64)(statement2, 1, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 2, mysrvc->status); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 3, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 4, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 5, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 6, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 7, mysrvc->max_latency_us/1000); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement2, 8, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 9, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement2, 10, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 11, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement2); + rc=(*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, mydb); + } + } + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement2); + } + if (resultset) { delete resultset; resultset=NULL; } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM pgsql_servers_incoming\n"); + mydb->execute("DELETE FROM pgsql_servers_incoming"); + + string global_checksum_v2 {}; + if (only_commit_runtime_pgsql_servers == false) { + // replication + if (incoming_replication_hostgroups) { // this IF is extremely important, otherwise replication hostgroups may disappear + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM pgsql_replication_hostgroups\n"); + mydb->execute("DELETE FROM pgsql_replication_hostgroups"); + generate_pgsql_replication_hostgroups_table(); + } + + // hostgroup attributes + if (incoming_hostgroup_attributes) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM pgsql_hostgroup_attributes\n"); + mydb->execute("DELETE FROM pgsql_hostgroup_attributes"); + generate_pgsql_hostgroup_attributes_table(); + } + + uint64_t new_hash = commit_update_checksum_from_pgsql_servers_v2(peer_pgsql_servers_v2.resultset); + + { + const string new_checksum { get_checksum_from_hash(new_hash) }; + proxy_info("Checksum for table %s is %s\n", "pgsql_servers_v2", new_checksum.c_str()); + } + + global_checksum_v2 = gen_global_pgsql_servers_v2_checksum(new_hash); + proxy_info("New computed global checksum for 'pgsql_servers_v2' is '%s'\n", global_checksum_v2.c_str()); + } + + // Update 'pgsql_servers' and global checksums + { + uint64_t new_hash = commit_update_checksum_from_pgsql_servers(peer_runtime_pgsql_servers.resultset); + const string new_checksum { get_checksum_from_hash(new_hash) }; + proxy_info("Checksum for table %s is %s\n", "pgsql_servers", new_checksum.c_str()); + + pthread_mutex_lock(&GloVars.checksum_mutex); + if (only_commit_runtime_pgsql_servers == false) { + update_glovars_pgsql_servers_v2_checksum(global_checksum_v2, peer_pgsql_servers_v2.checksum, true); + } + update_glovars_pgsql_servers_checksum(new_checksum, peer_runtime_pgsql_servers.checksum, update_version); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + + // fill Hostgroup_Manager_Mapping with latest records + update_hostgroup_manager_mappings(); + + //ev_async_send(gtid_ev_loop, gtid_ev_async); + + __sync_fetch_and_add(&status.servers_table_version,1); + + // We completely reset read_only_set1. It will generated (completely) again in read_only_action() + // Note: read_only_set1 will be regenerated all at once + read_only_set1.erase(read_only_set1.begin(), read_only_set1.end()); + // We completely reset read_only_set2. It will be again written in read_only_action() + // Note: read_only_set2 will be regenerated one server at the time + read_only_set2.erase(read_only_set2.begin(), read_only_set2.end()); + + this->status.p_counter_array[PgSQL_p_hg_counter::servers_table_version]->Increment(); + pthread_cond_broadcast(&status.servers_table_version_cond); + pthread_mutex_unlock(&status.servers_table_version_lock); + + // NOTE: In order to guarantee the latest generated version, this should be kept after all the + // calls to 'generate_pgsql_servers'. + update_table_pgsql_servers_for_monitor(false); + + wrunlock(); + unsigned long long curtime2=monotonic_time(); + curtime1 = curtime1/1000; + curtime2 = curtime2/1000; + proxy_info("PgSQL_HostGroups_Manager::commit() locked for %llums\n", curtime2-curtime1); + + if (GloMTH) { + GloMTH->signal_all_threads(1); + } + + return true; +} + +/** + * @brief Calculate the checksum for the runtime pgsql_servers record, after excluding all the rows + * with the status OFFLINE_HARD from the result set + * + * @details The runtime pgsql_servers is now considered as a distinct module and have a separate checksum calculation. + * This is because the records in the runtime module may differ from those in the admin pgsql_servers module, which + * can cause synchronization issues within the cluster. + * + * @param runtime_pgsql_servers resultset of runtime pgsql_servers or can be a nullptr. +*/ +uint64_t PgSQL_HostGroups_Manager::get_pgsql_servers_checksum(SQLite3_result* runtime_pgsql_servers) { + + //Note: GloVars.checksum_mutex needs to be locked + SQLite3_result* resultset = nullptr; + + if (runtime_pgsql_servers == nullptr) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + + mydb->execute_statement(PGHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS, &error, &cols, &affected_rows, &resultset); + + if (resultset) { + save_runtime_pgsql_servers(resultset); + } else { + proxy_info("Checksum for table %s is 0x%lX\n", "pgsql_servers", (long unsigned int)0); + } + } else { + resultset = runtime_pgsql_servers; + save_runtime_pgsql_servers(runtime_pgsql_servers); + } + + table_resultset_checksum[HGM_TABLES::PgSQL_SERVERS] = resultset != nullptr ? resultset->raw_checksum() : 0; + proxy_info("Checksum for table %s is 0x%lX\n", "pgsql_servers", table_resultset_checksum[HGM_TABLES::PgSQL_SERVERS]); + + return table_resultset_checksum[HGM_TABLES::PgSQL_SERVERS]; +} + +void PgSQL_HostGroups_Manager::purge_pgsql_servers_table() { + for (unsigned int i=0; ilen; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + PgSQL_SrvC *mysrvc=NULL; + for (unsigned int j=0; jmysrvs->servers->len; j++) { + mysrvc=myhgc->mysrvs->idx(j); + if (mysrvc->status==MYSQL_SERVER_STATUS_OFFLINE_HARD) { + if (mysrvc->ConnectionsUsed->conns_length()==0 && mysrvc->ConnectionsFree->conns_length()==0) { + // no more connections for OFFLINE_HARD server, removing it + mysrvc=(PgSQL_SrvC *)myhgc->mysrvs->servers->remove_index_fast(j); + j--; + delete mysrvc; + } + } + } + } +} + + + +void PgSQL_HostGroups_Manager::generate_pgsql_servers_table(int *_onlyhg) { + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + + PtrArray *lst=new PtrArray(); + //sqlite3 *mydb3=mydb->get_db(); + char *query1=(char *)"INSERT INTO pgsql_servers VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = mydb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, mydb); + std::string query32s = "INSERT INTO pgsql_servers VALUES " + generate_multi_rows_query(32,12); + char *query32 = (char *)query32s.c_str(); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = mydb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, mydb); + + if (pgsql_thread___hostgroup_manager_verbose) { + if (_onlyhg==NULL) { + proxy_info("Dumping current PgSQL Servers structures for hostgroup ALL\n"); + } else { + int hidonly=*_onlyhg; + proxy_info("Dumping current PgSQL Servers structures for hostgroup %d\n", hidonly); + } + } + for (unsigned int i=0; ilen; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + if (_onlyhg) { + int hidonly=*_onlyhg; + if (myhgc->hid!=(unsigned int)hidonly) { + // skipping this HG + continue; + } + } + PgSQL_SrvC *mysrvc=NULL; + for (unsigned int j=0; jmysrvs->servers->len; j++) { + mysrvc=myhgc->mysrvs->idx(j); + if (pgsql_thread___hostgroup_manager_verbose) { + char *st; + switch (mysrvc->status) { + case 0: + st=(char *)"ONLINE"; + break; + case 2: + st=(char *)"OFFLINE_SOFT"; + break; + case 3: + st=(char *)"OFFLINE_HARD"; + break; + default: + case 1: + case 4: + st=(char *)"SHUNNED"; + break; + } + fprintf(stderr,"HID: %d , address: %s , port: %d , weight: %ld , status: %s , max_connections: %ld , max_replication_lag: %u , use_ssl: %u , max_latency_ms: %u , comment: %s\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->weight, st, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us*1000, mysrvc->comment); + } + lst->add(mysrvc); + if (lst->len==32) { + while (lst->len) { + int i=lst->len; + i--; + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)lst->remove_index_fast(0); + uintptr_t ptr=(uintptr_t)mysrvc; + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+1, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement32, (i*12)+2, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+3, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+4, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+5, mysrvc->status); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+6, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+7, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+8, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+9, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+10, mysrvc->max_latency_us/1000); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement32, (i*12)+11, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (i*12)+12, ptr); ASSERT_SQLITE_OK(rc, mydb); + } + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, mydb); + } + } + } + while (lst->len) { + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)lst->remove_index_fast(0); + uintptr_t ptr=(uintptr_t)mysrvc; + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, mysrvc->myhgc->hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, mysrvc->address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, mysrvc->port); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, mysrvc->weight); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, mysrvc->status); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, mysrvc->compression); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, mysrvc->max_connections); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, mysrvc->max_replication_lag); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, mysrvc->use_ssl); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, mysrvc->max_latency_us/1000); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement1, 11, mysrvc->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 12, ptr); ASSERT_SQLITE_OK(rc, mydb); + + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, mydb); + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + if (pgsql_thread___hostgroup_manager_verbose) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + if (_onlyhg==NULL) { + mydb->execute_statement((char *)"SELECT hostgroup_id hid, hostname, port, weight, status, compression cmp, max_connections max_conns, max_replication_lag max_lag, use_ssl ssl, max_latency_ms max_lat, comment, mem_pointer FROM pgsql_servers", &error , &cols , &affected_rows , &resultset); + } else { + int hidonly=*_onlyhg; + char *q1 = (char *)malloc(256); + sprintf(q1,"SELECT hostgroup_id hid, hostname, port, weight, status, compression cmp, max_connections max_conns, max_replication_lag max_lag, use_ssl ssl, max_latency_ms max_lat, comment, mem_pointer FROM pgsql_servers WHERE hostgroup_id=%d" , hidonly); + mydb->execute_statement(q1, &error , &cols , &affected_rows , &resultset); + free(q1); + } + if (error) { + proxy_error("Error on read from pgsql_servers : %s\n", error); + } else { + if (resultset) { + if (_onlyhg==NULL) { + proxy_info("Dumping pgsql_servers: ALL\n"); + } else { + int hidonly=*_onlyhg; + proxy_info("Dumping pgsql_servers: HG %d\n", hidonly); + } + resultset->dump_to_stderr(); + } + } + if (resultset) { delete resultset; resultset=NULL; } + } + delete lst; +} + +void PgSQL_HostGroups_Manager::generate_pgsql_replication_hostgroups_table() { + if (incoming_replication_hostgroups==NULL) + return; + if (pgsql_thread___hostgroup_manager_verbose) { + proxy_info("New pgsql_replication_hostgroups table\n"); + } + for (std::vector::iterator it = incoming_replication_hostgroups->rows.begin() ; it != incoming_replication_hostgroups->rows.end(); ++it) { + SQLite3_row *r=*it; + char *o=NULL; + int comment_length=0; // #issue #643 + //if (r->fields[3]) { // comment is not null + o=escape_string_single_quotes(r->fields[3],false); + comment_length=strlen(o); + //} + char *query=(char *)malloc(256+comment_length); + //if (r->fields[3]) { // comment is not null + sprintf(query,"INSERT INTO pgsql_replication_hostgroups VALUES(%s,%s,'%s','%s')",r->fields[0], r->fields[1], r->fields[2], o); + if (o!=r->fields[3]) { // there was a copy + free(o); + } + //} else { + //sprintf(query,"INSERT INTO pgsql_replication_hostgroups VALUES(%s,%s,NULL)",r->fields[0],r->fields[1]); + //} + mydb->execute(query); + if (pgsql_thread___hostgroup_manager_verbose) { + fprintf(stderr,"writer_hostgroup: %s , reader_hostgroup: %s, check_type %s, comment: %s\n", r->fields[0],r->fields[1], r->fields[2], r->fields[3]); + } + free(query); + } + incoming_replication_hostgroups=NULL; +} + +void PgSQL_HostGroups_Manager::update_table_pgsql_servers_for_monitor(bool lock) { + if (lock) { + wrlock(); + } + + std::lock_guard pgsql_servers_lock(this->pgsql_servers_to_monitor_mutex); + + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + char* query = const_cast("SELECT hostname, port, status, use_ssl FROM pgsql_servers WHERE status != 3 GROUP BY hostname, port"); + + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + + if (error != nullptr) { + proxy_error("Error on read from pgsql_servers : %s\n", error); + } else { + if (resultset != nullptr) { + delete this->pgsql_servers_to_monitor; + this->pgsql_servers_to_monitor = resultset; + } + } + + if (lock) { + wrunlock(); + } + + MySQL_Monitor::trigger_dns_cache_update(); +} + +SQLite3_result * PgSQL_HostGroups_Manager::dump_table_pgsql(const string& name) { + char * query = (char *)""; + if (name == "pgsql_replication_hostgroups") { + query=(char *)"SELECT writer_hostgroup, reader_hostgroup, check_type, comment FROM pgsql_replication_hostgroups"; + } else if (name == "pgsql_hostgroup_attributes") { + query=(char *)"SELECT hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, hostgroup_settings, servers_defaults, comment FROM pgsql_hostgroup_attributes ORDER BY hostgroup_id"; + } else if (name == "pgsql_servers") { + query = (char *)PGHGM_GEN_ADMIN_RUNTIME_SERVERS; + } else if (name == "cluster_pgsql_servers") { + query = (char *)PGHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS; + } else { + assert(0); + } + wrlock(); + if (name == "pgsql_servers") { + purge_pgsql_servers_table(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM pgsql_servers\n"); + mydb->execute("DELETE FROM pgsql_servers"); + generate_pgsql_servers_table(); + } + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + wrunlock(); + return resultset; +} + +void PgSQL_HostGroups_Manager::increase_reset_counter() { + wrlock(); + status.pgconnpoll_reset++; + wrunlock(); +} +void PgSQL_HostGroups_Manager::push_MyConn_to_pool(PgSQL_Connection *c, bool _lock) { + assert(c->parent); + PgSQL_SrvC *mysrvc=NULL; + if (_lock) + wrlock(); + c->auto_increment_delay_token = 0; + status.pgconnpoll_push++; + mysrvc=(PgSQL_SrvC *)c->parent; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PgSQL_Connection %p, server %s:%d with status %d\n", c, mysrvc->address, mysrvc->port, mysrvc->status); + mysrvc->ConnectionsUsed->remove(c); + if (GloMTH == NULL) { goto __exit_push_MyConn_to_pool; } + if (c->largest_query_length > (unsigned int)GloMTH->variables.threshold_query_length) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying PgSQL_Connection %p, server %s:%d with status %d . largest_query_length = %lu\n", c, mysrvc->address, mysrvc->port, mysrvc->status, c->largest_query_length); + delete c; + goto __exit_push_MyConn_to_pool; + } + if (mysrvc->status==MYSQL_SERVER_STATUS_ONLINE) { + if (c->async_state_machine==ASYNC_IDLE) { + if (GloMTH == NULL) { goto __exit_push_MyConn_to_pool; } + if (c->local_stmts->get_num_backend_stmts() > (unsigned int)GloMTH->variables.max_stmts_per_connection) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying PgSQL_Connection %p, server %s:%d with status %d because has too many prepared statements\n", c, mysrvc->address, mysrvc->port, mysrvc->status); +// delete c; + mysrvc->ConnectionsUsed->add(c); // Add the connection back to the list of used connections + destroy_MyConn_from_pool(c, false); // Destroy the connection from the pool + } else { + c->optimize(); + mysrvc->ConnectionsFree->add(c); + } + } else { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying PgSQL_Connection %p, server %s:%d with status %d\n", c, mysrvc->address, mysrvc->port, mysrvc->status); + delete c; + } + } else { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying PgSQL_Connection %p, server %s:%d with status %d\n", c, mysrvc->address, mysrvc->port, mysrvc->status); + delete c; + } +__exit_push_MyConn_to_pool: + if (_lock) + wrunlock(); +} + +void PgSQL_HostGroups_Manager::push_MyConn_to_pool_array(PgSQL_Connection **ca, unsigned int cnt) { + unsigned int i=0; + PgSQL_Connection *c=NULL; + c=ca[i]; + wrlock(); + while (icnt(); + static time_t last_hg_log = 0; +#ifdef TEST_AURORA + unsigned long long a1 = array_mysrvc_total/10000; + array_mysrvc_total += l; + unsigned long long a2 = array_mysrvc_total/10000; + if (a2 > a1) { + fprintf(stderr, "Total: %llu, Candidates: %llu\n", array_mysrvc_total-l, array_mysrvc_cands); + } +#endif // TEST_AURORA + PgSQL_SrvC *mysrvcCandidates_static[32]; + PgSQL_SrvC **mysrvcCandidates = mysrvcCandidates_static; + unsigned int num_candidates = 0; + bool max_connections_reached = false; + if (l>32) { + mysrvcCandidates = (PgSQL_SrvC **)malloc(sizeof(PgSQL_SrvC *)*l); + } + if (l) { + //int j=0; + for (j=0; jidx(j); + if (mysrvc->status==MYSQL_SERVER_STATUS_ONLINE) { // consider this server only if ONLINE + if (mysrvc->ConnectionsUsed->conns_length() < mysrvc->max_connections) { // consider this server only if didn't reach max_connections + if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : pgsql_thread___default_max_latency_ms *1000 ) ) { // consider the host only if not too far + if (gtid_trxid) { +#if 0 + if (PgHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } +#endif // 0 + } else { + if (max_lag_ms >= 0) { + if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } else { + sess->thread->status_variables.stvar[st_var_aws_aurora_replicas_skipped_during_query]++; + } + } else { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } + } + } else { + max_connections_reached = true; + } + } else { + if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED) { + // try to recover shunned servers + if (mysrvc->shunned_automatic && pgsql_thread___shun_recovery_time_sec) { + time_t t; + t=time(NULL); + // we do all these changes without locking . We assume the server is not used from long + // even if the server is still in used and any of the follow command fails it is not critical + // because this is only an attempt to recover a server that is probably dead anyway + + // the next few lines of code try to solve issue #530 + int max_wait_sec = (pgsql_thread___shun_recovery_time_sec * 1000 >= pgsql_thread___connect_timeout_server_max ? pgsql_thread___connect_timeout_server_max /1000 - 1 : pgsql_thread___shun_recovery_time_sec); + if (max_wait_sec < 1) { // min wait time should be at least 1 second + max_wait_sec = 1; + } + if (t > mysrvc->time_last_detected_error && (t - mysrvc->time_last_detected_error) > max_wait_sec) { + if ( + (mysrvc->shunned_and_kill_all_connections==false) // it is safe to bring it back online + || + (mysrvc->shunned_and_kill_all_connections==true && mysrvc->ConnectionsUsed->conns_length()==0 && mysrvc->ConnectionsFree->conns_length()==0) // if shunned_and_kill_all_connections is set, ensure all connections are already dropped + ) { +#ifdef DEBUG + if (GloMTH->variables.hostgroup_manager_verbose >= 3) { + proxy_info("Unshunning server %s:%d.\n", mysrvc->address, mysrvc->port); + } +#endif + mysrvc->status=MYSQL_SERVER_STATUS_ONLINE; + mysrvc->shunned_automatic=false; + mysrvc->shunned_and_kill_all_connections=false; + mysrvc->connect_ERR_at_time_last_detected_error=0; + mysrvc->time_last_detected_error=0; + // note: the following function scans all the hostgroups. + // This is ok for now because we only have a global mutex. + // If one day we implement a mutex per hostgroup (unlikely, + // but possible), this must be taken into consideration + if (pgsql_thread___unshun_algorithm == 1) { + PgHGM->unshun_server_all_hostgroups(mysrvc->address, mysrvc->port, t, max_wait_sec, &mysrvc->myhgc->hid); + } + // if a server is taken back online, consider it immediately + if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : pgsql_thread___default_max_latency_ms *1000 ) ) { // consider the host only if not too far + if (gtid_trxid) { +#if 0 + if (PgHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } +#endif // 0 + } else { + if (max_lag_ms >= 0) { + if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } else { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } + } + } + } + } + } + } + } + if (max_lag_ms > 0) { // we are using AWS Aurora, as this logic is implemented only here + unsigned int min_num_replicas = sess->thread->variables.aurora_max_lag_ms_only_read_from_replicas; + if (min_num_replicas) { + if (num_candidates >= min_num_replicas) { // there are at least N replicas + // we try to remove the writer + unsigned int total_aws_aurora_current_lag_us=0; + for (j=0; jaws_aurora_current_lag_us; + } + if (total_aws_aurora_current_lag_us) { // we are just double checking that we don't have all servers with aws_aurora_current_lag_us==0 + for (j=0; jaws_aurora_current_lag_us==0) { + sum-=mysrvc->weight; + TotalUsedConn-=mysrvc->ConnectionsUsed->conns_length(); + if (j < num_candidates-1) { + mysrvcCandidates[j]=mysrvcCandidates[num_candidates-1]; + } + num_candidates--; + } + } + } + } + } + } + if (sum==0) { + // per issue #531 , we try a desperate attempt to bring back online any shunned server + // we do this lowering the maximum wait time to 10% + // most of the follow code is copied from few lines above + time_t t; + t=time(NULL); + int max_wait_sec = (pgsql_thread___shun_recovery_time_sec * 1000 >= pgsql_thread___connect_timeout_server_max ? pgsql_thread___connect_timeout_server_max /10000 - 1 : pgsql_thread___shun_recovery_time_sec /10 ); + if (max_wait_sec < 1) { // min wait time should be at least 1 second + max_wait_sec = 1; + } + if (t - last_hg_log > 1) { // log this at most once per second to avoid spamming the logs + last_hg_log = time(NULL); + + if (gtid_trxid) { + proxy_error("Hostgroup %u has no servers ready for GTID '%s:%ld'. Waiting for replication...\n", hid, gtid_uuid, gtid_trxid); + } else { + proxy_error("Hostgroup %u has no servers available%s! Checking servers shunned for more than %u second%s\n", hid, + (max_connections_reached ? " or max_connections reached for all servers" : ""), max_wait_sec, max_wait_sec == 1 ? "" : "s"); + } + } + for (j=0; jidx(j); + if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED && mysrvc->shunned_automatic==true) { + if ((t - mysrvc->time_last_detected_error) > max_wait_sec) { + mysrvc->status=MYSQL_SERVER_STATUS_ONLINE; + mysrvc->shunned_automatic=false; + mysrvc->connect_ERR_at_time_last_detected_error=0; + mysrvc->time_last_detected_error=0; + // if a server is taken back online, consider it immediately + if ( mysrvc->current_latency_us < ( mysrvc->max_latency_us ? mysrvc->max_latency_us : pgsql_thread___default_max_latency_ms *1000 ) ) { // consider the host only if not too far + if (gtid_trxid) { +#if 0 + if (PgHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid)) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } +#endif // 0 + } else { + if (max_lag_ms >= 0) { + if ((unsigned int)max_lag_ms >= mysrvc->aws_aurora_current_lag_us/1000) { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } else { + sum+=mysrvc->weight; + TotalUsedConn+=mysrvc->ConnectionsUsed->conns_length(); + mysrvcCandidates[num_candidates]=mysrvc; + num_candidates++; + } + } + } + } + } + } + } + if (sum==0) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PgSQL_SrvC NULL because no backend ONLINE or with weight\n"); + if (l>32) { + free(mysrvcCandidates); + } +#ifdef TEST_AURORA + array_mysrvc_cands += num_candidates; +#endif // TEST_AURORA + return NULL; // if we reach here, we couldn't find any target + } + +/* + unsigned int New_sum=0; + unsigned int New_TotalUsedConn=0; + // we will now scan again to ignore overloaded servers + for (j=0; jConnectionsUsed->conns_length(); + if ((len * sum) <= (TotalUsedConn * mysrvc->weight * 1.5 + 1)) { + + New_sum+=mysrvc->weight; + New_TotalUsedConn+=len; + } else { + // remove the candidate + if (j+1 < num_candidates) { + mysrvcCandidates[j] = mysrvcCandidates[num_candidates-1]; + } + j--; + num_candidates--; + } + } +*/ + + unsigned int New_sum=sum; + + if (New_sum==0) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PgSQL_SrvC NULL because no backend ONLINE or with weight\n"); + if (l>32) { + free(mysrvcCandidates); + } +#ifdef TEST_AURORA + array_mysrvc_cands += num_candidates; +#endif // TEST_AURORA + return NULL; // if we reach here, we couldn't find any target + } + + // latency awareness algorithm is enabled only when compiled with USE_MYSRVC_ARRAY + if (sess && sess->thread->variables.min_num_servers_lantency_awareness) { + if ((int) num_candidates >= sess->thread->variables.min_num_servers_lantency_awareness) { + unsigned int servers_with_latency = 0; + unsigned int total_latency_us = 0; + // scan and verify that all servers have some latency + for (j=0; jcurrent_latency_us) { + servers_with_latency++; + total_latency_us += mysrvc->current_latency_us; + } + } + if (servers_with_latency == num_candidates) { + // all servers have some latency. + // That is good. If any server have no latency, something is wrong + // and we will skip this algorithm + sess->thread->status_variables.stvar[st_var_ConnPool_get_conn_latency_awareness]++; + unsigned int avg_latency_us = 0; + avg_latency_us = total_latency_us/num_candidates; + for (j=0; jcurrent_latency_us > avg_latency_us) { + // remove the candidate + if (j+1 < num_candidates) { + mysrvcCandidates[j] = mysrvcCandidates[num_candidates-1]; + } + j--; + num_candidates--; + } + } + // we scan again to adjust weight + New_sum = 0; + for (j=0; jweight; + } + } + } + } + + + unsigned int k; + if (New_sum > 32768) { + k=rand()%New_sum; + } else { + k=fastrand()%New_sum; + } + k++; + New_sum=0; + + for (j=0; jweight; + if (k<=New_sum) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PgSQL_SrvC %p, server %s:%d\n", mysrvc, mysrvc->address, mysrvc->port); + if (l>32) { + free(mysrvcCandidates); + } +#ifdef TEST_AURORA + array_mysrvc_cands += num_candidates; +#endif // TEST_AURORA + return mysrvc; + } + } + } else { + time_t t = time(NULL); + + if (t - last_hg_log > 1) { + last_hg_log = time(NULL); + proxy_error("Hostgroup %u has no servers available!\n", hid); + } + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PgSQL_SrvC NULL\n"); + if (l>32) { + free(mysrvcCandidates); + } +#ifdef TEST_AURORA + array_mysrvc_cands += num_candidates; +#endif // TEST_AURORA + return NULL; // if we reach here, we couldn't find any target +} + +//unsigned int PgSQL_SrvList::cnt() { +// return servers->len; +//} + +//PgSQL_SrvC * PgSQL_SrvList::idx(unsigned int i) { return (PgSQL_SrvC *)servers->index(i); } + +void PgSQL_SrvConnList::get_random_MyConn_inner_search(unsigned int start, unsigned int end, unsigned int& conn_found_idx, unsigned int& connection_quality_level, unsigned int& number_of_matching_session_variables, const PgSQL_Connection * client_conn) { + PgSQL_Connection * conn=NULL; + unsigned int k; + for (k = start; k < end; k++) { + conn = (PgSQL_Connection *)conns->index(k); + if (conn->has_same_connection_options(client_conn)) { + if (connection_quality_level == 0) { + // this is our best candidate so far + connection_quality_level = 1; + conn_found_idx = k; + } + if (conn->requires_RESETTING_CONNECTION(client_conn)==false) { + if (connection_quality_level == 1) { + // this is our best candidate so far + connection_quality_level = 2; + conn_found_idx = k; + } + unsigned int cnt_match = 0; // number of matching session variables + unsigned int not_match = 0; // number of not matching session variables + cnt_match = conn->number_of_matching_session_variables(client_conn, not_match); + + if (not_match==0) { + // it seems we found the perfect connection + number_of_matching_session_variables = cnt_match; + connection_quality_level = 3; + conn_found_idx = k; + return; // exit immediately, we found the perfect connection + } else { + // we didn't find the perfect connection + // but maybe is better than what we have so far? + if (cnt_match > number_of_matching_session_variables) { + // this is our best candidate so far + number_of_matching_session_variables = cnt_match; + conn_found_idx = k; + } + } + } else { + /*if (connection_quality_level == 1) { + int rca = pgsql_thread___reset_connection_algorithm; + if (rca==1) { + int ql = GloMTH->variables.connpoll_reset_queue_length; + if (ql==0) { + // if: + // pgsql-reset_connection_algorithm=1 and + // pgsql-connpoll_reset_queue_length=0 + // we will not return a connection with connection_quality_level == 1 + // because we want to run COM_CHANGE_USER + // This change was introduced to work around Galera bug + // https://github.com/codership/galera/issues/613 + connection_quality_level = 0; + } + } + } + */ + } + } + } +} + +PgSQL_Connection * PgSQL_SrvConnList::get_random_MyConn(PgSQL_Session *sess, bool ff) { + PgSQL_Connection * conn=NULL; + unsigned int i; + unsigned int conn_found_idx; + unsigned int l=conns_length(); + unsigned int connection_quality_level = 0; + bool needs_warming = false; + // connection_quality_level: + // 0 : not found any good connection, tracked options are not OK + // 1 : tracked options are OK , but RESETTING SESSION is required + // 2 : tracked options are OK , RESETTING SESSION is not required, but some SET statement or INIT_DB needs to be executed + // 3 : tracked options are OK , RESETTING SESSION is not required, and it seems that SET statements or INIT_DB ARE not required + unsigned int number_of_matching_session_variables = 0; // this includes session variables AND schema + bool connection_warming = pgsql_thread___connection_warming; + int free_connections_pct = pgsql_thread___free_connections_pct; + if (mysrvc->myhgc->attributes.configured == true) { + // pgsql_hostgroup_attributes takes priority + connection_warming = mysrvc->myhgc->attributes.connection_warming; + free_connections_pct = mysrvc->myhgc->attributes.free_connections_pct; + } + if (connection_warming == true) { + unsigned int total_connections = mysrvc->ConnectionsFree->conns_length()+mysrvc->ConnectionsUsed->conns_length(); + unsigned int expected_warm_connections = free_connections_pct*mysrvc->max_connections/100; + if (total_connections < expected_warm_connections) { + needs_warming = true; + } + } + if (l && ff==false && needs_warming==false) { + if (l>32768) { + i=rand()%l; + } else { + i=fastrand()%l; + } + if (sess && sess->client_myds && sess->client_myds->myconn && sess->client_myds->myconn->userinfo) { + PgSQL_Connection * client_conn = sess->client_myds->myconn; + get_random_MyConn_inner_search(i, l, conn_found_idx, connection_quality_level, number_of_matching_session_variables, client_conn); + if (connection_quality_level !=3 ) { // we didn't find the perfect connection + get_random_MyConn_inner_search(0, i, conn_found_idx, connection_quality_level, number_of_matching_session_variables, client_conn); + } + // connection_quality_level: + // 1 : tracked options are OK , but RESETTING SESSION is required + // 2 : tracked options are OK , RESETTING SESSION is not required, but some SET statement or INIT_DB needs to be executed + switch (connection_quality_level) { + case 0: // not found any good connection, tracked options are not OK + // we must check if connections need to be freed before + // creating a new connection + { + unsigned int conns_free = mysrvc->ConnectionsFree->conns_length(); + unsigned int conns_used = mysrvc->ConnectionsUsed->conns_length(); + unsigned int pct_max_connections = (3 * mysrvc->max_connections) / 4; + unsigned int connections_to_free = 0; + + if (conns_free >= 1) { + // connection cleanup is triggered when connections exceed 3/4 of the total + // allowed max connections, this cleanup ensures that at least *one connection* + // will be freed. + if (pct_max_connections <= (conns_free + conns_used)) { + connections_to_free = (conns_free + conns_used) - pct_max_connections; + if (connections_to_free == 0) connections_to_free = 1; + } + + while (conns_free && connections_to_free) { + PgSQL_Connection* conn = mysrvc->ConnectionsFree->remove(0); + delete conn; + + conns_free = mysrvc->ConnectionsFree->conns_length(); + connections_to_free -= 1; + } + } + + // we must create a new connection + conn = new PgSQL_Connection(); + conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + __sync_fetch_and_add(&PgHGM->status.server_connections_created, 1); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PostgreSQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); + } + break; + case 1: //tracked options are OK , but RESETTING SESSION is required + // we may consider creating a new connection + { + unsigned int conns_free = mysrvc->ConnectionsFree->conns_length(); + unsigned int conns_used = mysrvc->ConnectionsUsed->conns_length(); + if ((conns_used > conns_free) && (mysrvc->max_connections > (conns_free/2 + conns_used/2)) ) { + conn = new PgSQL_Connection(); + conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + __sync_fetch_and_add(&PgHGM->status.server_connections_created, 1); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PostgreSQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); + } else { + conn=(PgSQL_Connection *)conns->remove_index_fast(conn_found_idx); + } + } + break; + case 2: // tracked options are OK , RESETTING SESSION is not required, but some SET statement or INIT_DB needs to be executed + case 3: // tracked options are OK , RESETTING SESSION is not required, and it seems that SET statements or INIT_DB ARE not required + // here we return the best connection we have, no matter if connection_quality_level is 2 or 3 + conn=(PgSQL_Connection *)conns->remove_index_fast(conn_found_idx); + break; + default: // this should never happen + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + } else { + conn=(PgSQL_Connection *)conns->remove_index_fast(i); + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PostgreSQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); + return conn; + } else { + unsigned long long curtime = monotonic_time(); + curtime = curtime / 1000 / 1000; // convert to second + PgSQL_HGC *_myhgc = mysrvc->myhgc; + if (curtime > _myhgc->current_time_now) { + _myhgc->current_time_now = curtime; + _myhgc->new_connections_now = 0; + } + _myhgc->new_connections_now++; + unsigned int throttle_connections_per_sec_to_hostgroup = (unsigned int) pgsql_thread___throttle_connections_per_sec_to_hostgroup; + if (_myhgc->attributes.configured == true) { + // pgsql_hostgroup_attributes takes priority + throttle_connections_per_sec_to_hostgroup = _myhgc->attributes.throttle_connections_per_sec; + } + if (_myhgc->new_connections_now > (unsigned int) throttle_connections_per_sec_to_hostgroup) { + __sync_fetch_and_add(&PgHGM->status.server_connections_delayed, 1); + return NULL; + } else { + conn = new PgSQL_Connection(); + conn->parent=mysrvc; + // if attributes.multiplex == true , STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG is set to false. And vice-versa + conn->set_status(!conn->parent->myhgc->attributes.multiplex, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + __sync_fetch_and_add(&PgHGM->status.server_connections_created, 1); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning PostgreSQL Connection %p, server %s:%d\n", conn, conn->parent->address, conn->parent->port); + return conn; + } + } + return NULL; // never reach here +} + +void PgSQL_HostGroups_Manager::unshun_server_all_hostgroups(const char * address, uint16_t port, time_t t, int max_wait_sec, unsigned int *skip_hid) { + // we scan all hostgroups looking for a specific server to unshun + // if skip_hid is not NULL , the specific hostgroup is skipped + if (GloMTH->variables.hostgroup_manager_verbose >= 3) { + char buf[64]; + if (skip_hid == NULL) { + sprintf(buf,"NULL"); + } else { + sprintf(buf,"%u", *skip_hid); + } + proxy_info("Calling unshun_server_all_hostgroups() for server %s:%d . Arguments: %lu , %d , %s\n" , address, port, t, max_wait_sec, buf); + } + int i, j; + for (i=0; i<(int)MyHostGroups->len; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + if (skip_hid != NULL && myhgc->hid == *skip_hid) { + // if skip_hid is not NULL, we skip that specific hostgroup + continue; + } + bool found = false; // was this server already found in this hostgroup? + for (j=0; found==false && j<(int)myhgc->mysrvs->cnt(); j++) { + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)myhgc->mysrvs->servers->index(j); + if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED) { + // we only care for SHUNNED nodes + // Note that we check for address and port only for status==MYSQL_SERVER_STATUS_SHUNNED , + // that means that potentially we will pass by the matching node and still looping . + // This is potentially an optimization because we only check status and do not perform any strcmp() + if (strcmp(mysrvc->address,address)==0 && mysrvc->port==port) { + // we found the server in this hostgroup + // no need to process more servers in the same hostgroup + found = true; + if (t > mysrvc->time_last_detected_error && (t - mysrvc->time_last_detected_error) > max_wait_sec) { + if ( + (mysrvc->shunned_and_kill_all_connections==false) // it is safe to bring it back online + || + (mysrvc->shunned_and_kill_all_connections==true && mysrvc->ConnectionsUsed->conns_length()==0 && mysrvc->ConnectionsFree->conns_length()==0) // if shunned_and_kill_all_connections is set, ensure all connections are already dropped + ) { + if (GloMTH->variables.hostgroup_manager_verbose >= 3) { + proxy_info("Unshunning server %d:%s:%d . time_last_detected_error=%lu\n", mysrvc->myhgc->hid, address, port, mysrvc->time_last_detected_error); + } + mysrvc->status=MYSQL_SERVER_STATUS_ONLINE; + mysrvc->shunned_automatic=false; + mysrvc->shunned_and_kill_all_connections=false; + mysrvc->connect_ERR_at_time_last_detected_error=0; + mysrvc->time_last_detected_error=0; + } + } + } + } + } + } +} + +PgSQL_Connection * PgSQL_HostGroups_Manager::get_MyConn_from_pool(unsigned int _hid, PgSQL_Session *sess, bool ff, char * gtid_uuid, uint64_t gtid_trxid, int max_lag_ms) { + PgSQL_Connection * conn=NULL; + wrlock(); + status.pgconnpoll_get++; + PgSQL_HGC *myhgc=MyHGC_lookup(_hid); + PgSQL_SrvC *mysrvc = NULL; +#ifdef TEST_AURORA + for (int i=0; i<10; i++) +#endif // TEST_AURORA + mysrvc = myhgc->get_random_MySrvC(gtid_uuid, gtid_trxid, max_lag_ms, sess); + if (mysrvc) { // a PgSQL_SrvC exists. If not, we return NULL = no targets + conn=mysrvc->ConnectionsFree->get_random_MyConn(sess, ff); + if (conn) { + mysrvc->ConnectionsUsed->add(conn); + status.pgconnpoll_get_ok++; + mysrvc->update_max_connections_used(); + } + } + wrunlock(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning MySQL Connection %p, server %s:%d\n", conn, (conn ? conn->parent->address : "") , (conn ? conn->parent->port : 0 )); + return conn; +} + +void PgSQL_HostGroups_Manager::destroy_MyConn_from_pool(PgSQL_Connection *c, bool _lock) { + bool to_del=true; // the default, legacy behavior + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)c->parent; + if (mysrvc->status==MYSQL_SERVER_STATUS_ONLINE && c->send_quit) { + if (c->async_state_machine!=ASYNC_IDLE) { + // the connection seems health, but we are trying to destroy it + // probably because there is a long running query + // therefore we will try to kill the connection + + /* KILL BACKEND CONNECTION IS NOT IMPLEMENTED YET + if (pgsql_thread___kill_backend_connection_when_disconnect) { + + int myerr = mysql_errno(c->pgsql); + switch (myerr) { + case 1231: + break; + default: + if (c->pgsql->thread_id) { + PgSQL_Connection_userinfo *ui=c->userinfo; + char *auth_password=NULL; + if (ui->password) { + if (ui->password[0]=='*') { // we don't have the real password, let's pass sha1 + auth_password=ui->sha1_pass; + } else { + auth_password=ui->password; + } + } + KillArgs *ka = new KillArgs(ui->username, auth_password, c->parent->address, c->parent->port, c->parent->myhgc->hid, c->pgsql->thread_id, KILL_CONNECTION, c->parent->use_ssl, NULL, c->connected_host_details.ip); + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + pthread_attr_setstacksize (&attr, 256*1024); + pthread_t pt; + if (pthread_create(&pt, &attr, &kill_query_thread, ka) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + break; + } + }*/ + } + } + if (to_del) { + // we lock only this part of the code because we need to remove the connection from ConnectionsUsed + if (_lock) { + wrlock(); + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Destroying PgSQL_Connection %p, server %s:%d Error %s\n", c, mysrvc->address, mysrvc->port, + c->get_error_code_with_message().c_str()); + mysrvc->ConnectionsUsed->remove(c); + status.pgconnpoll_destroy++; + if (_lock) { + wrunlock(); + } + delete c; + } +} + +inline double get_prometheus_counter_val( + std::map& counter_map, const std::string& endpoint_id +) { + const auto& counter_entry = counter_map.find(endpoint_id); + double current_val = 0; + + if (counter_entry != counter_map.end()) { + current_val = counter_entry->second->Value(); + } + + return current_val; +} + +void reset_hg_attrs_server_defaults(PgSQL_SrvC* mysrvc) { + mysrvc->weight = -1; + mysrvc->max_connections = -1; + mysrvc->use_ssl = -1; +} + +void update_hg_attrs_server_defaults(PgSQL_SrvC* mysrvc, PgSQL_HGC* myhgc) { + if (mysrvc->weight == -1) { + if (myhgc->servers_defaults.weight != -1) { + mysrvc->weight = myhgc->servers_defaults.weight; + } else { + // Same harcoded default as in 'CREATE TABLE pgsql_servers ...' + mysrvc->weight = 1; + } + } + if (mysrvc->max_connections == -1) { + if (myhgc->servers_defaults.max_connections != -1) { + mysrvc->max_connections = myhgc->servers_defaults.max_connections; + } else { + // Same harcoded default as in 'CREATE TABLE pgsql_servers ...' + mysrvc->max_connections = 1000; + } + } + if (mysrvc->use_ssl == -1) { + if (myhgc->servers_defaults.use_ssl != -1) { + mysrvc->use_ssl = myhgc->servers_defaults.use_ssl; + } else { + // Same harcoded default as in 'CREATE TABLE pgsql_servers ...' + mysrvc->use_ssl = 0; + } + } +} + +void PgSQL_HostGroups_Manager::add(PgSQL_SrvC *mysrvc, unsigned int _hid) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Adding PgSQL_SrvC %p (%s:%d) for hostgroup %d\n", mysrvc, mysrvc->address, mysrvc->port, _hid); + + // Since metrics for servers are stored per-endpoint; the metrics for a particular endpoint can live longer than the + // 'PgSQL_SrvC' itself. For example, a failover or a server config change could remove the server from a particular + // hostgroup, and a subsequent one bring it back to the original hostgroup. For this reason, everytime a 'mysrvc' is + // created and added to a particular hostgroup, we update the endpoint metrics for it. + std::string endpoint_id { std::to_string(_hid) + ":" + string { mysrvc->address } + ":" + std::to_string(mysrvc->port) }; + + mysrvc->bytes_recv = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_recv_map, endpoint_id); + mysrvc->bytes_sent = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_sent_map, endpoint_id); + mysrvc->connect_ERR = get_prometheus_counter_val(this->status.p_connection_pool_conn_err_map, endpoint_id); + mysrvc->connect_OK = get_prometheus_counter_val(this->status.p_connection_pool_conn_ok_map, endpoint_id); + mysrvc->queries_sent = get_prometheus_counter_val(this->status.p_connection_pool_queries_map, endpoint_id); + + PgSQL_HGC *myhgc=MyHGC_lookup(_hid); + update_hg_attrs_server_defaults(mysrvc, myhgc); + myhgc->mysrvs->add(mysrvc); +} + +void PgSQL_HostGroups_Manager::replication_lag_action_inner(PgSQL_HGC *myhgc, const char *address, unsigned int port, int current_replication_lag) { + int j; + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)myhgc->mysrvs->servers->index(j); + if (strcmp(mysrvc->address,address)==0 && mysrvc->port==port) { + if (mysrvc->status==MYSQL_SERVER_STATUS_ONLINE) { + if ( +// (current_replication_lag==-1 ) +// || + ( + current_replication_lag>=0 && + mysrvc->max_replication_lag > 0 && // see issue #4018 + ((unsigned int)current_replication_lag > mysrvc->max_replication_lag) + ) + ) { + // always increase the counter + mysrvc->cur_replication_lag_count += 1; + if (mysrvc->cur_replication_lag_count >= (unsigned int)mysql_thread___monitor_replication_lag_count) { + proxy_warning("Shunning server %s:%d from HG %u with replication lag of %d second, count number: '%d'\n", address, port, myhgc->hid, current_replication_lag, mysrvc->cur_replication_lag_count); + mysrvc->status=MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG; + } else { + proxy_info( + "Not shunning server %s:%d from HG %u with replication lag of %d second, count number: '%d' < replication_lag_count: '%d'\n", + address, + port, + myhgc->hid, + current_replication_lag, + mysrvc->cur_replication_lag_count, + mysql_thread___monitor_replication_lag_count + ); + } + } else { + mysrvc->cur_replication_lag_count = 0; + } + } else { + if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { + if ( + (current_replication_lag>=0 && ((unsigned int)current_replication_lag <= mysrvc->max_replication_lag)) + || + (current_replication_lag==-2) // see issue 959 + ) { + mysrvc->status=MYSQL_SERVER_STATUS_ONLINE; + proxy_warning("Re-enabling server %s:%d from HG %u with replication lag of %d second\n", address, port, myhgc->hid, current_replication_lag); + mysrvc->cur_replication_lag_count = 0; + } + } + } + return; + } + } +} + +void PgSQL_HostGroups_Manager::replication_lag_action(const std::list& pgsql_servers) { + + //this method does not use admin table, so this lock is not needed. + //GloAdmin->mysql_servers_wrlock(); + unsigned long long curtime1 = monotonic_time(); + wrlock(); + + for (const auto& server : pgsql_servers) { + + const int hid = std::get(server); + const std::string& address = std::get(server); + const unsigned int port = std::get(server); + const int current_replication_lag = std::get(server); + + if (mysql_thread___monitor_replication_lag_group_by_host == false) { + // legacy check. 1 check per server per hostgroup + PgSQL_HGC *myhgc = MyHGC_find(hid); + replication_lag_action_inner(myhgc,address.c_str(),port,current_replication_lag); + } + else { + // only 1 check per server, no matter the hostgroup + // all hostgroups must be searched + for (unsigned int i=0; ilen; i++) { + PgSQL_HGC*myhgc=(PgSQL_HGC*)MyHostGroups->index(i); + replication_lag_action_inner(myhgc,address.c_str(),port,current_replication_lag); + } + } + } + + wrunlock(); + //GloAdmin->mysql_servers_wrunlock(); + + unsigned long long curtime2 = monotonic_time(); + curtime1 = curtime1 / 1000; + curtime2 = curtime2 / 1000; + proxy_debug(PROXY_DEBUG_MONITOR, 7, "PgSQL_HostGroups_Manager::replication_lag_action() locked for %llums (server count:%ld)\n", curtime2 - curtime1, pgsql_servers.size()); +} + +void PgSQL_HostGroups_Manager::drop_all_idle_connections() { + // NOTE: the caller should hold wrlock + int i, j; + for (i=0; i<(int)MyHostGroups->len; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)myhgc->mysrvs->servers->index(j); + if (mysrvc->status!=MYSQL_SERVER_STATUS_ONLINE) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d is not online\n", mysrvc->address, mysrvc->port); + //__sync_fetch_and_sub(&status.server_connections_connected, mysrvc->ConnectionsFree->conns->len); + mysrvc->ConnectionsFree->drop_all_connections(); + } + + // Drop idle connections if beyond max_connection + while (mysrvc->ConnectionsFree->conns_length() && mysrvc->ConnectionsUsed->conns_length()+mysrvc->ConnectionsFree->conns_length() > mysrvc->max_connections) { + PgSQL_Connection *conn=mysrvc->ConnectionsFree->remove(0); + delete conn; + } + + //PtrArray *pa=mysrvc->ConnectionsFree->conns; + PgSQL_SrvConnList *mscl=mysrvc->ConnectionsFree; + int free_connections_pct = pgsql_thread___free_connections_pct; + if (mysrvc->myhgc->attributes.configured == true) { + // pgsql_hostgroup_attributes takes priority + free_connections_pct = mysrvc->myhgc->attributes.free_connections_pct; + } + while (mscl->conns_length() > free_connections_pct*mysrvc->max_connections/100) { + PgSQL_Connection *mc=mscl->remove(0); + delete mc; + } + + // drop all connections with life exceeding pgsql-connection_max_age + if (pgsql_thread___connection_max_age_ms) { + unsigned long long curtime=monotonic_time(); + int i=0; + for (i=0; i<(int)mscl->conns_length() ; i++) { + PgSQL_Connection *mc=mscl->index(i); + unsigned long long intv = pgsql_thread___connection_max_age_ms; + intv *= 1000; + if (curtime > mc->creation_time + intv) { + mc=mscl->remove(0); + delete mc; + i--; + } + } + } + + } + } +} + +/* + * Prepares at most num_conn idle connections in the given hostgroup for + * pinging. When -1 is passed as a hostgroup, all hostgroups are examined. + * + * The resulting idle connections are returned in conn_list. Note that not all + * currently idle connections will be returned (some might be purged). + * + * Connections are purged according to 2 criteria: + * - whenever the maximal number of connections for a server is hit, free + * connections will be purged + * - also, idle connections that cause the number of free connections to rise + * above a certain percentage of the maximal number of connections will be + * dropped as well + */ +int PgSQL_HostGroups_Manager::get_multiple_idle_connections(int _hid, unsigned long long _max_last_time_used, PgSQL_Connection **conn_list, int num_conn) { + wrlock(); + drop_all_idle_connections(); + int num_conn_current=0; + int j,k; + PgSQL_HGC* myhgc = NULL; + // Multimap holding the required info for accesing the oldest idle connections found. + std::multimap> oldest_idle_connections {}; + + for (int i=0; i<(int)MyHostGroups->len; i++) { + if (_hid == -1) { + // all hostgroups must be examined + // as of version 2.3.2 , this is always the case + myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + } else { + // only one hostgroup is examined + // as of version 2.3.2 , this never happen + // but the code support this functionality + myhgc = MyHGC_find(_hid); + i = (int)MyHostGroups->len; // to exit from this "for" loop + if (myhgc == NULL) + continue; // immediately exit + } + if (_hid >= 0 && _hid!=(int)myhgc->hid) continue; + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)myhgc->mysrvs->servers->index(j); + //PtrArray *pa=mysrvc->ConnectionsFree->conns; + PgSQL_SrvConnList *mscl=mysrvc->ConnectionsFree; + for (k=0; k<(int)mscl->conns_length(); k++) { + PgSQL_Connection *mc=mscl->index(k); + // If the connection is idle ... + if (mc->last_time_used && mc->last_time_used < _max_last_time_used) { + if ((int)oldest_idle_connections.size() < num_conn) { + oldest_idle_connections.insert({mc->last_time_used, { mysrvc, k }}); + } else if (num_conn != 0) { + auto last_elem_it = std::prev(oldest_idle_connections.end()); + + if (mc->last_time_used < last_elem_it->first) { + oldest_idle_connections.erase(last_elem_it); + oldest_idle_connections.insert({mc->last_time_used, { mysrvc, k }}); + } + } + } + } + } + } + + // In order to extract the found connections, the following actions must be performed: + // + // 1. Filter the found connections by 'PgSQL_SrvC'. + // 2. Order by indexes on 'ConnectionsFree' in desc order. + // 3. Move the conns from 'ConnectionsFree' into 'ConnectionsUsed'. + std::unordered_map> mysrvcs_conns_idxs {}; + + // 1. Filter the connections by 'PgSQL_SrvC'. + // + // We extract this for being able to later iterate through the obtained 'PgSQL_SrvC' using the conn indexes. + for (const auto& conn_info : oldest_idle_connections) { + PgSQL_SrvC* mysrvc = conn_info.second.first; + int32_t mc_idx = conn_info.second.second; + auto mysrcv_it = mysrvcs_conns_idxs.find(mysrvc); + + if (mysrcv_it == mysrvcs_conns_idxs.end()) { + mysrvcs_conns_idxs.insert({ mysrvc, { mc_idx }}); + } else { + mysrcv_it->second.push_back(mc_idx); + } + } + + // 2. Order by indexes on FreeConns in desc order. + // + // Since the conns are stored in 'ConnectionsFree', which holds the conns in a 'PtrArray', and we plan + // to remove multiple connections using the pre-stored indexes. We need to reorder the indexes in 'desc' + // order, otherwise we could be trashing the array while consuming it. See 'PtrArray::remove_index_fast'. + for (auto& mysrvc_conns_idxs : mysrvcs_conns_idxs) { + std::sort(std::begin(mysrvc_conns_idxs.second), std::end(mysrvc_conns_idxs.second), std::greater()); + } + + // 3. Move the conns from 'ConnectionsFree' into 'ConnectionsUsed'. + for (auto& conn_info : mysrvcs_conns_idxs) { + PgSQL_SrvC* mysrvc = conn_info.first; + + for (const int conn_idx : conn_info.second) { + PgSQL_SrvConnList* mscl = mysrvc->ConnectionsFree; + PgSQL_Connection* mc = mscl->remove(conn_idx); + mysrvc->ConnectionsUsed->add(mc); + + conn_list[num_conn_current] = mc; + num_conn_current++; + + // Left here as a safeguard + if (num_conn_current >= num_conn) { + goto __exit_get_multiple_idle_connections; + } + } + } + +__exit_get_multiple_idle_connections: + status.pgconnpoll_get_ping+=num_conn_current; + wrunlock(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Returning %d idle connections\n", num_conn_current); + return num_conn_current; +} + +void PgSQL_HostGroups_Manager::save_incoming_pgsql_table(SQLite3_result *s, const string& name) { + SQLite3_result ** inc = NULL; + if (name == "pgsql_replication_hostgroups") { + inc = &incoming_replication_hostgroups; + } else if (name == "pgsql_hostgroup_attributes") { + inc = &incoming_hostgroup_attributes; + } else { + assert(0); + } + if (*inc != nullptr) { + delete *inc; + *inc = nullptr; + } + *inc = s; +} + +void PgSQL_HostGroups_Manager::save_runtime_pgsql_servers(SQLite3_result *s) { + if (runtime_pgsql_servers) { + delete runtime_pgsql_servers; + runtime_pgsql_servers = nullptr; + } + runtime_pgsql_servers=s; +} + +void PgSQL_HostGroups_Manager::save_pgsql_servers_v2(SQLite3_result* s) { + if (incoming_pgsql_servers_v2) { + delete incoming_pgsql_servers_v2; + incoming_pgsql_servers_v2 = nullptr; + } + incoming_pgsql_servers_v2 = s; +} + +SQLite3_result* PgSQL_HostGroups_Manager::get_current_pgsql_table(const string& name) { + if (name == "pgsql_replication_hostgroups") { + return this->incoming_replication_hostgroups; + } else if (name == "pgsql_hostgroup_attributes") { + return this->incoming_hostgroup_attributes; + } else if (name == "cluster_pgsql_servers") { + return this->runtime_pgsql_servers; + } else if (name == "pgsql_servers_v2") { + return this->incoming_pgsql_servers_v2; + } else { + assert(0); + } + return NULL; +} + + + +SQLite3_result * PgSQL_HostGroups_Manager::SQL3_Free_Connections() { + const int colnum=12; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Free Connections in Pool\n"); + SQLite3_result *result=new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"fd"); + result->add_column_definition(SQLITE_TEXT,"hostgroup"); + result->add_column_definition(SQLITE_TEXT,"srv_host"); + result->add_column_definition(SQLITE_TEXT,"srv_port"); + result->add_column_definition(SQLITE_TEXT,"user"); + result->add_column_definition(SQLITE_TEXT,"dbname"); + result->add_column_definition(SQLITE_TEXT,"init_connect"); + result->add_column_definition(SQLITE_TEXT,"time_zone"); + result->add_column_definition(SQLITE_TEXT,"sql_mode"); + //result->add_column_definition(SQLITE_TEXT,"autocommit"); + result->add_column_definition(SQLITE_TEXT,"idle_ms"); + result->add_column_definition(SQLITE_TEXT,"statistics"); + result->add_column_definition(SQLITE_TEXT,"pgsql_info"); + unsigned long long curtime = monotonic_time(); + wrlock(); + int i,j, k, l; + for (i=0; i<(int)MyHostGroups->len; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)myhgc->mysrvs->servers->index(j); + if (mysrvc->status!=MYSQL_SERVER_STATUS_ONLINE) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d is not online\n", mysrvc->address, mysrvc->port); + mysrvc->ConnectionsFree->drop_all_connections(); + } + // drop idle connections if beyond max_connection + while (mysrvc->ConnectionsFree->conns_length() && mysrvc->ConnectionsUsed->conns_length()+mysrvc->ConnectionsFree->conns_length() > mysrvc->max_connections) { + //PgSQL_Connection *conn=(PgSQL_Connection *)mysrvc->ConnectionsFree->conns->remove_index_fast(0); + PgSQL_Connection *conn=mysrvc->ConnectionsFree->remove(0); + delete conn; + } + char buf[1024]; + for (l=0; l < (int) mysrvc->ConnectionsFree->conns_length(); l++) { + char **pta=(char **)malloc(sizeof(char *)*colnum); + PgSQL_Connection *conn = mysrvc->ConnectionsFree->index(l); + sprintf(buf,"%d", conn->fd); + pta[0]=strdup(buf); + sprintf(buf,"%d", (int)myhgc->hid); + pta[1]=strdup(buf); + pta[2]=strdup(mysrvc->address); + sprintf(buf,"%d", mysrvc->port); + pta[3]=strdup(buf); + pta[4] = strdup(conn->userinfo->username); + pta[5] = strdup(conn->userinfo->dbname); + pta[6] = NULL; + if (conn->options.init_connect) { + pta[6] = strdup(conn->options.init_connect); + } + pta[7] = NULL; + /*if (conn->variables[SQL_TIME_ZONE].value) { + pta[7] = strdup(conn->variables[SQL_TIME_ZONE].value); + }*/ + pta[8] = NULL; + /*if (conn->variables[SQL_SQL_MODE].value) { + pta[8] = strdup(conn->variables[SQL_SQL_MODE].value); + }*/ + //sprintf(buf,"%d", conn->options.autocommit); + //pta[9]=strdup(buf); + sprintf(buf,"%llu", (curtime-conn->last_time_used)/1000); + pta[9]=strdup(buf); + { + json j; + char buff[32]; + sprintf(buff,"%p",conn); + j["address"] = buff; + uint64_t age_ms = (curtime - conn->creation_time)/1000; + j["age_ms"] = age_ms; + j["bytes_recv"] = conn->bytes_info.bytes_recv; + j["bytes_sent"] = conn->bytes_info.bytes_sent; + j["pgconnpoll_get"] = conn->statuses.pgconnpoll_get; + j["pgconnpoll_put"] = conn->statuses.pgconnpoll_put; + j["questions"] = conn->statuses.questions; + const string s = j.dump(); + pta[10] = strdup(s.c_str()); + } + { + json j; + char buff[32]; + sprintf(buff, "%p", conn->get_pg_connection()); + j["address"] = buff; + j["host"] = conn->get_pg_host(); + j["host_addr"] = conn->get_pg_hostaddr(); + j["port"] = conn->get_pg_port(); + j["user"] = conn->get_pg_user(); + j["database"] = conn->get_pg_dbname(); + j["backend_pid"] = conn->get_pg_backend_pid(); + j["using_ssl"] = conn->get_pg_ssl_in_use() ? "YES" : "NO"; + j["error_msg"] = conn->get_pg_error_message(); + j["options"] = conn->get_pg_options(); + j["fd"] = conn->get_pg_socket_fd(); + j["protocol_version"] = conn->get_pg_protocol_version(); + j["server_version"] = conn->get_pg_server_version_str(buff, sizeof(buff)); + j["transaction_status"] = conn->get_pg_transaction_status_str(); + j["connection_status"] = conn->get_pg_connection_status_str(); + j["client_encoding"] = conn->get_pg_client_encoding(); + j["is_nonblocking"] = conn->get_pg_is_nonblocking() ? "YES" : "NO"; + const string s = j.dump(); + pta[11] = strdup(s.c_str()); + } + result->add_row(pta); + for (k=0; k& labels, std::map& m_map, unsigned long long value, PgSQL_p_hg_dyn_counter::metric idx +) { + const auto& counter_id = m_map.find(endpoint_id); + if (counter_id != m_map.end()) { + const auto& cur_val = counter_id->second->Value(); + counter_id->second->Increment(value - cur_val); + } else { + auto& new_counter = status.p_dyn_counter_array[idx]; + m_map.insert( + { + endpoint_id, + std::addressof(new_counter->Add(labels)) + } + ); + } +} + +void PgSQL_HostGroups_Manager::p_update_connection_pool_update_gauge( + const std::string& endpoint_id, const std::map& labels, + std::map& m_map, unsigned long long value, PgSQL_p_hg_dyn_gauge::metric idx +) { + const auto& counter_id = m_map.find(endpoint_id); + if (counter_id != m_map.end()) { + counter_id->second->Set(value); + } else { + auto& new_counter = status.p_dyn_gauge_array[idx]; + m_map.insert( + { + endpoint_id, + std::addressof(new_counter->Add(labels)) + } + ); + } +} + +void PgSQL_HostGroups_Manager::p_update_connection_pool() { + std::vector cur_servers_ids {}; + wrlock(); + for (int i = 0; i < static_cast(MyHostGroups->len); i++) { + PgSQL_HGC *myhgc = static_cast(MyHostGroups->index(i)); + for (int j = 0; j < static_cast(myhgc->mysrvs->cnt()); j++) { + PgSQL_SrvC *mysrvc = static_cast(myhgc->mysrvs->servers->index(j)); + std::string endpoint_addr = mysrvc->address; + std::string endpoint_port = std::to_string(mysrvc->port); + std::string hostgroup_id = std::to_string(myhgc->hid); + std::string endpoint_id = hostgroup_id + ":" + endpoint_addr + ":" + endpoint_port; + const std::map common_labels { + {"endpoint", endpoint_addr + ":" + endpoint_port}, + {"hostgroup", hostgroup_id } + }; + cur_servers_ids.push_back(endpoint_id); + + // proxysql_connection_pool_bytes_data_recv metric + std::map recv_pool_bytes_labels = common_labels; + recv_pool_bytes_labels.insert({"traffic_flow", "recv"}); + p_update_connection_pool_update_counter(endpoint_id, recv_pool_bytes_labels, + status.p_conn_pool_bytes_data_recv_map, mysrvc->bytes_recv, PgSQL_p_hg_dyn_counter::conn_pool_bytes_data_recv); + + // proxysql_connection_pool_bytes_data_sent metric + std::map sent_pool_bytes_labels = common_labels; + sent_pool_bytes_labels.insert({"traffic_flow", "sent"}); + p_update_connection_pool_update_counter(endpoint_id, sent_pool_bytes_labels, + status.p_conn_pool_bytes_data_sent_map, mysrvc->bytes_sent, PgSQL_p_hg_dyn_counter::conn_pool_bytes_data_sent); + + // proxysql_connection_pool_conn_err metric + std::map pool_conn_err_labels = common_labels; + pool_conn_err_labels.insert({"status", "err"}); + p_update_connection_pool_update_counter(endpoint_id, pool_conn_err_labels, + status.p_connection_pool_conn_err_map, mysrvc->connect_ERR, PgSQL_p_hg_dyn_counter::connection_pool_conn_err); + + // proxysql_connection_pool_conn_ok metric + std::map pool_conn_ok_labels = common_labels; + pool_conn_ok_labels.insert({"status", "ok"}); + p_update_connection_pool_update_counter(endpoint_id, pool_conn_ok_labels, + status.p_connection_pool_conn_ok_map, mysrvc->connect_OK, PgSQL_p_hg_dyn_counter::connection_pool_conn_ok); + + // proxysql_connection_pool_conn_free metric + std::map pool_conn_free_labels = common_labels; + pool_conn_free_labels.insert({"status", "free"}); + p_update_connection_pool_update_gauge(endpoint_id, pool_conn_free_labels, + status.p_connection_pool_conn_free_map, mysrvc->ConnectionsFree->conns_length(), PgSQL_p_hg_dyn_gauge::connection_pool_conn_free); + + // proxysql_connection_pool_conn_used metric + std::map pool_conn_used_labels = common_labels; + pool_conn_used_labels.insert({"status", "used"}); + p_update_connection_pool_update_gauge(endpoint_id, pool_conn_used_labels, + status.p_connection_pool_conn_used_map, mysrvc->ConnectionsUsed->conns_length(), PgSQL_p_hg_dyn_gauge::connection_pool_conn_used); + + // proxysql_connection_pool_latency_us metric + p_update_connection_pool_update_gauge(endpoint_id, common_labels, + status.p_connection_pool_latency_us_map, mysrvc->current_latency_us, PgSQL_p_hg_dyn_gauge::connection_pool_latency_us); + + // proxysql_connection_pool_queries metric + p_update_connection_pool_update_counter(endpoint_id, common_labels, + status.p_connection_pool_queries_map, mysrvc->queries_sent, PgSQL_p_hg_dyn_counter::connection_pool_queries); + + // proxysql_connection_pool_status metric + p_update_connection_pool_update_gauge(endpoint_id, common_labels, + status.p_connection_pool_status_map, mysrvc->status + 1, PgSQL_p_hg_dyn_gauge::connection_pool_status); + } + } + + // Remove the non-present servers for the gauge metrics + vector missing_server_keys {}; + + for (const auto& key : status.p_connection_pool_status_map) { + if (std::find(cur_servers_ids.begin(), cur_servers_ids.end(), key.first) == cur_servers_ids.end()) { + missing_server_keys.push_back(key.first); + } + } + + for (const auto& key : missing_server_keys) { + auto gauge = status.p_connection_pool_status_map[key]; + status.p_dyn_gauge_array[PgSQL_p_hg_dyn_gauge::connection_pool_status]->Remove(gauge); + status.p_connection_pool_status_map.erase(key); + + gauge = status.p_connection_pool_conn_used_map[key]; + status.p_dyn_gauge_array[PgSQL_p_hg_dyn_gauge::connection_pool_conn_free]->Remove(gauge); + status.p_connection_pool_conn_used_map.erase(key); + + gauge = status.p_connection_pool_conn_free_map[key]; + status.p_dyn_gauge_array[PgSQL_p_hg_dyn_gauge::connection_pool_conn_used]->Remove(gauge); + status.p_connection_pool_conn_free_map.erase(key); + + gauge = status.p_connection_pool_latency_us_map[key]; + status.p_dyn_gauge_array[PgSQL_p_hg_dyn_gauge::connection_pool_latency_us]->Remove(gauge); + status.p_connection_pool_latency_us_map.erase(key); + } + + wrunlock(); +} + +SQLite3_result * PgSQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset, int *hid) { + const int colnum=13; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Connection Pool\n"); + SQLite3_result *result=new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"hostgroup"); + result->add_column_definition(SQLITE_TEXT,"srv_host"); + result->add_column_definition(SQLITE_TEXT,"srv_port"); + result->add_column_definition(SQLITE_TEXT,"status"); + result->add_column_definition(SQLITE_TEXT,"ConnUsed"); + result->add_column_definition(SQLITE_TEXT,"ConnFree"); + result->add_column_definition(SQLITE_TEXT,"ConnOK"); + result->add_column_definition(SQLITE_TEXT,"ConnERR"); + result->add_column_definition(SQLITE_TEXT,"MaxConnUsed"); + result->add_column_definition(SQLITE_TEXT,"Queries"); + result->add_column_definition(SQLITE_TEXT,"Bytes_sent"); + result->add_column_definition(SQLITE_TEXT,"Bytes_recv"); + result->add_column_definition(SQLITE_TEXT,"Latency_us"); + wrlock(); + int i,j, k; + for (i=0; i<(int)MyHostGroups->len; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + for (j=0; j<(int)myhgc->mysrvs->cnt(); j++) { + PgSQL_SrvC *mysrvc=(PgSQL_SrvC *)myhgc->mysrvs->servers->index(j); + if (hid == NULL) { + if (mysrvc->status!=MYSQL_SERVER_STATUS_ONLINE) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d is not online\n", mysrvc->address, mysrvc->port); + //__sync_fetch_and_sub(&status.server_connections_connected, mysrvc->ConnectionsFree->conns->len); + mysrvc->ConnectionsFree->drop_all_connections(); + } + // drop idle connections if beyond max_connection + while (mysrvc->ConnectionsFree->conns_length() && mysrvc->ConnectionsUsed->conns_length()+mysrvc->ConnectionsFree->conns_length() > mysrvc->max_connections) { + //PgSQL_Connection *conn=(PgSQL_Connection *)mysrvc->ConnectionsFree->conns->remove_index_fast(0); + PgSQL_Connection *conn=mysrvc->ConnectionsFree->remove(0); + delete conn; + //__sync_fetch_and_sub(&status.server_connections_connected, 1); + } + } else { + if (*hid != (int)myhgc->hid) { + continue; + } + } + char buf[1024]; + char **pta=(char **)malloc(sizeof(char *)*colnum); + sprintf(buf,"%d", (int)myhgc->hid); + pta[0]=strdup(buf); + pta[1]=strdup(mysrvc->address); + sprintf(buf,"%d", mysrvc->port); + pta[2]=strdup(buf); + switch (mysrvc->status) { + case 0: + pta[3]=strdup("ONLINE"); + break; + case 1: + pta[3]=strdup("SHUNNED"); + break; + case 2: + pta[3]=strdup("OFFLINE_SOFT"); + break; + case 3: + pta[3]=strdup("OFFLINE_HARD"); + break; + case 4: + pta[3]=strdup("SHUNNED_REPLICATION_LAG"); + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + sprintf(buf,"%u", mysrvc->ConnectionsUsed->conns_length()); + pta[4]=strdup(buf); + sprintf(buf,"%u", mysrvc->ConnectionsFree->conns_length()); + pta[5]=strdup(buf); + sprintf(buf,"%u", mysrvc->connect_OK); + pta[6]=strdup(buf); + if (_reset) { + mysrvc->connect_OK=0; + } + sprintf(buf,"%u", mysrvc->connect_ERR); + pta[7]=strdup(buf); + if (_reset) { + mysrvc->connect_ERR=0; + } + sprintf(buf,"%u", mysrvc->max_connections_used); + pta[8]=strdup(buf); + if (_reset) { + mysrvc->max_connections_used=0; + } + sprintf(buf,"%llu", mysrvc->queries_sent); + pta[9]=strdup(buf); + if (_reset) { + mysrvc->queries_sent=0; + } + sprintf(buf,"%llu", mysrvc->bytes_sent); + pta[10]=strdup(buf); + if (_reset) { + mysrvc->bytes_sent=0; + } + sprintf(buf,"%llu", mysrvc->bytes_recv); + pta[11]=strdup(buf); + if (_reset) { + mysrvc->bytes_recv=0; + } + sprintf(buf,"%u", mysrvc->current_latency_us); + pta[12]=strdup(buf); + result->add_row(pta); + for (k=0; k3"; + mydb->execute_statement((char *)q1, &error , &cols , &affected_rows , &res_set1); + for (std::vector::iterator it = res_set1->rows.begin() ; it != res_set1->rows.end(); ++it) { + SQLite3_row *r=*it; + std::string s = r->fields[0]; + s += ":::"; + s += r->fields[1]; + read_only_set1.insert(s); + } + proxy_info("Regenerating read_only_set1 with %lu servers\n", read_only_set1.size()); + if (read_only_set1.empty()) { + // to avoid regenerating this set always with 0 entries, we generate a fake entry + read_only_set1.insert("----:::----"); + } + delete res_set1; + } + wrunlock(); + std::string ser = hostname; + ser += ":::"; + ser += std::to_string(port); + std::set::iterator it; + it = read_only_set1.find(ser); + if (it != read_only_set1.end()) { + num_rows=1; + } + + if (admindb==NULL) { // we initialize admindb only if needed + admindb=new SQLite3DB(); + admindb->open((char *)"file:mem_admindb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + } + + switch (read_only) { + case 0: + if (num_rows==0) { + // the server has read_only=0 , but we can't find any writer, so we perform a swap + GloAdmin->mysql_servers_wrlock(); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 phase 1 : Dumping pgsql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->save_proxysql_servers_runtime_to_database(false); // SAVE PgSQL SERVERS FROM RUNTIME + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 phase 2 : Dumping pgsql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + sprintf(query,Q2A,hostname,port); + admindb->execute(query); + sprintf(query,Q2B,hostname,port); + admindb->execute(query); + if (mysql_thread___monitor_writer_is_also_reader) { + sprintf(query,Q3A,hostname,port); + } else { + sprintf(query,Q3B,hostname,port); + } + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 phase 3 : Dumping pgsql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->load_proxysql_servers_to_runtime(); // LOAD PgSQL SERVERS TO RUNTIME + GloAdmin->mysql_servers_wrunlock(); + } else { + // there is a server in writer hostgroup, let check the status of present and not present hosts + bool act=false; + wrlock(); + std::set::iterator it; + // read_only_set2 acts as a cache + // if the server was RO=0 on the previous check and no action was needed, + // it will be here + it = read_only_set2.find(ser); + if (it != read_only_set2.end()) { + // the server was already detected as RO=0 + // no action required + } else { + // it is the first time that we detect RO on this server + sprintf(query,Q1B,hostname,port,hostname,port,hostname,port); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int status=MYSQL_SERVER_STATUS_OFFLINE_HARD; // default status, even for missing + if (r->fields[1]) { // has status + status=atoi(r->fields[1]); + } + if (status==MYSQL_SERVER_STATUS_OFFLINE_HARD) { + act=true; + } + } + if (act == false) { + // no action required, therefore we write in read_only_set2 + proxy_info("read_only_action() detected RO=0 on server %s:%d for the first time after commit(), but no need to reconfigure\n", hostname, port); + read_only_set2.insert(ser); + } + } + wrunlock(); + if (act==true) { // there are servers either missing, or with stats=OFFLINE_HARD + GloAdmin->mysql_servers_wrlock(); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 , rows=%d , phase 1 : Dumping pgsql_servers for %s:%d\n", num_rows, hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->save_proxysql_servers_runtime_to_database(false); // SAVE PgSQL SERVERS FROM RUNTIME + sprintf(query,Q2A,hostname,port); + admindb->execute(query); + sprintf(query,Q2B,hostname,port); + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 , rows=%d , phase 2 : Dumping pgsql_servers for %s:%d\n", num_rows, hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + if (mysql_thread___monitor_writer_is_also_reader) { + sprintf(query,Q3A,hostname,port); + } else { + sprintf(query,Q3B,hostname,port); + } + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=0 , rows=%d , phase 3 : Dumping pgsql_servers for %s:%d\n", num_rows, hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->load_proxysql_servers_to_runtime(); // LOAD PgSQL SERVERS TO RUNTIME + GloAdmin->mysql_servers_wrunlock(); + } + } + break; + case 1: + if (num_rows) { + // the server has read_only=1 , but we find it as writer, so we perform a swap + GloAdmin->mysql_servers_wrlock(); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=1 phase 1 : Dumping pgsql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->save_proxysql_servers_runtime_to_database(false); // SAVE PgSQL SERVERS FROM RUNTIME + sprintf(query,Q4,hostname,port); + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=1 phase 2 : Dumping pgsql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + sprintf(query,Q5,hostname,port); + admindb->execute(query); + if (GloMTH->variables.hostgroup_manager_verbose) { + char *error2=NULL; + int cols2=0; + int affected_rows2=0; + SQLite3_result *resultset2=NULL; + char * query2 = NULL; + char *q = (char *)"SELECT * FROM pgsql_servers WHERE hostname=\"%s\" AND port=%d"; + query2 = (char *)malloc(strlen(q)+strlen(hostname)+32); + sprintf(query2,q,hostname,port); + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + if (error2) { + proxy_error("Error on read from pgsql_servers : %s\n", error2); + } else { + if (resultset2) { + proxy_info("read_only_action RO=1 phase 3 : Dumping pgsql_servers for %s:%d\n", hostname, port); + resultset2->dump_to_stderr(); + } + } + if (resultset2) { delete resultset2; resultset2=NULL; } + free(query2); + } + GloAdmin->load_proxysql_servers_to_runtime(); // LOAD PgSQL SERVERS TO RUNTIME + GloAdmin->mysql_servers_wrunlock(); + } + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + + pthread_mutex_unlock(&readonly_mutex); + if (resultset) { + delete resultset; + } + free(query); +} + +/** + * @brief New implementation of the read_only_action method that does not depend on the admin table. + * The method checks each server in the provided list and adjusts the servers according to their corresponding read_only value. + * If any change has occured, checksum is calculated. + * + * @param pgsql_servers List of servers having hostname, port and read only value. + * + */ +void PgSQL_HostGroups_Manager::read_only_action_v2(const std::list& pgsql_servers) { + + bool update_pgsql_servers_table = false; + + unsigned long long curtime1 = monotonic_time(); + wrlock(); + for (const auto& server : pgsql_servers) { + bool is_writer = false; + const std::string& hostname = std::get(server); + const int port = std::get(server); + const int read_only = std::get(server); + const std::string& srv_id = hostname + ":::" + std::to_string(port); + + auto itr = hostgroup_server_mapping.find(srv_id); + + if (itr == hostgroup_server_mapping.end()) { + proxy_warning("Server %s:%d not found\n", hostname.c_str(), port); + continue; + } + + HostGroup_Server_Mapping* host_server_mapping = itr->second.get(); + + if (!host_server_mapping) + assert(0); + + const std::vector& writer_map = host_server_mapping->get(HostGroup_Server_Mapping::Type::WRITER); + + is_writer = !writer_map.empty(); + + if (read_only == 0) { + if (is_writer == false) { + // the server has read_only=0 (writer), but we can't find any writer, + // so we copy all reader nodes to writer + proxy_info("Server '%s:%d' found with 'read_only=0', but not found as writer\n", hostname.c_str(), port); + proxy_debug(PROXY_DEBUG_MONITOR, 5, "Server '%s:%d' found with 'read_only=0', but not found as writer\n", hostname.c_str(), port); + host_server_mapping->copy_if_not_exists(HostGroup_Server_Mapping::Type::WRITER, HostGroup_Server_Mapping::Type::READER); + + if (mysql_thread___monitor_writer_is_also_reader == false) { + // remove node from reader + host_server_mapping->clear(HostGroup_Server_Mapping::Type::READER); + } + + update_pgsql_servers_table = true; + proxy_info("Regenerating table 'pgsql_servers' due to actions on server '%s:%d'\n", hostname.c_str(), port); + } else { + bool act = false; + + // if the server was RO=0 on the previous check then no action is needed + if (host_server_mapping->get_readonly_flag() != 0) { + // it is the first time that we detect RO on this server + const std::vector& reader_map = host_server_mapping->get(HostGroup_Server_Mapping::Type::READER); + + for (const auto& reader_node : reader_map) { + for (const auto& writer_node : writer_map) { + + if (reader_node.writer_hostgroup_id == writer_node.writer_hostgroup_id) { + goto __writer_found; + } + } + act = true; + break; + __writer_found: + continue; + } + + if (act == false) { + // no action required, therefore we set readonly_flag to 0 + proxy_info("read_only_action_v2() detected RO=0 on server %s:%d for the first time after commit(), but no need to reconfigure\n", hostname.c_str(), port); + host_server_mapping->set_readonly_flag(0); + } + } else { + // the server was already detected as RO=0 + // no action required + } + + if (act == true) { // there are servers either missing, or with stats=OFFLINE_HARD + + proxy_info("Server '%s:%d' with 'read_only=0' found missing at some 'writer_hostgroup'\n", hostname.c_str(), port); + proxy_debug(PROXY_DEBUG_MONITOR, 5, "Server '%s:%d' with 'read_only=0' found missing at some 'writer_hostgroup'\n", hostname.c_str(), port); + + // copy all reader nodes to writer + host_server_mapping->copy_if_not_exists(HostGroup_Server_Mapping::Type::WRITER, HostGroup_Server_Mapping::Type::READER); + + if (mysql_thread___monitor_writer_is_also_reader == false) { + // remove node from reader + host_server_mapping->clear(HostGroup_Server_Mapping::Type::READER); + } + + update_pgsql_servers_table = true; + proxy_info("Regenerating table 'pgsql_servers' due to actions on server '%s:%d'\n", hostname.c_str(), port); + } + } + } else if (read_only == 1) { + if (is_writer) { + // the server has read_only=1 (reader), but we find it as writer, so we copy all writer nodes to reader (previous reader nodes will be reused) + proxy_info("Server '%s:%d' found with 'read_only=1', but not found as reader\n", hostname.c_str(), port); + proxy_debug(PROXY_DEBUG_MONITOR, 5, "Server '%s:%d' found with 'read_only=1', but not found as reader\n", hostname.c_str(), port); + host_server_mapping->copy_if_not_exists(HostGroup_Server_Mapping::Type::READER, HostGroup_Server_Mapping::Type::WRITER); + + // clearing all writer nodes + host_server_mapping->clear(HostGroup_Server_Mapping::Type::WRITER); + + update_pgsql_servers_table = true; + proxy_info("Regenerating table 'pgsql_servers' due to actions on server '%s:%d'\n", hostname.c_str(), port); + } + } else { + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + } + + if (update_pgsql_servers_table) { + purge_pgsql_servers_table(); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM pgsql_servers\n"); + mydb->execute("DELETE FROM pgsql_servers"); + generate_pgsql_servers_table(); + + // Update the global checksums after 'pgsql_servers' regeneration + { + unique_ptr resultset { get_admin_runtime_pgsql_servers(mydb) }; + uint64_t raw_checksum = resultset ? resultset->raw_checksum() : 0; + + // This is required to be updated to avoid extra rebuilding member 'hostgroup_server_mapping' + // during 'commit'. For extra details see 'hgsm_pgsql_servers_checksum' @details. + hgsm_pgsql_servers_checksum = raw_checksum; + + string mysrvs_checksum { get_checksum_from_hash(raw_checksum) }; + save_runtime_pgsql_servers(resultset.release()); + proxy_info("Checksum for table %s is %s\n", "pgsql_servers", mysrvs_checksum.c_str()); + + pthread_mutex_lock(&GloVars.checksum_mutex); + update_glovars_pgsql_servers_checksum(mysrvs_checksum); + pthread_mutex_unlock(&GloVars.checksum_mutex); + } + } + wrunlock(); + unsigned long long curtime2 = monotonic_time(); + curtime1 = curtime1 / 1000; + curtime2 = curtime2 / 1000; + proxy_debug(PROXY_DEBUG_MONITOR, 7, "PgSQL_HostGroups_Manager::read_only_action_v2() locked for %llums (server count:%ld)\n", curtime2 - curtime1, pgsql_servers.size()); +} + +// shun_and_killall +// this function is called only from MySQL_Monitor::monitor_ping() +// it temporary disables a host that is not responding to pings, and mark the host in a way that when used the connection will be dropped +// return true if the status was changed +bool PgSQL_HostGroups_Manager::shun_and_killall(char *hostname, int port) { + time_t t = time(NULL); + bool ret = false; + wrlock(); + PgSQL_SrvC *mysrvc=NULL; + for (unsigned int i=0; ilen; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + unsigned int j; + unsigned int l=myhgc->mysrvs->cnt(); + if (l) { + for (j=0; jmysrvs->idx(j); + if (mysrvc->port==port && strcmp(mysrvc->address,hostname)==0) { + switch (mysrvc->status) { + case MYSQL_SERVER_STATUS_SHUNNED: + if (mysrvc->shunned_automatic==false) { + break; + } + case MYSQL_SERVER_STATUS_ONLINE: + if (mysrvc->status == MYSQL_SERVER_STATUS_ONLINE) { + ret = true; + } + mysrvc->status=MYSQL_SERVER_STATUS_SHUNNED; + case MYSQL_SERVER_STATUS_OFFLINE_SOFT: + mysrvc->shunned_automatic=true; + mysrvc->shunned_and_kill_all_connections=true; + mysrvc->ConnectionsFree->drop_all_connections(); + break; + default: + break; + } + // if Monitor is enabled and pgsql-monitor_ping_interval is + // set too high, ProxySQL will unshun hosts that are not + // available. For this reason time_last_detected_error will + // be tuned in the future + if (mysql_thread___monitor_enabled) { + int a = pgsql_thread___shun_recovery_time_sec; + int b = mysql_thread___monitor_ping_interval; + b = b/1000; + if (b > a) { + t = t + (b - a); + } + } + mysrvc->time_last_detected_error = t; + } + } + } + } + wrunlock(); + return ret; +} + +// set_server_current_latency_us +// this function is called only from MySQL_Monitor::monitor_ping() +// it set the average latency for a host in the last 3 pings +// the connection pool will use this information to evaluate or exclude a specific hosts +// note that this variable is in microsecond, while user defines it in millisecond +void PgSQL_HostGroups_Manager::set_server_current_latency_us(char *hostname, int port, unsigned int _current_latency_us) { + wrlock(); + PgSQL_SrvC *mysrvc=NULL; + for (unsigned int i=0; ilen; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + unsigned int j; + unsigned int l=myhgc->mysrvs->cnt(); + if (l) { + for (j=0; jmysrvs->idx(j); + if (mysrvc->port==port && strcmp(mysrvc->address,hostname)==0) { + mysrvc->current_latency_us=_current_latency_us; + } + } + } + } + wrunlock(); +} + +void PgSQL_HostGroups_Manager::p_update_metrics() { + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::servers_table_version], status.servers_table_version); + // Update *server_connections* related metrics + status.p_gauge_array[PgSQL_p_hg_gauge::server_connections_connected]->Set(status.server_connections_connected); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::server_connections_aborted], status.server_connections_aborted); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::server_connections_created], status.server_connections_created); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::server_connections_delayed], status.server_connections_delayed); + + // Update *client_connections* related metrics + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::client_connections_created], status.client_connections_created); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::client_connections_aborted], status.client_connections_aborted); + status.p_gauge_array[PgSQL_p_hg_gauge::client_connections_connected]->Set(status.client_connections); + + // Update *acess_denied* related metrics + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::access_denied_wrong_password], status.access_denied_wrong_password); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::access_denied_max_connections], status.access_denied_max_connections); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::access_denied_max_user_connections], status.access_denied_max_user_connections); + + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::selects_for_update__autocommit0], status.select_for_update_or_equivalent); + + // Update *com_* related metrics + //p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_autocommit], status.autocommit_cnt); + //p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_autocommit_filtered], status.autocommit_cnt_filtered); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_commit_cnt], status.commit_cnt); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_commit_cnt_filtered], status.commit_cnt_filtered); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_rollback], status.rollback_cnt); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_rollback_filtered], status.rollback_cnt_filtered); + //p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_backend_init_db], status.backend_init_db); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_backend_reset_connection], status.backend_reset_connection); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_backend_set_client_encoding], status.backend_set_client_encoding); + //p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_frontend_init_db], status.frontend_init_db); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_frontend_set_client_encoding], status.frontend_set_client_encoding); + //p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::com_frontend_use_db], status.frontend_use_db); + + // Update *myconnpoll* related metrics + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::pghgm_pgconnpool_get], status.pgconnpoll_get); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::pghgm_pgconnpool_get_ok], status.pgconnpoll_get_ok); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::pghgm_pgconnpool_get_ping], status.pgconnpoll_get_ping); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::pghgm_pgconnpool_push], status.pgconnpoll_push); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::pghgm_pgconnpool_reset], status.pgconnpoll_reset); + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::pghgm_pgconnpool_destroy], status.pgconnpoll_destroy); + + p_update_counter(status.p_counter_array[PgSQL_p_hg_counter::auto_increment_delay_multiplex], status.auto_increment_delay_multiplex); + + // Update the *connection_pool* metrics + this->p_update_connection_pool(); +} + +SQLite3_result * PgSQL_HostGroups_Manager::SQL3_Get_ConnPool_Stats() { + const int colnum=2; + char buf[256]; + char **pta=(char **)malloc(sizeof(char *)*colnum); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping PgSQL Global Status\n"); + SQLite3_result *result=new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"Variable_Name"); + result->add_column_definition(SQLITE_TEXT,"Variable_Value"); + wrlock(); + // NOTE: as there is no string copy, we do NOT free pta[0] and pta[1] + { + pta[0]=(char *)"PgHGM_pgconnpoll_get"; + sprintf(buf,"%lu",status.pgconnpoll_get); + pta[1]=buf; + result->add_row(pta); + } + { + pta[0]=(char *)"PgHGM_pgconnpoll_get_ok"; + sprintf(buf,"%lu",status.pgconnpoll_get_ok); + pta[1]=buf; + result->add_row(pta); + } + { + pta[0]=(char *)"PgHGM_pgconnpoll_push"; + sprintf(buf,"%lu",status.pgconnpoll_push); + pta[1]=buf; + result->add_row(pta); + } + { + pta[0]=(char *)"PgHGM_pgconnpoll_destroy"; + sprintf(buf,"%lu",status.pgconnpoll_destroy); + pta[1]=buf; + result->add_row(pta); + } + { + pta[0]=(char *)"PgHGM_pgconnpoll_reset"; + sprintf(buf,"%lu",status.pgconnpoll_reset); + pta[1]=buf; + result->add_row(pta); + } + wrunlock(); + free(pta); + return result; +} + + +unsigned long long PgSQL_HostGroups_Manager::Get_Memory_Stats() { + unsigned long long intsize=0; + wrlock(); + PgSQL_SrvC *mysrvc=NULL; + for (unsigned int i=0; ilen; i++) { + intsize+=sizeof(PgSQL_HGC); + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + unsigned int j,k; + unsigned int l=myhgc->mysrvs->cnt(); + if (l) { + for (j=0; jmysrvs->idx(j); + intsize+=((mysrvc->ConnectionsUsed->conns_length())*sizeof(PgSQL_Connection *)); + for (k=0; kConnectionsFree->conns_length(); k++) { + //PgSQL_Connection *myconn=(PgSQL_Connection *)mysrvc->ConnectionsFree->conns->index(k); + PgSQL_Connection *myconn=mysrvc->ConnectionsFree->index(k); + intsize+= sizeof(PgSQL_Connection); + intsize+=myconn->get_memory_usage(); + //intsize+=(4096*15); // ASYNC_CONTEXT_DEFAULT_STACK_SIZE + if (myconn->query_result) { + intsize+=myconn->query_result->current_size(); + } + } + intsize+=((mysrvc->ConnectionsUsed->conns_length())*sizeof(PgSQL_Connection *)); + } + } + } + wrunlock(); + return intsize; +} + +class PgSQL_Errors_stats { +public: + PgSQL_Errors_stats(int _hostgroup, const char *_hostname, int _port, const char *_username, const char *_address, const char *_dbname, + const char* _sqlstate, const char *_errmsg, time_t tn) { + hostgroup = _hostgroup; + if (_hostname) { + hostname = strdup(_hostname); + } else { + hostname = strdup((char *)""); + } + port = _port; + if (_username) { + username = strdup(_username); + } else { + username = strdup((char *)""); + } + if (_address) { + client_address = strdup(_address); + } else { + client_address = strdup((char *)""); + } + if (_dbname) { + dbname = strdup(_dbname); + } else { + dbname = strdup((char *)""); + } + if (_sqlstate) { + strncpy(sqlstate, _sqlstate, 5); + sqlstate[5] = '\0'; + } else { + sqlstate[0] = '\0'; + } + if (_errmsg) { + errmsg = strdup(_errmsg); + } else { + _errmsg = strdup((char *)""); + } + last_seen = tn; + first_seen = tn; + count_star = 1; + } + ~PgSQL_Errors_stats() { + if (hostname) { + free(hostname); + hostname=NULL; + } + if (username) { + free(username); + username=NULL; + } + if (client_address) { + free(client_address); + client_address=NULL; + } + if (dbname) { + free(dbname); + dbname=NULL; + } + if (errmsg) { + free(errmsg); + errmsg=NULL; + } + } + char **get_row() { + char buf[128]; + char **pta=(char **)malloc(sizeof(char *)*PgSQL_ERRORS_STATS_FIELD_NUM); + sprintf(buf,"%d",hostgroup); + pta[0]=strdup(buf); + assert(hostname); + pta[1]=strdup(hostname); + sprintf(buf,"%d",port); + pta[2]=strdup(buf); + assert(username); + pta[3]=strdup(username); + assert(client_address); + pta[4]=strdup(client_address); + assert(dbname); + pta[5]=strdup(dbname); + pta[6]=strdup(sqlstate); + sprintf(buf,"%llu",count_star); + pta[7]=strdup(buf); + sprintf(buf,"%ld", first_seen); + pta[8]=strdup(buf); + sprintf(buf,"%ld", last_seen); + pta[9]=strdup(buf); + assert(errmsg); + pta[10]=strdup(errmsg); + return pta; + } + void add_time(unsigned long long n, const char *le) { + count_star++; + if (first_seen==0) { + first_seen=n; + } + last_seen=n; + if (strcmp(errmsg,le)){ + free(errmsg); + errmsg=strdup(le); + } + } + void free_row(char **pta) { + int i; + for (i=0;i::iterator it; + pthread_mutex_lock(&pgsql_errors_mutex); + + it=pgsql_errors_umap.find(hash1); + + if (it != pgsql_errors_umap.end()) { + // found + PgSQL_Errors_stats* err_stats = it->second; + err_stats->add_time(tn, errmsg); + } else { + PgSQL_Errors_stats* err_stats = new PgSQL_Errors_stats(hostgroup, hostname, port, username, address, dbname, sqlstate, errmsg, tn); + pgsql_errors_umap.insert(std::make_pair(hash1, err_stats)); + } + pthread_mutex_unlock(&pgsql_errors_mutex); +} + +SQLite3_result* PgSQL_HostGroups_Manager::get_pgsql_errors(bool reset) { + SQLite3_result *result=new SQLite3_result(PgSQL_ERRORS_STATS_FIELD_NUM); + pthread_mutex_lock(&pgsql_errors_mutex); + result->add_column_definition(SQLITE_TEXT,"hid"); + result->add_column_definition(SQLITE_TEXT,"hostname"); + result->add_column_definition(SQLITE_TEXT,"port"); + result->add_column_definition(SQLITE_TEXT,"username"); + result->add_column_definition(SQLITE_TEXT,"client_address"); + result->add_column_definition(SQLITE_TEXT,"database"); + result->add_column_definition(SQLITE_TEXT,"sqlstate"); + result->add_column_definition(SQLITE_TEXT,"count_star"); + result->add_column_definition(SQLITE_TEXT,"first_seen"); + result->add_column_definition(SQLITE_TEXT,"last_seen"); + result->add_column_definition(SQLITE_TEXT,"last_error"); + for (std::unordered_map::iterator it=pgsql_errors_umap.begin(); it!=pgsql_errors_umap.end(); ++it) { + PgSQL_Errors_stats *err_stats=it->second; + char **pta= err_stats->get_row(); + result->add_row(pta); + err_stats->free_row(pta); + if (reset) { + delete err_stats; + } + } + if (reset) { + pgsql_errors_umap.erase(pgsql_errors_umap.begin(),pgsql_errors_umap.end()); + } + pthread_mutex_unlock(&pgsql_errors_mutex); + return result; +} + +/** + * @brief Initializes the supplied 'PgSQL_HGC' with the specified 'hostgroup_settings'. + * @details Input verification is performed in the supplied 'hostgroup_settings'. It's expected to be a valid + * JSON that may contain the following fields: + * - handle_warnings: Value must be >= 0. + * + * In case input verification fails for a field, supplied 'PgSQL_HGC' is NOT updated for that field. An error + * message is logged specifying the source of the error. + * + * @param hostgroup_settings String containing a JSON defined in 'pgsql_hostgroup_attributes'. + * @param myhgc The 'PgSQL_HGC' of the target hostgroup of the supplied 'hostgroup_settings'. + */ +void init_myhgc_hostgroup_settings(const char* hostgroup_settings, PgSQL_HGC* myhgc) { + const uint32_t hid = myhgc->hid; + + if (hostgroup_settings[0] != '\0') { + try { + nlohmann::json j = nlohmann::json::parse(hostgroup_settings); + + const auto handle_warnings_check = [](int8_t handle_warnings) -> bool { return handle_warnings == 0 || handle_warnings == 1; }; + int8_t handle_warnings = PgSQL_j_get_srv_default_int_val(j, hid, "handle_warnings", handle_warnings_check); + myhgc->attributes.handle_warnings = handle_warnings; + } + catch (const json::exception& e) { + proxy_error( + "JSON parsing for 'pgsql_hostgroup_attributes.hostgroup_settings' for hostgroup %d failed with exception `%s`.\n", + hid, e.what() + ); + } + } +} + +/** + * @brief Initializes the supplied 'PgSQL_HGC' with the specified 'servers_defaults'. + * @details Input verification is performed in the supplied 'server_defaults'. It's expected to be a valid + * JSON that may contain the following fields: + * - weight: Must be an unsigned integer >= 0. + * - max_connections: Must be an unsigned integer >= 0. + * - use_ssl: Must be a integer with either value 0 or 1. + * + * In case input verification fails for a field, supplied 'PgSQL_HGC' is NOT updated for that field. An error + * message is logged specifying the source of the error. + * + * @param servers_defaults String containing a JSON defined in 'pgsql_hostgroup_attributes'. + * @param myhgc The 'PgSQL_HGC' of the target hostgroup of the supplied 'servers_defaults'. + */ +void init_myhgc_servers_defaults(char* servers_defaults, PgSQL_HGC* myhgc) { + uint32_t hid = myhgc->hid; + + if (strcmp(servers_defaults, "") != 0) { + try { + nlohmann::json j = nlohmann::json::parse(servers_defaults); + + const auto weight_check = [] (int64_t weight) -> bool { return weight >= 0; }; + int64_t weight = PgSQL_j_get_srv_default_int_val(j, hid, "weight", weight_check); + + myhgc->servers_defaults.weight = weight; + + const auto max_conns_check = [] (int64_t max_conns) -> bool { return max_conns >= 0; }; + int64_t max_conns = PgSQL_j_get_srv_default_int_val(j, hid, "max_connections", max_conns_check); + + myhgc->servers_defaults.max_connections = max_conns; + + const auto use_ssl_check = [] (int32_t use_ssl) -> bool { return use_ssl == 0 || use_ssl == 1; }; + int32_t use_ssl = PgSQL_j_get_srv_default_int_val(j, hid, "use_ssl", use_ssl_check); + + myhgc->servers_defaults.use_ssl = use_ssl; + } catch (const json::exception& e) { + proxy_error( + "JSON parsing for 'pgsql_hostgroup_attributes.servers_defaults' for hostgroup %d failed with exception `%s`.\n", + hid, e.what() + ); + } + } +} + +void PgSQL_HostGroups_Manager::generate_pgsql_hostgroup_attributes_table() { + if (incoming_hostgroup_attributes==NULL) { + return; + } + int rc; + sqlite3_stmt *statement=NULL; + + const char * query=(const char *)"INSERT INTO pgsql_hostgroup_attributes ( " + "hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, " + "init_connect, multiplex, connection_warming, throttle_connections_per_sec, " + "ignore_session_variables, hostgroup_settings, servers_defaults, comment) VALUES " + "(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query, -1, &statement, 0); + rc = mydb->prepare_v2(query, &statement); + ASSERT_SQLITE_OK(rc, mydb); + proxy_info("New pgsql_hostgroup_attributes table\n"); + bool current_configured[MyHostGroups->len]; + // set configured = false to all + // in this way later we can known which HG were updated + for (unsigned int i=0; ilen; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + current_configured[i] = myhgc->attributes.configured; + myhgc->attributes.configured = false; + } + + /** + * @brief We iterate the whole resultset incoming_hostgroup_attributes and configure + * both the hostgroup in memory, but also pupulate table pgsql_hostgroup_attributes + * connection errors. + * @details for each row in incoming_hostgroup_attributes: + * 1. it finds (or create) the hostgroup + * 2. it writes the in pgsql_hostgroup_attributes + * 3. it finds (or create) the attributes of the hostgroup + */ + for (std::vector::iterator it = incoming_hostgroup_attributes->rows.begin() ; it != incoming_hostgroup_attributes->rows.end(); ++it) { + SQLite3_row *r=*it; + unsigned int hid = (unsigned int)atoi(r->fields[0]); + PgSQL_HGC *myhgc = MyHGC_lookup(hid); // note: MyHGC_lookup() will create the HG if doesn't exist! + int max_num_online_servers = atoi(r->fields[1]); + int autocommit = atoi(r->fields[2]); + int free_connections_pct = atoi(r->fields[3]); + char * init_connect = r->fields[4]; + int multiplex = atoi(r->fields[5]); + int connection_warming = atoi(r->fields[6]); + int throttle_connections_per_sec = atoi(r->fields[7]); + char * ignore_session_variables = r->fields[8]; + char * hostgroup_settings = r->fields[9]; + char * servers_defaults = r->fields[10]; + char * comment = r->fields[11]; + proxy_info("Loading MySQL Hostgroup Attributes info for (%d,%d,%d,%d,\"%s\",%d,%d,%d,\"%s\",\"%s\",\"%s\",\"%s\")\n", + hid, max_num_online_servers, autocommit, free_connections_pct, + init_connect, multiplex, connection_warming, throttle_connections_per_sec, + ignore_session_variables, hostgroup_settings, servers_defaults, comment + ); + rc=(*proxy_sqlite3_bind_int64)(statement, 1, hid); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, max_num_online_servers); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 3, autocommit); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 4, free_connections_pct); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 5, init_connect, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 6, multiplex); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 7, connection_warming); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_int64)(statement, 8, throttle_connections_per_sec); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 9, ignore_session_variables, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 10, hostgroup_settings, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 11, servers_defaults, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_bind_text)(statement, 12, comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb); + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb); + rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mydb); + myhgc->attributes.configured = true; + myhgc->attributes.max_num_online_servers = max_num_online_servers; + myhgc->attributes.autocommit = autocommit; + myhgc->attributes.free_connections_pct = free_connections_pct; + myhgc->attributes.multiplex = multiplex; + myhgc->attributes.connection_warming = connection_warming; + myhgc->attributes.throttle_connections_per_sec = throttle_connections_per_sec; + if (myhgc->attributes.init_connect != NULL) + free(myhgc->attributes.init_connect); + myhgc->attributes.init_connect = strdup(init_connect); + if (myhgc->attributes.comment != NULL) + free(myhgc->attributes.comment); + myhgc->attributes.comment = strdup(comment); + // for ignore_session_variables we store 2 versions: + // 1. the text + // 2. the JSON + // Because calling JSON functions is expensive, we first verify if it changes + if (myhgc->attributes.ignore_session_variables_text == NULL) { + myhgc->attributes.ignore_session_variables_text = strdup(ignore_session_variables); + if (strlen(ignore_session_variables) != 0) { // only if there is a valid JSON + if (myhgc->attributes.ignore_session_variables_json != nullptr) { delete myhgc->attributes.ignore_session_variables_json; } + myhgc->attributes.ignore_session_variables_json = new json(json::parse(ignore_session_variables)); + } + } else { + if (strcmp(myhgc->attributes.ignore_session_variables_text, ignore_session_variables) != 0) { + free(myhgc->attributes.ignore_session_variables_text); + myhgc->attributes.ignore_session_variables_text = strdup(ignore_session_variables); + if (strlen(ignore_session_variables) != 0) { // only if there is a valid JSON + if (myhgc->attributes.ignore_session_variables_json != nullptr) { delete myhgc->attributes.ignore_session_variables_json; } + myhgc->attributes.ignore_session_variables_json = new json(json::parse(ignore_session_variables)); + } + // TODO: assign the variables + } + } + init_myhgc_hostgroup_settings(hostgroup_settings, myhgc); + init_myhgc_servers_defaults(servers_defaults, myhgc); + } + for (unsigned int i=0; ilen; i++) { + PgSQL_HGC *myhgc=(PgSQL_HGC *)MyHostGroups->index(i); + if (myhgc->attributes.configured == false) { + if (current_configured[i] == true) { + // if configured == false and previously it was configured == true , reset to defaults + proxy_info("Resetting hostgroup attributes for hostgroup %u\n", myhgc->hid); + myhgc->reset_attributes(); + } + } + } + + delete incoming_hostgroup_attributes; + incoming_hostgroup_attributes=NULL; +} + +int PgSQL_HostGroups_Manager::create_new_server_in_hg( + uint32_t hid, const PgSQL_srv_info_t& srv_info, const PgSQL_srv_opts_t& srv_opts +) { + int32_t res = -1; + PgSQL_SrvC* mysrvc = find_server_in_hg(hid, srv_info.addr, srv_info.port); + + if (mysrvc == nullptr) { + char* c_hostname { const_cast(srv_info.addr.c_str()) }; + PgSQL_SrvC* mysrvc = new PgSQL_SrvC( + c_hostname, srv_info.port, srv_opts.weigth, MYSQL_SERVER_STATUS_ONLINE, 0, srv_opts.max_conns, 0, + srv_opts.use_ssl, 0, const_cast("") + ); + add(mysrvc,hid); + proxy_info( + "Adding new discovered %s node %s:%d with: hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", + srv_info.kind.c_str(), c_hostname, srv_info.port, hid, mysrvc->weight, mysrvc->max_connections, + mysrvc->use_ssl + ); + + res = 0; + } else { + // If the server is found as 'OFFLINE_HARD' we reset the 'PgSQL_SrvC' values corresponding with the + // 'servers_defaults' (as in a new 'PgSQL_SrvC' creation). We then later update these values with the + // 'servers_defaults' attributes from its corresponding 'PgSQL_HGC'. This way we ensure uniform behavior + // of new servers, and 'OFFLINE_HARD' ones when a user update 'servers_defaults' values, and reloads + // the servers to runtime. + if (mysrvc && mysrvc->status == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + reset_hg_attrs_server_defaults(mysrvc); + update_hg_attrs_server_defaults(mysrvc, mysrvc->myhgc); + mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; + + proxy_info( + "Found healthy previously discovered %s node %s:%d as 'OFFLINE_HARD', setting back as 'ONLINE' with:" + " hostgroup=%d, weight=%ld, max_connections=%ld, use_ssl=%d\n", + srv_info.kind.c_str(), srv_info.addr.c_str(), srv_info.port, hid, mysrvc->weight, + mysrvc->max_connections, mysrvc->use_ssl + ); + + res = 0; + } + } + + return res; +} + +int PgSQL_HostGroups_Manager::remove_server_in_hg(uint32_t hid, const string& addr, uint16_t port) { + PgSQL_SrvC* mysrvc = find_server_in_hg(hid, addr, port); + if (mysrvc == nullptr) { + return -1; + } + + uint64_t mysrvc_addr = reinterpret_cast(mysrvc); + + proxy_warning( + "Removed server at address %ld, hostgroup %d, address %s port %d." + " Setting status OFFLINE HARD and immediately dropping all free connections." + " Used connections will be dropped when trying to use them\n", + mysrvc_addr, hid, mysrvc->address, mysrvc->port + ); + + // Set the server status + mysrvc->status=MYSQL_SERVER_STATUS_OFFLINE_HARD; + mysrvc->ConnectionsFree->drop_all_connections(); + + // TODO-NOTE: This is only required in case the caller isn't going to perform: + // - Full deletion of servers in the target 'hid'. + // - Table regeneration for the servers in the target 'hid'. + // This is a very common pattern when further operations have been performed over the + // servers, e.g. a set of servers additions and deletions over the target hostgroups. + // //////////////////////////////////////////////////////////////////////// + + // Remove the server from the table + const string del_srv_query { "DELETE FROM pgsql_servers WHERE mem_pointer=" + std::to_string(mysrvc_addr) }; + mydb->execute(del_srv_query.c_str()); + + // //////////////////////////////////////////////////////////////////////// + + return 0; +} + +PgSQL_SrvC* PgSQL_HostGroups_Manager::find_server_in_hg(unsigned int _hid, const std::string& addr, int port) { + PgSQL_SrvC* f_server = nullptr; + + PgSQL_HGC* myhgc = nullptr; + for (uint32_t i = 0; i < MyHostGroups->len; i++) { + myhgc = static_cast(MyHostGroups->index(i)); + + if (myhgc->hid == _hid) { + break; + } + } + + if (myhgc != nullptr) { + for (uint32_t j = 0; j < myhgc->mysrvs->cnt(); j++) { + PgSQL_SrvC* mysrvc = static_cast(myhgc->mysrvs->servers->index(j)); + + if (strcmp(mysrvc->address, addr.c_str()) == 0 && mysrvc->port == port) { + f_server = mysrvc; + } + } + } + + return f_server; +} + +void PgSQL_HostGroups_Manager::HostGroup_Server_Mapping::copy_if_not_exists(Type dest_type, Type src_type) { + + assert(dest_type != src_type); + + const std::vector& src_nodes = mapping[src_type]; + + if (src_nodes.empty()) return; + + std::vector& dest_nodes = mapping[dest_type]; + std::list append; + + for (const auto& src_node : src_nodes) { + + for (const auto& dest_node : dest_nodes) { + + if (src_node.reader_hostgroup_id == dest_node.reader_hostgroup_id && + src_node.writer_hostgroup_id == dest_node.writer_hostgroup_id) { + goto __skip; + } + } + + append.push_back(src_node); + + __skip: + continue; + } + + if (append.empty()) { + return; + } + + if (dest_nodes.capacity() < (dest_nodes.size() + append.size())) + dest_nodes.reserve(dest_nodes.size() + append.size()); + + for (auto& node : append) { + + if (node.srv->status == MYSQL_SERVER_STATUS_SHUNNED || + node.srv->status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { + // Status updated from "*SHUNNED" to "ONLINE" as "read_only" value was successfully + // retrieved from the backend server, indicating server is now online. + node.srv->status = MYSQL_SERVER_STATUS_ONLINE; + } + + PgSQL_SrvC* new_srv = insert_HGM(get_hostgroup_id(dest_type, node), node.srv); + + if (!new_srv) assert(0); + + node.srv = new_srv; + dest_nodes.push_back(node); + } +} + +void PgSQL_HostGroups_Manager::HostGroup_Server_Mapping::remove(Type type, size_t index) { + + std::vector& nodes = mapping[type]; + + // ensure that we're not attempting to access out of the bounds of the container. + assert(index < nodes.size()); + + remove_HGM(nodes[index].srv); + + //Swap the element with the back element, except in the case when we're the last element. + if (index + 1 != nodes.size()) + std::swap(nodes[index], nodes.back()); + + //Pop the back of the container, deleting our old element. + nodes.pop_back(); +} + +void PgSQL_HostGroups_Manager::HostGroup_Server_Mapping::clear(Type type) { + + for (const auto& node : mapping[type]) { + remove_HGM(node.srv); + } + + mapping[type].clear(); +} + +unsigned int PgSQL_HostGroups_Manager::HostGroup_Server_Mapping::get_hostgroup_id(Type type, const Node& node) const { + + if (type == Type::WRITER) + return node.writer_hostgroup_id; + else if (type == Type::READER) + return node.reader_hostgroup_id; + else + assert(0); +} + +PgSQL_SrvC* PgSQL_HostGroups_Manager::HostGroup_Server_Mapping::insert_HGM(unsigned int hostgroup_id, const PgSQL_SrvC* srv) { + + PgSQL_HGC* myhgc = myHGM->MyHGC_lookup(hostgroup_id); + + if (!myhgc) + return NULL; + + PgSQL_SrvC* ret_srv = NULL; + + for (uint32_t j = 0; j < myhgc->mysrvs->cnt(); j++) { + PgSQL_SrvC* mysrvc = static_cast(myhgc->mysrvs->servers->index(j)); + if (strcmp(mysrvc->address, srv->address) == 0 && mysrvc->port == srv->port) { + if (mysrvc->status == MYSQL_SERVER_STATUS_OFFLINE_HARD) { + + mysrvc->weight = srv->weight; + mysrvc->compression = srv->compression; + mysrvc->max_connections = srv->max_connections; + mysrvc->max_replication_lag = srv->max_replication_lag; + mysrvc->use_ssl = srv->use_ssl; + mysrvc->max_latency_us = srv->max_latency_us; + mysrvc->comment = strdup(srv->comment); + mysrvc->status = MYSQL_SERVER_STATUS_ONLINE; + + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info( + "Found server node in Host Group Container %s:%d as 'OFFLINE_HARD', setting back as 'ONLINE' with:" + " hostgroup_id=%d, weight=%ld, compression=%d, max_connections=%ld, use_ssl=%d," + " max_replication_lag=%d, max_latency_ms=%d, comment=%s\n", + mysrvc->address, mysrvc->port, hostgroup_id, mysrvc->weight, mysrvc->compression, + mysrvc->max_connections, mysrvc->use_ssl, mysrvc->max_replication_lag, (mysrvc->max_latency_us / 1000), + mysrvc->comment + ); + } + ret_srv = mysrvc; + break; + } + } + } + + if (!ret_srv) { + if (GloMTH->variables.hostgroup_manager_verbose) { + proxy_info("Creating new server in HG %d : %s:%d , weight=%ld, status=%d\n", hostgroup_id, srv->address, srv->port, srv->weight, srv->status); + } + + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%ld, status=%d, mem_ptr=%p into hostgroup=%d\n", srv->address, srv->port, srv->weight, srv->status, srv, hostgroup_id); + + ret_srv = new PgSQL_SrvC(srv->address, srv->port, srv->weight, srv->status, srv->compression, + srv->max_connections, srv->max_replication_lag, srv->use_ssl, (srv->max_latency_us / 1000), srv->comment); + + myhgc->mysrvs->add(ret_srv); + } + + return ret_srv; +} + +void PgSQL_HostGroups_Manager::HostGroup_Server_Mapping::remove_HGM(PgSQL_SrvC* srv) { + proxy_warning("Removed server at address %p, hostgroup %d, address %s port %d. Setting status OFFLINE HARD and immediately dropping all free connections. Used connections will be dropped when trying to use them\n", (void*)srv, srv->myhgc->hid, srv->address, srv->port); + srv->status = MYSQL_SERVER_STATUS_OFFLINE_HARD; + srv->ConnectionsFree->drop_all_connections(); +} diff --git a/lib/PgSQL_Logger.cpp b/lib/PgSQL_Logger.cpp new file mode 100644 index 0000000000..3b906bafe8 --- /dev/null +++ b/lib/PgSQL_Logger.cpp @@ -0,0 +1,1055 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include +#include "proxysql.h" +#include "cpp.h" + +#include "PgSQL_Data_Stream.h" +#include "PgSQL_Query_Processor.h" +#include "MySQL_PreparedStatement.h" +#include "PgSQL_Logger.hpp" + +#include +#include + + +#ifdef DEBUG +#define DEB "_DEBUG" +#else +#define DEB "" +#endif /* DEBUG */ +#define PROXYSQL_MYSQL_LOGGER_VERSION "2.5.0421" DEB + +extern PgSQL_Logger *GloPgSQL_Logger; + +static uint8_t mysql_encode_length(uint64_t len, unsigned char *hd) { + if (len < 251) return 1; + if (len < 65536) { if (hd) { *hd=0xfc; }; return 3; } + if (len < 16777216) { if (hd) { *hd=0xfd; }; return 4; } + if (hd) { *hd=0xfe; } + return 9; +} + +static inline int write_encoded_length(unsigned char *p, uint64_t val, uint8_t len, char prefix) { + if (len==1) { + *p=(char)val; + return 1; + } + *p=prefix; + p++; + memcpy(p,&val,len-1); + return len; +} + +PgSQL_Event::PgSQL_Event (log_event_type _et, uint32_t _thread_id, char * _username, char * _schemaname , uint64_t _start_time , uint64_t _end_time , uint64_t _query_digest, char *_client, size_t _client_len) { + thread_id=_thread_id; + username=_username; + schemaname=_schemaname; + start_time=_start_time; + end_time=_end_time; + query_digest=_query_digest; + client=_client; + client_len=_client_len; + et=_et; + hid=UINT64_MAX; + server=NULL; + extra_info = NULL; + have_affected_rows=false; + affected_rows=0; + last_insert_id = 0; + have_rows_sent=false; + rows_sent=0; + client_stmt_id=0; +} + +void PgSQL_Event::set_client_stmt_id(uint32_t client_stmt_id) { + this->client_stmt_id = client_stmt_id; +} + +// if affected rows is set, last_insert_id is set too. +// They are part of the same OK packet +void PgSQL_Event::set_affected_rows(uint64_t ar, uint64_t lid) { + have_affected_rows=true; + affected_rows=ar; + last_insert_id=lid; +} + +void PgSQL_Event::set_rows_sent(uint64_t rs) { + have_rows_sent=true; + rows_sent=rs; +} + +void PgSQL_Event::set_extra_info(char *_err) { + extra_info = _err; +} + +void PgSQL_Event::set_query(const char *ptr, int len) { + query_ptr=(char *)ptr; + query_len=len; +} + +void PgSQL_Event::set_server(int _hid, const char *ptr, int len) { + server=(char *)ptr; + server_len=len; + hid=_hid; +} + +uint64_t PgSQL_Event::write(std::fstream *f, PgSQL_Session *sess) { + uint64_t total_bytes=0; + switch (et) { + case PROXYSQL_COM_QUERY: + case PROXYSQL_COM_STMT_EXECUTE: + case PROXYSQL_COM_STMT_PREPARE: + if (pgsql_thread___eventslog_format==1) { // format 1 , binary + total_bytes=write_query_format_1(f); + } else { // format 2 , json + total_bytes=write_query_format_2_json(f); + } + break; + case PROXYSQL_MYSQL_AUTH_OK: + case PROXYSQL_MYSQL_AUTH_ERR: + case PROXYSQL_MYSQL_AUTH_CLOSE: + case PROXYSQL_MYSQL_AUTH_QUIT: + case PROXYSQL_MYSQL_INITDB: + case PROXYSQL_ADMIN_AUTH_OK: + case PROXYSQL_ADMIN_AUTH_ERR: + case PROXYSQL_ADMIN_AUTH_CLOSE: + case PROXYSQL_ADMIN_AUTH_QUIT: + case PROXYSQL_SQLITE_AUTH_OK: + case PROXYSQL_SQLITE_AUTH_ERR: + case PROXYSQL_SQLITE_AUTH_CLOSE: + case PROXYSQL_SQLITE_AUTH_QUIT: + write_auth(f, sess); + break; + default: + break; + } + return total_bytes; +} + +void PgSQL_Event::write_auth(std::fstream *f, PgSQL_Session *sess) { + json j = {}; + j["timestamp"] = start_time/1000; + { + time_t timer=start_time/1000/1000; + struct tm* tm_info; + tm_info = localtime(&timer); + char buffer1[36]; + char buffer2[64]; + strftime(buffer1, 32, "%Y-%m-%d %H:%M:%S", tm_info); + sprintf(buffer2,"%s.%03u", buffer1, (unsigned)(start_time%1000000)/1000); + j["time"] = buffer2; + } + j["thread_id"] = thread_id; + if (username) { + j["username"] = username; + } else { + j["username"] = ""; + } + if (schemaname) { + j["schemaname"] = schemaname; + } else { + j["schemaname"] = ""; + } + if (client) { + j["client_addr"] = client; + } else { + j["client_addr"] = ""; + } + if (server) { + j["server_addr"] = server; + } + if (extra_info) { + j["extra_info"] = extra_info; + } + switch (et) { + case PROXYSQL_MYSQL_AUTH_OK: + j["event"]="MySQL_Client_Connect_OK"; + break; + case PROXYSQL_MYSQL_AUTH_ERR: + j["event"]="MySQL_Client_Connect_ERR"; + break; + case PROXYSQL_MYSQL_AUTH_CLOSE: + j["event"]="MySQL_Client_Close"; + break; + case PROXYSQL_MYSQL_AUTH_QUIT: + j["event"]="MySQL_Client_Quit"; + break; + case PROXYSQL_MYSQL_INITDB: + j["event"]="MySQL_Client_Init_DB"; + break; + case PROXYSQL_ADMIN_AUTH_OK: + j["event"]="Admin_Connect_OK"; + break; + case PROXYSQL_ADMIN_AUTH_ERR: + j["event"]="Admin_Connect_ERR"; + break; + case PROXYSQL_ADMIN_AUTH_CLOSE: + j["event"]="Admin_Close"; + break; + case PROXYSQL_ADMIN_AUTH_QUIT: + j["event"]="Admin_Quit"; + break; + case PROXYSQL_SQLITE_AUTH_OK: + j["event"]="SQLite3_Connect_OK"; + break; + case PROXYSQL_SQLITE_AUTH_ERR: + j["event"]="SQLite3_Connect_ERR"; + break; + case PROXYSQL_SQLITE_AUTH_CLOSE: + j["event"]="SQLite3_Close"; + break; + case PROXYSQL_SQLITE_AUTH_QUIT: + j["event"]="SQLite3_Quit"; + break; + default: + break; + } + switch (et) { + case PROXYSQL_MYSQL_AUTH_CLOSE: + case PROXYSQL_ADMIN_AUTH_CLOSE: + case PROXYSQL_SQLITE_AUTH_CLOSE: + { + uint64_t curtime_real=realtime_time(); + uint64_t curtime_mono=sess->thread->curtime; + uint64_t timediff = curtime_mono - sess->start_time; + uint64_t orig_time = curtime_real - timediff; + time_t timer= (orig_time)/1000/1000; + struct tm* tm_info; + tm_info = localtime(&timer); + char buffer1[36]; + char buffer2[64]; + strftime(buffer1, 32, "%Y-%m-%d %H:%M:%S", tm_info); + sprintf(buffer2,"%s.%03u", buffer1, (unsigned)(orig_time%1000000)/1000); + j["creation_time"] = buffer2; + //unsigned long long life = sess->thread->curtime - sess->start_time; + //life/=1000; + float f = timediff; + f /= 1000; + sprintf(buffer1, "%.3fms", f); + j["duration"] = buffer1; + } + break; + default: + break; + } + if (sess->client_myds) { + if (sess->client_myds->proxy_addr.addr) { + std::string s = sess->client_myds->proxy_addr.addr; + s += ":" + std::to_string(sess->client_myds->proxy_addr.port); + j["proxy_addr"] = s; + } + j["ssl"] = sess->client_myds->encrypted; + } + // for performance reason, we are moving the write lock + // right before the write to disk + //GloPgSQL_Logger->wrlock(); + //move wrlock() function to log_audit_entry() function, avoid to get a null pointer in a multithreaded environment + *f << j.dump(-1, ' ', false, json::error_handler_t::replace) << std::endl; +} + +uint64_t PgSQL_Event::write_query_format_1(std::fstream *f) { + uint64_t total_bytes=0; + total_bytes+=1; // et + total_bytes+=mysql_encode_length(thread_id, NULL); + username_len=strlen(username); + total_bytes+=mysql_encode_length(username_len,NULL)+username_len; + schemaname_len=strlen(schemaname); + total_bytes+=mysql_encode_length(schemaname_len,NULL)+schemaname_len; + + total_bytes+=mysql_encode_length(client_len,NULL)+client_len; + + total_bytes+=mysql_encode_length(hid, NULL); + if (hid!=UINT64_MAX) { + total_bytes+=mysql_encode_length(server_len,NULL)+server_len; + } + + total_bytes+=mysql_encode_length(start_time,NULL); + total_bytes+=mysql_encode_length(end_time,NULL); + total_bytes+=mysql_encode_length(client_stmt_id,NULL); + total_bytes+=mysql_encode_length(affected_rows,NULL); + total_bytes+=mysql_encode_length(last_insert_id,NULL); // as in MySQL Protocol, last_insert_id is immediately after affected_rows + total_bytes+=mysql_encode_length(rows_sent,NULL); + + total_bytes+=mysql_encode_length(query_digest,NULL); + + total_bytes+=mysql_encode_length(query_len,NULL)+query_len; + + // for performance reason, we are moving the write lock + // right before the write to disk + //GloPgSQL_Logger->wrlock(); + //move wrlock() function to log_request() function, avoid to get a null pointer in a multithreaded environment + + // write total length , fixed size + f->write((const char *)&total_bytes,sizeof(uint64_t)); + //char prefix; + uint8_t len; + + f->write((char *)&et,1); + + len=mysql_encode_length(thread_id,buf); + write_encoded_length(buf,thread_id,len,buf[0]); + f->write((char *)buf,len); + + len=mysql_encode_length(username_len,buf); + write_encoded_length(buf,username_len,len,buf[0]); + f->write((char *)buf,len); + f->write(username,username_len); + + len=mysql_encode_length(schemaname_len,buf); + write_encoded_length(buf,schemaname_len,len,buf[0]); + f->write((char *)buf,len); + f->write(schemaname,schemaname_len); + + len=mysql_encode_length(client_len,buf); + write_encoded_length(buf,client_len,len,buf[0]); + f->write((char *)buf,len); + f->write(client,client_len); + + len=mysql_encode_length(hid,buf); + write_encoded_length(buf,hid,len,buf[0]); + f->write((char *)buf,len); + + if (hid!=UINT64_MAX) { + len=mysql_encode_length(server_len,buf); + write_encoded_length(buf,server_len,len,buf[0]); + f->write((char *)buf,len); + f->write(server,server_len); + } + + len=mysql_encode_length(start_time,buf); + write_encoded_length(buf,start_time,len,buf[0]); + f->write((char *)buf,len); + + len=mysql_encode_length(end_time,buf); + write_encoded_length(buf,end_time,len,buf[0]); + f->write((char *)buf,len); + + if (et == PROXYSQL_COM_STMT_PREPARE || et == PROXYSQL_COM_STMT_EXECUTE) { + len=mysql_encode_length(client_stmt_id,buf); + write_encoded_length(buf,client_stmt_id,len,buf[0]); + f->write((char *)buf,len); + } + + len=mysql_encode_length(affected_rows,buf); + write_encoded_length(buf,affected_rows,len,buf[0]); + f->write((char *)buf,len); + + len=mysql_encode_length(last_insert_id,buf); + write_encoded_length(buf,last_insert_id,len,buf[0]); + f->write((char *)buf,len); + + len=mysql_encode_length(rows_sent,buf); + write_encoded_length(buf,rows_sent,len,buf[0]); + f->write((char *)buf,len); + + len=mysql_encode_length(query_digest,buf); + write_encoded_length(buf,query_digest,len,buf[0]); + f->write((char *)buf,len); + + len=mysql_encode_length(query_len,buf); + write_encoded_length(buf,query_len,len,buf[0]); + f->write((char *)buf,len); + if (query_len) { + f->write(query_ptr,query_len); + } + + return total_bytes; +} + +uint64_t PgSQL_Event::write_query_format_2_json(std::fstream *f) { + json j = {}; + uint64_t total_bytes=0; + if (hid!=UINT64_MAX) { + j["hostgroup_id"] = hid; + } else { + j["hostgroup_id"] = -1; + } + j["thread_id"] = thread_id; + switch (et) { + case PROXYSQL_COM_STMT_EXECUTE: + j["event"]="COM_STMT_EXECUTE"; + break; + case PROXYSQL_COM_STMT_PREPARE: + j["event"]="COM_STMT_PREPARE"; + break; + default: + j["event"]="COM_QUERY"; + break; + } + if (username) { + j["username"] = username; + //} else { + // j["username"] = ""; + } + if (schemaname) { + j["schemaname"] = schemaname; + //} else { + // j["schemaname"] = ""; + } + if (client) { + j["client"] = client; + //} else { + // j["client"] = ""; + } + if (hid!=UINT64_MAX) { + if (server) { + j["server"] = server; + } + } + if (have_affected_rows == true) { + // in JSON format we only log rows_affected and last_insert_id + // if they are present. + // rows_affected is logged also if 0, while + // last_insert_id is log logged if 0 + j["rows_affected"] = affected_rows; + if (last_insert_id != 0) { + j["last_insert_id"] = last_insert_id; + } + } + if (have_rows_sent == true) { + j["rows_sent"] = rows_sent; + } + j["query"] = string(query_ptr,query_len); + j["starttime_timestamp_us"] = start_time; + { + time_t timer=start_time/1000/1000; + struct tm* tm_info; + tm_info = localtime(&timer); + char buffer1[36]; + char buffer2[64]; + strftime(buffer1, 32, "%Y-%m-%d %H:%M:%S", tm_info); + sprintf(buffer2,"%s.%06u", buffer1, (unsigned)(start_time%1000000)); + j["starttime"] = buffer2; + } + j["endtime_timestamp_us"] = end_time; + { + time_t timer=end_time/1000/1000; + struct tm* tm_info; + tm_info = localtime(&timer); + char buffer1[36]; + char buffer2[64]; + strftime(buffer1, 32, "%Y-%m-%d %H:%M:%S", tm_info); + sprintf(buffer2,"%s.%06u", buffer1, (unsigned)(end_time%1000000)); + j["endtime"] = buffer2; + } + j["duration_us"] = end_time-start_time; + char digest_hex[20]; + sprintf(digest_hex,"0x%016llX", (long long unsigned int)query_digest); + j["digest"] = digest_hex; + + if (et == PROXYSQL_COM_STMT_PREPARE || et == PROXYSQL_COM_STMT_EXECUTE) { + j["client_stmt_id"] = client_stmt_id; + } + + // for performance reason, we are moving the write lock + // right before the write to disk + //GloPgSQL_Logger->wrlock(); + //move wrlock() function to log_request() function, avoid to get a null pointer in a multithreaded environment + + *f << j.dump(-1, ' ', false, json::error_handler_t::replace) << std::endl; + return total_bytes; // always 0 +} + +extern PgSQL_Query_Processor* GloPgQPro; + +PgSQL_Logger::PgSQL_Logger() { + events.enabled=false; + events.base_filename=NULL; + events.datadir=NULL; + events.base_filename=strdup((char *)""); + audit.enabled=false; + audit.base_filename=NULL; + audit.datadir=NULL; + audit.base_filename=strdup((char *)""); +#ifdef PROXYSQL_LOGGER_PTHREAD_MUTEX + pthread_mutex_init(&wmutex,NULL); +#else + spinlock_rwlock_init(&rwlock); +#endif + events.logfile=NULL; + events.log_file_id=0; + events.max_log_file_size=100*1024*1024; + audit.logfile=NULL; + audit.log_file_id=0; + audit.max_log_file_size=100*1024*1024; +}; + +PgSQL_Logger::~PgSQL_Logger() { + if (events.datadir) { + free(events.datadir); + } + free(events.base_filename); + if (audit.datadir) { + free(audit.datadir); + } + free(audit.base_filename); +}; + +void PgSQL_Logger::wrlock() { +#ifdef PROXYSQL_LOGGER_PTHREAD_MUTEX + pthread_mutex_lock(&wmutex); +#else + spin_wrlock(&rwlock); +#endif +}; + +void PgSQL_Logger::wrunlock() { +#ifdef PROXYSQL_LOGGER_PTHREAD_MUTEX + pthread_mutex_unlock(&wmutex); +#else + spin_wrunlock(&rwlock); +#endif +}; + +void PgSQL_Logger::flush_log() { + if (audit.enabled==false && events.enabled==false) return; + wrlock(); + events_flush_log_unlocked(); + audit_flush_log_unlocked(); + wrunlock(); +} + + +void PgSQL_Logger::events_close_log_unlocked() { + if (events.logfile) { + events.logfile->flush(); + events.logfile->close(); + delete events.logfile; + events.logfile=NULL; + } +} + +void PgSQL_Logger::audit_close_log_unlocked() { + if (audit.logfile) { + audit.logfile->flush(); + audit.logfile->close(); + delete audit.logfile; + audit.logfile=NULL; + } +} + +void PgSQL_Logger::events_flush_log_unlocked() { + if (events.enabled==false) return; + events_close_log_unlocked(); + events_open_log_unlocked(); +} + +void PgSQL_Logger::audit_flush_log_unlocked() { + if (audit.enabled==false) return; + audit_close_log_unlocked(); + audit_open_log_unlocked(); +} + +void PgSQL_Logger::events_open_log_unlocked() { + events.log_file_id=events_find_next_id(); + if (events.log_file_id!=0) { + events.log_file_id=events_find_next_id()+1; + } else { + events.log_file_id++; + } + char *filen=NULL; + if (events.base_filename[0]=='/') { // absolute path + filen=(char *)malloc(strlen(events.base_filename)+11); + sprintf(filen,"%s.%08d",events.base_filename,events.log_file_id); + } else { // relative path + filen=(char *)malloc(strlen(events.datadir)+strlen(events.base_filename)+11); + sprintf(filen,"%s/%s.%08d",events.datadir,events.base_filename,events.log_file_id); + } + events.logfile=new std::fstream(); + events.logfile->exceptions ( std::ofstream::failbit | std::ofstream::badbit ); + try { + events.logfile->open(filen , std::ios::out | std::ios::binary); + proxy_info("Starting new mysql event log file %s\n", filen); + } + catch (const std::ofstream::failure&) { + proxy_error("Error creating new mysql event log file %s\n", filen); + delete events.logfile; + events.logfile=NULL; + } + free(filen); +}; + +void PgSQL_Logger::audit_open_log_unlocked() { + audit.log_file_id=audit_find_next_id(); + if (audit.log_file_id!=0) { + audit.log_file_id=audit_find_next_id()+1; + } else { + audit.log_file_id++; + } + char *filen=NULL; + if (audit.base_filename[0]=='/') { // absolute path + filen=(char *)malloc(strlen(audit.base_filename)+11); + sprintf(filen,"%s.%08d",audit.base_filename,audit.log_file_id); + } else { // relative path + filen=(char *)malloc(strlen(audit.datadir)+strlen(audit.base_filename)+11); + sprintf(filen,"%s/%s.%08d",audit.datadir,audit.base_filename,audit.log_file_id); + } + audit.logfile=new std::fstream(); + audit.logfile->exceptions ( std::ofstream::failbit | std::ofstream::badbit ); + try { + audit.logfile->open(filen , std::ios::out | std::ios::binary); + proxy_info("Starting new audit log file %s\n", filen); + } + catch (const std::ofstream::failure&) { + proxy_error("Error creating new audit log file %s\n", filen); + delete audit.logfile; + audit.logfile=NULL; + } + free(filen); +}; + +void PgSQL_Logger::events_set_base_filename() { + // if filename is the same, return + wrlock(); + events.max_log_file_size=pgsql_thread___eventslog_filesize; + if (strcmp(events.base_filename,pgsql_thread___eventslog_filename)==0) { + wrunlock(); + return; + } + // close current log + events_close_log_unlocked(); + // set file id to 0 , so that find_next_id() will be called + events.log_file_id=0; + free(events.base_filename); + events.base_filename=strdup(pgsql_thread___eventslog_filename); + if (strlen(events.base_filename)) { + events.enabled=true; + events_open_log_unlocked(); + } else { + events.enabled=false; + } + wrunlock(); +} + +void PgSQL_Logger::events_set_datadir(char *s) { + if (events.datadir) + free(events.datadir); + events.datadir=strdup(s); + flush_log(); +}; + +void PgSQL_Logger::audit_set_base_filename() { + // if filename is the same, return + wrlock(); + audit.max_log_file_size=pgsql_thread___auditlog_filesize; + if (strcmp(audit.base_filename,pgsql_thread___auditlog_filename)==0) { + wrunlock(); + return; + } + // close current log + audit_close_log_unlocked(); + // set file id to 0 , so that find_next_id() will be called + audit.log_file_id=0; + free(audit.base_filename); + audit.base_filename=strdup(pgsql_thread___auditlog_filename); + if (strlen(audit.base_filename)) { + audit.enabled=true; + audit_open_log_unlocked(); + } else { + audit.enabled=false; + } + wrunlock(); +} + +void PgSQL_Logger::audit_set_datadir(char *s) { + if (audit.datadir) + free(audit.datadir); + audit.datadir=strdup(s); + flush_log(); +}; + +void PgSQL_Logger::log_request(PgSQL_Session *sess, PgSQL_Data_Stream *myds) { + if (events.enabled==false) return; + if (events.logfile==NULL) return; + // 'PgSQL_Session::client_myds' could be NULL in case of 'RequestEnd' being called over a freshly created session + // due to a failed 'CONNECTION_RESET'. Because this scenario isn't a client request, we just return. + if (sess->client_myds==NULL || sess->client_myds->myconn== NULL) return; + + PgSQL_Connection_userinfo *ui=sess->client_myds->myconn->userinfo; + + uint64_t curtime_real=realtime_time(); + uint64_t curtime_mono=sess->thread->curtime; + int cl=0; + char *ca=(char *)""; // default + if (sess->client_myds->addr.addr) { + ca=sess->client_myds->addr.addr; + } + cl+=strlen(ca); + if (cl && sess->client_myds->addr.port) { + ca=(char *)malloc(cl+9); + sprintf(ca,"%s:%d",sess->client_myds->addr.addr,sess->client_myds->addr.port); + } + cl=strlen(ca); + enum log_event_type let = PROXYSQL_COM_QUERY; // default + switch (sess->status) { + case PROCESSING_STMT_EXECUTE: + let = PROXYSQL_COM_STMT_EXECUTE; + break; + case PROCESSING_STMT_PREPARE: + let = PROXYSQL_COM_STMT_PREPARE; + break; + case WAITING_CLIENT_DATA: + { + unsigned char c=*((unsigned char *)sess->pkt.ptr+sizeof(mysql_hdr)); + switch ((enum_mysql_command)c) { + case _MYSQL_COM_STMT_PREPARE: + // proxysql is responding to COM_STMT_PREPARE without + // preparing on any backend + let = PROXYSQL_COM_STMT_PREPARE; + break; + default: + break; + } + } + break; + default: + break; + } + + uint64_t query_digest = 0; + + if (sess->status != PROCESSING_STMT_EXECUTE) { + query_digest = GloPgQPro->get_digest(&sess->CurrentQuery.QueryParserArgs); + } else { + query_digest = sess->CurrentQuery.stmt_info->digest; + } + + PgSQL_Event me(let, + sess->thread_session_id,ui->username,ui->dbname, + sess->CurrentQuery.start_time + curtime_real - curtime_mono, + sess->CurrentQuery.end_time + curtime_real - curtime_mono, + query_digest, + ca, cl + ); + char *c = NULL; + int ql = 0; + switch (sess->status) { + case PROCESSING_STMT_EXECUTE: + c = (char *)sess->CurrentQuery.stmt_info->query; + ql = sess->CurrentQuery.stmt_info->query_length; + me.set_client_stmt_id(sess->CurrentQuery.stmt_client_id); + break; + case PROCESSING_STMT_PREPARE: + default: + c = (char *)sess->CurrentQuery.QueryPointer; + ql = sess->CurrentQuery.QueryLength; + // NOTE: This needs to be located in the 'default' case because otherwise will miss state + // 'WAITING_CLIENT_DATA'. This state is possible when the prepared statement is found in the + // global cache and due to that we immediately reply to the client and session doesn't reach + // 'PROCESSING_STMT_PREPARE' state. 'stmt_client_id' is expected to be '0' for anything that isn't + // a prepared statement, still, logging should rely 'log_event_type' instead of this value. + me.set_client_stmt_id(sess->CurrentQuery.stmt_client_id); + break; + } + if (c) { + me.set_query(c,ql); + } else { + me.set_query("",0); + } + + if (sess->CurrentQuery.have_affected_rows) { + me.set_affected_rows(sess->CurrentQuery.affected_rows, sess->CurrentQuery.last_insert_id); + } + me.set_rows_sent(sess->CurrentQuery.rows_sent); + + int sl=0; + char *sa=(char *)""; // default + if (myds) { + if (myds->myconn) { + sa=myds->myconn->parent->address; + } + } + sl+=strlen(sa); + if (sl && myds->myconn->parent->port) { + sa=(char *)malloc(sl+9); + sprintf(sa,"%s:%d", myds->myconn->parent->address, myds->myconn->parent->port); + } + sl=strlen(sa); + if (sl) { + int hid=-1; + hid=myds->myconn->parent->myhgc->hid; + me.set_server(hid,sa,sl); + } + + // for performance reason, we are moving the write lock + // right before the write to disk + //wrlock(); + + //add a mutex lock in a multithreaded environment, avoid to get a null pointer of events.logfile that leads to the program coredump + GloPgSQL_Logger->wrlock(); + + me.write(events.logfile, sess); + + + unsigned long curpos=events.logfile->tellp(); + if (curpos > events.max_log_file_size) { + events_flush_log_unlocked(); + } + wrunlock(); + + if (cl && sess->client_myds->addr.port) { + free(ca); + } + if (sl && myds->myconn->parent->port) { + free(sa); + } +} + +void PgSQL_Logger::log_audit_entry(log_event_type _et, PgSQL_Session *sess, PgSQL_Data_Stream *myds, char *xi) { + if (audit.enabled==false) return; + if (audit.logfile==NULL) return; + + if (sess == NULL) return; + if (sess->client_myds == NULL) return; + + PgSQL_Connection_userinfo *ui= NULL; + if (sess) { + if (sess->client_myds) { + if (sess->client_myds->myconn) { + ui = sess->client_myds->myconn->userinfo; + } + } + } + if (sess) { + // to reduce complexing in the calling function, we do some changes here + switch (_et) { + case PROXYSQL_MYSQL_AUTH_OK: + switch (sess->session_type) { + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + _et = PROXYSQL_ADMIN_AUTH_OK; + break; + case PROXYSQL_SESSION_SQLITE: + _et = PROXYSQL_SQLITE_AUTH_OK; + default: + break; + } + break; + case PROXYSQL_MYSQL_AUTH_ERR: + switch (sess->session_type) { + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + _et = PROXYSQL_ADMIN_AUTH_ERR; + break; + case PROXYSQL_SESSION_SQLITE: + _et = PROXYSQL_SQLITE_AUTH_ERR; + default: + break; + } + break; + case PROXYSQL_MYSQL_AUTH_QUIT: + switch (sess->session_type) { + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + _et = PROXYSQL_ADMIN_AUTH_QUIT; + break; + case PROXYSQL_SESSION_SQLITE: + _et = PROXYSQL_SQLITE_AUTH_QUIT; + default: + break; + } + break; + case PROXYSQL_MYSQL_AUTH_CLOSE: + switch (sess->session_type) { + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + _et = PROXYSQL_ADMIN_AUTH_CLOSE; + break; + case PROXYSQL_SESSION_SQLITE: + _et = PROXYSQL_SQLITE_AUTH_CLOSE; + default: + break; + } + break; + default: + break; + } + } + + uint64_t curtime_real=realtime_time(); + int cl=0; + char *ca=(char *)""; // default + if (sess->client_myds->addr.addr) { + ca=sess->client_myds->addr.addr; + } + cl+=strlen(ca); + if (cl && sess->client_myds->addr.port) { + ca=(char *)malloc(cl+9); + sprintf(ca,"%s:%d",sess->client_myds->addr.addr,sess->client_myds->addr.port); + } + cl=strlen(ca); + + char *un = (char *)""; + char *sn = (char *)""; + if (ui) { + if (ui->username) { + un = ui->username; + } + if (ui->dbname) { + sn = ui->dbname; + } + } + PgSQL_Event me(_et, sess->thread_session_id, + un, sn, + curtime_real, 0, 0, + ca, cl + ); +/* + char *c=(char *)sess->CurrentQuery.QueryPointer; + if (c) { + me.set_query(c,sess->CurrentQuery.QueryLength); + } else { + me.set_query("",0); + } +*/ + int sl=0; + char *sa=(char *)""; // default + if (myds) { + if (myds->myconn) { + sa=myds->myconn->parent->address; + } + } + sl+=strlen(sa); + if (sl && myds->myconn->parent->port) { + sa=(char *)malloc(sl+9); + sprintf(sa,"%s:%d", myds->myconn->parent->address, myds->myconn->parent->port); + } + sl=strlen(sa); + + if (xi) { + me.set_extra_info(xi); + } + + // for performance reason, we are moving the write lock + // right before the write to disk + //wrlock(); + + //add a mutex lock in a multithreaded environment, avoid to get a null pointer of events.logfile that leads to the program coredump + GloPgSQL_Logger->wrlock(); + me.write(audit.logfile, sess); + + + unsigned long curpos=audit.logfile->tellp(); + if (curpos > audit.max_log_file_size) { + audit_flush_log_unlocked(); + } + wrunlock(); + + if (cl && sess->client_myds->addr.port) { + free(ca); + } + if (sl && myds->myconn->parent->port) { + free(sa); + } +} + +void PgSQL_Logger::flush() { + wrlock(); + if (events.logfile) { + events.logfile->flush(); + } + if (audit.logfile) { + audit.logfile->flush(); + } + wrunlock(); +} + +unsigned int PgSQL_Logger::events_find_next_id() { + int maxidx=0; + DIR *dir; + struct dirent *ent; + char *eval_filename = NULL; + char *eval_dirname = NULL; + char *eval_pathname = NULL; + assert(events.base_filename); + if (events.base_filename[0] == '/') { + eval_pathname = strdup(events.base_filename); + eval_filename = basename(eval_pathname); + eval_dirname = dirname(eval_pathname); + } else { + assert(events.datadir); + eval_filename = strdup(events.base_filename); + eval_dirname = strdup(events.datadir); + } + size_t efl=strlen(eval_filename); + if ((dir = opendir(eval_dirname)) != NULL) { + while ((ent = readdir (dir)) != NULL) { + if (strlen(ent->d_name)==efl+9) { + if (strncmp(ent->d_name,eval_filename,efl)==0) { + if (ent->d_name[efl]=='.') { + int idx=atoi(ent->d_name+efl+1); + if (idx>maxidx) maxidx=idx; + } + } + } + } + closedir (dir); + if (events.base_filename[0] != '/') { + free(eval_dirname); + free(eval_filename); + } + if (eval_pathname) { + free(eval_pathname); + } + return maxidx; + } else { + /* could not open directory */ + proxy_error("Unable to open datadir: %s\n", eval_dirname); + exit(EXIT_FAILURE); + } + return 0; +} + +unsigned int PgSQL_Logger::audit_find_next_id() { + int maxidx=0; + DIR *dir; + struct dirent *ent; + char *eval_filename = NULL; + char *eval_dirname = NULL; + char *eval_pathname = NULL; + assert(audit.base_filename); + if (audit.base_filename[0] == '/') { + eval_pathname = strdup(audit.base_filename); + eval_filename = basename(eval_pathname); + eval_dirname = dirname(eval_pathname); + } else { + assert(audit.datadir); + eval_filename = strdup(audit.base_filename); + eval_dirname = strdup(audit.datadir); + } + size_t efl=strlen(eval_filename); + if ((dir = opendir(eval_dirname)) != NULL) { + while ((ent = readdir (dir)) != NULL) { + if (strlen(ent->d_name)==efl+9) { + if (strncmp(ent->d_name,eval_filename,efl)==0) { + if (ent->d_name[efl]=='.') { + int idx=atoi(ent->d_name+efl+1); + if (idx>maxidx) maxidx=idx; + } + } + } + } + closedir (dir); + if (audit.base_filename[0] != '/') { + free(eval_dirname); + free(eval_filename); + } + if (eval_pathname) { + free(eval_pathname); + } + return maxidx; + } else { + /* could not open directory */ + proxy_error("Unable to open datadir: %s\n", eval_dirname); + exit(EXIT_FAILURE); + } + return 0; +} + +void PgSQL_Logger::print_version() { + fprintf(stderr,"Standard ProxySQL MySQL Logger rev. %s -- %s -- %s\n", PROXYSQL_MYSQL_LOGGER_VERSION, __FILE__, __TIMESTAMP__); +}; + diff --git a/lib/PgSQL_Protocol.cpp b/lib/PgSQL_Protocol.cpp new file mode 100644 index 0000000000..40ff303e95 --- /dev/null +++ b/lib/PgSQL_Protocol.cpp @@ -0,0 +1,1958 @@ +//#include "openssl/rand.h" +#include "proxysql.h" +#include "cpp.h" +/* +#include "re2/re2.h" +#include "re2/regexp.h" +#include "MySQL_PreparedStatement.h" + + +#include "MySQL_LDAP_Authentication.hpp" +#include "MySQL_Variables.h" +#include +*/ +#include "PgSQL_Authentication.h" +#include "PgSQL_Data_Stream.h" +#include "PgSQL_Protocol.h" +extern "C" { +#include "usual/time.h" +} +//#include "usual/time.c" + +extern PgSQL_Authentication* GloPgAuth; + +/* + * PgSQL type OIDs for result sets + */ +#define BYTEAOID 17 +#define INT8OID 20 +#define INT4OID 23 +#define TEXTOID 25 +#define NUMERICOID 1700 + + +void PG_pkt::make_space(unsigned int len) { + if (ownership == false) return; + + if ((size + len) <= capacity) { + return; + } else { + capacity = l_near_pow_2(size + len); + ptr = (char *)realloc(ptr, capacity); + } +} + +void PG_pkt::put_char(char val) { + make_space(sizeof(char)); + ptr[size++] = val; +} + +void PG_pkt::put_uint16(uint16_t val) { + make_space(4); + ptr[size++] = (val >> 8) & 255; + ptr[size++] = val & 255; +} + +void PG_pkt::put_uint32(uint32_t val) { + make_space(4); + ptr[size++] = (val >> 24) & 255; + ptr[size++] = (val >> 16) & 255; + ptr[size++] = (val >> 8) & 255; + ptr[size++] = val & 255; +} + +void PG_pkt::put_uint64(uint64_t val) { + put_uint32(val >> 32); + put_uint32((uint32_t)val); +} + +void PG_pkt::put_bytes(const void *data, int len) { + make_space(len); + memcpy(ptr + size, data, len); + size += len; +} + +void PG_pkt::put_string(const char *str) { + int len = strlen(str); + put_bytes(str, len + 1); +} + + +void PG_pkt::start_packet(int type) { + assert(type < 256); + put_char(type); + put_uint32(0); // this is a space reserved for the packet length +} + +void PG_pkt::finish_packet() { + uint8_t* pos = NULL; + unsigned len = 0; + + if (multiple_pkt_mode == false) { + pos = (uint8_t*)ptr + 1; // the first byte after the packet type + len = size - 1; // the length of the packet minus the packet type byte + } else { + + if (pkt_offset.empty() == false) { + const unsigned int offset = pkt_offset.back(); + pos = (uint8_t*)ptr + offset + 1; + len = (size - offset) - 1; + } + } + + *pos++ = (len >> 24) & 255; + *pos++ = (len >> 16) & 255; + *pos++ = (len >> 8) & 255; + *pos++ = len & 255; +} + +void PG_pkt::write_generic(int type, const char *pktdesc, ...) { + va_list ap; + const char *adesc = pktdesc; + + if (multiple_pkt_mode) + pkt_offset.push_back(size); + + start_packet(type); + va_start(ap, pktdesc); + while (*adesc) { + switch (*adesc) { + case 'c': // char/byte + put_char(va_arg(ap, int)); + break; + case 'h': // uint16 + put_uint16(va_arg(ap, int)); + break; + case 'i': // uint32 + put_uint32(va_arg(ap, int)); + break; + case 'q': // uint64 + put_uint64(va_arg(ap, uint64_t)); + break; + case 's': // Cstring + put_string(va_arg(ap, char *)); + break; + case 'b': // bytes + { + uint8_t *bin = va_arg(ap, uint8_t *); + int len = va_arg(ap, int); + put_bytes(bin, len); + } + break; + default: + assert(0); + break; + } + adesc++; + } + va_end(ap); + + finish_packet(); +} + +void PG_pkt::write_RowDescription(const char *tupdesc, ...) { + va_list ap; + int ncol = strlen(tupdesc); + + start_packet('T'); + + put_uint16(ncol); + + va_start(ap, tupdesc); + for (int i = 0; i < ncol; i++) { + char * name = va_arg(ap, char *); + + /* Fields: name, reloid, colnr, oid, typsize, typmod, fmt */ + put_string(name); + put_uint32(0); + put_uint16(0); + const char c = tupdesc[i]; + switch (c) { + case 's': + put_uint32(TEXTOID); + put_uint16(-1); + break; + case 'b': + put_uint32(BYTEAOID); + put_uint16(-1); + break; + case 'i': + put_uint32(INT4OID); + put_uint16(4); + break; + case 'q': + put_uint32(INT8OID); + put_uint16(8); + break; + case 'N': + put_uint32(NUMERICOID); + put_uint16(-1); + break; + case 'T': + put_uint32(TEXTOID); + put_uint16(-1); + break; + default: + assert(0); + break; + } + put_uint32(-1); + put_uint16(0); + } + va_end(ap); + + /* set correct length */ + finish_packet(); +} + + +void SQLite3_to_Postgres(PtrSizeArray *psa, SQLite3_result *result, char *error, int affected_rows, const char *query_type) { + assert(psa != NULL); + const char *fs = strchr(query_type, ' '); + int qtlen = strlen(query_type); + if (fs != NULL) { + qtlen = (fs - query_type) + 1; + } + char buf[qtlen]; + memcpy(buf,query_type, qtlen-1); + buf[qtlen-1] = 0; + { + char *s = buf; + while (*s) { + *s = toupper((unsigned char) *s); + s++; + } + } + if (result) { + int ncol = result->columns; + PG_pkt pkt(64); + pkt.start_packet('T'); + pkt.put_uint16(ncol); + for (int i=0; i < ncol ; i++) { + char *name = result->column_definition[i]->name; + pkt.put_string(name); + pkt.put_uint32(0); + pkt.put_uint16(0); + pkt.put_uint32(TEXTOID); // we add all columns as TEXT + pkt.put_uint16(-1); + pkt.put_uint32(-1); + pkt.put_uint16(0); + } + pkt.finish_packet(); + pkt.to_PtrSizeArray(psa); + for (int r=0; rrows_count; r++) { + //PG_pkt pkt(128); + pkt.start_packet('D'); + pkt.put_uint16(ncol); + for (int i=0; i < ncol; i++) { + const char *val = result->rows[r]->fields[i]; + if (val != NULL) { + int len = result->rows[r]->sizes[i]; + pkt.put_uint32(len); + pkt.put_bytes(val, len); + } else { + pkt.put_uint32(-1); // NULL + } + } + pkt.finish_packet(); + pkt.to_PtrSizeArray(psa); + } + + if (strcmp(buf,"SELECT") == 0) { + char tmpbuf[128]; + sprintf(tmpbuf,"%s %d", buf, result->rows_count); + pkt.write_generic('C', "s", tmpbuf); + } else { + pkt.write_CommandComplete(buf); + } + pkt.to_PtrSizeArray(psa); + pkt.write_ReadyForQuery(); + pkt.to_PtrSizeArray(psa); + } else { // no resultset + PG_pkt pkt(64); + if (error) { + // there was an error + pkt.write_generic('E', "cscscsc", + 'S', "ERROR", + 'C', "28000", + 'M', error, 0); +/* + if (strcmp(error,(char *)"database is locked")==0) { + pkt.write_generic('E', + myprot->generate_pkt_ERR(true,NULL,NULL,sid,1205,(char *)"HY000",error); + } else { + myprot->generate_pkt_ERR(true,NULL,NULL,sid,1045,(char *)"28000",error); + } +*/ + // see https://www.postgresql.org/docs/current/protocol-message-formats.html + } else { + char tmpbuf[128]; + if (strcmp(buf,"INSERT") == 0) { + sprintf(tmpbuf,"%s 0 %d", buf, affected_rows); + pkt.write_generic('C', "s", tmpbuf); + } else if (strcmp(buf,"UPDATE") == 0 || strcmp(buf,"DELETE") == 0) { + sprintf(tmpbuf,"%s %d", buf, affected_rows); + pkt.write_generic('C', "s", tmpbuf); + } else { + pkt.write_CommandComplete(buf); + } + } + pkt.to_PtrSizeArray(psa); + pkt.write_ReadyForQuery(); + pkt.to_PtrSizeArray(psa); + } +} +void PG_pkt::write_DataRow(const char *tupdesc, ...) { + int ncol = strlen(tupdesc); + va_list ap; + + start_packet('D'); + put_uint16(ncol); + + va_start(ap, tupdesc); + for (int i = 0; i < ncol; i++) { + char tmp[128]; + char *tmp2 = NULL; + const char *val = NULL; + + if (tupdesc[i] == 'i') { + snprintf(tmp, sizeof(tmp), "%d", va_arg(ap, int)); + val = tmp; + } else if (tupdesc[i] == 'q' || tupdesc[i] == 'N') { + snprintf(tmp, sizeof(tmp), "%" PRIu64, va_arg(ap, uint64_t)); + val = tmp; + } else if (tupdesc[i] == 's') { + val = va_arg(ap, char *); + } else if (tupdesc[i] == 'b') { + int blen = va_arg(ap, int); + if (blen >= 0) { + uint8_t *bval = va_arg(ap, uint8_t *); + size_t required = 2 + blen * 2 + 1; + tmp2 = (char *)malloc(required); + strcpy(tmp2, "\\x"); + for (int j = 0; j < blen; j++) + sprintf(tmp2 + (2 + j * 2), "%02x", bval[j]); + val = tmp2; + } else { + (void) va_arg(ap, uint8_t *); + val = NULL; + } + } else if (tupdesc[i] == 'T') { + usec_t time = va_arg(ap, usec_t); + val = format_time_s(time, tmp, sizeof(tmp)); + } else { + fprintf(stderr, "bad tupdesc: %s", tupdesc); + assert(0); + } + + if (val) { + int len = strlen(val); + put_uint32(len); + put_bytes(val, len); + if (tmp2 != NULL) { + free(tmp2); + tmp2 = NULL; + } + } else { + /* NULL */ + put_uint32(-1); + } + } + va_end(ap); + + /* set correct length */ + finish_packet(); +} + +PtrSize_t * PG_pkt::get_PtrSize(unsigned c) { + PtrSize_t * pkt = (PtrSize_t *)malloc(sizeof(PtrSize_t)); + pkt->ptr = ptr; + pkt->size = size; + capacity = l_near_pow_2(c); + size = 0; + ptr = (char *)malloc(capacity); + return pkt; +} + +void PG_pkt::to_PtrSizeArray(PtrSizeArray *psa, unsigned c) { + psa->add(ptr, size); + size = 0; + if (c != 0) { + capacity = l_near_pow_2(c); + ptr = (char *)malloc(capacity); + } else { + capacity = 0; + ptr = NULL; + } +} + +bool PgSQL_Protocol::generate_pkt_initial_handshake(bool send, void** _ptr, unsigned int* len, uint32_t* _thread_id, bool deprecate_eof_active) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Generating handshake pkt\n"); + + PG_pkt pgpkt{}; + + const int type = 'R'; + + uint32_t thread_id = __sync_fetch_and_add(&glovars.thread_id, 1); + if (thread_id == 0) { + thread_id = __sync_fetch_and_add(&glovars.thread_id, 1); // again! + } + *_thread_id = thread_id; + + switch ((AUTHENTICATION_METHOD)pgsql_thread___authentication_method) { + + case AUTHENTICATION_METHOD::NO_PASSWORD: + pgpkt.write_generic(type, "i", PG_PKT_AUTH_OK); + break; + case AUTHENTICATION_METHOD::CLEAR_TEXT_PASSWORD: + pgpkt.write_generic(type, "i", PG_PKT_AUTH_PLAIN); + break; + case AUTHENTICATION_METHOD::MD5_PASSWORD: + pgpkt.write_generic(type, "i", PG_PKT_AUTH_MD5); + break; + case AUTHENTICATION_METHOD::SASL_SCRAM_SHA_256: + pgpkt.write_generic(type, "iss", PG_PKT_AUTH_SASL, "SCRAM-SHA-256", ""); + break; + case AUTHENTICATION_METHOD::SASL_SCRAM_SHA_256_PLUS: + pgpkt.write_generic(type, "iss", PG_PKT_AUTH_SASL, "SCRAM-SHA-256-PLUS", ""); + break; + default: + assert(0); + } + + (*myds)->auth_method = (AUTHENTICATION_METHOD)pgsql_thread___authentication_method; + (*myds)->auth_next_pkt_type = 'p'; + + if (send == true) { + auto buff = pgpkt.detach(); + (*myds)->PSarrayOUT->add((void*)buff.first, buff.second); + (*myds)->DSS = STATE_SERVER_HANDSHAKE; + (*myds)->sess->status = CONNECTING_CLIENT; + } + //if (len) { *len = size; } + //if (_ptr) { *_ptr = (void*)ptr; } + + return true; +} + +/* + * @brief Reads and converts a big endian 32-bit unsigned integer from the provided packet buffer into the destination pointer. + * + * This function is used to extract the big endian 32-bit unsigned integer value at the specified position in a given + * packet buffer, and stores it in the destination pointer passed as an argument. + * + * @param[in] pkt A pointer to the start of the input packet buffer from which to read the 32-bit integer. + * + * @param[out] dst_p A pointer where the extracted big endian 32-bit unsigned integer value will be stored. + */ +static inline bool get_uint32be(unsigned char* pkt, uint32_t* dst_p) +{ + int read_pos = 0; + unsigned a, b, c, d; + + a = pkt[read_pos++]; + b = pkt[read_pos++]; + c = pkt[read_pos++]; + d = pkt[read_pos++]; + *dst_p = (a << 24) | (b << 16) | (c << 8) | d; + return true; +} + + +/** + * @brief Extracts a 16-bit unsigned integer from a packet and stores it in the provided destination pointer. + * + * This function reads two bytes from the packet `pkt` starting from the beginning, interprets them as a big-endian unsigned 16-bit integer, + * and stores the result into the memory location pointed to by `dst_p`. It consistently returns true to indicate successful execution. + * + * @param pkt Pointer to the packet data (array of unsigned chars) from which the 16-bit integer will be extracted. + * The caller must ensure this pointer is valid and points to at least two bytes of data. + * @param dst_p Pointer to a uint16_t variable where the extracted integer will be stored. The caller must ensure that + * this pointer is valid and points to a uint16_t variable. + * + * @return Always returns true to indicate success. + * + * @note This function uses big-endian byte order (network byte order) for interpreting the packet data. + * It is assumed that the packet buffer `pkt` contains at least two bytes (the size of a uint16_t). + * The function uses post-increment to move the reading position after extracting each byte. + */ +static inline bool get_uint16be(unsigned char* pkt, uint16_t* dst_p) +{ + int read_pos = 0; ///< Current read position in the buffer. + unsigned a, b; + + // Read the two bytes from the buffer + a = pkt[read_pos++]; ///< First byte read from the buffer. + b = pkt[read_pos++]; ///< Second byte read from the buffer. + *dst_p = (a << 8) | b; + return true; +} + +bool PgSQL_Protocol::get_header(unsigned char* pkt, unsigned int pkt_len, pgsql_hdr* hdr) { + unsigned int type; + uint32_t len; + unsigned int got; + unsigned int avail; + uint16_t len16; + uint8_t type8; + uint32_t code; + //const uint8_t* ptr; + + unsigned int read_pos = 0; + + if (pkt_len < NEW_HEADER_LEN) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Packet received is less than %d bytes\n", NEW_HEADER_LEN); + return false; + } + + // below check is not needed + //if (read_pos + 1 > pkt_len) { + // return false; + //} + // + + type8 = pkt[read_pos++]; + type = type8; + + if (type != 0) { + /* + * Regular (v3) packet, starts with type byte and + * 4-byte length. + */ + + if (read_pos + 4 > pkt_len) + return false; + + /* wire length does not include type byte */ + if (!get_uint32be(pkt + read_pos, &len)) + return false; + read_pos+=4; + len++; + got = NEW_HEADER_LEN; + } + else { + /* + * Startup/special (formerly v2) packet, formally + * starts with 4-byte length. We assume the first + * byte is zero because in current use they shouldn't + * be that long to have more than zero in the MSB. + */ + + // below check is not needed + //if (read_pos + 1 > pkt_len) { + // return false; + //} + // + + /* second byte should also be zero */ + type8 = pkt[read_pos++]; + + if (type8 != 0) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Unknown special packet\n"); + return false; + } + + /* don't tolerate partial pkt */ + if ((pkt_len - read_pos) < OLD_HEADER_LEN - 2) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Special packet is less than %d bytes\n", OLD_HEADER_LEN); + return false; + } + + if (read_pos + 2 > pkt_len) + return false; + + if (!get_uint16be(pkt + read_pos, &len16)) + return false; + + read_pos += 2; + len = len16; + + /* 4-byte code follows */ + if (!get_uint32be(pkt + read_pos, &code)) + return false; + + read_pos += 4; + + if (code == PG_PKT_CANCEL) { + type = PG_PKT_CANCEL; + } + else if (code == PG_PKT_SSLREQ) { + type = PG_PKT_SSLREQ; + } + else if (code == PG_PKT_GSSENCREQ) { + type = PG_PKT_GSSENCREQ; + } + else if ((code >> 16) == 3 && (code & 0xFFFF) < 2) { + type = PG_PKT_STARTUP; + } + else if (code == PG_PKT_STARTUP_V2) { + type = PG_PKT_STARTUP_V2; + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "unknown special pkt: len=%u code=%u\n", len, code); + return false; + } + got = OLD_HEADER_LEN; + } + + /* don't believe nonsense */ + if (len < got || len > 2147483647) + return false; + + /* store pkt info */ + hdr->type = type; + hdr->len = len; + + /* fill pkt with only data for this packet */ + if (len > pkt_len - read_pos) { + avail = pkt_len - read_pos; + } + else { + avail = len; + } + + hdr->data.ptr = pkt + read_pos; + hdr->data.size = avail; + read_pos += avail; + + if (read_pos > pkt_len) + return false; + + return true; +} + +unsigned int get_string(const char* data, unsigned int len, const char** dst_p) +{ + const char* res = data; + const char* nul = (const char*)memchr(res, 0, len); + if (!nul) + return 0; + *dst_p = res; + return (nul + 1 - data); +} + +void PgSQL_Protocol::load_conn_parameters(pgsql_hdr* pkt, bool startup) +{ + const char* key, * val; + unsigned int read_pos = 0; + + while (1) { + + int pos = get_string(((const char*)pkt->data.ptr) + read_pos, pkt->data.size - read_pos, &key); + if (pos == 0) return; + + read_pos += pos; + + pos = get_string(((const char*)pkt->data.ptr) + read_pos, pkt->data.size - read_pos, &val); + if (pos == 0) return; + + read_pos += pos; + + //slog_debug(server, "S: param: %s = %s", key, val); + (*myds)->myconn->conn_params.set_value(key, val); + } +} + +bool PgSQL_Protocol::process_startup_packet(unsigned char* pkt, unsigned int len, bool& ssl_request) { + + ssl_request = false; + pgsql_hdr hdr{}; + if (!get_header(pkt, len, &hdr)) { + return false; + } + + if (hdr.type == PG_PKT_SSLREQ) { + const bool have_ssl = pgsql_thread___have_ssl; + char* ssl_supported = (char*)malloc(1); + *ssl_supported = have_ssl ? 'S' : 'N'; + (*myds)->PSarrayOUT->add((void*)ssl_supported, 1); + (*myds)->sess->writeout(); + (*myds)->encrypted = have_ssl; + ssl_request = true; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 8, "Session=%p , DS=%p. SSL_REQUEST:'%c'\n", (*myds)->sess, (*myds), *ssl_supported); + return true; + } + + //PG_PKT_STARTUP_V2 not supported + if (hdr.type != PG_PKT_STARTUP) { + return false; + } + + load_conn_parameters(&hdr, true); + + const unsigned char* user = (unsigned char*)(*myds)->myconn->conn_params.get_value(PG_USER); + + if (!user || *user == '\0') { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p. no username supplied.\n", (*myds), (*myds)->sess); + generate_error_packet(true, false, "no username supplied", PGSQL_ERROR_CODES::ERRCODE_PROTOCOL_VIOLATION, true); + return false; + } + + (*myds)->DSS = STATE_SERVER_HANDSHAKE; + + return true; +} + +EXECUTION_STATE PgSQL_Protocol::process_handshake_response_packet(unsigned char* pkt, unsigned int len) { +#ifdef DEBUG + //if (dump_pkt) { __dump_pkt(__func__, pkt, len); } +#endif + + char* user = NULL; + char* pass = NULL; + + char* password = NULL; + //char* db = NULL; + char* attributes = NULL; + void* sha1_pass = NULL; + int max_connections; + int default_hostgroup = -1; + enum proxysql_session_type session_type = (*myds)->sess->session_type; + bool using_password = false; + bool transaction_persistent = true; + bool fast_forward = false; + bool _ret_use_ssl = false; + EXECUTION_STATE ret = EXECUTION_STATE::FAILED; + + pgsql_hdr hdr{}; + if (!get_header(pkt, len, &hdr)) { + return EXECUTION_STATE::FAILED; + } + + assert((hdr.data.size - 1) > 0); + + if (hdr.type != (*myds)->auth_next_pkt_type) { + return EXECUTION_STATE::FAILED; + } + + user = (char*)(*myds)->myconn->conn_params.get_value(PG_USER); + + if (!user || *user == '\0') { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. Client password pkt before startup packet.\n", (*myds), (*myds)->sess, user); + generate_error_packet(true, false, "client password pkt before startup packet", PGSQL_ERROR_CODES::ERRCODE_PROTOCOL_VIOLATION, true); + goto __exit_process_pkt_handshake_response; + } + + password = GloPgAuth->lookup((char*)user, USERNAME_FRONTEND, &_ret_use_ssl, &default_hostgroup, &transaction_persistent, &fast_forward, &max_connections, &sha1_pass, &attributes); + + if (password) { +#ifdef DEBUG + char* tmp_pass = strdup(password); + int lpass = strlen(tmp_pass); + for (int i = 2; i < lpass - 1; i++) { + tmp_pass[i] = '*'; + } + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , password='%s'\n", (*myds), (*myds)->sess, user, tmp_pass); + free(tmp_pass); +#endif // debug + (*myds)->sess->default_hostgroup = default_hostgroup; + //(*myds)->sess->default_schema = default_schema; // just the pointer is passed + (*myds)->sess->user_attributes = attributes; // just the pointer is passed + //(*myds)->sess->schema_locked = schema_locked; + (*myds)->sess->transaction_persistent = transaction_persistent; + (*myds)->sess->session_fast_forward = false; // default + if ((*myds)->sess->session_type == PROXYSQL_SESSION_PGSQL) { + (*myds)->sess->session_fast_forward = fast_forward; + } + (*myds)->sess->user_max_connections = max_connections; + } else { + + if ( + ((*myds)->sess->session_type == PROXYSQL_SESSION_ADMIN) + || + ((*myds)->sess->session_type == PROXYSQL_SESSION_STATS) + || + ((*myds)->sess->session_type == PROXYSQL_SESSION_SQLITE) + ) { + if (strcmp((const char*)user, mysql_thread___monitor_username) == 0) { + if (strcmp(password, mysql_thread___monitor_password) == 0) { + (*myds)->sess->default_hostgroup = STATS_HOSTGROUP; + (*myds)->sess->default_schema = strdup((char*)"main"); // just the pointer is passed + (*myds)->sess->schema_locked = false; + (*myds)->sess->transaction_persistent = false; + (*myds)->sess->session_fast_forward = false; + (*myds)->sess->user_max_connections = 0; + password = l_strdup(mysql_thread___monitor_password); + ret = EXECUTION_STATE::SUCCESSFUL; + } + } + } + } + + + if (password) { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s' , auth_method=%d\n", (*myds), (*myds)->sess, user, (int)(*myds)->auth_method); + switch ((*myds)->auth_method) { + case AUTHENTICATION_METHOD::CLEAR_TEXT_PASSWORD: + { + uint32_t pass_len = hdr.data.size; + pass = (char*)malloc(pass_len + 1); + memcpy(pass, hdr.data.ptr, pass_len); + pass[pass_len] = 0; + + using_password = (pass_len > 0); + + if (pass_len) { + if (pass[pass_len - 1] == 0) { + pass_len--; // remove the extra 0 if present + } + } + + if (!pass || *pass == '\0') { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. Empty password returned by client.\n", (*myds), (*myds)->sess, user); + generate_error_packet(true, false, "empty password returned by client", PGSQL_ERROR_CODES::ERRCODE_PROTOCOL_VIOLATION, true); + break; + } + + if (strcmp(password, pass) == 0) { + ret = EXECUTION_STATE::SUCCESSFUL; + } + } + break; + case AUTHENTICATION_METHOD::SASL_SCRAM_SHA_256: + { + const char* mech; + uint32_t length; + const unsigned char* data; + int read_pos = 0; + using_password = true; + + if ((*myds)->scram_state == NULL) { + (*myds)->scram_state = scram_state_init(); + } + + PgCredentials stored_user_info{ '\0' }; + strncpy(stored_user_info.name, user, MAX_USERNAME); + strncpy(stored_user_info.passwd, password, MAX_PASSWORD); + + if (!(*myds)->scram_state->server_nonce) { + /* process as SASLInitialResponse */ + int pos = get_string((const char*)hdr.data.ptr, hdr.data.size, &mech); + + if (pos == 0) { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. SASL mechanism not found.\n", (*myds), (*myds)->sess, user); + break; + } + + read_pos = pos; + + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. Selected SASL mechanism: %s.\n", (*myds), (*myds)->sess, user, mech); + if (strcmp(mech, "SCRAM-SHA-256") != 0) { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. Client selected an invalid SASL authentication mechanism: %s.\n", (*myds), (*myds)->sess, user, mech); + generate_error_packet(true, false, "client selected an invalid SASL authentication mechanism", + PGSQL_ERROR_CODES::ERRCODE_PROTOCOL_VIOLATION, true); + break; + } + + if (get_uint32be(((unsigned char*)hdr.data.ptr) + read_pos, &length) == false) { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. Malformed packet.\n", (*myds), (*myds)->sess, user); + break; + } + + read_pos += 4; + + if ((hdr.data.size - read_pos) < length) { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. Malformed packet.\n", (*myds), (*myds)->sess, user); + break; + } + + // check mem boundry + + if (!scram_handle_client_first((*myds)->scram_state, &stored_user_info, ((const unsigned char*)hdr.data.ptr) + read_pos, length)) { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. SASL authentication failed\n", (*myds), (*myds)->sess, user); + generate_error_packet(true, false, "SASL authentication failed", PGSQL_ERROR_CODES::ERRCODE_PROTOCOL_VIOLATION, true); + break; + } + + ret = EXECUTION_STATE::PENDING; + } + else { + /* process as SASLResponse */ + //length = mbuf_avail_for_read(&pkt->data); + //if (!mbuf_get_bytes(&pkt->data, length, &data)) + // return false; + + data = (const unsigned char*)hdr.data.ptr; + length = hdr.data.size; + + if (scram_handle_client_final((*myds)->scram_state, &stored_user_info, data, length)) { + /* save SCRAM keys for user */ + if (!(*myds)->scram_state->adhoc) { + memcpy(stored_user_info.scram_ClientKey, + (*myds)->scram_state->ClientKey, + sizeof((*myds)->scram_state->ClientKey)); + memcpy(stored_user_info.scram_ServerKey, + (*myds)->scram_state->ServerKey, + sizeof((*myds)->scram_state->ServerKey)); + stored_user_info.has_scram_keys = true; + } + + free_scram_state((*myds)->scram_state); + (*myds)->scram_state = NULL; + //if (!finish_client_login(client)) + // return false; + //welcome_client(); + ret = EXECUTION_STATE::SUCCESSFUL; + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. SASL authentication failed.\n", (*myds), (*myds)->sess, user); + //generate_error_packet(false, "SASL authentication failed", NULL, true); + } + } + } + break; + default: + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s' . goto __exit_process_pkt_handshake_response . Unknown auth method\n", (*myds), (*myds)->sess, user); + break; + } + } else { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. User not found in the database.\n", (*myds), (*myds)->sess, user); + generate_error_packet(true, false, "User not found", PGSQL_ERROR_CODES::ERRCODE_PROTOCOL_VIOLATION, true); + } + // set the default session charset + //(*myds)->sess->default_charset = charset; + + /*if (pass_len == 0 && strlen(password) == 0) { + ret = true; + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , username='%s' , password=''\n", (*myds), (*myds)->sess, user); + }*/ + + assert(sess); + assert(sess->client_myds); + //assert(sess->client_myds->myconn); + /*myconn->set_charset(charset, CONNECT_START); + { + std::stringstream ss; + ss << charset; + + mysql_variables.client_set_value(sess, SQL_CHARACTER_SET_RESULTS, ss.str().c_str()); + mysql_variables.client_set_value(sess, SQL_CHARACTER_SET_CLIENT, ss.str().c_str()); + mysql_variables.client_set_value(sess, SQL_CHARACTER_SET_CONNECTION, ss.str().c_str()); + mysql_variables.client_set_value(sess, SQL_COLLATION_CONNECTION, ss.str().c_str()); + } +*/ + + + if (ret == EXECUTION_STATE::SUCCESSFUL) { + + (*myds)->DSS = STATE_CLIENT_HANDSHAKE; + + if (userinfo->username) free(userinfo->username); + if (userinfo->password) free(userinfo->password); + + userinfo->username = strdup((const char*)user); + userinfo->password = strdup((const char*)password); + + const char* db = (*myds)->myconn->conn_params.get_value(PG_DATABASE); + userinfo->set_dbname(db ? db : userinfo->username); + + const char* charset = (*myds)->myconn->conn_params.get_value(PG_CLIENT_ENCODING); + + //if (charset) + // (*myds)->sess->default_charset = charset; + } + else { + // we always duplicate username and password, or crashes happen + if (!userinfo->username) // if set already, ignore + userinfo->username = strdup((const char*)user); + if (using_password) + userinfo->password = strdup((const char*)""); + } + userinfo->set(NULL, NULL, NULL, NULL); // just to call compute_hash() + +__exit_process_pkt_handshake_response: + free(pass); + if (password) { + free(password); + password = NULL; + } + if (sha1_pass) { + free(sha1_pass); + sha1_pass = NULL; + } + + if (ret == EXECUTION_STATE::SUCCESSFUL) { + //ret = verify_user_attributes(__LINE__, __func__, user); + } + return ret; +} + +void PgSQL_Protocol::welcome_client() { + PG_pkt pgpkt(128); + + pgpkt.set_multi_pkt_mode(true); + pgpkt.write_AuthenticationOk(); + + if (sess->session_type == PROXYSQL_SESSION_ADMIN) + pgpkt.write_ParameterStatus("is_superuser", "on"); // only for admin + + const char* application_name = (*myds)->myconn->conn_params.get_value(PG_APPLICATION_NAME); + if (application_name) + pgpkt.write_ParameterStatus("application_name", application_name); + + const char* client_encoding = (*myds)->myconn->conn_params.get_value(PG_CLIENT_ENCODING); + if (client_encoding) + pgpkt.write_ParameterStatus("client_encoding", client_encoding); + // if client does not provide client_encoding, PostgreSQL uses the default client encoding. + // We need to save the default client encoding to send it to the client in case client doesn't provide one. + else if (pgsql_thread___default_client_encoding) + pgpkt.write_ParameterStatus("client_encoding", pgsql_thread___default_client_encoding); + + if (pgsql_thread___server_version) + pgpkt.write_ParameterStatus("server_version", pgsql_thread___server_version); + + pgpkt.write_ParameterStatus("server_encoding", "UTF8"); + + pgpkt.write_ReadyForQuery(); + pgpkt.set_multi_pkt_mode(false); + + auto buff = pgpkt.detach(); + (*myds)->PSarrayOUT->add((void*)buff.first, buff.second); + //(*myds)->DSS = STATE_CLIENT_AUTH_OK; + //(*myds)->sess->status = WAITING_CLIENT_DATA; +} + +void PgSQL_Protocol::generate_error_packet(bool send, bool ready, const char* msg, PGSQL_ERROR_CODES code, bool fatal, bool track, PtrSize_t* _ptr) { + // to avoid memory leak + assert(send == true || _ptr); + + if (send) { + // in case of fatal error we dont generate ready packets + ready = !fatal; + } + + PG_pkt pgpkt{}; + + if (ready) + pgpkt.set_multi_pkt_mode(true); + + pgpkt.write_generic('E', "cscscscsc", + 'S', fatal ? "FATAL" : "ERROR", + 'V', fatal ? "FATAL" : "ERROR", + 'C', PgSQL_Error_Helper::get_error_code(code), 'M', msg, 0); + + if (ready == true) { + pgpkt.write_ReadyForQuery(); + pgpkt.set_multi_pkt_mode(false); + } + + + auto buff = pgpkt.detach(); + if (send) { + (*myds)->PSarrayOUT->add((void*)buff.first, buff.second); + switch ((*myds)->DSS) { + case STATE_SERVER_HANDSHAKE: + case STATE_CLIENT_HANDSHAKE: + case STATE_QUERY_SENT_DS: + case STATE_QUERY_SENT_NET: + case STATE_ERR: + (*myds)->DSS = STATE_ERR; + break; + case STATE_OK: + break; + case STATE_SLEEP: + if ((*myds)->sess->session_fast_forward == true) { // see issue #733 + break; + } + default: + // LCOV_EXCL_START + assert(0); + // LCOV_EXCL_STOP + } + } + + if (_ptr) { + _ptr->ptr = buff.first; + _ptr->size = buff.second; + } + + if (track) { + if (*myds && (*myds)->sess && (*myds)->sess->thread) { + (*myds)->sess->thread->status_variables.stvar[st_var_generated_pkt_err]++; + } + } +} + +bool PgSQL_Protocol::scram_handle_client_first(ScramState* scram_state, PgCredentials* user, const unsigned char* data, uint32_t datalen) +{ + char* ibuf; + char* input; + + scram_reset_error(); + + ibuf = (char*)malloc(datalen + 1); + if (ibuf == NULL) + return false; + memcpy(ibuf, data, datalen); + ibuf[datalen] = '\0'; + + input = ibuf; + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. SCRAM client-first-message = \"%s\"\n", (*myds), (*myds)->sess, user->name, input); + if (!read_client_first_message(input, + &scram_state->cbind_flag, + &scram_state->client_first_message_bare, + &scram_state->client_nonce)) + goto failed; + + if (!user->mock_auth) { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. stored secret = \"%s\"\n", (*myds), (*myds)->sess, user->name, user->passwd); + switch (get_password_type(user->passwd)) { + case PASSWORD_TYPE_MD5: + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. SCRAM authentication failed: user has MD5 secret\n", (*myds), (*myds)->sess, user->name); + goto failed; + case PASSWORD_TYPE_PLAINTEXT: + case PASSWORD_TYPE_SCRAM_SHA_256: + break; + } + } + + if (!build_server_first_message(scram_state, user->name, user->mock_auth ? NULL : user->passwd)) + goto failed; + + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. SCRAM server-first-message = \"%s\"\n", (*myds), (*myds)->sess, user->name, scram_state->server_first_message); + { + PG_pkt pgpkt{}; + pgpkt.write_AuthenticationRequest(PG_PKT_AUTH_SASL_CONT, (const uint8_t*)scram_state->server_first_message, strlen(scram_state->server_first_message)); + auto buff = pgpkt.detach(); + (*myds)->PSarrayOUT->add((void*)buff.first, buff.second); + } + + free(ibuf); + return true; +failed: + free(ibuf); + return false; +} + +bool PgSQL_Protocol::scram_handle_client_final(ScramState* scram_state, PgCredentials* user, const unsigned char* data, uint32_t datalen) +{ + char* ibuf; + char* input; + const char* client_final_nonce = NULL; + char* proof = NULL; + char* server_final_message; + + scram_reset_error(); + + ibuf = (char*)malloc(datalen + 1); + if (ibuf == NULL) + return false; + memcpy(ibuf, data, datalen); + ibuf[datalen] = '\0'; + + input = ibuf; + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. SCRAM client-final-message = \"%s\"\n", (*myds), (*myds)->sess, user->name, input); + if (!read_client_final_message(scram_state, data, input, + &client_final_nonce, + &proof)) + goto failed; + + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s'. SCRAM client-final-message-without-proof = \"%s\"\n", (*myds), + (*myds)->sess, user->name, scram_state->client_final_message_without_proof); + + if (!verify_final_nonce(scram_state, client_final_nonce)) { + proxy_error("Invalid SCRAM response (nonce does not match)\n"); + goto failed; + } + + if (!verify_client_proof(scram_state, proof)) { + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s. Password authentication failed\n", (*myds), + (*myds)->sess, user->name); + goto failed; + } + + server_final_message = build_server_final_message(scram_state); + if (!server_final_message) + goto failed; + proxy_debug(PROXY_DEBUG_MYSQL_AUTH, 5, "Session=%p , DS=%p , user='%s. SCRAM server-final-message = \"%s\"\n", (*myds), + (*myds)->sess, user->name, server_final_message); + + { + PG_pkt pgpkt{}; + pgpkt.write_AuthenticationRequest(PG_PKT_AUTH_SASL_FIN, (const uint8_t*)server_final_message, strlen(server_final_message)); + auto buff = pgpkt.detach(); + (*myds)->PSarrayOUT->add((void*)buff.first, buff.second); + } + + free(server_final_message); + free(proof); + free(ibuf); + return true; +failed: + free(proof); + free(ibuf); + return false; +} + +char* extract_tag_from_query(const char* query) { + + constexpr size_t crete_table_len = sizeof("CREATE TABLE AS") - 1; + + size_t qtlen = strlen(query); + if ((qtlen > crete_table_len) && strncasecmp(query, "CREATE TABLE AS", crete_table_len) == 0) { + return strdup("SELECT"); + } + else { + const char* fs = strchr(query, ' '); + + if (fs != NULL) { + qtlen = (fs - query) + 1; + } + char buf[qtlen]; + memcpy(buf, query, qtlen - 1); + buf[qtlen - 1] = 0; + { + char* s = buf; + while (*s) { + *s = toupper((unsigned char)*s); + s++; + } + } + + return strdup(buf); + } +} + + +bool PgSQL_Protocol::generate_ok_packet(bool send, bool ready, const char* msg, int rows, const char* query, PtrSize_t* _ptr) { + // to avoid memory leak + assert(send == true || _ptr); + + PG_pkt pgpkt{}; + + if (ready == true) { + pgpkt.set_multi_pkt_mode(true); + } + + char* tag = extract_tag_from_query(query); + assert(tag); + + char tmpbuf[128]; + if (strcmp(tag, "INSERT") == 0) { + sprintf(tmpbuf, "%s 0 %d", tag, rows); + pgpkt.write_CommandComplete(tmpbuf); + } else if (strcmp(tag, "UPDATE") == 0 || + strcmp(tag, "DELETE") == 0 || + strcmp(tag, "MERGE") == 0 || + strcmp(tag, "MOVE") == 0 || + strcmp(tag, "FETCH") == 0 || + strcmp(tag, "COPY") == 0 || + strcmp(tag, "SELECT") == 0 || + strcmp(tag, "COPY") == 0 ) { + sprintf(tmpbuf, "%s %d", tag, rows); + pgpkt.write_CommandComplete(tmpbuf); + } else { + pgpkt.write_CommandComplete(tag); + } + + if (ready == true) { + pgpkt.write_ReadyForQuery(); + pgpkt.set_multi_pkt_mode(false); + } + + auto buff = pgpkt.detach(); + if (send == true) { + (*myds)->PSarrayOUT->add((void*)buff.first, buff.second); + } else { + _ptr->ptr = buff.first; + _ptr->size = buff.second; + } + free(tag); + return true; +} + +//bool PgSQL_Protocol::generate_row_description(bool send, PgSQL_Query_Result* rs, const PG_Fields& fields, unsigned int size) { +// if ((*myds)->sess->mirror == true) { +// return true; +// } +// +// unsigned char* _ptr = NULL; +// +// if (rs) { +// if (size <= (PGSQL_RESULTSET_BUFLEN - rs->buffer_used)) { +// // there is space in the buffer, add the data to it +// _ptr = rs->buffer + rs->buffer_used; +// rs->buffer_used += size; +// } else { +// // there is no space in the buffer, we flush the buffer and recreate it +// rs->buffer_to_PSarrayOut(); +// // now we can check again if there is space in the buffer +// if (size <= (PGSQL_RESULTSET_BUFLEN - rs->buffer_used)) { +// // there is space in the NEW buffer, add the data to it +// _ptr = rs->buffer + rs->buffer_used; +// rs->buffer_used += size; +// } else { +// // a new buffer is not enough to store the new row +// _ptr = (unsigned char*)l_alloc(size); +// } +// } +// } else { +// _ptr = (unsigned char*)l_alloc(size); +// } +// +// PG_pkt pgpkt(_ptr, 0); +// +// pgpkt.put_char('T'); +// pgpkt.put_uint32(size ); +// pgpkt.put_uint16(fields.size()); +// +// for (unsigned int i = 0; i < fields.size(); i++) { +// pgpkt.put_string(fields[i].name); +// pgpkt.put_uint32(fields[i].tbl_oid); +// pgpkt.put_uint16(fields[i].col_idx); +// pgpkt.put_uint32(fields[i].type_oid); +// pgpkt.put_uint16(fields[i].col_len); +// pgpkt.put_uint32(fields[i].type_mod); +// pgpkt.put_uint16(fields[i].fmt); +// } +// +// if (send == true) { (*myds)->PSarrayOUT->add((void*)_ptr, size); } +// +////#ifdef DEBUG +//// if (dump_pkt) { __dump_pkt(__func__, _ptr, size); } +////#endif +// if (rs) { +// if (_ptr >= rs->buffer && _ptr < rs->buffer + PGSQL_RESULTSET_BUFLEN) { +// // we are writing within the buffer, do not add to PSarrayOUT +// } else { +// // we are writing outside the buffer, add to PSarrayOUT +// rs->PSarrayOUT.add(_ptr, size); +// } +// } +// return true; +//} + + +unsigned int PgSQL_Protocol::copy_row_description_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result) { + assert(pg_query_result); + assert(result); + + unsigned int fields_cnt = PQnfields(result); + unsigned int size = 1 + 4 + 2; + for (unsigned int i = 0; i < fields_cnt; i++) { + size += strlen(PQfname(result, i)) + 1 + 18; // null terminator, name, reloid, colnr, oid, typsize, typmod, fmt + } + + bool alloced_new_buffer = false; + unsigned char* _ptr = pg_query_result->buffer_reserve_space(size); + + // buffer is not enough to store the new row description. Remember we have already pushed data to PSarrayOUT + if (_ptr == NULL) { + _ptr = (unsigned char*)l_alloc(size); + alloced_new_buffer = true; + } + + PG_pkt pgpkt(_ptr, size); + + pgpkt.put_char('T'); + pgpkt.put_uint32(size - 1); + pgpkt.put_uint16(fields_cnt); + + for (unsigned int i = 0; i < fields_cnt; i++) { + pgpkt.put_string(PQfname(result, i)); + pgpkt.put_uint32(PQftable(result, i)); + pgpkt.put_uint16(PQftablecol(result, i)); + pgpkt.put_uint32(PQftype(result, i)); + pgpkt.put_uint16(PQfsize(result, i)); + pgpkt.put_uint32(PQfmod(result, i)); + pgpkt.put_uint16(PQfformat(result, i)); + } + + if (send == true) { + // not supported + //(*myds)->PSarrayOUT->add((void*)_ptr, size); + } + +//#ifdef DEBUG +// if (dump_pkt) { __dump_pkt(__func__, _ptr, size); } +//#endif + + pg_query_result->resultset_size = size; + + if (alloced_new_buffer) { + // we created new buffer + //pg_query_result->buffer_to_PSarrayOut(); + pg_query_result->PSarrayOUT.add(_ptr, size); + } + + pg_query_result->num_fields = fields_cnt; + pg_query_result->pkt_count++; + return size; +} + +unsigned int PgSQL_Protocol::copy_row_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result) { + assert(pg_query_result); + assert(result); + assert(pg_query_result->num_fields); + + const unsigned int numRows = PQntuples(result); + unsigned int total_size = 0; + for (unsigned int i = 0; i < numRows; i++) { + unsigned int size = 1 + 4 + 2; // 'D', length, field count + for (unsigned int j = 0; j < pg_query_result->num_fields; j++) { + size += PQgetlength(result, i, j) + 4; // length, value + } + total_size += size; + + bool alloced_new_buffer = false; + unsigned char* _ptr = pg_query_result->buffer_reserve_space(size); + + // buffer is not enough to store the new row. Remember we have already pushed data to PSarrayOUT + if (_ptr == NULL) { + _ptr = (unsigned char*)l_alloc(size); + alloced_new_buffer = true; + } + + PG_pkt pgpkt(_ptr, size); + + pgpkt.put_char('D'); + pgpkt.put_uint32(size - 1); + pgpkt.put_uint16(pg_query_result->num_fields); + int column_value_len = 0; + for (unsigned int j = 0; j < pg_query_result->num_fields; j++) { + column_value_len = PQgetlength(result, i, j); + if (column_value_len == 0 && PQgetisnull(result, i, j) == 1) { + column_value_len = -1; /*0xFFFFFFFF*/ + } + pgpkt.put_uint32(column_value_len); + if (column_value_len > 0) { + pgpkt.put_bytes(PQgetvalue(result, i, j), column_value_len); + } + } + + if (send == true) { + // not supported + //(*myds)->PSarrayOUT->add((void*)_ptr, size); + } + + pg_query_result->resultset_size += size; + + if (alloced_new_buffer) { + // we created new buffer + //pg_query_result->buffer_to_PSarrayOut(); + pg_query_result->PSarrayOUT.add(_ptr, size); + } + + pg_query_result->pkt_count++; + } + + pg_query_result->num_rows += numRows; + + return total_size; +} + +unsigned int PgSQL_Protocol::copy_command_completion_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result, + bool extract_affected_rows) { + assert(pg_query_result); + assert(result); + + const char* tag = PQcmdStatus((PGresult*)result); + if (!tag) assert(0); // for testing it should not be null + + const unsigned int size = strlen(tag) + 1 + 1 + 4; // tag length, null byte, 'C', length, tag + bool alloced_new_buffer = false; + + unsigned char* _ptr = pg_query_result->buffer_reserve_space(size); + + // buffer is not enough to store the new row. Remember we have already pushed data to PSarrayOUT + if (_ptr == NULL) { + _ptr = (unsigned char*)l_alloc(size); + alloced_new_buffer = true; + } + + PG_pkt pgpkt(_ptr, size); + + pgpkt.put_char('C'); + pgpkt.put_uint32(size - 1); + pgpkt.put_string(tag); + + if (send == true) { + // not supported + //(*myds)->PSarrayOUT->add((void*)_ptr, size); + } + + pg_query_result->resultset_size += size; + + if (alloced_new_buffer) { + // we created new buffer + //pg_query_result->buffer_to_PSarrayOut(); + pg_query_result->PSarrayOUT.add(_ptr, size); + } + pg_query_result->pkt_count++; + + // To prevent rows sent from being considered as affected rows, + // we avoid extracting affected rows for SELECT queries. + if (extract_affected_rows) { + const char* extracted_affect_rows = PQcmdTuples(const_cast(result)); + if (*extracted_affect_rows) + pg_query_result->affected_rows = strtoull(extracted_affect_rows, NULL, 10); + } + return size; +} + +unsigned int PgSQL_Protocol::copy_error_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result) { + assert(pg_query_result); + assert(result); + + const char* severity = PQresultErrorField(result, PG_DIAG_SEVERITY); + const char* text = PQresultErrorField(result, PG_DIAG_SEVERITY_NONLOCALIZED); + const char* sqlstate = PQresultErrorField(result, PG_DIAG_SQLSTATE); + const char* primary = PQresultErrorField(result, PG_DIAG_MESSAGE_PRIMARY); + const char* detail = PQresultErrorField(result, PG_DIAG_MESSAGE_DETAIL); + const char* hint = PQresultErrorField(result, PG_DIAG_MESSAGE_HINT); + const char* position = PQresultErrorField(result, PG_DIAG_STATEMENT_POSITION); + const char* internal_position = PQresultErrorField(result, PG_DIAG_INTERNAL_POSITION); + const char* internal_query = PQresultErrorField(result, PG_DIAG_INTERNAL_QUERY); + const char* context = PQresultErrorField(result, PG_DIAG_CONTEXT); + const char* schema_name = PQresultErrorField(result, PG_DIAG_SCHEMA_NAME); + const char* table_name = PQresultErrorField(result, PG_DIAG_TABLE_NAME); + const char* column_name = PQresultErrorField(result, PG_DIAG_COLUMN_NAME); + const char* datatype_name = PQresultErrorField(result, PG_DIAG_DATATYPE_NAME); + const char* constraint_name = PQresultErrorField(result, PG_DIAG_CONSTRAINT_NAME); + const char* source_file = PQresultErrorField(result, PG_DIAG_SOURCE_FILE); + const char* source_line = PQresultErrorField(result, PG_DIAG_SOURCE_LINE); + const char* source_function = PQresultErrorField(result, PG_DIAG_SOURCE_FUNCTION); + + unsigned int size = 1 + 4 + 1; // 'E', length, null byte + + if (severity) size += strlen(severity) + 1 + 1; + if (text) size += strlen(text) + 1 + 1; + if (sqlstate) size += strlen(sqlstate) + 1 + 1; + if (primary) size += strlen(primary) + 1 + 1; + if (detail) size += strlen(detail) + 1 + 1; + if (hint) size += strlen(hint) + 1 + 1; + if (position) size += strlen(position) + 1 + 1; + if (internal_position) size += strlen(internal_position) + 1 + 1; + if (internal_query) size += strlen(internal_query) + 1 + 1; + if (context) size += strlen(context) + 1 + 1; + if (schema_name) size += strlen(schema_name) + 1 + 1; + if (table_name) size += strlen(table_name) + 1 + 1; + if (column_name) size += strlen(column_name) + 1 + 1; + if (datatype_name) size += strlen(datatype_name) + 1 + 1; + if (constraint_name) size += strlen(constraint_name) + 1 + 1; + if (source_file) size += strlen(source_file) + 1 + 1; + if (source_line) size += strlen(source_line) + 1 + 1; + if (source_function) size += strlen(source_function) + 1 + 1; + + bool alloced_new_buffer = false; + unsigned char* _ptr = pg_query_result->buffer_reserve_space(size); + + // buffer is not enough to store the new row. Remember we have already pushed data to PSarrayOUT + if (_ptr == NULL) { + _ptr = (unsigned char*)l_alloc(size); + alloced_new_buffer = true; + } + + PG_pkt pgpkt(_ptr, size); + + pgpkt.put_char('E'); + pgpkt.put_uint32(size - 1); + if (severity) { + pgpkt.put_char('S'); + pgpkt.put_string(severity); + } + if (text) { + pgpkt.put_char('V'); + pgpkt.put_string(text); + } + if (sqlstate) { + pgpkt.put_char('C'); + pgpkt.put_string(sqlstate); + } + if (primary) { + pgpkt.put_char('M'); + pgpkt.put_string(primary); + } + if (detail) { + pgpkt.put_char('D'); + pgpkt.put_string(detail); + } + if (hint) { + pgpkt.put_char('H'); + pgpkt.put_string(hint); + } + if (position) { + pgpkt.put_char('P'); + pgpkt.put_string(position); + } + if (internal_position) { + pgpkt.put_char('p'); + pgpkt.put_string(internal_position); + } + if (internal_query) { + pgpkt.put_char('q'); + pgpkt.put_string(internal_query); + } + if (context) { + pgpkt.put_char('W'); + pgpkt.put_string(context); + } + if (schema_name) { + pgpkt.put_char('s'); + pgpkt.put_string(schema_name); + } + if (table_name) { + pgpkt.put_char('t'); + pgpkt.put_string(table_name); + } + if (column_name) { + pgpkt.put_char('c'); + pgpkt.put_string(column_name); + } + if (datatype_name) { + pgpkt.put_char('d'); + pgpkt.put_string(datatype_name); + } + if (constraint_name) { + pgpkt.put_char('n'); + pgpkt.put_string(constraint_name); + } + if (source_file) { + pgpkt.put_char('F'); + pgpkt.put_string(source_file); + } + if (source_line) { + pgpkt.put_char('L'); + pgpkt.put_string(source_line); + } + if (source_function) { + pgpkt.put_char('R'); + pgpkt.put_string(source_function); + } + pgpkt.put_char('\0'); + + if (send == true) { + // not supported + //(*myds)->PSarrayOUT->add((void*)_ptr, size); + } + + pg_query_result->resultset_size += size; + + if (alloced_new_buffer) { + // we created new buffer + //pg_query_result->buffer_to_PSarrayOut(); + pg_query_result->PSarrayOUT.add(_ptr, size); + } + pg_query_result->pkt_count++; + return size; +} + +unsigned int PgSQL_Protocol::copy_empty_query_response_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PGresult* result) { + assert(pg_query_result); + // we are currently not using result. It is just for future use + + const unsigned int size = 1 + 4; // I, length + bool alloced_new_buffer = false; + + unsigned char* _ptr = pg_query_result->buffer_reserve_space(size); + + // buffer is not enough to store the new row. Remember we have already pushed data to PSarrayOUT + if (_ptr == NULL) { + _ptr = (unsigned char*)l_alloc(size); + alloced_new_buffer = true; + } + + PG_pkt pgpkt(_ptr, size); + + pgpkt.put_char('I'); + pgpkt.put_uint32(size - 1); + + if (send == true) { + // not supported + //(*myds)->PSarrayOUT->add((void*)_ptr, size); + } + + pg_query_result->resultset_size += size; + + if (alloced_new_buffer) { + // we created new buffer + //pg_query_result->buffer_to_PSarrayOut(); + pg_query_result->PSarrayOUT.add(_ptr, size); + } + pg_query_result->pkt_count++; + return size; +} + +unsigned int PgSQL_Protocol::copy_ready_status_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, PGTransactionStatusType txn_status) { + assert(pg_query_result); + + char txn_state = 'I'; + if (txn_status == PQTRANS_INTRANS) + txn_state = 'T'; + else if (txn_status == PQTRANS_INERROR) + txn_state = 'E'; + + const unsigned int size = 1 + 4 + 1; // Z, length, I/T/E + bool alloced_new_buffer = false; + + unsigned char* _ptr = pg_query_result->buffer_reserve_space(size); + + // buffer is not enough to store the new row. Remember we have already pushed data to PSarrayOUT + if (_ptr == NULL) { + _ptr = (unsigned char*)l_alloc(size); + alloced_new_buffer = true; + } + + PG_pkt pgpkt(_ptr, size); + + pgpkt.put_char('Z'); + pgpkt.put_uint32(size - 1); + pgpkt.put_char(txn_state); + + if (send == true) { + // not supported + //(*myds)->PSarrayOUT->add((void*)_ptr, size); + } + + pg_query_result->resultset_size += size; + + if (alloced_new_buffer) { + // we created new buffer + //pg_query_result->buffer_to_PSarrayOut(); + pg_query_result->PSarrayOUT.add(_ptr, size); + } + pg_query_result->pkt_count++; + return size; +} + +unsigned int PgSQL_Protocol::copy_buffer_to_PgSQL_Query_Result(bool send, PgSQL_Query_Result* pg_query_result, const PSresult* result) { + assert(pg_query_result); + assert(result && result->len && result->data); + + bool alloced_new_buffer = false; + + const unsigned int size = result->len; + unsigned char* _ptr = pg_query_result->buffer_reserve_space(size); + + // buffer is not enough to store the new row. Remember we have already pushed data to PSarrayOUT + if (_ptr == NULL) { + _ptr = (unsigned char*)l_alloc(size); + alloced_new_buffer = true; + } + + memcpy(_ptr, result->data, size); + + if (send == true) { + // not supported + //(*myds)->PSarrayOUT->add((void*)_ptr, size); + } + + pg_query_result->resultset_size += size; + + if (alloced_new_buffer) { + // we created new buffer + //pg_query_result->buffer_to_PSarrayOut(); + pg_query_result->PSarrayOUT.add(_ptr, size); + } + pg_query_result->pkt_count++; + + // assuming single-row result + if (result->id == 'D') + pg_query_result->num_rows += 1; + + return size; +} + +PgSQL_Query_Result::PgSQL_Query_Result() { + buffer = NULL; + transfer_started = false; + buffer_used = 0; + resultset_size = 0; + num_fields = 0; + num_rows = 0; + pkt_count = 0; + affected_rows = -1; + result_packet_type = PGSQL_QUERY_RESULT_NO_DATA; +} + +PgSQL_Query_Result::~PgSQL_Query_Result() { + PtrSize_t pkt; + while (PSarrayOUT.len) { + PSarrayOUT.remove_index_fast(0, &pkt); + l_free(pkt.size, pkt.ptr); + } + + if (buffer) { + free(buffer); + buffer = NULL; + } +} + +void PgSQL_Query_Result::buffer_init() { + if (buffer == NULL) { + buffer = (unsigned char*)malloc(PGSQL_RESULTSET_BUFLEN); + } + buffer_used = 0; +} + +void PgSQL_Query_Result::init(PgSQL_Protocol* _proto, PgSQL_Data_Stream* _myds, PgSQL_Connection* _conn) { + PROXY_TRACE2(); + transfer_started = false; + proto = _proto; + conn = _conn; + myds = _myds; + buffer_init(); + reset(); + + if (proto == NULL) { + return; // this is a mirror + } +} + +unsigned int PgSQL_Query_Result::add_row_description(const PGresult* result) { + const unsigned int res = proto->copy_row_description_to_PgSQL_Query_Result(false, this, result); + result_packet_type |= PGSQL_QUERY_RESULT_TUPLE; + return res; +} + +unsigned int PgSQL_Query_Result::add_row(const PGresult* result) { + + return proto->copy_row_to_PgSQL_Query_Result(false,this, result); +} + +unsigned int PgSQL_Query_Result::add_row(const PSresult* result) { + + const unsigned int res = proto->copy_buffer_to_PgSQL_Query_Result(false, this, result); + result_packet_type |= PGSQL_QUERY_RESULT_TUPLE; // temporary + return res; +} + +unsigned int PgSQL_Query_Result::add_error(const PGresult* result) { + unsigned int size = 0; + + if (result) { + size = proto->copy_error_to_PgSQL_Query_Result(false, this, result); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, conn->parent->myhgc->hid, conn->parent->address, conn->parent->port, 1907); + } + else { + PtrSize_t pkt; + if (myds && myds->killed_at) { // see case #750 + if (myds->kill_type == 0) { + proto->generate_error_packet(false, false, (char*)"Query execution was interrupted, query_timeout exceeded", + PGSQL_ERROR_CODES::ERRCODE_QUERY_CANCELED, false, false, &pkt); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, conn->parent->myhgc->hid, conn->parent->address, conn->parent->port, 1907); + } else { + proto->generate_error_packet(false, false, (char*)"Query execution was interrupted", + PGSQL_ERROR_CODES::ERRCODE_QUERY_CANCELED, false, false, &pkt); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, conn->parent->myhgc->hid, conn->parent->address, conn->parent->port, 1317); + } + } else if (conn->is_error_present()) { + proto->generate_error_packet(false, false, conn->get_error_message().c_str(), conn->get_error_code(), false, false, &pkt); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, conn->parent->myhgc->hid, conn->parent->address, conn->parent->port, 1907); + } else { + assert(0); // should never reach here + } + + PSarrayOUT.add(pkt.ptr, pkt.size); + resultset_size += pkt.size; + size = pkt.size; + } + + result_packet_type |= PGSQL_QUERY_RESULT_ERROR; + return size; +} + +unsigned int PgSQL_Query_Result::add_empty_query_response(const PGresult* result) { + const unsigned int bytes = proto->copy_empty_query_response_to_PgSQL_Query_Result(false, this, result); + result_packet_type |= PGSQL_QUERY_RESULT_EMPTY; + return bytes; +} + +unsigned int PgSQL_Query_Result::add_ready_status(PGTransactionStatusType txn_status) { + const unsigned int bytes = proto->copy_ready_status_to_PgSQL_Query_Result(false, this, txn_status); + buffer_to_PSarrayOut(); + result_packet_type |= PGSQL_QUERY_RESULT_READY; + return bytes; +} + +bool PgSQL_Query_Result::get_resultset(PtrSizeArray* PSarrayFinal) { + transfer_started = true; + // Ready packet confirms that the result is complete + const bool result_complete = (result_packet_type & PGSQL_QUERY_RESULT_READY); + if (result_complete == true) { + assert(buffer_used == 0); // we still have data in the buffer + } else { + buffer_to_PSarrayOut(); + } + + if (proto) { + PSarrayFinal->copy_add(&PSarrayOUT, 0, PSarrayOUT.len); + while (PSarrayOUT.len) + PSarrayOUT.remove_index(PSarrayOUT.len - 1, NULL); + } + if (result_complete) + reset(); // reset only if result is complete + return result_complete; +} + +void PgSQL_Query_Result::buffer_to_PSarrayOut() { + if (buffer_used == 0) + return; // exit immediately if the buffer is empty + if (buffer_used < PGSQL_RESULTSET_BUFLEN / 2) { + buffer = (unsigned char*)realloc(buffer, buffer_used); + } + PSarrayOUT.add(buffer, buffer_used); + buffer = (unsigned char*)malloc(PGSQL_RESULTSET_BUFLEN); + buffer_used = 0; +} + +unsigned long long PgSQL_Query_Result::current_size() { + unsigned long long intsize = 0; + intsize += sizeof(PgSQL_Query_Result); + intsize += PGSQL_RESULTSET_BUFLEN; // size of buffer + if (PSarrayOUT.len == 0) // see bug #699 + return intsize; + intsize += sizeof(PtrSizeArray); + intsize += (PSarrayOUT.size * sizeof(PtrSize_t*)); + unsigned int i; + for (i = 0; i < PSarrayOUT.len; i++) { + PtrSize_t* pkt = PSarrayOUT.index(i); + if (pkt->size > PGSQL_RESULTSET_BUFLEN) { + intsize += pkt->size; + } + else { + intsize += PGSQL_RESULTSET_BUFLEN; + } + } + return intsize; +} + +unsigned int PgSQL_Query_Result::add_command_completion(const PGresult* result, bool extract_affected_rows) { + const unsigned int bytes = proto->copy_command_completion_to_PgSQL_Query_Result(false, this, result, extract_affected_rows); + result_packet_type |= PGSQL_QUERY_RESULT_COMMAND; + /*if (affected_rows) { + myds->sess->CurrentQuery.have_affected_rows = true; // if affected rows is set, last_insert_id is set too + myds->sess->CurrentQuery.affected_rows = affected_rows; + myds->sess->CurrentQuery.last_insert_id = 0; // not supported + }*/ + return bytes; +} + +unsigned char* PgSQL_Query_Result::buffer_reserve_space(unsigned int size) { + unsigned char* ret_buffer = NULL; + if (size <= buffer_available_capacity()) { + // there is space in the buffer, add the data to it + ret_buffer = buffer + buffer_used; + buffer_used += size; + } + else { + // there is no space in the buffer, we flush the buffer and recreate it + buffer_to_PSarrayOut(); + // now we can check again if there is space in the buffer + if (size <= buffer_available_capacity()) { + // there is space in the NEW buffer, add the data to it + ret_buffer = buffer + buffer_used; + buffer_used += size; + } + } + return ret_buffer; +} + +void PgSQL_Query_Result::reset() { + resultset_size = 0; + num_fields = 0; + num_rows = 0; + pkt_count = 0; + affected_rows = -1; + result_packet_type = PGSQL_QUERY_RESULT_NO_DATA; +} diff --git a/lib/PgSQL_Query_Processor.cpp b/lib/PgSQL_Query_Processor.cpp new file mode 100644 index 0000000000..e3427e7123 --- /dev/null +++ b/lib/PgSQL_Query_Processor.cpp @@ -0,0 +1,1144 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include // std::cout +#include // std::sort +#include // std::vector +#include "proxysql.h" +#include "cpp.h" +#include "Command_Counter.h" +#include "PgSQL_Query_Processor.h" + +extern PgSQL_Threads_Handler* GloPTH; +extern ProxySQL_Admin *GloAdmin; + +static __thread Command_Counter* _thr_commands_counters[PGSQL_QUERY___NONE]; + +static char* commands_counters_desc[PGSQL_QUERY___NONE] = { + [PGSQL_QUERY_SELECT] = (char*)"SELECT", + [PGSQL_QUERY_INSERT] = (char*)"INSERT", + [PGSQL_QUERY_UPDATE] = (char*)"UPDATE", + [PGSQL_QUERY_DELETE] = (char*)"DELETE", + [PGSQL_QUERY_MERGE] = (char*)"MERGE", + [PGSQL_QUERY_CREATE_TABLE] = (char*)"CREATE_TABLE", + [PGSQL_QUERY_ALTER_TABLE] = (char*)"ALTER_TABLE", + [PGSQL_QUERY_DROP_TABLE] = (char*)"DROP_TABLE", + [PGSQL_QUERY_TRUNCATE] = (char*)"TRUNCATE", + [PGSQL_QUERY_COPY] = (char*)"COPY", + [PGSQL_QUERY_CREATE_INDEX] = (char*)"CREATE_INDEX", + [PGSQL_QUERY_DROP_INDEX] = (char*)"DROP_INDEX", + [PGSQL_QUERY_ALTER_INDEX] = (char*)"ALTER_INDEX", + [PGSQL_QUERY_CREATE_VIEW] = (char*)"CREATE_VIEW", + [PGSQL_QUERY_DROP_VIEW] = (char*)"DROP_VIEW", + [PGSQL_QUERY_ALTER_VIEW] = (char*)"ALTER_VIEW", + [PGSQL_QUERY_CREATE_MATERIALIZED_VIEW] = (char*)"CREATE_MATERIALIZED_VIEW", + [PGSQL_QUERY_ALTER_MATERIALIZED_VIEW] = (char*)"ALTER_MATERIALIZED_VIEW", + [PGSQL_QUERY_REFRESH_MATERIALIZED_VIEW] = (char*)"REFRESH_MATERIALIZED_VIEW", + [PGSQL_QUERY_DROP_MATERIALIZED_VIEW] = (char*)"DROP_MATERIALIZED_VIEW", + [PGSQL_QUERY_CREATE_SEQUENCE] = (char*)"CREATE_SEQUENCE", + [PGSQL_QUERY_ALTER_SEQUENCE] = (char*)"ALTER_SEQUENCE", + [PGSQL_QUERY_DROP_SEQUENCE] = (char*)"DROP_SEQUENCE", + [PGSQL_QUERY_CREATE_SCHEMA] = (char*)"CREATE_SCHEMA", + [PGSQL_QUERY_DROP_SCHEMA] = (char*)"DROP_SCHEMA", + [PGSQL_QUERY_ALTER_SCHEMA] = (char*)"ALTER_SCHEMA", + [PGSQL_QUERY_CREATE_FUNCTION] = (char*)"CREATE_FUNCTION", + [PGSQL_QUERY_ALTER_FUNCTION] = (char*)"ALTER_FUNCTION", + [PGSQL_QUERY_DROP_FUNCTION] = (char*)"DROP_FUNCTION", + [PGSQL_QUERY_CREATE_PROCEDURE] = (char*)"CREATE_PROCEDURE", + [PGSQL_QUERY_ALTER_PROCEDURE] = (char*)"ALTER_PROCEDURE", + [PGSQL_QUERY_CALL] = (char*)"CALL", + [PGSQL_QUERY_DROP_PROCEDURE] = (char*)"DROP_PROCEDURE", + [PGSQL_QUERY_CREATE_AGGREGATE] = (char*)"CREATE_AGGREGATE", + [PGSQL_QUERY_ALTER_AGGREGATE] = (char*)"ALTER_AGGREGATE", + [PGSQL_QUERY_DROP_AGGREGATE] = (char*)"DROP_AGGREGATE", + [PGSQL_QUERY_CREATE_OPERATOR] = (char*)"CREATE_OPERATOR", + [PGSQL_QUERY_ALTER_OPERATOR] = (char*)"ALTER_OPERATOR", + [PGSQL_QUERY_DROP_OPERATOR] = (char*)"DROP_OPERATOR", + [PGSQL_QUERY_CREATE_TYPE] = (char*)"CREATE_TYPE", + [PGSQL_QUERY_ALTER_TYPE] = (char*)"ALTER_TYPE", + [PGSQL_QUERY_DROP_TYPE] = (char*)"DROP_TYPE", + [PGSQL_QUERY_CREATE_DOMAIN] = (char*)"CREATE_DOMAIN", + [PGSQL_QUERY_ALTER_DOMAIN] = (char*)"ALTER_DOMAIN", + [PGSQL_QUERY_DROP_DOMAIN] = (char*)"DROP_DOMAIN", + [PGSQL_QUERY_CREATE_TRIGGER] = (char*)"CREATE_TRIGGER", + [PGSQL_QUERY_ALTER_TRIGGER] = (char*)"ALTER_TRIGGER", + [PGSQL_QUERY_DROP_TRIGGER] = (char*)"DROP_TRIGGER", + [PGSQL_QUERY_CREATE_RULE] = (char*)"CREATE_RULE", + [PGSQL_QUERY_ALTER_RULE] = (char*)"ALTER_RULE", + [PGSQL_QUERY_DROP_RULE] = (char*)"DROP_RULE", + [PGSQL_QUERY_CREATE_EXTENSION] = (char*)"CREATE_EXTENSION", + [PGSQL_QUERY_ALTER_EXTENSION] = (char*)"ALTER_EXTENSION", + [PGSQL_QUERY_DROP_EXTENSION] = (char*)"DROP_EXTENSION", + [PGSQL_QUERY_CREATE_POLICY] = (char*)"CREATE_POLICY", + [PGSQL_QUERY_ALTER_POLICY] = (char*)"ALTER_POLICY", + [PGSQL_QUERY_DROP_POLICY] = (char*)"DROP_POLICY", + [PGSQL_QUERY_CREATE_ROLE] = (char*)"CREATE_ROLE", + [PGSQL_QUERY_ALTER_ROLE] = (char*)"ALTER_ROLE", + [PGSQL_QUERY_DROP_ROLE] = (char*)"DROP_ROLE", + [PGSQL_QUERY_CREATE_USER] = (char*)"CREATE_USER", + [PGSQL_QUERY_ALTER_USER] = (char*)"ALTER_USER", + [PGSQL_QUERY_DROP_USER] = (char*)"DROP_USER", + [PGSQL_QUERY_GRANT] = (char*)"GRANT", + [PGSQL_QUERY_REVOKE] = (char*)"REVOKE", + [PGSQL_QUERY_COMMENT] = (char*)"COMMENT", + [PGSQL_QUERY_NOTIFY] = (char*)"NOTIFY", + [PGSQL_QUERY_LISTEN] = (char*)"LISTEN", + [PGSQL_QUERY_UNLISTEN] = (char*)"UNLISTEN", + [PGSQL_QUERY_LOCK] = (char*)"LOCK", + [PGSQL_QUERY_CHECKPOINT] = (char*)"CHECKPOINT", + [PGSQL_QUERY_REINDEX] = (char*)"REINDEX", + [PGSQL_QUERY_VACUUM] = (char*)"VACUUM", + [PGSQL_QUERY_ANALYZE] = (char*)"ANALYZE", + [PGSQL_QUERY_EXPLAIN] = (char*)"EXPLAIN", + [PGSQL_QUERY_EXECUTE] = (char*)"EXECUTE", + [PGSQL_QUERY_PREPARE] = (char*)"PREPARE", + [PGSQL_QUERY_DEALLOCATE] = (char*)"DEALLOCATE", + [PGSQL_QUERY_FETCH] = (char*)"FETCH", + [PGSQL_QUERY_MOVE] = (char*)"MOVE", + [PGSQL_QUERY_SAVEPOINT] = (char*)"SAVEPOINT", + [PGSQL_QUERY_ROLLBACK_TO_SAVEPOINT] = (char*)"ROLLBACK_TO_SAVEPOINT", + [PGSQL_QUERY_RELEASE_SAVEPOINT] = (char*)"RELEASE_SAVEPOINT", + [PGSQL_QUERY_BEGIN] = (char*)"BEGIN", + [PGSQL_QUERY_COMMIT] = (char*)"COMMIT", + [PGSQL_QUERY_ROLLBACK] = (char*)"ROLLBACK", + [PGSQL_QUERY_DECLARE_CURSOR] = (char*)"DECLARE_CURSOR", + [PGSQL_QUERY_CLOSE_CURSOR] = (char*)"CLOSE_CURSOR", + [PGSQL_QUERY_DISCARD] = (char*)"DISCARD", + [PGSQL_QUERY_SHOW] = (char*)"SHOW", + [PGSQL_QUERY_SET] = (char*)"SET", + [PGSQL_QUERY_RESET] = (char*)"RESET", + [PGSQL_QUERY_ALTER_DATABASE] = (char*)"ALTER_DATABASE", + [PGSQL_QUERY_CREATE_DATABASE] = (char*)"CREATE_DATABASE", + [PGSQL_QUERY_DROP_DATABASE] = (char*)"DROP_DATABASE", + [PGSQL_QUERY_CREATE_COLLATION] = (char*)"CREATE_COLLATION", + [PGSQL_QUERY_ALTER_COLLATION] = (char*)"ALTER_COLLATION", + [PGSQL_QUERY_DROP_COLLATION] = (char*)"DROP_COLLATION", + [PGSQL_QUERY_CREATE_TEXT_SEARCH_CONFIGURATION] = (char*)"CREATE_TEXT_SEARCH_CONFIGURATION", + [PGSQL_QUERY_ALTER_TEXT_SEARCH_CONFIGURATION] = (char*)"ALTER_TEXT_SEARCH_CONFIGURATION", + [PGSQL_QUERY_DROP_TEXT_SEARCH_CONFIGURATION] = (char*)"DROP_TEXT_SEARCH_CONFIGURATION", + [PGSQL_QUERY_CREATE_TEXT_SEARCH_DICTIONARY] = (char*)"CREATE_TEXT_SEARCH_DICTIONARY", + [PGSQL_QUERY_ALTER_TEXT_SEARCH_DICTIONARY] = (char*)"ALTER_TEXT_SEARCH_DICTIONARY", + [PGSQL_QUERY_DROP_TEXT_SEARCH_DICTIONARY] = (char*)"DROP_TEXT_SEARCH_DICTIONARY", + [PGSQL_QUERY_CREATE_TEXT_SEARCH_TEMPLATE] = (char*)"CREATE_TEXT_SEARCH_TEMPLATE", + [PGSQL_QUERY_ALTER_TEXT_SEARCH_TEMPLATE] = (char*)"ALTER_TEXT_SEARCH_TEMPLATE", + [PGSQL_QUERY_DROP_TEXT_SEARCH_TEMPLATE] = (char*)"DROP_TEXT_SEARCH_TEMPLATE", + [PGSQL_QUERY_CREATE_TEXT_SEARCH_PARSER] = (char*)"CREATE_TEXT_SEARCH_PARSER", + [PGSQL_QUERY_ALTER_TEXT_SEARCH_PARSER] = (char*)"ALTER_TEXT_SEARCH_PARSER", + [PGSQL_QUERY_DROP_TEXT_SEARCH_PARSER] = (char*)"DROP_TEXT_SEARCH_PARSER", + [PGSQL_QUERY_CREATE_FOREIGN_TABLE] = (char*)"CREATE_FOREIGN_TABLE", + [PGSQL_QUERY_ALTER_FOREIGN_TABLE] = (char*)"ALTER_FOREIGN_TABLE", + [PGSQL_QUERY_DROP_FOREIGN_TABLE] = (char*)"DROP_FOREIGN_TABLE", + [PGSQL_QUERY_IMPORT_FOREIGN_SCHEMA] = (char*)"IMPORT_FOREIGN_SCHEMA", + [PGSQL_QUERY_CREATE_SERVER] = (char*)"CREATE_SERVER", + [PGSQL_QUERY_ALTER_SERVER] = (char*)"ALTER_SERVER", + [PGSQL_QUERY_DROP_SERVER] = (char*)"DROP_SERVER", + [PGSQL_QUERY_CREATE_USER_MAPPING] = (char*)"CREATE_USER_MAPPING", + [PGSQL_QUERY_ALTER_USER_MAPPING] = (char*)"ALTER_USER_MAPPING", + [PGSQL_QUERY_DROP_USER_MAPPING] = (char*)"DROP_USER_MAPPING", + [PGSQL_QUERY_CREATE_PUBLICATION] = (char*)"CREATE_PUBLICATION", + [PGSQL_QUERY_ALTER_PUBLICATION] = (char*)"ALTER_PUBLICATION", + [PGSQL_QUERY_DROP_PUBLICATION] = (char*)"DROP_PUBLICATION", + [PGSQL_QUERY_CREATE_SUBSCRIPTION] = (char*)"CREATE_SUBSCRIPTION", + [PGSQL_QUERY_ALTER_SUBSCRIPTION] = (char*)"ALTER_SUBSCRIPTION", + [PGSQL_QUERY_DROP_SUBSCRIPTION] = (char*)"DROP_SUBSCRIPTION", + [PGSQL_QUERY_CREATE_ACCESS_METHOD] = (char*)"CREATE_ACCESS_METHOD", + [PGSQL_QUERY_ALTER_ACCESS_METHOD] = (char*)"ALTER_ACCESS_METHOD", + [PGSQL_QUERY_DROP_ACCESS_METHOD] = (char*)"DROP_ACCESS_METHOD", + [PGSQL_QUERY_CREATE_EVENT_TRIGGER] = (char*)"CREATE_EVENT_TRIGGER", + [PGSQL_QUERY_ALTER_EVENT_TRIGGER] = (char*)"ALTER_EVENT_TRIGGER", + [PGSQL_QUERY_DROP_EVENT_TRIGGER] = (char*)"DROP_EVENT_TRIGGER", + [PGSQL_QUERY_CREATE_TRANSFORM] = (char*)"CREATE_TRANSFORM", + [PGSQL_QUERY_ALTER_TRANSFORM] = (char*)"ALTER_TRANSFORM", + [PGSQL_QUERY_DROP_TRANSFORM] = (char*)"DROP_TRANSFORM", + [PGSQL_QUERY_CREATE_CAST] = (char*)"CREATE_CAST", + [PGSQL_QUERY_ALTER_CAST] = (char*)"ALTER_CAST", + [PGSQL_QUERY_DROP_CAST] = (char*)"DROP_CAST", + [PGSQL_QUERY_CREATE_OPERATOR_CLASS] = (char*)"CREATE_OPERATOR_CLASS", + [PGSQL_QUERY_ALTER_OPERATOR_CLASS] = (char*)"ALTER_OPERATOR_CLASS", + [PGSQL_QUERY_DROP_OPERATOR_CLASS] = (char*)"DROP_OPERATOR_CLASS", + [PGSQL_QUERY_CREATE_OPERATOR_FAMILY] = (char*)"CREATE_OPERATOR_FAMILY", + [PGSQL_QUERY_ALTER_OPERATOR_FAMILY] = (char*)"ALTER_OPERATOR_FAMILY", + [PGSQL_QUERY_DROP_OPERATOR_FAMILY] = (char*)"DROP_OPERATOR_FAMILY", + [PGSQL_QUERY_CREATE_TABLESPACE] = (char*)"CREATE_TABLESPACE", + [PGSQL_QUERY_ALTER_TABLESPACE] = (char*)"ALTER_TABLESPACE", + [PGSQL_QUERY_DROP_TABLESPACE] = (char*)"DROP_TABLESPACE", + [PGSQL_QUERY_CLUSTER] = (char*)"PGSQL_QUERY_CLUSTER", + [PGSQL_QUERY_UNKNOWN] = (char*)"UNKNOWN", +}; + +PgSQL_Rule_Text::PgSQL_Rule_Text(const PgSQL_Query_Processor_Rule_t* pqr) { + num_fields = 35; // this count the number of fields + pta = NULL; + pta = (char**)malloc(sizeof(char*) * num_fields); + itostr(pta[0], (long long)pqr->rule_id); + itostr(pta[1], (long long)pqr->active); + pta[2] = strdup_null(pqr->username); + pta[3] = strdup_null(pqr->schemaname); + itostr(pta[4], (long long)pqr->flagIN); + + pta[5] = strdup_null(pqr->client_addr); + pta[6] = strdup_null(pqr->proxy_addr); + itostr(pta[7], (long long)pqr->proxy_port); + + char buf[20]; + if (pqr->digest) { + sprintf(buf, "0x%016llX", (long long unsigned int)pqr->digest); + pta[8] = strdup(buf); + } + else { + pta[8] = NULL; + } + + pta[9] = strdup_null(pqr->match_digest); + pta[10] = strdup_null(pqr->match_pattern); + itostr(pta[11], (long long)pqr->negate_match_pattern); + std::string re_mod; + re_mod = ""; + if ((pqr->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) re_mod = "CASELESS"; + if ((pqr->re_modifiers & QP_RE_MOD_GLOBAL) == QP_RE_MOD_GLOBAL) { + if (re_mod.length()) { + re_mod = re_mod + ","; + } + re_mod = re_mod + "GLOBAL"; + } + pta[12] = strdup_null((char*)re_mod.c_str()); // re_modifiers + itostr(pta[13], (long long)pqr->flagOUT); + pta[14] = strdup_null(pqr->replace_pattern); + itostr(pta[15], (long long)pqr->destination_hostgroup); + itostr(pta[16], (long long)pqr->cache_ttl); + itostr(pta[17], (long long)pqr->cache_empty_result); + itostr(pta[18], (long long)pqr->cache_timeout); + itostr(pta[19], (long long)pqr->reconnect); + itostr(pta[20], (long long)pqr->timeout); + itostr(pta[21], (long long)pqr->retries); + itostr(pta[22], (long long)pqr->delay); + itostr(pta[23], (long long)pqr->next_query_flagIN); + itostr(pta[24], (long long)pqr->mirror_flagOUT); + itostr(pta[25], (long long)pqr->mirror_hostgroup); + pta[26] = strdup_null(pqr->error_msg); + pta[27] = strdup_null(pqr->OK_msg); + itostr(pta[28], (long long)pqr->sticky_conn); + itostr(pta[29], (long long)pqr->multiplex); + + itostr(pta[30], (long long)pqr->log); + itostr(pta[31], (long long)pqr->apply); + pta[32] = strdup_null(pqr->attributes); + pta[33] = strdup_null(pqr->comment); // issue #643 + itostr(pta[34], (long long)pqr->hits); +} + +PgSQL_Query_Processor::PgSQL_Query_Processor() : + Query_Processor(GloPTH->get_variable_int("query_rules_fast_routing_algorithm")) { + + for (int i = 0; i < PGSQL_QUERY___NONE; i++) commands_counters[i] = new Command_Counter(i,15,commands_counters_desc); +} + +PgSQL_Query_Processor::~PgSQL_Query_Processor() { + for (int i = 0; i < PGSQL_QUERY___NONE; i++) delete commands_counters[i]; +} + +void PgSQL_Query_Processor::update_query_processor_stats() { + Query_Processor::update_query_processor_stats(); + for (int i = 0; i < PGSQL_QUERY___NONE; i++) commands_counters[i]->add_and_reset(_thr_commands_counters[i]); +} + +void PgSQL_Query_Processor::init_thread() { + Query_Processor::init_thread(); + for (int i = 0; i < PGSQL_QUERY___NONE; i++) _thr_commands_counters[i] = new Command_Counter(i,15,commands_counters_desc); +} + +void PgSQL_Query_Processor::end_thread() { + Query_Processor::end_thread(); + for (int i = 0; i < PGSQL_QUERY___NONE; i++) delete _thr_commands_counters[i]; +}; + +unsigned long long PgSQL_Query_Processor::query_parser_update_counters(PgSQL_Session* sess, enum PGSQL_QUERY_command c, SQP_par_t* qp, unsigned long long t) { + if (c >= PGSQL_QUERY___NONE) return 0; + unsigned long long ret = _thr_commands_counters[c]->add_time(t); + Query_Processor::query_parser_update_counters(sess, qp->digest_total, qp->digest, qp->digest_text, t); + return ret; +} + +PgSQL_Query_Processor_Output* PgSQL_Query_Processor::process_query(PgSQL_Session* sess, void* ptr, unsigned int size, PgSQL_Query_Info* qi) { + // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE + // to avoid unnecssary deallocation/allocation, we initialize qpo witout new allocation + PgSQL_Query_Processor_Output* ret = sess->qpo; + ret->init(); + + SQP_par_t stmt_exec_qp; + SQP_par_t* qp = NULL; + if (qi) { + // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE + if (ptr) { + qp = (SQP_par_t*)&qi->QueryParserArgs; + } else { + qp = &stmt_exec_qp; + //qp->digest = qi->stmt_info->digest; + //qp->digest_text = qi->stmt_info->digest_text; + //qp->first_comment = qi->stmt_info->first_comment; + } + } +#define stackbuffer_size 128 + char stackbuffer[stackbuffer_size]; + unsigned int len = 0; + char* query = NULL; + // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE + if (ptr) { + len = size - sizeof(mysql_hdr) - 1; + if (len < stackbuffer_size) { + query = stackbuffer; + } else { + query = (char*)l_alloc(len + 1); + } + memcpy(query, (char*)ptr + sizeof(mysql_hdr) + 1, len); + query[len] = 0; + } + else { + //query = qi->stmt_info->query; + //len = qi->stmt_info->query_length; + } + + Query_Processor::process_query(sess, ptr == NULL, query, len, ret, qp); + + // FIXME : there is too much data being copied around + if (len < stackbuffer_size) { + // query is in the stack + } else { + if (ptr) { + l_free(len + 1, query); + } + } + + return ret; +} + +PgSQL_Query_Processor_Rule_t* PgSQL_Query_Processor::new_query_rule(int rule_id, bool active, const char* username, const char* schemaname, int flagIN, const char* client_addr, + const char* proxy_addr, int proxy_port, const char* digest, const char* match_digest, const char* match_pattern, bool negate_match_pattern, + const char* re_modifiers, int flagOUT, const char* replace_pattern, int destination_hostgroup, int cache_ttl, int cache_empty_result, + int cache_timeout, int reconnect, int timeout, int retries, int delay, int next_query_flagIN, int mirror_hostgroup, + int mirror_flagOUT, const char* error_msg, const char* OK_msg, int sticky_conn, int multiplex, int log, + bool apply, const char* attributes, const char* comment) { + + PgSQL_Query_Processor_Rule_t* newQR = (PgSQL_Query_Processor_Rule_t*)malloc(sizeof(PgSQL_Query_Processor_Rule_t)); + newQR->rule_id = rule_id; + newQR->active = active; + newQR->username = (username ? strdup(username) : NULL); + newQR->schemaname = (schemaname ? strdup(schemaname) : NULL); + newQR->flagIN = flagIN; + newQR->match_digest = (match_digest ? strdup(match_digest) : NULL); + newQR->match_pattern = (match_pattern ? strdup(match_pattern) : NULL); + newQR->negate_match_pattern = negate_match_pattern; + newQR->re_modifiers = 0; + { + tokenizer_t tok; + tokenizer(&tok, re_modifiers, ",", TOKENIZER_NO_EMPTIES); + const char* token; + for (token = tokenize(&tok); token; token = tokenize(&tok)) { + if (strncasecmp(token, (char*)"CASELESS", strlen((char*)"CASELESS")) == 0) { + newQR->re_modifiers |= QP_RE_MOD_CASELESS; + } + if (strncasecmp(token, (char*)"GLOBAL", strlen((char*)"GLOBAL")) == 0) { + newQR->re_modifiers |= QP_RE_MOD_GLOBAL; + } + } + free_tokenizer(&tok); + } + newQR->flagOUT = flagOUT; + newQR->replace_pattern = (replace_pattern ? strdup(replace_pattern) : NULL); + newQR->destination_hostgroup = destination_hostgroup; + newQR->cache_ttl = cache_ttl; + newQR->cache_empty_result = cache_empty_result; + newQR->cache_timeout = cache_timeout; + newQR->reconnect = reconnect; + newQR->timeout = timeout; + newQR->retries = retries; + newQR->delay = delay; + newQR->next_query_flagIN = next_query_flagIN; + newQR->mirror_flagOUT = mirror_flagOUT; + newQR->mirror_hostgroup = mirror_hostgroup; + newQR->error_msg = (error_msg ? strdup(error_msg) : NULL); + newQR->OK_msg = (OK_msg ? strdup(OK_msg) : NULL); + newQR->sticky_conn = sticky_conn; + newQR->multiplex = multiplex; + newQR->apply = apply; + newQR->attributes = (attributes ? strdup(attributes) : NULL); + newQR->comment = (comment ? strdup(comment) : NULL); // see issue #643 + newQR->regex_engine1 = NULL; + newQR->regex_engine2 = NULL; + newQR->hits = 0; + + newQR->client_addr_wildcard_position = -1; // not existing by default + newQR->client_addr = (client_addr ? strdup(client_addr) : NULL); + if (newQR->client_addr) { + char* pct = strchr(newQR->client_addr, '%'); + if (pct) { // there is a wildcard . We assume Admin did already all the input validation + if (pct == newQR->client_addr) { + // client_addr == '%' + // % is at the end of the string, but also at the beginning + // becoming a catch all + newQR->client_addr_wildcard_position = 0; + } + else { + // this math is valid also if (pct == newQR->client_addr) + // but we separate it to clarify that client_addr_wildcard_position is a match all + newQR->client_addr_wildcard_position = strlen(newQR->client_addr) - strlen(pct); + } + } + } + newQR->proxy_addr = (proxy_addr ? strdup(proxy_addr) : NULL); + newQR->proxy_port = proxy_port; + newQR->log = log; + newQR->digest = 0; + if (digest) { + unsigned long long num = strtoull(digest, NULL, 0); + if (num != ULLONG_MAX && num != 0) { + newQR->digest = num; + } + else { + proxy_error("Incorrect digest for rule_id %d : %s\n", rule_id, digest); + } + } + newQR->flagOUT_weights_total = 0; + newQR->flagOUT_ids = NULL; + newQR->flagOUT_weights = NULL; + if (newQR->attributes != NULL) { + if (strlen(newQR->attributes)) { + nlohmann::json j_attributes = nlohmann::json::parse(newQR->attributes); + if (j_attributes.find("flagOUTs") != j_attributes.end()) { + newQR->flagOUT_ids = new vector; + newQR->flagOUT_weights = new vector; + const nlohmann::json& flagOUTs = j_attributes["flagOUTs"]; + if (flagOUTs.type() == nlohmann::json::value_t::array) { + for (auto it = flagOUTs.begin(); it != flagOUTs.end(); it++) { + bool parsed = false; + const nlohmann::json& j = *it; + if (j.find("id") != j.end() && j.find("weight") != j.end()) { + if (j["id"].type() == nlohmann::json::value_t::number_unsigned && j["weight"].type() == nlohmann::json::value_t::number_unsigned) { + int id = j["id"]; + int weight = j["weight"]; + newQR->flagOUT_ids->push_back(id); + newQR->flagOUT_weights->push_back(weight); + newQR->flagOUT_weights_total += weight; + parsed = true; + } + } + if (parsed == false) { + proxy_error("Failed to parse flagOUTs in JSON on attributes for rule_id %d : %s\n", newQR->rule_id, j.dump().c_str()); + } + } + } + else { + proxy_error("Failed to parse flagOUTs attributes for rule_id %d : %s\n", newQR->rule_id, flagOUTs.dump().c_str()); + } + } + } + } + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Creating new rule in %p : rule_id:%d, active:%d, username=%s, schemaname=%s, flagIN:%d, %smatch_digest=\"%s\", %smatch_pattern=\"%s\", flagOUT:%d replace_pattern=\"%s\", destination_hostgroup:%d, apply:%d\n", newQR, newQR->rule_id, newQR->active, newQR->username, newQR->schemaname, newQR->flagIN, (newQR->negate_match_pattern ? "(!)" : ""), newQR->match_digest, (newQR->negate_match_pattern ? "(!)" : ""), newQR->match_pattern, newQR->flagOUT, newQR->replace_pattern, newQR->destination_hostgroup, newQR->apply); + return newQR; +} + +PgSQL_Query_Processor_Rule_t* PgSQL_Query_Processor::new_query_rule(const PgSQL_Query_Processor_Rule_t* pqr) { + + char buf[20]; + if (pqr->digest) { // not 0 + sprintf(buf, "0x%016llX", (long long unsigned int)pqr->digest); + } + + std::string re_mod; + re_mod = ""; + if ((pqr->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) re_mod = "CASELESS"; + if ((pqr->re_modifiers & QP_RE_MOD_GLOBAL) == QP_RE_MOD_GLOBAL) { + if (re_mod.length()) { + re_mod = re_mod + ","; + } + re_mod = re_mod + "GLOBAL"; + } + + PgSQL_Query_Processor_Rule_t* newQR = (PgSQL_Query_Processor_Rule_t*)malloc(sizeof(PgSQL_Query_Processor_Rule_t)); + newQR->rule_id = pqr->rule_id; + newQR->active = pqr->active; + newQR->username = (pqr->username ? strdup(pqr->username) : NULL); + newQR->schemaname = (pqr->schemaname ? strdup(pqr->schemaname) : NULL); + newQR->flagIN = pqr->flagIN; + newQR->match_digest = (pqr->match_digest ? strdup(pqr->match_digest) : NULL); + newQR->match_pattern = (pqr->match_pattern ? strdup(pqr->match_pattern) : NULL); + newQR->negate_match_pattern = pqr->negate_match_pattern; + newQR->re_modifiers = 0; + { + tokenizer_t tok; + tokenizer(&tok, re_mod.c_str(), ",", TOKENIZER_NO_EMPTIES); + const char* token; + for (token = tokenize(&tok); token; token = tokenize(&tok)) { + if (strncasecmp(token, (char*)"CASELESS", strlen((char*)"CASELESS")) == 0) { + newQR->re_modifiers |= QP_RE_MOD_CASELESS; + } + if (strncasecmp(token, (char*)"GLOBAL", strlen((char*)"GLOBAL")) == 0) { + newQR->re_modifiers |= QP_RE_MOD_GLOBAL; + } + } + free_tokenizer(&tok); + } + newQR->flagOUT = pqr->flagOUT; + newQR->replace_pattern = (pqr->replace_pattern ? strdup(pqr->replace_pattern) : NULL); + newQR->destination_hostgroup = pqr->destination_hostgroup; + newQR->cache_ttl = pqr->cache_ttl; + newQR->cache_empty_result = pqr->cache_empty_result; + newQR->cache_timeout = pqr->cache_timeout; + newQR->reconnect = pqr->reconnect; + newQR->timeout = pqr->timeout; + newQR->retries = pqr->retries; + newQR->delay = pqr->delay; + newQR->next_query_flagIN = pqr->next_query_flagIN; + newQR->mirror_flagOUT = pqr->mirror_flagOUT; + newQR->mirror_hostgroup = pqr->mirror_hostgroup; + newQR->error_msg = (pqr->error_msg ? strdup(pqr->error_msg) : NULL); + newQR->OK_msg = (pqr->OK_msg ? strdup(pqr->OK_msg) : NULL); + newQR->sticky_conn = pqr->sticky_conn; + newQR->multiplex = pqr->multiplex; + newQR->apply = pqr->apply; + newQR->attributes = (pqr->attributes ? strdup(pqr->attributes) : NULL); + newQR->comment = (pqr->comment ? strdup(pqr->comment) : NULL); // see issue #643 + newQR->regex_engine1 = NULL; + newQR->regex_engine2 = NULL; + newQR->hits = 0; + + newQR->client_addr_wildcard_position = -1; // not existing by default + newQR->client_addr = (pqr->client_addr ? strdup(pqr->client_addr) : NULL); + if (newQR->client_addr) { + char* pct = strchr(newQR->client_addr, '%'); + if (pct) { // there is a wildcard . We assume Admin did already all the input validation + if (pct == newQR->client_addr) { + // client_addr == '%' + // % is at the end of the string, but also at the beginning + // becoming a catch all + newQR->client_addr_wildcard_position = 0; + } + else { + // this math is valid also if (pct == newQR->client_addr) + // but we separate it to clarify that client_addr_wildcard_position is a match all + newQR->client_addr_wildcard_position = strlen(newQR->client_addr) - strlen(pct); + } + } + } + newQR->proxy_addr = (pqr->proxy_addr ? strdup(pqr->proxy_addr) : NULL); + newQR->proxy_port = pqr->proxy_port; + newQR->log = pqr->log; + newQR->digest = 0; + if (pqr->digest) { + unsigned long long num = strtoull(buf, NULL, 0); + if (num != ULLONG_MAX && num != 0) { + newQR->digest = num; + } + else { + proxy_error("Incorrect digest for rule_id %d : %s\n", pqr->rule_id, buf); + } + } + newQR->flagOUT_weights_total = 0; + newQR->flagOUT_ids = NULL; + newQR->flagOUT_weights = NULL; + if (newQR->attributes != NULL) { + if (strlen(newQR->attributes)) { + nlohmann::json j_attributes = nlohmann::json::parse(newQR->attributes); + if (j_attributes.find("flagOUTs") != j_attributes.end()) { + newQR->flagOUT_ids = new vector; + newQR->flagOUT_weights = new vector; + const nlohmann::json& flagOUTs = j_attributes["flagOUTs"]; + if (flagOUTs.type() == nlohmann::json::value_t::array) { + for (auto it = flagOUTs.begin(); it != flagOUTs.end(); it++) { + bool parsed = false; + const nlohmann::json& j = *it; + if (j.find("id") != j.end() && j.find("weight") != j.end()) { + if (j["id"].type() == nlohmann::json::value_t::number_unsigned && j["weight"].type() == nlohmann::json::value_t::number_unsigned) { + int id = j["id"]; + int weight = j["weight"]; + newQR->flagOUT_ids->push_back(id); + newQR->flagOUT_weights->push_back(weight); + newQR->flagOUT_weights_total += weight; + parsed = true; + } + } + if (parsed == false) { + proxy_error("Failed to parse flagOUTs in JSON on attributes for rule_id %d : %s\n", newQR->rule_id, j.dump().c_str()); + } + } + } + else { + proxy_error("Failed to parse flagOUTs attributes for rule_id %d : %s\n", newQR->rule_id, flagOUTs.dump().c_str()); + } + } + } + } + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Creating new rule in %p : rule_id:%d, active:%d, username=%s, schemaname=%s, flagIN:%d, %smatch_digest=\"%s\", %smatch_pattern=\"%s\", flagOUT:%d replace_pattern=\"%s\", destination_hostgroup:%d, apply:%d\n", newQR, newQR->rule_id, newQR->active, newQR->username, newQR->schemaname, newQR->flagIN, (newQR->negate_match_pattern ? "(!)" : ""), newQR->match_digest, (newQR->negate_match_pattern ? "(!)" : ""), newQR->match_pattern, newQR->flagOUT, newQR->replace_pattern, newQR->destination_hostgroup, newQR->apply); + return newQR; +} + +SQLite3_result* PgSQL_Query_Processor::get_current_query_rules() { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query rules, using Global version %d\n", version); + SQLite3_result* result = new SQLite3_result(34); + PgSQL_Query_Processor_Rule_t* qr1; + rdlock(); + result->add_column_definition(SQLITE_TEXT, "rule_id"); + result->add_column_definition(SQLITE_TEXT, "active"); + result->add_column_definition(SQLITE_TEXT, "username"); + result->add_column_definition(SQLITE_TEXT, "database"); + result->add_column_definition(SQLITE_TEXT, "flagIN"); + result->add_column_definition(SQLITE_TEXT, "client_addr"); + result->add_column_definition(SQLITE_TEXT, "proxy_addr"); + result->add_column_definition(SQLITE_TEXT, "proxy_port"); + result->add_column_definition(SQLITE_TEXT, "digest"); + result->add_column_definition(SQLITE_TEXT, "match_digest"); + result->add_column_definition(SQLITE_TEXT, "match_pattern"); + result->add_column_definition(SQLITE_TEXT, "negate_match_pattern"); + result->add_column_definition(SQLITE_TEXT, "re_modifiers"); + result->add_column_definition(SQLITE_TEXT, "flagOUT"); + result->add_column_definition(SQLITE_TEXT, "replace_pattern"); + result->add_column_definition(SQLITE_TEXT, "destination_hostgroup"); + result->add_column_definition(SQLITE_TEXT, "cache_ttl"); + result->add_column_definition(SQLITE_TEXT, "cache_empty_result"); + result->add_column_definition(SQLITE_TEXT, "cache_timeout"); + result->add_column_definition(SQLITE_TEXT, "reconnect"); + result->add_column_definition(SQLITE_TEXT, "timeout"); + result->add_column_definition(SQLITE_TEXT, "retries"); + result->add_column_definition(SQLITE_TEXT, "delay"); + result->add_column_definition(SQLITE_TEXT, "next_query_flagIN"); + result->add_column_definition(SQLITE_TEXT, "mirror_flagOUT"); + result->add_column_definition(SQLITE_TEXT, "mirror_hostgroup"); + result->add_column_definition(SQLITE_TEXT, "error_msg"); + result->add_column_definition(SQLITE_TEXT, "OK_msg"); + result->add_column_definition(SQLITE_TEXT, "sticky_conn"); + result->add_column_definition(SQLITE_TEXT, "multiplex"); + result->add_column_definition(SQLITE_TEXT, "log"); + result->add_column_definition(SQLITE_TEXT, "apply"); + result->add_column_definition(SQLITE_TEXT, "attributes"); + result->add_column_definition(SQLITE_TEXT, "comment"); // issue #643 + result->add_column_definition(SQLITE_TEXT, "hits"); + for (std::vector::iterator it = rules.begin(); it != rules.end(); ++it) { + qr1 = static_cast(*it); + PgSQL_Rule_Text* qt = new PgSQL_Rule_Text(qr1); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping Query Rule id: %d\n", qr1->rule_id); + result->add_row(qt->pta); + delete qt; + } + wrunlock(); + return result; +} + +SQLite3_result* PgSQL_Query_Processor::get_stats_commands_counters() { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping commands counters\n"); + SQLite3_result* result = new SQLite3_result(15); + result->add_column_definition(SQLITE_TEXT, "Command"); + result->add_column_definition(SQLITE_TEXT, "Total_Cnt"); + result->add_column_definition(SQLITE_TEXT, "Total_Time_us"); + result->add_column_definition(SQLITE_TEXT, "cnt_100us"); + result->add_column_definition(SQLITE_TEXT, "cnt_500us"); + result->add_column_definition(SQLITE_TEXT, "cnt_1ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_5ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_10ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_50ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_100ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_500ms"); + result->add_column_definition(SQLITE_TEXT, "cnt_1s"); + result->add_column_definition(SQLITE_TEXT, "cnt_5s"); + result->add_column_definition(SQLITE_TEXT, "cnt_10s"); + result->add_column_definition(SQLITE_TEXT, "cnt_INFs"); + for (int i = 0; i < MYSQL_COM_QUERY__UNINITIALIZED; i++) { + char** pta = commands_counters[i]->get_row(); + result->add_row(pta); + commands_counters[i]->free_row(pta); + } + return result; +} + +enum PGSQL_QUERY_command PgSQL_Query_Processor::query_parser_command_type(SQP_par_t* qp) { + char* text = NULL; + if (qp->digest_text) { + text = qp->digest_text; + } + else { + text = qp->query_prefix; + } + + enum PGSQL_QUERY_command ret = PGSQL_QUERY_UNKNOWN; + char c1; + + tokenizer_t tok; + tokenizer(&tok, text, " ", TOKENIZER_NO_EMPTIES); + char* token = NULL; +__get_token: + token = (char*)tokenize(&tok); + if (token == NULL) { + goto __exit__query_parser_command_type; + } +__remove_parenthesis: + if (token[0] == '(') { + if (strlen(token) > 1) { + token++; + goto __remove_parenthesis; + } + else { + goto __get_token; + } + } + c1 = token[0]; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Command:%s Prefix:%c\n", token, c1); + switch (c1) { + case 'a': + case 'A': + if (!strcasecmp("ALTER", token)) { + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) ret = PGSQL_QUERY_ALTER_TABLE; + else if (!strcasecmp("INDEX", token)) ret = PGSQL_QUERY_ALTER_INDEX; + else if (!strcasecmp("VIEW", token)) ret = PGSQL_QUERY_ALTER_VIEW; + else if (!strcasecmp("MATERIALIZED", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("VIEW", token)) ret = PGSQL_QUERY_ALTER_MATERIALIZED_VIEW; + } + else if (!strcasecmp("SEQUENCE", token)) ret = PGSQL_QUERY_ALTER_SEQUENCE; + else if (!strcasecmp("SCHEMA", token)) ret = PGSQL_QUERY_ALTER_SCHEMA; + else if (!strcasecmp("FUNCTION", token)) ret = PGSQL_QUERY_ALTER_FUNCTION; + else if (!strcasecmp("PROCEDURE", token)) ret = PGSQL_QUERY_ALTER_PROCEDURE; + else if (!strcasecmp("AGGREGATE", token)) ret = PGSQL_QUERY_ALTER_AGGREGATE; + else if (!strcasecmp("OPERATOR", token)) ret = PGSQL_QUERY_ALTER_OPERATOR; + else if (!strcasecmp("TYPE", token)) ret = PGSQL_QUERY_ALTER_TYPE; + else if (!strcasecmp("DOMAIN", token)) ret = PGSQL_QUERY_ALTER_DOMAIN; + else if (!strcasecmp("TRIGGER", token)) ret = PGSQL_QUERY_ALTER_TRIGGER; + else if (!strcasecmp("RULE", token)) ret = PGSQL_QUERY_ALTER_RULE; + else if (!strcasecmp("EXTENSION", token)) ret = PGSQL_QUERY_ALTER_EXTENSION; + else if (!strcasecmp("POLICY", token)) ret = PGSQL_QUERY_ALTER_POLICY; + else if (!strcasecmp("ROLE", token)) ret = PGSQL_QUERY_ALTER_ROLE; + else if (!strcasecmp("USER", token)) ret = PGSQL_QUERY_ALTER_USER; + else if (!strcasecmp("DATABASE", token)) ret = PGSQL_QUERY_ALTER_DATABASE; + else if (!strcasecmp("COLLATION", token)) ret = PGSQL_QUERY_ALTER_COLLATION; + else if (!strcasecmp("TEXT", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("SEARCH", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("CONFIGURATION", token)) ret = PGSQL_QUERY_ALTER_TEXT_SEARCH_CONFIGURATION; + else if (!strcasecmp("DICTIONARY", token)) ret = PGSQL_QUERY_ALTER_TEXT_SEARCH_DICTIONARY; + else if (!strcasecmp("TEMPLATE", token)) ret = PGSQL_QUERY_ALTER_TEXT_SEARCH_TEMPLATE; + else if (!strcasecmp("PARSER", token)) ret = PGSQL_QUERY_ALTER_TEXT_SEARCH_PARSER; + } + } + } + } + else if (!strcasecmp("FOREIGN", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("TABLE", token)) ret = PGSQL_QUERY_ALTER_FOREIGN_TABLE; + } + else if (!strcasecmp("SERVER", token)) ret = PGSQL_QUERY_ALTER_SERVER; + else if (!strcasecmp("USER", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("MAPPING", token)) ret = PGSQL_QUERY_ALTER_USER_MAPPING; + } + else if (!strcasecmp("PUBLICATION", token)) ret = PGSQL_QUERY_ALTER_PUBLICATION; + else if (!strcasecmp("SUBSCRIPTION", token)) ret = PGSQL_QUERY_ALTER_SUBSCRIPTION; + else if (!strcasecmp("ACCESS", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("METHOD", token)) ret = PGSQL_QUERY_ALTER_ACCESS_METHOD; + } + else if (!strcasecmp("EVENT", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("TRIGGER", token)) ret = PGSQL_QUERY_ALTER_EVENT_TRIGGER; + } + else if (!strcasecmp("TRANSFORM", token)) ret = PGSQL_QUERY_ALTER_TRANSFORM; + else if (!strcasecmp("CAST", token)) ret = PGSQL_QUERY_ALTER_CAST; + else if (!strcasecmp("OPERATOR", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("CLASS", token)) ret = PGSQL_QUERY_ALTER_OPERATOR_CLASS; + else if (!strcasecmp("FAMILY", token)) ret = PGSQL_QUERY_ALTER_OPERATOR_FAMILY; + } + } + else if (!strcasecmp("TABLESPACE", token)) ret = PGSQL_QUERY_ALTER_TABLESPACE; + break; + } + if (!strcasecmp("ANALYZE", token)) { + ret = PGSQL_QUERY_ANALYZE; + break; + } + break; + case 'b': + case 'B': + if (!strcasecmp("BEGIN", token)) { + ret = PGSQL_QUERY_BEGIN; + break; + } + break; + case 'c': + case 'C': + if (!strcasecmp("CALL", token)) { + ret = PGSQL_QUERY_CALL; + break; + } + if (!strcasecmp("CHECKPOINT", token)) { + ret = PGSQL_QUERY_CHECKPOINT; + break; + } + if (!strcasecmp("CLOSE", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("CURSOR", token)) ret = PGSQL_QUERY_CLOSE_CURSOR; + break; + } + if (!strcasecmp("CLUSTER", token)) { + ret = PGSQL_QUERY_CLUSTER; + break; + } + if (!strcasecmp("COMMENT", token)) { + ret = PGSQL_QUERY_COMMENT; + break; + } + if (!strcasecmp("COMMIT", token)) { + ret = PGSQL_QUERY_COMMIT; + break; + } + if (!strcasecmp("COPY", token)) { + ret = PGSQL_QUERY_COPY; + break; + } + if (!strcasecmp("CREATE", token)) { + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) ret = PGSQL_QUERY_CREATE_TABLE; + else if (!strcasecmp("INDEX", token)) ret = PGSQL_QUERY_CREATE_INDEX; + else if (!strcasecmp("VIEW", token)) ret = PGSQL_QUERY_CREATE_VIEW; + else if (!strcasecmp("MATERIALIZED", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("VIEW", token)) ret = PGSQL_QUERY_CREATE_MATERIALIZED_VIEW; + } + else if (!strcasecmp("SEQUENCE", token)) ret = PGSQL_QUERY_CREATE_SEQUENCE; + else if (!strcasecmp("SCHEMA", token)) ret = PGSQL_QUERY_CREATE_SCHEMA; + else if (!strcasecmp("FUNCTION", token)) ret = PGSQL_QUERY_CREATE_FUNCTION; + else if (!strcasecmp("PROCEDURE", token)) ret = PGSQL_QUERY_CREATE_PROCEDURE; + else if (!strcasecmp("AGGREGATE", token)) ret = PGSQL_QUERY_CREATE_AGGREGATE; + else if (!strcasecmp("OPERATOR", token)) ret = PGSQL_QUERY_CREATE_OPERATOR; + else if (!strcasecmp("TYPE", token)) ret = PGSQL_QUERY_CREATE_TYPE; + else if (!strcasecmp("DOMAIN", token)) ret = PGSQL_QUERY_CREATE_DOMAIN; + else if (!strcasecmp("TRIGGER", token)) ret = PGSQL_QUERY_CREATE_TRIGGER; + else if (!strcasecmp("RULE", token)) ret = PGSQL_QUERY_CREATE_RULE; + else if (!strcasecmp("EXTENSION", token)) ret = PGSQL_QUERY_CREATE_EXTENSION; + else if (!strcasecmp("POLICY", token)) ret = PGSQL_QUERY_CREATE_POLICY; + else if (!strcasecmp("ROLE", token)) ret = PGSQL_QUERY_CREATE_ROLE; + else if (!strcasecmp("USER", token)) ret = PGSQL_QUERY_CREATE_USER; + else if (!strcasecmp("DATABASE", token)) ret = PGSQL_QUERY_CREATE_DATABASE; + else if (!strcasecmp("COLLATION", token)) ret = PGSQL_QUERY_CREATE_COLLATION; + else if (!strcasecmp("TEXT", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("SEARCH", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("CONFIGURATION", token)) ret = PGSQL_QUERY_CREATE_TEXT_SEARCH_CONFIGURATION; + else if (!strcasecmp("DICTIONARY", token)) ret = PGSQL_QUERY_CREATE_TEXT_SEARCH_DICTIONARY; + else if (!strcasecmp("TEMPLATE", token)) ret = PGSQL_QUERY_CREATE_TEXT_SEARCH_TEMPLATE; + else if (!strcasecmp("PARSER", token)) ret = PGSQL_QUERY_CREATE_TEXT_SEARCH_PARSER; + } + } + } + } + else if (!strcasecmp("FOREIGN", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("TABLE", token)) ret = PGSQL_QUERY_CREATE_FOREIGN_TABLE; + } + else if (!strcasecmp("SERVER", token)) ret = PGSQL_QUERY_CREATE_SERVER; + else if (!strcasecmp("USER", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("MAPPING", token)) ret = PGSQL_QUERY_CREATE_USER_MAPPING; + } + else if (!strcasecmp("PUBLICATION", token)) ret = PGSQL_QUERY_CREATE_PUBLICATION; + else if (!strcasecmp("SUBSCRIPTION", token)) ret = PGSQL_QUERY_CREATE_SUBSCRIPTION; + else if (!strcasecmp("ACCESS", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("METHOD", token)) ret = PGSQL_QUERY_CREATE_ACCESS_METHOD; + } + else if (!strcasecmp("EVENT", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("TRIGGER", token)) ret = PGSQL_QUERY_CREATE_EVENT_TRIGGER; + } + else if (!strcasecmp("TRANSFORM", token)) ret = PGSQL_QUERY_CREATE_TRANSFORM; + else if (!strcasecmp("CAST", token)) ret = PGSQL_QUERY_CREATE_CAST; + else if (!strcasecmp("OPERATOR", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("CLASS", token)) ret = PGSQL_QUERY_CREATE_OPERATOR_CLASS; + else if (!strcasecmp("FAMILY", token)) ret = PGSQL_QUERY_CREATE_OPERATOR_FAMILY; + } + } + else if (!strcasecmp("TABLESPACE", token)) ret = PGSQL_QUERY_CREATE_TABLESPACE; + break; + } + break; + case 'd': + case 'D': + if (!strcasecmp("DEALLOCATE", token)) { + ret = PGSQL_QUERY_DEALLOCATE; + break; + } + if (!strcasecmp("DECLARE", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("CURSOR", token)) ret = PGSQL_QUERY_DECLARE_CURSOR; + break; + } + if (!strcasecmp("DELETE", token)) { + ret = PGSQL_QUERY_DELETE; + break; + } + if (!strcasecmp("DISCARD", token)) { + ret = PGSQL_QUERY_DISCARD; + break; + } + if (!strcasecmp("DROP", token)) { + token = (char*)tokenize(&tok); + if (token == NULL) break; + if (!strcasecmp("TABLE", token)) ret = PGSQL_QUERY_DROP_TABLE; + else if (!strcasecmp("INDEX", token)) ret = PGSQL_QUERY_DROP_INDEX; + else if (!strcasecmp("VIEW", token)) ret = PGSQL_QUERY_DROP_VIEW; + else if (!strcasecmp("MATERIALIZED", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("VIEW", token)) ret = PGSQL_QUERY_DROP_MATERIALIZED_VIEW; + } + else if (!strcasecmp("SEQUENCE", token)) ret = PGSQL_QUERY_DROP_SEQUENCE; + else if (!strcasecmp("SCHEMA", token)) ret = PGSQL_QUERY_DROP_SCHEMA; + else if (!strcasecmp("FUNCTION", token)) ret = PGSQL_QUERY_DROP_FUNCTION; + else if (!strcasecmp("PROCEDURE", token)) ret = PGSQL_QUERY_DROP_PROCEDURE; + else if (!strcasecmp("AGGREGATE", token)) ret = PGSQL_QUERY_DROP_AGGREGATE; + else if (!strcasecmp("OPERATOR", token)) ret = PGSQL_QUERY_DROP_OPERATOR; + else if (!strcasecmp("TYPE", token)) ret = PGSQL_QUERY_DROP_TYPE; + else if (!strcasecmp("DOMAIN", token)) ret = PGSQL_QUERY_DROP_DOMAIN; + else if (!strcasecmp("TRIGGER", token)) ret = PGSQL_QUERY_DROP_TRIGGER; + else if (!strcasecmp("RULE", token)) ret = PGSQL_QUERY_DROP_RULE; + else if (!strcasecmp("EXTENSION", token)) ret = PGSQL_QUERY_DROP_EXTENSION; + else if (!strcasecmp("POLICY", token)) ret = PGSQL_QUERY_DROP_POLICY; + else if (!strcasecmp("ROLE", token)) ret = PGSQL_QUERY_DROP_ROLE; + else if (!strcasecmp("USER", token)) ret = PGSQL_QUERY_DROP_USER; + else if (!strcasecmp("DATABASE", token)) ret = PGSQL_QUERY_DROP_DATABASE; + else if (!strcasecmp("COLLATION", token)) ret = PGSQL_QUERY_DROP_COLLATION; + else if (!strcasecmp("TEXT", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("SEARCH", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("CONFIGURATION", token)) ret = PGSQL_QUERY_DROP_TEXT_SEARCH_CONFIGURATION; + else if (!strcasecmp("DICTIONARY", token)) ret = PGSQL_QUERY_DROP_TEXT_SEARCH_DICTIONARY; + else if (!strcasecmp("TEMPLATE", token)) ret = PGSQL_QUERY_DROP_TEXT_SEARCH_TEMPLATE; + else if (!strcasecmp("PARSER", token)) ret = PGSQL_QUERY_DROP_TEXT_SEARCH_PARSER; + } + } + } + } + else if (!strcasecmp("FOREIGN", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("TABLE", token)) ret = PGSQL_QUERY_DROP_FOREIGN_TABLE; + } + else if (!strcasecmp("SERVER", token)) ret = PGSQL_QUERY_DROP_SERVER; + else if (!strcasecmp("USER", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("MAPPING", token)) ret = PGSQL_QUERY_DROP_USER_MAPPING; + } + else if (!strcasecmp("PUBLICATION", token)) ret = PGSQL_QUERY_DROP_PUBLICATION; + else if (!strcasecmp("SUBSCRIPTION", token)) ret = PGSQL_QUERY_DROP_SUBSCRIPTION; + else if (!strcasecmp("ACCESS", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("METHOD", token)) ret = PGSQL_QUERY_DROP_ACCESS_METHOD; + } + else if (!strcasecmp("EVENT", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("TRIGGER", token)) ret = PGSQL_QUERY_DROP_EVENT_TRIGGER; + } + else if (!strcasecmp("TRANSFORM", token)) ret = PGSQL_QUERY_DROP_TRANSFORM; + else if (!strcasecmp("CAST", token)) ret = PGSQL_QUERY_DROP_CAST; + else if (!strcasecmp("OPERATOR", token)) { + token = (char*)tokenize(&tok); + if (token != NULL) { + if (!strcasecmp("CLASS", token)) ret = PGSQL_QUERY_DROP_OPERATOR_CLASS; + else if (!strcasecmp("FAMILY", token)) ret = PGSQL_QUERY_DROP_OPERATOR_FAMILY; + } + } + else if (!strcasecmp("TABLESPACE", token)) ret = PGSQL_QUERY_DROP_TABLESPACE; + break; + } + break; + case 'e': + case 'E': + if (!strcasecmp("EXECUTE", token)) { + ret = PGSQL_QUERY_EXECUTE; + break; + } + if (!strcasecmp("EXPLAIN", token)) { + ret = PGSQL_QUERY_EXPLAIN; + break; + } + break; + case 'f': + case 'F': + if (!strcasecmp("FETCH", token)) { + ret = PGSQL_QUERY_FETCH; + break; + } + break; + case 'g': + case 'G': + if (!strcasecmp("GRANT", token)) { + ret = PGSQL_QUERY_GRANT; + break; + } + break; + case 'i': + case 'I': + if (!strcasecmp("INSERT", token)) { + ret = PGSQL_QUERY_INSERT; + break; + } + if (!strcasecmp("IMPORT", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("FOREIGN", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("SCHEMA", token)) ret = PGSQL_QUERY_IMPORT_FOREIGN_SCHEMA; + } + break; + } + break; + case 'l': + case 'L': + if (!strcasecmp("LISTEN", token)) { + ret = PGSQL_QUERY_LISTEN; + break; + } + if (!strcasecmp("LOAD", token)) { + ret = PGSQL_QUERY_UNKNOWN; // Not in the enum, but exists in PostgreSQL + break; + } + if (!strcasecmp("LOCK", token)) { + ret = PGSQL_QUERY_LOCK; + break; + } + break; + case 'm': + case 'M': + if (!strcasecmp("MERGE", token)) { + ret = PGSQL_QUERY_MERGE; + break; + } + if (!strcasecmp("MOVE", token)) { + ret = PGSQL_QUERY_MOVE; + break; + } + break; + case 'n': + case 'N': + if (!strcasecmp("NOTIFY", token)) { + ret = PGSQL_QUERY_NOTIFY; + break; + } + break; + case 'p': + case 'P': + if (!strcasecmp("PREPARE", token)) { + ret = PGSQL_QUERY_PREPARE; + break; + } + break; + case 'r': + case 'R': + if (!strcasecmp("REFRESH", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("MATERIALIZED", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("VIEW", token)) ret = PGSQL_QUERY_REFRESH_MATERIALIZED_VIEW; + } + break; + } + if (!strcasecmp("REINDEX", token)) { + ret = PGSQL_QUERY_REINDEX; + break; + } + if (!strcasecmp("RELEASE", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("SAVEPOINT", token)) ret = PGSQL_QUERY_RELEASE_SAVEPOINT; + break; + } + if (!strcasecmp("RESET", token)) { + ret = PGSQL_QUERY_RESET; + break; + } + if (!strcasecmp("REVOKE", token)) { + ret = PGSQL_QUERY_REVOKE; + break; + } + if (!strcasecmp("ROLLBACK", token)) { + token = (char*)tokenize(&tok); + if (token == NULL) { + ret = PGSQL_QUERY_ROLLBACK; + } + else if (!strcasecmp("TO", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("SAVEPOINT", token)) ret = PGSQL_QUERY_ROLLBACK_TO_SAVEPOINT; + } + break; + } + break; + case 's': + case 'S': + if (!strcasecmp("SAVEPOINT", token)) { + ret = PGSQL_QUERY_SAVEPOINT; + break; + } + if (!strcasecmp("SELECT", token)) { + ret = PGSQL_QUERY_SELECT; + break; + } + if (!strcasecmp("SET", token)) { + ret = PGSQL_QUERY_SET; + break; + } + if (!strcasecmp("SHOW", token)) { + ret = PGSQL_QUERY_SHOW; + break; + } + if (!strcasecmp("START", token)) { + token = (char*)tokenize(&tok); + if (token != NULL && !strcasecmp("TRANSACTION", token)) ret = PGSQL_QUERY_BEGIN; + break; + } + break; + case 't': + case 'T': + if (!strcasecmp("TRUNCATE", token)) { + ret = PGSQL_QUERY_TRUNCATE; + break; + } + break; + case 'u': + case 'U': + if (!strcasecmp("UNLISTEN", token)) { + ret = PGSQL_QUERY_UNLISTEN; + break; + } + if (!strcasecmp("UPDATE", token)) { + ret = PGSQL_QUERY_UPDATE; + break; + } + break; + case 'v': + case 'V': + if (!strcasecmp("VACUUM", token)) { + ret = PGSQL_QUERY_VACUUM; + break; + } + break; + default: + break; + } + +__exit__query_parser_command_type: + free_tokenizer(&tok); + if (qp->query_prefix) { + free(qp->query_prefix); + qp->query_prefix = NULL; + } + return ret; +} diff --git a/lib/PgSQL_Session.cpp b/lib/PgSQL_Session.cpp new file mode 100644 index 0000000000..edac2a69d2 --- /dev/null +++ b/lib/PgSQL_Session.cpp @@ -0,0 +1,7738 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +#include "PgSQL_HostGroups_Manager.h" +#include "PgSQL_Thread.h" +#include "proxysql.h" +#include "cpp.h" +#include "proxysql_utils.h" +#include "re2/re2.h" +#include "re2/regexp.h" +#include "mysqld_error.h" + +#include "PgSQL_Data_Stream.h" +#include "MySQL_Data_Stream.h" +#include "PgSQL_Query_Processor.h" +#include "MySQL_PreparedStatement.h" +#include "PgSQL_Logger.hpp" +#include "StatCounters.h" +#include "PgSQL_Authentication.h" +#include "MySQL_LDAP_Authentication.hpp" +#include "MySQL_Protocol.h" +#include "SQLite3_Server.h" +#include "MySQL_Variables.h" +#include "ProxySQL_Cluster.hpp" + + +#include "libinjection.h" +#include "libinjection_sqli.h" + +#define SELECT_VERSION_COMMENT "select @@version_comment limit 1" +#define SELECT_VERSION_COMMENT_LEN 32 +//#define SELECT_DB_USER "select DATABASE(), USER() limit 1" +#define SELECT_DB_USER_LEN 33 +//#define SELECT_CHARSET_STATUS "select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1" +#define SELECT_CHARSET_STATUS_LEN 115 +#define PROXYSQL_VERSION_COMMENT "\x01\x00\x00\x01\x01\x27\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x11\x40\x40\x76\x65\x72\x73\x69\x6f\x6e\x5f\x63\x6f\x6d\x6d\x65\x6e\x74\x00\x0c\x21\x00\x18\x00\x00\x00\xfd\x00\x00\x1f\x00\x00\x05\x00\x00\x03\xfe\x00\x00\x02\x00\x0b\x00\x00\x04\x0a(ProxySQL)\x05\x00\x00\x05\xfe\x00\x00\x02\x00" +#define PROXYSQL_VERSION_COMMENT_LEN 81 + +// PROXYSQL_VERSION_COMMENT_WITH_OK is sent instead of PROXYSQL_VERSION_COMMENT +// if Client supports CLIENT_DEPRECATE_EOF +#define PROXYSQL_VERSION_COMMENT_WITH_OK "\x01\x00\x00\x01\x01" \ +"\x27\x00\x00\x02\x03\x64\x65\x66\x00\x00\x00\x11\x40\x40\x76\x65\x72\x73\x69\x6f\x6e\x5f\x63\x6f\x6d\x6d\x65\x6e\x74\x00\x0c\x21\x00\x18\x00\x00\x00\xfd\x00\x00\x1f\x00\x00" \ +"\x0b\x00\x00\x03\x0a(ProxySQL)" \ +"\x07\x00\x00\x04\xfe\x00\x00\x02\x00\x00\x00" +#define PROXYSQL_VERSION_COMMENT_WITH_OK_LEN 74 + +#define SELECT_CONNECTION_ID "SELECT CONNECTION_ID()" +#define SELECT_CONNECTION_ID_LEN 22 +#define SELECT_LAST_INSERT_ID "SELECT LAST_INSERT_ID()" +#define SELECT_LAST_INSERT_ID_LEN 23 +#define SELECT_LAST_INSERT_ID_LIMIT1 "SELECT LAST_INSERT_ID() LIMIT 1" +#define SELECT_LAST_INSERT_ID_LIMIT1_LEN 31 +#define SELECT_VARIABLE_IDENTITY "SELECT @@IDENTITY" +#define SELECT_VARIABLE_IDENTITY_LEN 17 +#define SELECT_VARIABLE_IDENTITY_LIMIT1 "SELECT @@IDENTITY LIMIT 1" +#define SELECT_VARIABLE_IDENTITY_LIMIT1_LEN 25 + +#define EXPMARIA + +using std::function; +using std::vector; + +static inline char is_digit(char c) { + if (c >= '0' && c <= '9') + return 1; + return 0; +} +static inline char is_normal_char(char c) { + if (c >= 'a' && c <= 'z') + return 1; + if (c >= 'A' && c <= 'Z') + return 1; + if (c >= '0' && c <= '9') + return 1; + if (c == '$' || c == '_') + return 1; + return 0; +} + +static const std::set pgsql_variables_boolean = { + "aurora_read_replica_read_committed", + "foreign_key_checks", + "innodb_strict_mode", + "innodb_table_locks", + "sql_auto_is_null", + "sql_big_selects", + "sql_generate_invisible_primary_key", + "sql_log_bin", + "sql_quote_show_create", + "sql_require_primary_key", + "sql_safe_updates", + "unique_checks", +}; + +static const std::set pgsql_variables_numeric = { + "auto_increment_increment", + "auto_increment_offset", + "group_concat_max_len", + "innodb_lock_wait_timeout", + "join_buffer_size", + "lock_wait_timeout", + "long_query_time", + "max_execution_time", + "max_heap_table_size", + "max_join_size", + "max_sort_length", + "max_statement_time", + "optimizer_prune_level", + "optimizer_search_depth", + "optimizer_use_condition_selectivity", + "query_cache_type", + "sort_buffer_size", + "sql_select_limit", + "timestamp", + "tmp_table_size", + "wsrep_sync_wait" +}; +static const std::set pgsql_variables_strings = { + "default_storage_engine", + "default_tmp_storage_engine", + "group_replication_consistency", + "lc_messages", + "lc_time_names", + "log_slow_filter", + "optimizer_switch", + "wsrep_osu_method", +}; + +#include "proxysql_find_charset.h" + +extern PgSQL_Authentication* GloPgAuth; +extern MySQL_LDAP_Authentication* GloMyLdapAuth; +extern ProxySQL_Admin* GloAdmin; +extern PgSQL_Logger* GloPgSQL_Logger; +extern MySQL_STMT_Manager_v14* GloMyStmt; + +extern SQLite3_Server* GloSQLite3Server; + +#ifdef PROXYSQLCLICKHOUSE +extern ClickHouse_Authentication* GloClickHouseAuth; +extern ClickHouse_Server* GloClickHouseServer; +#endif /* PROXYSQLCLICKHOUSE */ + +/* +std::string proxysql_session_type_str(enum proxysql_session_type session_type) { + if (session_type == PROXYSQL_SESSION_MYSQL) { + return "PROXYSQL_SESSION_MYSQL"; + } else if (session_type == PROXYSQL_SESSION_ADMIN) { + return "PROXYSQL_SESSION_ADMIN"; + } else if (session_type == PROXYSQL_SESSION_STATS) { + return "PROXYSQL_SESSION_STATS"; + } else if (session_type == PROXYSQL_SESSION_SQLITE) { + return "PROXYSQL_SESSION_SQLITE"; + } else if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { + return "PROXYSQL_SESSION_CLICKHOUSE"; + } else if (session_type == PROXYSQL_SESSION_MYSQL_EMU) { + return "PROXYSQL_SESSION_MYSQL_EMU"; + } else { + return "PROXYSQL_SESSION_NONE"; + } +};*/ + +/* +Session_Regex::Session_Regex(char *p) { + s=strdup(p); + re2::RE2::Options *opt2=new re2::RE2::Options(RE2::Quiet); + opt2->set_case_sensitive(false); + opt=(void *)opt2; + re=(RE2 *)new RE2(s, *opt2); +} + +PgSQL_Session_Regex::~PgSQL_Session_Regex() { + free(s); + delete (RE2 *)re; + delete (re2::RE2::Options *)opt; +} + +bool PgSQL_Session_Regex::match(char *m) { + bool rc=false; + rc=RE2::PartialMatch(m,*(RE2 *)re); + return rc; +} +*/ + +PgSQL_KillArgs::PgSQL_KillArgs(char* u, char* p, char* h, unsigned int P, unsigned int _hid, unsigned long i, int kt, int _use_ssl, PgSQL_Thread* _mt) : + PgSQL_KillArgs(u, p, h, P, _hid, i, kt, _use_ssl, _mt, NULL) { + // resolving DNS if available in Cache + if (h && P) { + const std::string& ip = MySQL_Monitor::dns_lookup(h, false); + + if (ip.empty() == false) { + ip_addr = strdup(ip.c_str()); + } + } +} +PgSQL_KillArgs::PgSQL_KillArgs(char* u, char* p, char* h, unsigned int P, unsigned int _hid, unsigned long i, int kt, int _use_ssl, PgSQL_Thread* _mt, char* ip) { + username = strdup(u); + password = strdup(p); + hostname = strdup(h); + ip_addr = NULL; + if (ip) + ip_addr = strdup(ip); + port = P; + hid = _hid; + id = i; + kill_type = kt; + use_ssl = _use_ssl; + mt = _mt; +} + +PgSQL_KillArgs::~PgSQL_KillArgs() { + free(username); + free(password); + free(hostname); + if (ip_addr) + free(ip_addr); +} + +const char* PgSQL_KillArgs::get_host_address() const { + const char* host_address = hostname; + + if (ip_addr) + host_address = ip_addr; + + return host_address; +} + +void* PgSQL_kill_query_thread(void* arg) { + PgSQL_KillArgs* ka = (PgSQL_KillArgs*)arg; + std::unique_ptr mysql_thr(new MySQL_Thread()); + mysql_thr->curtime = monotonic_time(); + mysql_thr->refresh_variables(); + MYSQL* pgsql = mysql_init(NULL); + mysql_options4(pgsql, MYSQL_OPT_CONNECT_ATTR_ADD, "program_name", "proxysql_killer"); + mysql_options4(pgsql, MYSQL_OPT_CONNECT_ATTR_ADD, "_server_host", ka->hostname); + + if (ka->use_ssl && ka->port) { + mysql_ssl_set(pgsql, + pgsql_thread___ssl_p2s_key, + pgsql_thread___ssl_p2s_cert, + pgsql_thread___ssl_p2s_ca, + pgsql_thread___ssl_p2s_capath, + pgsql_thread___ssl_p2s_cipher); + mysql_options(pgsql, MYSQL_OPT_SSL_CRL, pgsql_thread___ssl_p2s_crl); + mysql_options(pgsql, MYSQL_OPT_SSL_CRLPATH, pgsql_thread___ssl_p2s_crlpath); + mysql_options(pgsql, MARIADB_OPT_SSL_KEYLOG_CALLBACK, (void*)proxysql_keylog_write_line_callback); + } + + if (!pgsql) { + goto __exit_kill_query_thread; + } + MYSQL* ret; + if (ka->port) { + switch (ka->kill_type) { + case KILL_QUERY: + proxy_warning("KILL QUERY %lu on %s:%d\n", ka->id, ka->hostname, ka->port); + if (ka->mt) { + ka->mt->status_variables.stvar[st_var_killed_queries]++; + } + break; + case KILL_CONNECTION: + proxy_warning("KILL CONNECTION %lu on %s:%d\n", ka->id, ka->hostname, ka->port); + if (ka->mt) { + ka->mt->status_variables.stvar[st_var_killed_connections]++; + } + break; + default: + break; + } + ret = mysql_real_connect(pgsql, ka->get_host_address(), ka->username, ka->password, NULL, ka->port, NULL, 0); + } + else { + switch (ka->kill_type) { + case KILL_QUERY: + proxy_warning("KILL QUERY %lu on localhost\n", ka->id); + break; + case KILL_CONNECTION: + proxy_warning("KILL CONNECTION %lu on localhost\n", ka->id); + break; + default: + break; + } + ret = mysql_real_connect(pgsql, "localhost", ka->username, ka->password, NULL, 0, ka->hostname, 0); + } + if (!ret) { + proxy_error("Failed to connect to server %s:%d to run KILL %s %lu: Error: %s\n", ka->hostname, ka->port, (ka->kill_type == KILL_QUERY ? "QUERY" : "CONNECTION"), ka->id, mysql_error(pgsql)); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::pgsql, ka->hid, ka->hostname, ka->port, mysql_errno(pgsql)); + goto __exit_kill_query_thread; + } + + MySQL_Monitor::update_dns_cache_from_mysql_conn(pgsql); + + char buf[100]; + switch (ka->kill_type) { + case KILL_QUERY: + sprintf(buf, "KILL QUERY %lu", ka->id); + break; + case KILL_CONNECTION: + sprintf(buf, "KILL CONNECTION %lu", ka->id); + break; + default: + sprintf(buf, "KILL %lu", ka->id); + break; + } + // FIXME: these 2 calls are blocking, fortunately on their own thread + mysql_query(pgsql, buf); +__exit_kill_query_thread: + if (pgsql) + mysql_close(pgsql); + delete ka; + return NULL; +} + +extern PgSQL_Query_Processor* GloPgQPro; +extern Query_Cache* GloQC; +extern ProxySQL_Admin* GloAdmin; +extern PgSQL_Threads_Handler* GloPTH; + +PgSQL_Query_Info::PgSQL_Query_Info() { + PgQueryCmd=PGSQL_QUERY___NONE; + QueryPointer=NULL; + QueryLength=0; + QueryParserArgs.digest_text=NULL; + QueryParserArgs.first_comment=NULL; + stmt_info=NULL; + bool_is_select_NOT_for_update=false; + bool_is_select_NOT_for_update_computed=false; + have_affected_rows=false; // if affected rows is set, last_insert_id is set too + waiting_since = 0; + affected_rows=0; + last_insert_id = 0; + rows_sent=0; + start_time=0; + end_time=0; + stmt_client_id=0; +} + +PgSQL_Query_Info::~PgSQL_Query_Info() { + GloPgQPro->query_parser_free(&QueryParserArgs); + if (stmt_info) { + stmt_info=NULL; + } +} + +void PgSQL_Query_Info::begin(unsigned char *_p, int len, bool mysql_header) { + PgQueryCmd=PGSQL_QUERY___NONE; + QueryPointer=NULL; + QueryLength=0; + mysql_stmt=NULL; + stmt_meta=NULL; + QueryParserArgs.digest_text=NULL; + QueryParserArgs.first_comment=NULL; + start_time=sess->thread->curtime; + init(_p, len, mysql_header); + if (pgsql_thread___commands_stats || pgsql_thread___query_digests) { + query_parser_init(); + if (pgsql_thread___commands_stats) + query_parser_command_type(); + } + bool_is_select_NOT_for_update=false; + bool_is_select_NOT_for_update_computed=false; + have_affected_rows=false; // if affected rows is set, last_insert_id is set too + waiting_since = 0; + affected_rows=0; + last_insert_id = 0; + rows_sent=0; + stmt_client_id=0; +} + +void PgSQL_Query_Info::end() { + query_parser_update_counters(); + query_parser_free(); + if ((end_time-start_time) > (unsigned int)pgsql_thread___long_query_time *1000) { + __sync_add_and_fetch(&sess->thread->status_variables.stvar[st_var_queries_slow],1); + } + assert(mysql_stmt==NULL); + if (stmt_info) { + stmt_info=NULL; + } + if (stmt_meta) { // fix bug #796: memory is not freed in case of error during STMT_EXECUTE + if (stmt_meta->pkt) { + uint32_t stmt_global_id=0; + memcpy(&stmt_global_id,(char *)(stmt_meta->pkt)+5,sizeof(uint32_t)); + sess->SLDH->reset(stmt_global_id); + free(stmt_meta->pkt); + stmt_meta->pkt=NULL; + } + stmt_meta = NULL; + } +} + +void PgSQL_Query_Info::init(unsigned char *_p, int len, bool mysql_header) { + QueryLength=(mysql_header ? len-5 : len); + QueryPointer=(mysql_header ? _p+5 : _p); + PgQueryCmd = PGSQL_QUERY__UNINITIALIZED; + bool_is_select_NOT_for_update=false; + bool_is_select_NOT_for_update_computed=false; + have_affected_rows=false; // if affected rows is set, last_insert_id is set too + waiting_since = 0; + affected_rows=0; + last_insert_id = 0; + rows_sent=0; +} + +void PgSQL_Query_Info::query_parser_init() { + GloPgQPro->query_parser_init(&QueryParserArgs,(char *)QueryPointer,QueryLength,0); +} + +enum PGSQL_QUERY_command PgSQL_Query_Info::query_parser_command_type() { + PgQueryCmd = GloPgQPro->query_parser_command_type(&QueryParserArgs); + return PgQueryCmd; +} + +void PgSQL_Query_Info::query_parser_free() { + GloPgQPro->query_parser_free(&QueryParserArgs); +} + +unsigned long long PgSQL_Query_Info::query_parser_update_counters() { + if (stmt_info) { + //PgQueryCmd=stmt_info->MyComQueryCmd; + } + if (PgQueryCmd==PGSQL_QUERY___NONE) return 0; // this means that it was never initialized + if (PgQueryCmd==PGSQL_QUERY__UNINITIALIZED) return 0; // this means that it was never initialized + unsigned long long ret=GloPgQPro->query_parser_update_counters(sess, PgQueryCmd, &QueryParserArgs, end_time-start_time); + PgQueryCmd=PGSQL_QUERY___NONE; + QueryPointer=NULL; + QueryLength=0; + return ret; +} + +char * PgSQL_Query_Info::get_digest_text() { + return GloPgQPro->get_digest_text(&QueryParserArgs); +} + +bool PgSQL_Query_Info::is_select_NOT_for_update() { + if (stmt_info) { // we are processing a prepared statement. We already have the information + return stmt_info->is_select_NOT_for_update; + } + if (QueryPointer==NULL) { + return false; + } + if (bool_is_select_NOT_for_update_computed) { + return bool_is_select_NOT_for_update; + } + bool_is_select_NOT_for_update_computed=true; + if (QueryLength<7) { + return false; + } + char *QP = (char *)QueryPointer; + size_t ql = QueryLength; + // we try to use the digest, if avaiable + if (QueryParserArgs.digest_text) { + QP = QueryParserArgs.digest_text; + ql = strlen(QP); + } + if (strncasecmp(QP,(char *)"SELECT ",7)) { + return false; + } + // if we arrive till here, it is a SELECT + if (ql>=17) { + char *p=QP; + p+=ql-11; + if (strncasecmp(p," FOR UPDATE",11)==0) { + __sync_fetch_and_add(&PgHGM->status.select_for_update_or_equivalent, 1); + return false; + } + p=QP; + p+=ql-10; + if (strncasecmp(p," FOR SHARE",10)==0) { + __sync_fetch_and_add(&PgHGM->status.select_for_update_or_equivalent, 1); + return false; + } + if (ql>=25) { + char *p=QP; + p+=ql-19; + if (strncasecmp(p," LOCK IN SHARE MODE",19)==0) { + __sync_fetch_and_add(&PgHGM->status.select_for_update_or_equivalent, 1); + return false; + } + p=QP; + p+=ql-7; + if (strncasecmp(p," NOWAIT",7)==0) { + // let simplify. If NOWAIT is used, we assume FOR UPDATE|SHARE is used + __sync_fetch_and_add(&PgHGM->status.select_for_update_or_equivalent, 1); + return false; + } + p=QP; + p+=ql-12; + if (strncasecmp(p," SKIP LOCKED",12)==0) { + // let simplify. If SKIP LOCKED is used, we assume FOR UPDATE|SHARE is used + __sync_fetch_and_add(&PgHGM->status.select_for_update_or_equivalent, 1); + return false; + } + p=QP; + char buf[129]; + if (ql>=128) { // for long query, just check the last 128 bytes + p+=ql-128; + memcpy(buf,p,128); + buf[128]=0; + } else { + memcpy(buf,p,ql); + buf[ql]=0; + } + if (strcasestr(buf," FOR ")) { + if (strcasestr(buf," FOR UPDATE ")) { + __sync_fetch_and_add(&PgHGM->status.select_for_update_or_equivalent, 1); + return false; + } + if (strcasestr(buf," FOR SHARE ")) { + __sync_fetch_and_add(&PgHGM->status.select_for_update_or_equivalent, 1); + return false; + } + } + } + } + bool_is_select_NOT_for_update=true; + return true; +} + +void PgSQL_Session::set_status(enum session_status e) { + if (e == session_status___NONE) { + if (mybe) { + if (mybe->server_myds) { + assert(mybe->server_myds->myconn == 0); + if (mybe->server_myds->myconn) { + assert(mybe->server_myds->myconn->async_state_machine == ASYNC_IDLE); + } + } + } + } + status = e; +} + + +PgSQL_Session::PgSQL_Session() { + thread_session_id = 0; + //handler_ret = 0; + pause_until = 0; + qpo = new PgSQL_Query_Processor_Output(); + qpo->init(); + start_time = 0; + command_counters = new StatCounters(15, 10); + healthy = 1; + autocommit = true; + autocommit_handled = false; + sending_set_autocommit = false; + autocommit_on_hostgroup = -1; + killed = false; + session_type = PROXYSQL_SESSION_PGSQL; + //admin=false; + connections_handler = false; + max_connections_reached = false; + //stats=false; + client_authenticated = false; + default_schema = NULL; + user_attributes = NULL; + schema_locked = false; + session_fast_forward = false; + started_sending_data_to_client = false; + handler_function = NULL; + client_myds = NULL; + to_process = 0; + mybe = NULL; + mirror = false; + mirrorPkt.ptr = NULL; + mirrorPkt.size = 0; + set_status(session_status___NONE); + warning_in_hg = -1; + + idle_since = 0; + transaction_started_at = 0; + + CurrentQuery.sess = this; + CurrentQuery.mysql_stmt = NULL; + CurrentQuery.stmt_meta = NULL; + CurrentQuery.stmt_global_id = 0; + CurrentQuery.stmt_client_id = 0; + CurrentQuery.stmt_info = NULL; + + current_hostgroup = -1; + default_hostgroup = -1; + locked_on_hostgroup = -1; + locked_on_hostgroup_and_all_variables_set = false; + next_query_flagIN = -1; + mirror_hostgroup = -1; + mirror_flagOUT = -1; + active_transactions = 0; + + use_ssl = false; + change_user_auth_switch = false; + + match_regexes = NULL; + + init(); // we moved this out to allow CHANGE_USER + + last_insert_id = 0; // #1093 + + last_HG_affected_rows = -1; // #1421 : advanced support for LAST_INSERT_ID() + proxysql_node_address = NULL; + use_ldap_auth = false; +} + +void PgSQL_Session::reset() { + autocommit = true; + autocommit_handled = false; + sending_set_autocommit = false; + autocommit_on_hostgroup = -1; + warning_in_hg = -1; + current_hostgroup = -1; + default_hostgroup = -1; + locked_on_hostgroup = -1; + locked_on_hostgroup_and_all_variables_set = false; + if (sess_STMTs_meta) { + delete sess_STMTs_meta; + sess_STMTs_meta = NULL; + } + if (SLDH) { + delete SLDH; + SLDH = NULL; + } + if (mybes) { + reset_all_backends(); + delete mybes; + mybes = NULL; + } + mybe = NULL; + + if (session_type == PROXYSQL_SESSION_SQLITE) { + SQLite3_Session* sqlite_sess = (SQLite3_Session*)thread->gen_args; + if (sqlite_sess && sqlite_sess->sessdb) { + sqlite3* db = sqlite_sess->sessdb->get_db(); + if ((*proxy_sqlite3_get_autocommit)(db) == 0) { + sqlite_sess->sessdb->execute((char*)"COMMIT"); + } + } + } + if (client_myds) { + if (client_myds->myconn) { + client_myds->myconn->reset(); + } + } +} + +PgSQL_Session::~PgSQL_Session() { + + reset(); // we moved this out to allow CHANGE_USER + + if (locked_on_hostgroup >= 0) { + thread->status_variables.stvar[st_var_hostgroup_locked]--; + } + + if (client_myds) { + if (client_authenticated) { + switch (session_type) { +#ifdef PROXYSQLCLICKHOUSE + case PROXYSQL_SESSION_CLICKHOUSE: + GloClickHouseAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->username); + break; +#endif /* PROXYSQLCLICKHOUSE */ + default: + if (use_ldap_auth == false) { + GloPgAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->username); + } + else { + GloMyLdapAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->fe_username); + } + break; + } + } + delete client_myds; + } + if (default_schema) { + free(default_schema); + } + if (user_attributes) { + free(user_attributes); + user_attributes = NULL; + } + proxy_debug(PROXY_DEBUG_NET, 1, "Thread=%p, Session=%p -- Shutdown Session %p\n", this->thread, this, this); + delete command_counters; + if (session_type == PROXYSQL_SESSION_PGSQL && connections_handler == false && mirror == false) { + __sync_fetch_and_sub(&PgHGM->status.client_connections, 1); + } + assert(qpo); + delete qpo; + match_regexes = NULL; + if (mirror) { + __sync_sub_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1); + //GloPTH->status_variables.p_gauge_array[p_th_gauge::mirror_concurrency]->Decrement(); + } + if (proxysql_node_address) { + delete proxysql_node_address; + proxysql_node_address = NULL; + } +} + +bool PgSQL_Session::handler_CommitRollback(PtrSize_t* pkt) { + if (pkt->size <= 5) { return false; } + char c = ((char*)pkt->ptr)[5]; + bool ret = false; + if (c == 'c' || c == 'C') { + if (pkt->size >= sizeof("commit") + 5) { + if (strncasecmp((char*)"commit", (char*)pkt->ptr + 5, 6) == 0) { + __sync_fetch_and_add(&PgHGM->status.commit_cnt, 1); + ret = true; + } + } + } + else { + if (c == 'r' || c == 'R') { + if (pkt->size >= sizeof("rollback") + 5) { + if (strncasecmp((char*)"rollback", (char*)pkt->ptr + 5, 8) == 0) { + __sync_fetch_and_add(&PgHGM->status.rollback_cnt, 1); + ret = true; + } + } + } + } + + if (ret == false) { + return false; // quick exit + } + // in this part of the code (as at release 2.4.3) where we call + // NumActiveTransactions() with the check_savepoint flag . + // This to try to handle MySQL bug https://bugs.pgsql.com/bug.php?id=107875 + // + // Since we are limited to forwarding just one 'COMMIT|ROLLBACK', we work under the assumption that we + // only have one active transaction. Under this premise, we should execute this command under that + // specific connection, for that, we update 'current_hostgroup' with the first active transaction we are + // able to find. If more transactions are simultaneously open for the session, more 'COMMIT|ROLLBACK' + // commands are required to be issued by the client to continue ending transactions. + int hg = FindOneActiveTransaction(true); + if (hg != -1) { + // there is an active transaction, we must forward the request + current_hostgroup = hg; + return false; + } + else { + // there is no active transaction, we will just reply OK + client_myds->DSS = STATE_QUERY_SENT_NET; + //uint16_t setStatus = 0; + //if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + //client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + client_myds->myprot.generate_ok_packet(true, true, NULL, 0, (const char*)pkt->ptr + 5); + if (mirror == false) { + RequestEnd(NULL); + } else { + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + } + l_free(pkt->size, pkt->ptr); + if (c == 'c' || c == 'C') { + __sync_fetch_and_add(&PgHGM->status.commit_cnt_filtered, 1); + } else { + __sync_fetch_and_add(&PgHGM->status.rollback_cnt_filtered, 1); + } + return true; + } + return false; +} + + +void PgSQL_Session::generate_proxysql_internal_session_json(json& j) { + char buff[32]; + sprintf(buff, "%p", this); + j["address"] = buff; + if (thread) { + sprintf(buff, "%p", thread); + j["thread"] = buff; + } + const uint64_t age_ms = (thread->curtime - start_time) / 1000; + j["age_ms"] = age_ms; + j["status"] = status; + j["thread_session_id"] = thread_session_id; + j["current_hostgroup"] = current_hostgroup; + j["default_hostgroup"] = default_hostgroup; + j["locked_on_hostgroup"] = locked_on_hostgroup; + j["active_transactions"] = active_transactions; + j["transaction_time_ms"] = thread->curtime - transaction_started_at; + j["qpo"]["create_new_connection"] = qpo->create_new_conn; + j["qpo"]["reconnect"] = qpo->reconnect; + j["qpo"]["sticky_conn"] = qpo->sticky_conn; + j["qpo"]["cache_timeout"] = qpo->cache_timeout; + j["qpo"]["cache_ttl"] = qpo->cache_ttl; + j["qpo"]["delay"] = qpo->delay; + j["qpo"]["destination_hostgroup"] = qpo->destination_hostgroup; + j["qpo"]["firewall_whitelist_mode"] = qpo->firewall_whitelist_mode; + j["qpo"]["multiplex"] = qpo->multiplex; + j["qpo"]["timeout"] = qpo->timeout; + j["qpo"]["retries"] = qpo->retries; + j["qpo"]["max_lag_ms"] = qpo->max_lag_ms; + j["user_attributes"] = (user_attributes ? user_attributes : ""); + j["transaction_persistent"] = transaction_persistent; + if (client_myds != NULL) { // only if client_myds is defined + j["client"]["stream"]["pkts_recv"] = client_myds->pkts_recv; + j["client"]["stream"]["pkts_sent"] = client_myds->pkts_sent; + j["client"]["stream"]["bytes_recv"] = client_myds->bytes_info.bytes_recv; + j["client"]["stream"]["bytes_sent"] = client_myds->bytes_info.bytes_sent; + j["client"]["client_addr"]["address"] = (client_myds->addr.addr ? client_myds->addr.addr : ""); + j["client"]["client_addr"]["port"] = client_myds->addr.port; + j["client"]["proxy_addr"]["address"] = (client_myds->proxy_addr.addr ? client_myds->proxy_addr.addr : ""); + j["client"]["proxy_addr"]["port"] = client_myds->proxy_addr.port; + j["client"]["encrypted"] = client_myds->encrypted; + if (client_myds->encrypted) { + const SSL_CIPHER* cipher = SSL_get_current_cipher(client_myds->ssl); + if (cipher) { + const char* name = SSL_CIPHER_get_name(cipher); + if (name) { + j["client"]["ssl_cipher"] = name; + } + } + } + j["client"]["DSS"] = client_myds->DSS; + j["client"]["auth_method"] = AUTHENTICATION_METHOD_STR[(int)client_myds->auth_method]; + if (client_myds->myconn != NULL) { // only if myconn is defined + if (client_myds->myconn->userinfo != NULL) { // only if userinfo is defined + j["client"]["userinfo"]["username"] = (client_myds->myconn->userinfo->username ? client_myds->myconn->userinfo->username : ""); + j["client"]["userinfo"]["dbname"] = (client_myds->myconn->userinfo->dbname ? client_myds->myconn->userinfo->dbname : ""); +#ifdef DEBUG + j["client"]["userinfo"]["password"] = (client_myds->myconn->userinfo->password ? client_myds->myconn->userinfo->password : ""); +#endif + } + for (auto idx = 0; idx < SQL_NAME_LAST_LOW_WM; idx++) { + client_myds->myconn->variables[idx].fill_client_internal_session(j, idx); + } + { + PgSQL_Connection* c = client_myds->myconn; + for (std::vector::const_iterator it_c = c->dynamic_variables_idx.begin(); it_c != c->dynamic_variables_idx.end(); it_c++) { + c->variables[*it_c].fill_client_internal_session(j, *it_c); + } + } + //j["conn"]["autocommit"] = (client_myds->myconn->options.autocommit ? "ON" : "OFF"); + //j["conn"]["client_flag"]["value"] = client_myds->myconn->options.client_flag; + //j["conn"]["client_flag"]["client_found_rows"] = (client_myds->myconn->options.client_flag & CLIENT_FOUND_ROWS ? 1 : 0); + //j["conn"]["client_flag"]["client_multi_statements"] = (client_myds->myconn->options.client_flag & CLIENT_MULTI_STATEMENTS ? 1 : 0); + //j["conn"]["client_flag"]["client_multi_results"] = (client_myds->myconn->options.client_flag & CLIENT_MULTI_RESULTS ? 1 : 0); + //j["conn"]["client_flag"]["client_deprecate_eof"] = (client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF ? 1 : 0); + //j["conn"]["no_backslash_escapes"] = client_myds->myconn->options.no_backslash_escapes; + //j["conn"]["status"]["compression"] = client_myds->myconn->get_status(STATUS_MYSQL_CONNECTION_COMPRESSION); + //j["conn"]["ps"]["client_stmt_to_global_ids"] = client_myds->myconn->local_stmts->client_stmt_to_global_ids; + { + const PgSQL_Conn_Param& c = client_myds->myconn->conn_params; + + for (size_t i = 0; i < c.param_set.size(); i++) { + + if (c.param_value[c.param_set[i]] != NULL) { + + j["client"]["conn"]["connection_options"][PgSQL_Param_Name_Str[c.param_set[i]]] = c.param_value[c.param_set[i]]; + } + } + } + } + } + for (unsigned int i = 0; i < mybes->len; i++) { + PgSQL_Backend* _mybe = NULL; + _mybe = (PgSQL_Backend*)mybes->index(i); + j["backends"][i]["hostgroup_id"] = _mybe->hostgroup_id; + if (_mybe->server_myds) { + PgSQL_Data_Stream* _myds = _mybe->server_myds; + sprintf(buff, "%p", _myds); + j["backends"][i]["stream"]["address"] = buff; + j["backends"][i]["stream"]["questions"] = _myds->statuses.questions; + j["backends"][i]["stream"]["pgconnpoll_get"] = _myds->statuses.pgconnpoll_get; + j["backends"][i]["stream"]["pgconnpoll_put"] = _myds->statuses.pgconnpoll_put; + /* when fast_forward is not used, these metrics are always 0. Explicitly disabled + j["backend"][i]["stream"]["pkts_recv"] = _myds->pkts_recv; + j["backend"][i]["stream"]["pkts_sent"] = _myds->pkts_sent; + */ + j["backends"][i]["stream"]["bytes_recv"] = _myds->bytes_info.bytes_recv; + j["backends"][i]["stream"]["bytes_sent"] = _myds->bytes_info.bytes_sent; + j["backends"][i]["stream"]["DSS"] = _myds->DSS; + if (_myds->myconn) { + PgSQL_Connection* _myconn = _myds->myconn; + for (auto idx = 0; idx < SQL_NAME_LAST_LOW_WM; idx++) { + _myconn->variables[idx].fill_server_internal_session(j, i, idx); + } + for (std::vector::const_iterator it_c = _myconn->dynamic_variables_idx.begin(); it_c != _myconn->dynamic_variables_idx.end(); it_c++) { + _myconn->variables[*it_c].fill_server_internal_session(j, i, *it_c); + } + sprintf(buff, "%p", _myconn); + j["backends"][i]["conn"]["address"] = buff; + j["backends"][i]["conn"]["auto_increment_delay_token"] = _myconn->auto_increment_delay_token; + j["backends"][i]["conn"]["bytes_recv"] = _myconn->bytes_info.bytes_recv; + j["backends"][i]["conn"]["bytes_sent"] = _myconn->bytes_info.bytes_sent; + j["backends"][i]["conn"]["questions"] = _myconn->statuses.questions; + j["backends"][i]["conn"]["pgconnpoll_get"] = _myconn->statuses.pgconnpoll_get; + j["backends"][i]["conn"]["pgconnpoll_put"] = _myconn->statuses.pgconnpoll_put; + //j["backend"][i]["conn"]["charset"] = _myds->myconn->options.charset; // not used for backend + //j["backends"][i]["conn"]["session_track_gtids"] = (_myconn->options.session_track_gtids ? _myconn->options.session_track_gtids : ""); + j["backends"][i]["conn"]["init_connect"] = (_myconn->options.init_connect ? _myconn->options.init_connect : ""); + j["backends"][i]["conn"]["init_connect_sent"] = _myds->myconn->options.init_connect_sent; + j["backends"][i]["conn"]["standard_conforming_strings"] = _myconn->options.no_backslash_escapes; + //j["backends"][i]["conn"]["status"]["get_lock"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_GET_LOCK); + //j["backends"][i]["conn"]["status"]["lock_tables"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_LOCK_TABLES); + j["backends"][i]["conn"]["status"]["has_savepoint"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_HAS_SAVEPOINT); + //j["backends"][i]["conn"]["status"]["temporary_table"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_TEMPORARY_TABLE); + j["backends"][i]["conn"]["status"]["user_variable"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_USER_VARIABLE); + //j["backends"][i]["conn"]["status"]["found_rows"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_FOUND_ROWS); + j["backends"][i]["conn"]["status"]["no_multiplex"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_NO_MULTIPLEX); + j["backends"][i]["conn"]["status"]["no_multiplex_HG"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_NO_MULTIPLEX_HG); + //j["backends"][i]["conn"]["status"]["compression"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_COMPRESSION); + //j["backends"][i]["conn"]["status"]["prepared_statement"] = _myconn->get_status(STATUS_MYSQL_CONNECTION_PREPARED_STATEMENT); + { + // MultiplexDisabled : status returned by PgSQL_Connection::MultiplexDisabled(); + // MultiplexDisabled_ext : status returned by PgSQL_Connection::MultiplexDisabled() || PgSQL_Connection::isActiveTransaction() + bool multiplex_disabled = _myconn->MultiplexDisabled(); + j["backends"][i]["conn"]["MultiplexDisabled"] = multiplex_disabled; + if (multiplex_disabled == false) { + if (_myconn->IsActiveTransaction() == true) { + multiplex_disabled = true; + } + } + j["backends"][i]["conn"]["MultiplexDisabled_ext"] = multiplex_disabled; + } + //j["backends"][i]["conn"]["ps"]["backend_stmt_to_global_ids"] = _myconn->local_stmts->backend_stmt_to_global_ids; + //j["backends"][i]["conn"]["ps"]["global_stmt_to_backend_ids"] = _myconn->local_stmts->global_stmt_to_backend_ids; + //j["backends"][i]["conn"]["client_flag"]["value"] = _myconn->options.client_flag; + //j["backends"][i]["conn"]["client_flag"]["client_found_rows"] = (_myconn->options.client_flag & CLIENT_FOUND_ROWS ? 1 : 0); + //j["backends"][i]["conn"]["client_flag"]["client_multi_statements"] = (_myconn->options.client_flag & CLIENT_MULTI_STATEMENTS ? 1 : 0); + //j["backends"][i]["conn"]["client_flag"]["client_deprecate_eof"] = (_myconn->options.client_flag & CLIENT_DEPRECATE_EOF ? 1 : 0); + if (_myconn->is_connected()) { + sprintf(buff, "%p", _myconn->get_pg_connection()); + j["backends"][i]["conn"]["pgsql"]["address"] = buff; + j["backends"][i]["conn"]["pgsql"]["host"] = _myconn->get_pg_host(); + j["backends"][i]["conn"]["pgsql"]["host_addr"] = _myconn->get_pg_hostaddr(); + j["backends"][i]["conn"]["pgsql"]["port"] = _myconn->get_pg_port(); + j["backends"][i]["conn"]["pgsql"]["user"] = _myconn->get_pg_user(); +#ifdef DEBUG + j["backends"][i]["conn"]["pgsql"]["password"] = _myconn->get_pg_password(); +#endif + j["backends"][i]["conn"]["pgsql"]["database"] = _myconn->get_pg_dbname(); + j["backends"][i]["conn"]["pgsql"]["backend_pid"] = _myconn->get_pg_backend_pid(); + j["backends"][i]["conn"]["pgsql"]["using_ssl"] = _myconn->get_pg_ssl_in_use() ? "YES" : "NO"; + j["backends"][i]["conn"]["pgsql"]["error_msg"] = _myconn->get_pg_error_message(); + j["backends"][i]["conn"]["pgsql"]["options"] = _myconn->get_pg_options(); + j["backends"][i]["conn"]["pgsql"]["fd"] = _myconn->get_pg_socket_fd(); + j["backends"][i]["conn"]["pgsql"]["protocol_version"] = _myconn->get_pg_protocol_version(); + j["backends"][i]["conn"]["pgsql"]["server_version"] = _myconn->get_pg_server_version_str(buff, sizeof(buff)); + j["backends"][i]["conn"]["pgsql"]["transaction_status"] = _myconn->get_pg_transaction_status_str(); + j["backends"][i]["conn"]["pgsql"]["connection_status"] = _myconn->get_pg_connection_status_str(); + j["backends"][i]["conn"]["pgsql"]["client_encoding"] = _myconn->get_pg_client_encoding(); + j["backends"][i]["conn"]["pgsql"]["is_nonblocking"] = _myconn->get_pg_is_nonblocking() ? "YES" : "NO"; + } + } + } + } +} + +bool PgSQL_Session::handler_special_queries(PtrSize_t* pkt) { + bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + + if ( + (pkt->size == (SELECT_DB_USER_LEN + 5)) + || + (pkt->size == (SELECT_CHARSET_STATUS_LEN + 5)) + ) { + if (handler_special_queries_STATUS(pkt) == true) { + return true; + } + } + if (pkt->size > (5 + 18) && strncasecmp((char*)"PROXYSQL INTERNAL ", (char*)pkt->ptr + 5, 18) == 0) { + return_proxysql_internal(pkt); + return true; + } + if (locked_on_hostgroup == -1) { + //if (handler_SetAutocommit(pkt) == true) { + // return true; + //} + if (handler_CommitRollback(pkt) == true) { + return true; + } + } + + //handle 2564 + if (pkt->size == SELECT_VERSION_COMMENT_LEN + 5 && *((char*)(pkt->ptr) + 4) == (char)0x03 && strncmp((char*)SELECT_VERSION_COMMENT, (char*)pkt->ptr + 5, pkt->size - 5) == 0) { + // FIXME: this doesn't return AUTOCOMMIT or IN_TRANS + PtrSize_t pkt_2; + if (deprecate_eof_active) { + pkt_2.size = PROXYSQL_VERSION_COMMENT_WITH_OK_LEN; + pkt_2.ptr = l_alloc(pkt_2.size); + memcpy(pkt_2.ptr, PROXYSQL_VERSION_COMMENT_WITH_OK, pkt_2.size); + } + else { + pkt_2.size = PROXYSQL_VERSION_COMMENT_LEN; + pkt_2.ptr = l_alloc(pkt_2.size); + memcpy(pkt_2.ptr, PROXYSQL_VERSION_COMMENT, pkt_2.size); + } + status = WAITING_CLIENT_DATA; + client_myds->DSS = STATE_SLEEP; + client_myds->PSarrayOUT->add(pkt_2.ptr, pkt_2.size); + if (mirror == false) { + RequestEnd(NULL); + } + l_free(pkt->size, pkt->ptr); + return true; + } + if (pkt->size == strlen((char*)"select USER()") + 5 && strncmp((char*)"select USER()", (char*)pkt->ptr + 5, pkt->size - 5) == 0) { + // FIXME: this doesn't return AUTOCOMMIT or IN_TRANS + char* query1 = (char*)"SELECT \"%s\" AS 'USER()'"; + char* query2 = (char*)malloc(strlen(query1) + strlen(client_myds->myconn->userinfo->username) + 10); + sprintf(query2, query1, client_myds->myconn->userinfo->username); + char* error; + int cols; + int affected_rows; + SQLite3_result* resultset; + GloAdmin->admindb->execute_statement(query2, &error, &cols, &affected_rows, &resultset); + SQLite3_to_MySQL(resultset, error, affected_rows, &client_myds->myprot, false, deprecate_eof_active); + delete resultset; + free(query2); + if (mirror == false) { + RequestEnd(NULL); + } + l_free(pkt->size, pkt->ptr); + return true; + } + // MySQL client check command for dollars quote support, starting at version '8.1.0'. See #4300. + if ((pkt->size == strlen("SELECT $$") + 5) && strncasecmp("SELECT $$", (char*)pkt->ptr + 5, pkt->size - 5) == 0) { + pair err_info{ get_dollar_quote_error(pgsql_thread___server_version) }; + + client_myds->DSS = STATE_QUERY_SENT_NET; + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, err_info.first, (char*)"HY000", err_info.second, true); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + + if (mirror == false) { + RequestEnd(NULL); + } + l_free(pkt->size, pkt->ptr); + + return true; + } + if (locked_on_hostgroup >= 0 && (strncasecmp((char*)"SET ", (char*)pkt->ptr + 5, 4) == 0)) { + // this is a circuit breaker, we will send everything to the backend + // + // also note that in the current implementation we stop tracking variables: + // this becomes a problem if pgsql-set_query_lock_on_hostgroup is + // disabled while a session is already locked + return false; + } + if ((pkt->size < 60) && (pkt->size > 38) && (strncasecmp((char*)"SET SESSION character_set_server", (char*)pkt->ptr + 5, 32) == 0)) { // issue #601 + char* idx = NULL; + char* p = (char*)pkt->ptr + 37; + idx = (char*)memchr(p, '=', pkt->size - 37); + if (idx) { // we found = + PtrSize_t pkt_2; + pkt_2.size = 5 + strlen((char*)"SET NAMES ") + pkt->size - 1 - (idx - (char*)pkt->ptr); + pkt_2.ptr = l_alloc(pkt_2.size); + mysql_hdr Hdr; + memcpy(&Hdr, pkt->ptr, sizeof(mysql_hdr)); + Hdr.pkt_length = pkt_2.size - 5; + memcpy((char*)pkt_2.ptr + 4, (char*)pkt->ptr + 4, 1); + memcpy(pkt_2.ptr, &Hdr, sizeof(mysql_hdr)); + strcpy((char*)pkt_2.ptr + 5, (char*)"SET NAMES "); + memcpy((char*)pkt_2.ptr + 15, idx + 1, pkt->size - 1 - (idx - (char*)pkt->ptr)); + l_free(pkt->size, pkt->ptr); + pkt->size = pkt_2.size; + pkt->ptr = pkt_2.ptr; + // Fix 'use-after-free': To change the pointer of the 'PtrSize_t' being processed by + // 'PgSQL_Session::handler' we are forced to update 'PgSQL_Session::CurrentQuery'. + CurrentQuery.QueryPointer = static_cast(pkt_2.ptr); + CurrentQuery.QueryLength = pkt_2.size; + } + } + if ((pkt->size < 60) && (pkt->size > 39) && (strncasecmp((char*)"SET SESSION character_set_results", (char*)pkt->ptr + 5, 33) == 0)) { // like the above + char* idx = NULL; + char* p = (char*)pkt->ptr + 38; + idx = (char*)memchr(p, '=', pkt->size - 38); + if (idx) { // we found = + PtrSize_t pkt_2; + pkt_2.size = 5 + strlen((char*)"SET NAMES ") + pkt->size - 1 - (idx - (char*)pkt->ptr); + pkt_2.ptr = l_alloc(pkt_2.size); + mysql_hdr Hdr; + memcpy(&Hdr, pkt->ptr, sizeof(mysql_hdr)); + Hdr.pkt_length = pkt_2.size - 5; + memcpy((char*)pkt_2.ptr + 4, (char*)pkt->ptr + 4, 1); + memcpy(pkt_2.ptr, &Hdr, sizeof(mysql_hdr)); + strcpy((char*)pkt_2.ptr + 5, (char*)"SET NAMES "); + memcpy((char*)pkt_2.ptr + 15, idx + 1, pkt->size - 1 - (idx - (char*)pkt->ptr)); + l_free(pkt->size, pkt->ptr); + pkt->size = pkt_2.size; + pkt->ptr = pkt_2.ptr; + // Fix 'use-after-free': To change the pointer of the 'PtrSize_t' being processed by + // 'PgSQL_Session::handler' we are forced to update 'PgSQL_Session::CurrentQuery'. + CurrentQuery.QueryPointer = static_cast(pkt_2.ptr); + CurrentQuery.QueryLength = pkt_2.size; + } + } + if ( + (pkt->size < 100) && (pkt->size > 15) && (strncasecmp((char*)"SET NAMES ", (char*)pkt->ptr + 5, 10) == 0) + && + (memchr((const void*)((char*)pkt->ptr + 5), ',', pkt->size - 15) == NULL) // there is no comma + ) { + char* unstripped = strndup((char*)pkt->ptr + 15, pkt->size - 15); + char* csname = trim_spaces_and_quotes_in_place(unstripped); + //unsigned int charsetnr = 0; + const MARIADB_CHARSET_INFO* c; + char* collation_name_unstripped = NULL; + char* collation_name = NULL; + if (strcasestr(csname, " COLLATE ")) { + collation_name_unstripped = strcasestr(csname, " COLLATE ") + strlen(" COLLATE "); + collation_name = trim_spaces_and_quotes_in_place(collation_name_unstripped); + char* _s1 = index(csname, ' '); + char* _s2 = index(csname, '\''); + char* _s3 = index(csname, '"'); + char* _s = NULL; + if (_s1) { + _s = _s1; + } + if (_s2) { + if (_s) { + if (_s2 < _s) { + _s = _s2; + } + } + else { + _s = _s2; + } + } + if (_s3) { + if (_s) { + if (_s3 < _s) { + _s = _s3; + } + } + else { + _s = _s3; + } + } + if (_s) { + *_s = '\0'; + } + + _s1 = index(collation_name, ' '); + _s2 = index(collation_name, '\''); + _s3 = index(collation_name, '"'); + _s = NULL; + if (_s1) { + _s = _s1; + } + if (_s2) { + if (_s) { + if (_s2 < _s) { + _s = _s2; + } + } + else { + _s = _s2; + } + } + if (_s3) { + if (_s) { + if (_s3 < _s) { + _s = _s3; + } + } + else { + _s = _s3; + } + } + if (_s) { + *_s = '\0'; + } + + c = proxysql_find_charset_collate_names(csname, collation_name); + } + else { + c = proxysql_find_charset_name(csname); + } + free(unstripped); + if (c) { + client_myds->DSS = STATE_QUERY_SENT_NET; + //-- client_myds->myconn->set_charset(c->nr, NAMES); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + if (mirror == false) { + RequestEnd(NULL); + } + else { + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + } + l_free(pkt->size, pkt->ptr); + __sync_fetch_and_add(&PgHGM->status.frontend_set_client_encoding, 1); + return true; + } + } + // if query digest is disabled, warnings in ProxySQL are also deactivated, + // resulting in an empty response being sent to the client. + if ((pkt->size == 18) && (strncasecmp((char*)"SHOW WARNINGS", (char*)pkt->ptr + 5, 13) == 0) && + CurrentQuery.QueryParserArgs.digest_text == nullptr) { + SQLite3_result* resultset = new SQLite3_result(3); + resultset->add_column_definition(SQLITE_TEXT, "Level"); + resultset->add_column_definition(SQLITE_TEXT, "Code"); + resultset->add_column_definition(SQLITE_TEXT, "Message"); + SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); + delete resultset; + if (mirror == false) { + RequestEnd(NULL); + } + else { + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + } + l_free(pkt->size, pkt->ptr); + return true; + } + // if query digest is disabled, warnings in ProxySQL are also deactivated, + // resulting in zero warning count sent to the client. + if ((pkt->size == 27) && (strncasecmp((char*)"SHOW COUNT(*) WARNINGS", (char*)pkt->ptr + 5, 22) == 0) && + CurrentQuery.QueryParserArgs.digest_text == nullptr) { + SQLite3_result* resultset = new SQLite3_result(1); + resultset->add_column_definition(SQLITE_TEXT, "@@session.warning_count"); + char* pta[1]; + pta[0] = (char*)"0"; + resultset->add_row(pta); + SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot, false, deprecate_eof_active); + delete resultset; + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + if (mirror == false) { + RequestEnd(NULL); + } + l_free(pkt->size, pkt->ptr); + return true; + } + // 'LOAD DATA LOCAL INFILE' is unsupported. We report an specific error to inform clients about this fact. For more context see #833. + if ((pkt->size >= 22 + 5) && (strncasecmp((char*)"LOAD DATA LOCAL INFILE", (char*)pkt->ptr + 5, 22) == 0)) { + if (pgsql_thread___enable_load_data_local_infile == false) { + client_myds->DSS = STATE_QUERY_SENT_NET; + client_myds->myprot.generate_error_packet(true, true, "Unsupported 'LOAD DATA LOCAL INFILE' command", + PGSQL_ERROR_CODES::ERRCODE_FEATURE_NOT_SUPPORTED, false, true); + if (mirror == false) { + RequestEnd(NULL); + } + else { + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + } + l_free(pkt->size, pkt->ptr); + return true; + } + else { + if (pgsql_thread___verbose_query_error) { + proxy_warning( + "Command '%.*s' refers to file in ProxySQL instance, NOT on client side!\n", + static_cast(pkt->size - sizeof(mysql_hdr) - 1), + static_cast(pkt->ptr) + 5 + ); + } + else { + proxy_warning( + "Command 'LOAD DATA LOCAL INFILE' refers to file in ProxySQL instance, NOT on client side!\n" + ); + } + } + } + + return false; +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___create_mirror_session() { + if (pkt.size < 15 * 1024 * 1024 && (qpo->mirror_hostgroup >= 0 || qpo->mirror_flagOUT >= 0)) { + // check if there are too many mirror sessions in queue + if (thread->mirror_queue_mysql_sessions->len >= (unsigned int)pgsql_thread___mirror_max_queue_length) { + return; + } + // at this point, we will create the new session + // we will later decide if queue it or sent it immediately + +// int i=0; +// for (i=0;i<100;i++) { + PgSQL_Session* newsess = NULL; + if (thread->mirror_queue_mysql_sessions_cache->len == 0) { + newsess = new PgSQL_Session(); + newsess->client_myds = new PgSQL_Data_Stream(); + newsess->client_myds->DSS = STATE_SLEEP; + newsess->client_myds->sess = newsess; + newsess->client_myds->fd = 0; + newsess->client_myds->myds_type = MYDS_FRONTEND; + newsess->client_myds->PSarrayOUT = new PtrSizeArray(); + newsess->thread_session_id = __sync_fetch_and_add(&glovars.thread_id, 1); + if (newsess->thread_session_id == 0) { + newsess->thread_session_id = __sync_fetch_and_add(&glovars.thread_id, 1); + } + newsess->status = WAITING_CLIENT_DATA; + PgSQL_Connection* myconn = new PgSQL_Connection; + newsess->client_myds->attach_connection(myconn); + newsess->client_myds->myprot.init(&newsess->client_myds, newsess->client_myds->myconn->userinfo, newsess); + newsess->mirror = true; + newsess->client_myds->destroy_queues(); + } + else { + newsess = (PgSQL_Session*)thread->mirror_queue_mysql_sessions_cache->remove_index_fast(0); + } + newsess->client_myds->myconn->userinfo->set(client_myds->myconn->userinfo); + newsess->to_process = 1; + newsess->default_hostgroup = default_hostgroup; + if (qpo->mirror_hostgroup >= 0) { + newsess->mirror_hostgroup = qpo->mirror_hostgroup; // in the new session we copy the mirror hostgroup + } + else { + newsess->mirror_hostgroup = default_hostgroup; // copy the default + } + newsess->mirror_flagOUT = qpo->mirror_flagOUT; // in the new session we copy the mirror flagOUT + if (newsess->default_schema == NULL) { + newsess->default_schema = strdup(default_schema); + } + else { + if (strcmp(newsess->default_schema, default_schema)) { + free(newsess->default_schema); + newsess->default_schema = strdup(default_schema); + } + } + newsess->mirrorPkt.size = pkt.size; + newsess->mirrorPkt.ptr = l_alloc(newsess->mirrorPkt.size); + memcpy(newsess->mirrorPkt.ptr, pkt.ptr, pkt.size); + + if (thread->mirror_queue_mysql_sessions->len == 0) { + // there are no sessions in the queue, we try to execute immediately + // Only pgsql_thread___mirror_max_concurrency mirror session can run in parallel + if (__sync_add_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1) > (unsigned int)pgsql_thread___mirror_max_concurrency) { + // if the limit is reached, we queue it instead + __sync_sub_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1); + thread->mirror_queue_mysql_sessions->add(newsess); + } + else { + //GloPTH->status_variables.p_gauge_array[p_th_gauge::mirror_concurrency]->Increment(); + thread->register_session(thread,newsess); + newsess->handler(); // execute immediately + //newsess->to_process=0; + if (newsess->status == WAITING_CLIENT_DATA) { // the mirror session has completed + thread->unregister_session(thread->mysql_sessions->len - 1); + unsigned int l = (unsigned int)pgsql_thread___mirror_max_concurrency; + if (thread->mirror_queue_mysql_sessions->len * 0.3 > l) l = thread->mirror_queue_mysql_sessions->len * 0.3; + if (thread->mirror_queue_mysql_sessions_cache->len <= l) { + bool to_cache = true; + if (newsess->mybe) { + if (newsess->mybe->server_myds) { + to_cache = false; + } + } + if (to_cache) { + __sync_sub_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1); + //GloPTH->status_variables.p_gauge_array[p_th_gauge::mirror_concurrency]->Decrement(); + thread->mirror_queue_mysql_sessions_cache->add(newsess); + } + else { + delete newsess; + } + } + else { + delete newsess; + } + } + } + } + else { + thread->mirror_queue_mysql_sessions->add(newsess); + } + } +} + +int PgSQL_Session::handler_again___status_PINGING_SERVER() { + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + int rc = myconn->async_ping(myds->revents); + if (rc == 0) { + myconn->async_state_machine = ASYNC_IDLE; + myconn->compute_unknown_transaction_status(); + //if (pgsql_thread___multiplexing && (myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { + // due to issue #2096 we disable the global check on pgsql_thread___multiplexing + if ((myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + myds->return_MySQL_Connection_To_Pool(); + } else { + myds->destroy_MySQL_Connection_From_Pool(true); + } + delete mybe->server_myds; + mybe->server_myds = NULL; + set_status(session_status___NONE); + return -1; + } + else { + if (rc == -1 || rc == -2) { + if (rc == -2) { + unsigned long long us = pgsql_thread___ping_timeout_server * 1000; + us += thread->curtime; + us -= myds->wait_until; + proxy_error("Ping timeout during ping on %s:%d after %lluus (timeout %dms)\n", myconn->parent->address, myconn->parent->port, us, pgsql_thread___ping_timeout_server); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, ER_PROXYSQL_PING_TIMEOUT); + } + else { // rc==-1 + int myerr = mysql_errno(myconn->pgsql); + detected_broken_connection(__FILE__, __LINE__, __func__, "during ping", myconn, true); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::pgsql, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, myerr); + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + delete mybe->server_myds; + mybe->server_myds = NULL; + return -1; + } + else { + // rc==1 , nothing to do for now + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, myds->fd, myds, thread->curtime); + } + } + } + return 0; +} + +int PgSQL_Session::handler_again___status_RESETTING_CONNECTION() { + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, myds->fd, myds, thread->curtime); + } + myds->DSS = STATE_MARIADB_QUERY; + // we recreate local_stmts : see issue #752 + delete myconn->local_stmts; + myconn->local_stmts = new MySQL_STMTs_local_v14(false); // false by default, it is a backend + int rc = myconn->async_reset_session(myds->revents); + if (rc == 0) { + __sync_fetch_and_add(&PgHGM->status.backend_reset_connection, 1); + myds->myconn->reset(); + PgHGM->increase_reset_counter(); + myds->DSS = STATE_MARIADB_GENERIC; + myconn->async_state_machine = ASYNC_IDLE; + myds->return_MySQL_Connection_To_Pool(); + delete mybe->server_myds; + mybe->server_myds = NULL; + set_status(session_status___NONE); + return -1; + } else { + if (rc == -1 || rc == -2) { + if (rc == -2) { + proxy_error("Resetting Connection timeout during Reset Session on %s , %d\n", myconn->parent->address, myconn->parent->port); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::pgsql, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, ER_PROXYSQL_CHANGE_USER_TIMEOUT); + } else { // rc==-1 + const bool error_present = myconn->is_error_present(); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (error_present ? 9999 : ER_PROXYSQL_OFFLINE_SRV) // TOFIX: 9999 is a placeholder for the actual error code + ); + if (error_present) { + proxy_error("Detected an error during Reset Session on (%d,%s,%d) , FD (Conn:%d , MyDS:%d) : %s\n", myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, myds->fd, myds->myconn->fd, myconn->get_error_code_with_message().c_str()); + } else { + proxy_error( + "Detected an error during Reset Session on (%d,%s,%d) , FD (Conn:%d , MyDS:%d) : %d, %s\n", + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + myds->fd, + myds->myconn->fd, + ER_PROXYSQL_OFFLINE_SRV, + "Detected offline server prior to statement execution" + ); + } + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + RequestEnd(myds); //fix bug #682 + return -1; + } else { + // rc==1 , nothing to do for now + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, myds->fd, myds, thread->curtime); + } + } + } + return 0; +} + + +void PgSQL_Session::handler_again___new_thread_to_kill_connection() { + PgSQL_Data_Stream* myds = mybe->server_myds; + if (myds->myconn && myds->myconn->pgsql) { + if (myds->killed_at == 0) { + myds->wait_until = 0; + myds->killed_at = thread->curtime; + //fprintf(stderr,"Expired: %llu, %llu\n", mybe->server_myds->wait_until, thread->curtime); + PgSQL_Connection_userinfo* ui = client_myds->myconn->userinfo; + char* auth_password = NULL; + if (ui->password) { + if (ui->password[0] == '*') { // we don't have the real password, let's pass sha1 + auth_password = ui->sha1_pass; + } + else { + auth_password = ui->password; + } + } + + PgSQL_KillArgs* ka = new PgSQL_KillArgs(ui->username, auth_password, myds->myconn->parent->address, myds->myconn->parent->port, myds->myconn->parent->myhgc->hid, myds->myconn->pgsql->thread_id, KILL_QUERY, myds->myconn->parent->use_ssl, thread, myds->myconn->connected_host_details.ip); + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + pthread_attr_setstacksize(&attr, 256 * 1024); + pthread_t pt; + if (pthread_create(&pt, &attr, &PgSQL_kill_query_thread, ka) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + } +} + +// NEXT_IMMEDIATE is a legacy macro used inside handler() to immediately jump +// to handler_again +#define NEXT_IMMEDIATE(new_st) do { set_status(new_st); goto handler_again; } while (0) +// NEXT_IMMEDIATE_NEW is a new macro to use *outside* handler(). +// handler() should check the return code of the function it calls, and if +// true should jump to handler_again +#define NEXT_IMMEDIATE_NEW(new_st) do { set_status(new_st); return true; } while (0) + +#if 0 +bool PgSQL_Session::handler_again___verify_backend_multi_statement() { + if ((client_myds->myconn->options.client_flag & CLIENT_MULTI_STATEMENTS) != (mybe->server_myds->myconn->options.client_flag & CLIENT_MULTI_STATEMENTS)) { + + if (client_myds->myconn->options.client_flag & CLIENT_MULTI_STATEMENTS) + mybe->server_myds->myconn->options.client_flag |= CLIENT_MULTI_STATEMENTS; + else + mybe->server_myds->myconn->options.client_flag &= ~CLIENT_MULTI_STATEMENTS; + + switch (status) { // this switch can be replaced with a simple previous_status.push(status), but it is here for readibility + case PROCESSING_QUERY: + previous_status.push(PROCESSING_QUERY); + break; + case PROCESSING_STMT_PREPARE: + previous_status.push(PROCESSING_STMT_PREPARE); + break; + case PROCESSING_STMT_EXECUTE: + previous_status.push(PROCESSING_STMT_EXECUTE); + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + NEXT_IMMEDIATE_NEW(SETTING_MULTI_STMT); + } + return false; +} +#endif // 0 + +bool PgSQL_Session::handler_again___verify_init_connect() { + if (mybe->server_myds->myconn->options.init_connect_sent == false) { + // we needs to set it to true + mybe->server_myds->myconn->options.init_connect_sent = true; + char* tmp_init_connect = mysql_thread___init_connect; + char* init_connect_hg = mybe->server_myds->myconn->parent->myhgc->attributes.init_connect; + if (init_connect_hg != NULL && strlen(init_connect_hg) != 0) { + // mysql_hostgroup_attributes takes priority + tmp_init_connect = init_connect_hg; + } + if (tmp_init_connect) { + // we send init connect queries only if set + mybe->server_myds->myconn->options.init_connect = strdup(tmp_init_connect); + // Sets the previous status of the PgSQL session according to the current status. + set_previous_status_mode3(); + NEXT_IMMEDIATE_NEW(SETTING_INIT_CONNECT); + } + } + return false; +} + +#if 0 +bool PgSQL_Session::handler_again___verify_backend_session_track_gtids() { + bool ret = false; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session %p , client: %s , backend: %s\n", this, client_myds->myconn->options.session_track_gtids, mybe->server_myds->myconn->options.session_track_gtids); + // we first verify that the backend supports it + // if backend is old (or if it is not pgsql) ignore this setting + if ((mybe->server_myds->myconn->pgsql->server_capabilities & CLIENT_SESSION_TRACKING) == 0) { + // the backend doesn't support CLIENT_SESSION_TRACKING + return ret; // exit immediately + } + uint32_t b_int = mybe->server_myds->myconn->options.session_track_gtids_int; + uint32_t f_int = client_myds->myconn->options.session_track_gtids_int; + + // we need to precompute and hardcode the values for OFF and OWN_GTID + // for performance reason we hardcoded the values + // OFF = 114160514 + if ( + (b_int == 114160514) // OFF + || + (b_int == 0) // not configured yet + ) { + if (strcmp(mysql_thread___default_session_track_gtids, (char*)"OWN_GTID") == 0) { + // backend connection doesn't have session_track_gtids enabled + ret = true; + } + else { + if (f_int != 0 && f_int != 114160514) { + // client wants GTID + ret = true; + } + } + } + + if (ret) { + // we deprecated handler_again___verify_backend__generic_variable + // and moved the logic here + if (mybe->server_myds->myconn->options.session_track_gtids) { // reset current value + free(mybe->server_myds->myconn->options.session_track_gtids); + mybe->server_myds->myconn->options.session_track_gtids = NULL; + } + // because the only two possible values are OWN_GTID and OFF + // and because we don't mind receiving GTIDs , if we reach here + // it means we are setting it to OWN_GTID, either because the client + // wants it, or because it is the default + // therefore we hardcode "OWN_GTID" + mybe->server_myds->myconn->options.session_track_gtids = strdup((char*)"OWN_GTID"); + mybe->server_myds->myconn->options.session_track_gtids_int = + SpookyHash::Hash32((char*)"OWN_GTID", strlen((char*)"OWN_GTID"), 10); + // we now switch status to set session_track_gtids + // Sets the previous status of the PgSQL session according to the current status. + set_previous_status_mode3(); + NEXT_IMMEDIATE_NEW(SETTING_SESSION_TRACK_GTIDS); + } + return ret; +} + +bool PgSQL_Session::handler_again___verify_ldap_user_variable() { + bool ret = false; + if (mybe->server_myds->myconn->options.ldap_user_variable_sent == false) { + ret = true; + } + if (mybe->server_myds->myconn->options.ldap_user_variable_value == NULL) { + ret = true; + } + if (ret == false) { + if (mybe->server_myds->myconn->options.ldap_user_variable_sent) { + if (client_myds && client_myds->myconn) { + if (client_myds->myconn->userinfo) { + if (client_myds->myconn->userinfo->fe_username) { + if (strcmp(mybe->server_myds->myconn->options.ldap_user_variable_value, client_myds->myconn->userinfo->fe_username)) { + ret = true; + free(mybe->server_myds->myconn->options.ldap_user_variable); + mybe->server_myds->myconn->options.ldap_user_variable = NULL; + free(mybe->server_myds->myconn->options.ldap_user_variable_value); + mybe->server_myds->myconn->options.ldap_user_variable_value = NULL; + mybe->server_myds->myconn->options.ldap_user_variable_sent = false; + } + } + } + } + } + } + if (ret) { + // we needs to set it to true + mybe->server_myds->myconn->options.ldap_user_variable_sent = true; + if (mysql_thread___ldap_user_variable) { + // we send ldap user variable query only if set + mybe->server_myds->myconn->options.ldap_user_variable = strdup(mysql_thread___ldap_user_variable); + switch (status) { // this switch can be replaced with a simple previous_status.push(status), but it is here for readibility + case PROCESSING_QUERY: + previous_status.push(PROCESSING_QUERY); + break; + case PROCESSING_STMT_PREPARE: + previous_status.push(PROCESSING_STMT_PREPARE); + break; + case PROCESSING_STMT_EXECUTE: + previous_status.push(PROCESSING_STMT_EXECUTE); + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + NEXT_IMMEDIATE_NEW(SETTING_LDAP_USER_VARIABLE); + } + } + return false; +} +#endif // 0 + +bool PgSQL_Session::handler_again___verify_backend_user_db() { + PgSQL_Data_Stream* myds = mybe->server_myds; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session %p , client: %s , backend: %s\n", this, client_myds->myconn->userinfo->username, mybe->server_myds->myconn->userinfo->username); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session %p , client: %s , backend: %s\n", this, client_myds->myconn->userinfo->dbname, mybe->server_myds->myconn->userinfo->dbname); + if (client_myds->myconn->userinfo->hash != mybe->server_myds->myconn->userinfo->hash) { + assert(strcmp(client_myds->myconn->userinfo->username, myds->myconn->userinfo->username) == 0); + assert(strcmp(client_myds->myconn->userinfo->dbname, myds->myconn->userinfo->dbname) == 0); + } + // if we reach here, the username is the same + if (myds->myconn->requires_RESETTING_CONNECTION(client_myds->myconn)) { + // if we reach here, even if the username is the same, + // the backend connection has some session variable set + // that the client never asked for + // because we can't unset variables, we will reset the connection + // + // Sets the previous status of the PgSQL session according to the current status. + set_previous_status_mode3(); + mybe->server_myds->wait_until = thread->curtime + pgsql_thread___connect_timeout_server * 1000; // max_timeout + NEXT_IMMEDIATE_NEW(RESETTING_CONNECTION_V2); + } + return false; +} + +bool PgSQL_Session::handler_again___status_SETTING_INIT_CONNECT(int* _rc) { + bool ret = false; + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + myds->DSS = STATE_MARIADB_QUERY; + enum session_status st = status; + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + int rc = myconn->async_send_simple_command(myds->revents, myconn->options.init_connect, strlen(myconn->options.init_connect)); + if (rc == 0) { + myds->revents |= POLLOUT; // we also set again POLLOUT to send a query immediately! + //myds->free_mysql_real_query(); + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + else { + if (rc == -1 || rc == -2) { + // the command failed + int myerr = mysql_errno(myconn->pgsql); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (myerr ? myerr : ER_PROXYSQL_OFFLINE_SRV) + ); + if (myerr >= 2000 || myerr == 0) { + bool retry_conn = false; + // client error, serious + detected_broken_connection(__FILE__, __LINE__, __func__, "while setting INIT CONNECT", myconn); + //if ((myds->myconn->reusable==true) && ((myds->myprot.prot_status & SERVER_STATUS_IN_TRANS)==0)) { + if (rc != -2) { // see PMC-10003 + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (rc == -2) { + // Here we handle PMC-10003 + // and we terminate the session + retry_conn = false; + } + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + //previous_status.push(PROCESSING_QUERY); + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; // an error happened, we should destroy the Session + return ret; + } + else { + proxy_warning("Error while setting INIT CONNECT on %s:%d hg %d : %d, %s\n", myconn->parent->address, myconn->parent->port, current_hostgroup, myerr, mysql_error(myconn->pgsql)); + // we won't go back to PROCESSING_QUERY + st = previous_status.top(); + previous_status.pop(); + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, mysql_errno(myconn->pgsql), sqlstate, mysql_error(myconn->pgsql)); + myds->destroy_MySQL_Connection_From_Pool(true); + myds->fd = 0; + status = WAITING_CLIENT_DATA; + client_myds->DSS = STATE_SLEEP; + } + } + else { + // rc==1 , nothing to do for now + } + } + return ret; +} + +#if 0 +bool PgSQL_Session::handler_again___status_SETTING_LDAP_USER_VARIABLE(int* _rc) { + bool ret = false; + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + myds->DSS = STATE_MARIADB_QUERY; + enum session_status st = status; + + if ( + (GloMyLdapAuth == NULL) || (use_ldap_auth == false) + || + (client_myds == NULL || client_myds->myconn == NULL || client_myds->myconn->userinfo == NULL) + ) { // nothing to do + myds->revents |= POLLOUT; // we also set again POLLOUT to send a query immediately! + //myds->free_mysql_real_query(); + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + int rc; + if (myconn->async_state_machine == ASYNC_IDLE) { + char* fe = client_myds->myconn->userinfo->fe_username; + char* a = (char*)"SET @%s:='%s'"; + if (fe == NULL) { + fe = (char*)"unknown"; + } + if (myconn->options.ldap_user_variable_value) { + free(myconn->options.ldap_user_variable_value); + } + myconn->options.ldap_user_variable_value = strdup(fe); + char* buf = (char*)malloc(strlen(fe) + strlen(a) + strlen(myconn->options.ldap_user_variable)); + sprintf(buf, a, myconn->options.ldap_user_variable, fe); + rc = myconn->async_send_simple_command(myds->revents, buf, strlen(buf)); + free(buf); + } + else { // if async_state_machine is not ASYNC_IDLE , arguments are ignored + rc = myconn->async_send_simple_command(myds->revents, (char*)"", 0); + } + if (rc == 0) { + myds->revents |= POLLOUT; // we also set again POLLOUT to send a query immediately! + //myds->free_mysql_real_query(); + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + else { + if (rc == -1) { + // the command failed + int myerr = mysql_errno(myconn->pgsql); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (myerr ? myerr : ER_PROXYSQL_OFFLINE_SRV) + ); + if (myerr >= 2000 || myerr == 0) { + bool retry_conn = false; + // client error, serious + detected_broken_connection(__FILE__, __LINE__, __func__, "while setting LDAP USER VARIABLE", myconn); + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; // an error happened, we should destroy the Session + return ret; + } + else { + proxy_warning("Error while setting LDAP USER VARIABLE: %s:%d hg %d : %d, %s\n", myconn->parent->address, myconn->parent->port, current_hostgroup, myerr, mysql_error(myconn->pgsql)); + // we won't go back to PROCESSING_QUERY + st = previous_status.top(); + previous_status.pop(); + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, mysql_errno(myconn->pgsql), sqlstate, mysql_error(myconn->pgsql)); + myds->destroy_MySQL_Connection_From_Pool(true); + myds->fd = 0; + status = WAITING_CLIENT_DATA; + client_myds->DSS = STATE_SLEEP; + } + } + else { + // rc==1 , nothing to do for now + } + } + return ret; +} + +bool PgSQL_Session::handler_again___status_SETTING_SQL_LOG_BIN(int* _rc) { + bool ret = false; + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + myds->DSS = STATE_MARIADB_QUERY; + enum session_status st = status; + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + char* query = NULL; + unsigned long query_length = 0; + if (myconn->async_state_machine == ASYNC_IDLE) { + char* q = (char*)"SET SQL_LOG_BIN=%s"; + query = (char*)malloc(strlen(q) + 8); + sprintf(query, q, pgsql_variables.client_get_value(this, SQL_SQL_LOG_BIN)); + query_length = strlen(query); + } + int rc = myconn->async_send_simple_command(myds->revents, query, query_length); + if (query) { + free(query); + query = NULL; + } + if (rc == 0) { + if (!strcmp("0", pgsql_variables.client_get_value(this, SQL_SQL_LOG_BIN)) || !strcasecmp("OFF", pgsql_variables.client_get_value(this, SQL_SQL_LOG_BIN))) { + // Pay attention here. STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0 sets sql_log_bin to ZERO: + // - sql_log_bin=0 => true + // - sql_log_bin=1 => false + myconn->set_status(true, STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0); + } + else if (!strcmp("1", pgsql_variables.client_get_value(this, SQL_SQL_LOG_BIN)) || !strcasecmp("ON", pgsql_variables.client_get_value(this, SQL_SQL_LOG_BIN))) { + myconn->set_status(false, STATUS_MYSQL_CONNECTION_SQL_LOG_BIN0); + } + myds->revents |= POLLOUT; // we also set again POLLOUT to send a query immediately! + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + else { + if (rc == -1) { + // the command failed + int myerr = mysql_errno(myconn->pgsql); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (myerr ? myerr : ER_PROXYSQL_OFFLINE_SRV) + ); + if (myerr >= 2000 || myerr == 0) { + bool retry_conn = false; + // client error, serious + detected_broken_connection(__FILE__, __LINE__, __func__, "while setting SQL_LOG_BIN", myconn); + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; // an error happened, we should destroy the Session + return ret; + } + else { + proxy_warning("Error while setting SQL_LOG_BIN: %s:%d hg %d : %d, %s\n", myconn->parent->address, myconn->parent->port, current_hostgroup, myerr, mysql_error(myconn->pgsql)); + // we won't go back to PROCESSING_QUERY + st = previous_status.top(); + previous_status.pop(); + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, mysql_errno(myconn->pgsql), sqlstate, mysql_error(myconn->pgsql)); + myds->destroy_MySQL_Connection_From_Pool(true); + myds->fd = 0; + RequestEnd(myds); + } + } + else { + // rc==1 , nothing to do for now + } + } + return ret; +} +#endif // 0 + +bool PgSQL_Session::handler_again___status_CHANGING_CHARSET(int* _rc) { + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + + /* Validate that server can support client's charset */ + if (!validate_charset(this, SQL_CHARACTER_SET_CLIENT, *_rc)) { + return false; + } + + myds->DSS = STATE_MARIADB_QUERY; + enum session_status st = status; + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + + pgsql_variables.client_set_value(this, SQL_CHARACTER_SET, pgsql_variables.client_get_value(this, SQL_CHARACTER_SET_CLIENT)); + int charset = atoi(pgsql_variables.client_get_value(this, SQL_CHARACTER_SET_CLIENT)); + int rc = myconn->async_set_names(myds->revents, charset); + + if (rc == 0) { + __sync_fetch_and_add(&PgHGM->status.backend_set_client_encoding, 1); + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + else { + if (rc == -1) { + // the command failed + int myerr = mysql_errno(myconn->pgsql); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (myerr ? myerr : ER_PROXYSQL_OFFLINE_SRV) + ); + if (myerr >= 2000 || myerr == 0) { + if (myerr == 2019) { + proxy_error( + "Client trying to set a charset/collation (%u) not supported by backend (%s:%d). Changing it to %s\n", + charset, myconn->parent->address, myconn->parent->port, mysql_tracked_variables[SQL_CHARACTER_SET].default_value + ); + } + bool retry_conn = false; + // client error, serious + detected_broken_connection(__FILE__, __LINE__, __func__, "during SET NAMES", myconn); + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + //previous_status.push(PROCESSING_QUERY); + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; + return false; + } + else { + proxy_warning("Error during SET NAMES: %d, %s\n", myerr, mysql_error(myconn->pgsql)); + // we won't go back to PROCESSING_QUERY + st = previous_status.top(); + previous_status.pop(); + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, mysql_errno(myconn->pgsql), sqlstate, mysql_error(myconn->pgsql)); + myds->destroy_MySQL_Connection_From_Pool(true); + myds->fd = 0; + RequestEnd(myds); + } + } + else { + // rc==1 , nothing to do for now + } + } + return false; +} + +bool PgSQL_Session::handler_again___status_SETTING_GENERIC_VARIABLE(int* _rc, const char* var_name, const char* var_value, bool no_quote, bool set_transaction) { + bool ret = false; + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + myds->DSS = STATE_MARIADB_QUERY; + enum session_status st = status; + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + char* query = NULL; + unsigned long query_length = 0; + if (myconn->async_state_machine == ASYNC_IDLE) { + char* q = NULL; + if (set_transaction == false) { + if (no_quote) { + q = (char*)"SET %s=%s"; + } + else { + q = (char*)"SET %s='%s'"; // default + if (var_value[0] && var_value[0] == '@') { + q = (char*)"SET %s=%s"; + } + if (strncasecmp(var_value, (char*)"CONCAT", 6) == 0) + q = (char*)"SET %s=%s"; + if (strncasecmp(var_value, (char*)"IFNULL", 6) == 0) + q = (char*)"SET %s=%s"; + if (strncasecmp(var_value, (char*)"REPLACE", 7) == 0) + q = (char*)"SET %s=%s"; + if (var_value[0] && var_value[0] == '(') { // the value is a subquery + q = (char*)"SET %s=%s"; + } + } + } + else { + // NOTE: for now, only SET SESSION is supported + // the calling function is already passing "SESSION TRANSACTION" + q = (char*)"SET %s %s"; + } + query = (char*)malloc(strlen(q) + strlen(var_name) + strlen(var_value)); + if (strncasecmp("tx_isolation", var_name, 12) == 0) { + char* sv = mybe->server_myds->myconn->pgsql->server_version; + if (strncmp(sv, (char*)"8", 1) == 0) { + sprintf(query, q, "transaction_isolation", var_value); + } + else { + sprintf(query, q, "tx_isolation", var_value); + } + } + else if (strncasecmp("tx_read_only", var_name, 12) == 0) { + char* sv = mybe->server_myds->myconn->pgsql->server_version; + if (strncmp(sv, (char*)"8", 1) == 0) { + sprintf(query, q, "transaction_read_only", var_value); + } + else { + sprintf(query, q, "tx_read_only", var_value); + } + } + else if (strncasecmp("aurora_read_replica_read_committed", var_name, 34) == 0) { + // If aurora_read_replica_read_committed is set, isolation level is + // internally reset so that it will be set again. + // This solves the weird behavior in AWS Aurora related to isolation level + // as described in + // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Reference.html#AuroraMySQL.Reference.IsolationLevels + // Basically, to change isolation level you must first set + // aurora_read_replica_read_committed , and then isolation level + pgsql_variables.server_reset_value(this, SQL_ISOLATION_LEVEL); + sprintf(query, q, var_name, var_value); + } + else { + sprintf(query, q, var_name, var_value); + } + query_length = strlen(query); + } + int rc = myconn->async_send_simple_command(myds->revents, query, query_length); + if (query) { + free(query); + query = NULL; + } + if (rc == 0) { + myds->revents |= POLLOUT; // we also set again POLLOUT to send a query immediately! + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + + if (strcasecmp("transaction isolation level", var_name) == 0) { + pgsql_variables.server_reset_value(this, SQL_NEXT_ISOLATION_LEVEL); + pgsql_variables.client_reset_value(this, SQL_NEXT_ISOLATION_LEVEL); + } + else if (strcasecmp("transaction read", var_name) == 0) { + pgsql_variables.server_reset_value(this, SQL_NEXT_TRANSACTION_READ); + pgsql_variables.client_reset_value(this, SQL_NEXT_TRANSACTION_READ); + } + + NEXT_IMMEDIATE_NEW(st); + } + else { + if (rc == -1) { + // the command failed + int myerr = mysql_errno(myconn->pgsql); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (myerr ? myerr : ER_PROXYSQL_OFFLINE_SRV) + ); + if (myerr >= 2000 || myerr == 0) { + bool retry_conn = false; + // client error, serious + std::string action = "while setting "; + action += var_name; + detected_broken_connection(__FILE__, __LINE__, __func__, action.c_str(), myconn); + //if ((myds->myconn->reusable==true) && ((myds->myprot.prot_status & SERVER_STATUS_IN_TRANS)==0)) { + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; // an error happened, we should destroy the Session + return ret; + } + else { + proxy_warning("Error while setting %s to \"%s\" on %s:%d hg %d : %d, %s\n", var_name, var_value, myconn->parent->address, myconn->parent->port, current_hostgroup, myerr, mysql_error(myconn->pgsql)); + if ( + (myerr == 1064) // You have an error in your SQL syntax + || + (myerr == 1193) // variable is not found + || + (myerr == 1651) // Query cache is disabled + ) { + int idx = SQL_NAME_LAST_HIGH_WM; + for (int i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + if (strcasecmp(mysql_tracked_variables[i].set_variable_name, var_name) == 0) { + idx = i; + break; + } + } + if (idx != SQL_NAME_LAST_LOW_WM) { + myconn->var_absent[idx] = true; + + myds->myconn->async_free_result(); + myconn->compute_unknown_transaction_status(); + + myds->revents |= POLLOUT; // we also set again POLLOUT to send a query immediately! + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + } + + // we won't go back to PROCESSING_QUERY + st = previous_status.top(); + previous_status.pop(); + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, mysql_errno(myconn->pgsql), sqlstate, mysql_error(myconn->pgsql)); + int myerr = mysql_errno(myconn->pgsql); + switch (myerr) { + case 1231: + /* + too complicated code? + if (pgsql_thread___multiplexing && (myconn->reusable==true) && myconn->IsActiveTransaction()==false && myconn->MultiplexDisabled()==false) { + myds->DSS=STATE_NOT_INITIALIZED; + if (mysql_thread___autocommit_false_not_reusable && myconn->IsAutoCommit()==false) { + if (pgsql_thread___reset_connection_algorithm == 2) { + create_new_session_and_reset_connection(myds); + } else { + myds->destroy_MySQL_Connection_From_Pool(true); + } + } else { + myds->return_MySQL_Connection_To_Pool(); + } + } else { + myconn->async_state_machine=ASYNC_IDLE; + myds->DSS=STATE_MARIADB_GENERIC; + } + break; + */ + default: + myds->destroy_MySQL_Connection_From_Pool(true); + break; + } + myds->fd = 0; + RequestEnd(myds); + ret = true; + } + } + else { + // rc==1 , nothing to do for now + } + } + return ret; +} + +#if 0 +bool PgSQL_Session::handler_again___status_SETTING_MULTI_STMT(int* _rc) { + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + enum session_status st = status; + bool ret = false; + + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + int rc = myconn->async_set_option(myds->revents, myconn->options.client_flag & CLIENT_MULTI_STATEMENTS); + if (rc == 0) { + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + else { + if (rc == -1) { + // the command failed + int myerr = mysql_errno(myconn->pgsql); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (myerr ? myerr : ER_PROXYSQL_OFFLINE_SRV) + ); + if (myerr >= 2000 || myerr == 0) { + bool retry_conn = false; + // client error, serious + detected_broken_connection(__FILE__, __LINE__, __func__, "while setting MYSQL_OPTION_MULTI_STATEMENTS", myconn); + //if ((myds->myconn->reusable==true) && ((myds->myprot.prot_status & SERVER_STATUS_IN_TRANS)==0)) { + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; // an error happened, we should destroy the Session + return ret; + } + else { + proxy_warning("Error during MYSQL_OPTION_MULTI_STATEMENTS : %d, %s\n", myerr, mysql_error(myconn->pgsql)); + // we won't go back to PROCESSING_QUERY + st = previous_status.top(); + previous_status.pop(); + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, mysql_errno(myconn->pgsql), sqlstate, mysql_error(myconn->pgsql)); + myds->destroy_MySQL_Connection_From_Pool(true); + myds->fd = 0; + RequestEnd(myds); + } + } + else { + // rc==1 , nothing to do for now + } + } + return ret; +} + +bool PgSQL_Session::handler_again___status_SETTING_SESSION_TRACK_GTIDS(int* _rc) { + bool ret = false; + assert(mybe->server_myds->myconn); + ret = handler_again___status_SETTING_GENERIC_VARIABLE(_rc, (char*)"SESSION_TRACK_GTIDS", mybe->server_myds->myconn->options.session_track_gtids, true); + return ret; +} + +bool PgSQL_Session::handler_again___status_CHANGING_SCHEMA(int* _rc) { + bool ret = false; + //fprintf(stderr,"CHANGING_SCHEMA\n"); + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + myds->DSS = STATE_MARIADB_QUERY; + enum session_status st = status; + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + int rc = myconn->async_select_db(myds->revents); + if (rc == 0) { + //__sync_fetch_and_add(&PgHGM->status.backend_init_db, 1); + myds->myconn->userinfo->set(client_myds->myconn->userinfo); + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + else { + if (rc == -1) { + // the command failed + int myerr = mysql_errno(myconn->pgsql); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (myerr ? myerr : ER_PROXYSQL_OFFLINE_SRV) + ); + if (myerr >= 2000 || myerr == 0) { + bool retry_conn = false; + // client error, serious + detected_broken_connection(__FILE__, __LINE__, __func__, "during INIT_DB", myconn); + //if ((myds->myconn->reusable==true) && ((myds->myprot.prot_status & SERVER_STATUS_IN_TRANS)==0)) { + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; // an error happened, we should destroy the Session + return ret; + } + else { + proxy_warning("Error during INIT_DB: %d, %s\n", myerr, mysql_error(myconn->pgsql)); + // we won't go back to PROCESSING_QUERY + st = previous_status.top(); + previous_status.pop(); + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, mysql_errno(myconn->pgsql), sqlstate, mysql_error(myconn->pgsql)); + myds->destroy_MySQL_Connection_From_Pool(true); + myds->fd = 0; + RequestEnd(myds); + } + } + else { + // rc==1 , nothing to do for now + } + } + return false; +} +#endif // 0 + +bool PgSQL_Session::handler_again___status_CONNECTING_SERVER(int* _rc) { + //fprintf(stderr,"CONNECTING_SERVER\n"); + unsigned long long curtime = monotonic_time(); + thread->atomic_curtime = curtime; + if (mirror) { + mybe->server_myds->connect_retries_on_failure = 0; // no try for mirror + mybe->server_myds->wait_until = thread->curtime + pgsql_thread___connect_timeout_server * 1000; + pause_until = 0; + } + if (mybe->server_myds->max_connect_time ) { + if (thread->curtime >= mybe->server_myds->max_connect_time) { + if (mirror) { + PROXY_TRACE(); + } + + string errmsg{}; + const string session_info{ session_fast_forward ? "for 'fast_forward' session " : "" }; + const uint64_t query_time = (thread->curtime - CurrentQuery.start_time) / 1000; + + string_format( + "Max connect timeout reached while reaching hostgroup %d %safter %llums", + errmsg, current_hostgroup, session_info.c_str(), query_time + ); + + if (thread) { + thread->status_variables.stvar[st_var_max_connect_timeout_err]++; + } + client_myds->myprot.generate_error_packet(true, true, errmsg.c_str(), PGSQL_ERROR_CODES::ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION, + false, true); + RequestEnd(mybe->server_myds); + + string hg_status{}; + generate_status_one_hostgroup(current_hostgroup, hg_status); + proxy_error("%s . HG status: %s\n", errmsg.c_str(), hg_status.c_str()); + + while (previous_status.size()) { + previous_status.pop(); + } + if (mybe->server_myds->myconn) { + // NOTE-3404: Created connection never reached 'connect_cont' phase, due to that internal + // structures of 'pgsql->net' are not fully initialized. This induces a leak of the 'fd' + // associated with the socket opened by the library. To prevent this, we need to call + // `mysql_real_connect_cont` through `connect_cont`. This way we ensure a proper cleanup of + // all the resources when 'mysql_close' is later called. For more context see issue #3404. + mybe->server_myds->myconn->connect_cont(MYSQL_WAIT_TIMEOUT); + mybe->server_myds->destroy_MySQL_Connection_From_Pool(false); + if (mirror) { + PROXY_TRACE(); + NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); + } + } + mybe->server_myds->max_connect_time = 0; + NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); + } + } + if (mybe->server_myds->myconn == NULL) { + handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED__get_connection(); + } + if (mybe->server_myds->myconn == NULL) { + if (mirror) { + PROXY_TRACE(); + NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); + } + } + + // NOTE-connect_retries_delay: This check alone is not enough for imposing + // 'pgsql_thread___connect_retries_delay'. In case of 'async_connect' failing, 'pause_until' should also + // be set to 'pgsql_thread___connect_retries_delay'. Complementary NOTE below. + if (mybe->server_myds->myconn == NULL) { + pause_until = thread->curtime + pgsql_thread___connect_retries_delay * 1000; + *_rc = 1; + return false; + } + else { + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + int rc; + if (default_hostgroup < 0) { + // we are connected to a Admin module backend + // we pretend to set a user variable to disable multiplexing + myconn->set_status(true, STATUS_MYSQL_CONNECTION_USER_VARIABLE); + } + enum session_status st = status; + if (mybe->server_myds->myconn->async_state_machine == ASYNC_IDLE) { + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } + assert(st == status); + unsigned long long curtime = monotonic_time(); + + assert(myconn->async_state_machine != ASYNC_IDLE); + if (mirror) { + PROXY_TRACE(); + } + rc = myconn->async_connect(myds->revents); + if (myds->mypolls == NULL) { + // connection yet not in mypolls + myds->assign_fd_from_mysql_conn(); + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, curtime); + if (mirror) { + PROXY_TRACE(); + } + } + switch (rc) { + case 0: + myds->myds_type = MYDS_BACKEND; + myds->DSS = STATE_MARIADB_GENERIC; + status = WAITING_CLIENT_DATA; + st = previous_status.top(); + previous_status.pop(); + myds->wait_until = 0; + if (session_fast_forward == true) { + // we have a successful connection and session_fast_forward enabled + // set DSS=STATE_SLEEP or it will believe it have to use MARIADB client library + myds->DSS = STATE_SLEEP; + myds->myconn->send_quit = false; + myds->myconn->reusable = false; + // In a 'fast_forward' session after we disable compression for the fronted connection + // after we have adquired a backend connection, this is, the 'FAST_FORWARD' session status + // is reached, and the 1-1 connection relationship is established. We can safely do this + // due two main reasons: + // 1. The client and backend have to agree on compression, i.e. if the client connected without + // compression using fast-forward to a backend connections expected to have compression, it results + // in a fallback to a connection without compression, as it's expected by protocol. In this case we do + // not require to compress the data received from the backend. + // 2. The client and backend have agreed in using compression, in this case, the data received from + // the backend is already compressed, so we are only required to forward the data to the client. + // In both cases, we do not require to perform any specials actions for the received data, + // so we completely disable the compression flag for the client connection. + client_myds->myconn->set_status(false, STATUS_MYSQL_CONNECTION_COMPRESSION); + } + NEXT_IMMEDIATE_NEW(st); + break; + case -1: + case -2: + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::pgsql, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, mysql_errno(myconn->pgsql)); + if (myds->connect_retries_on_failure > 0) { + myds->connect_retries_on_failure--; + int myerr = mysql_errno(myconn->pgsql); + switch (myerr) { + case 1226: // ER_USER_LIMIT_REACHED , User '%s' has exceeded the '%s' resource (current value: %ld) + goto __exit_handler_again___status_CONNECTING_SERVER_with_err; + break; + default: + break; + } + if (mirror) { + PROXY_TRACE(); + } + myds->destroy_MySQL_Connection_From_Pool(false); + // NOTE-connect_retries_delay: In case of failure to connect, if + // 'pgsql_thread___connect_retries_delay' is set, we impose a delay in the session + // processing via 'pause_until'. Complementary NOTE above. + if (pgsql_thread___connect_retries_delay) { + pause_until = thread->curtime + pgsql_thread___connect_retries_delay * 1000; + set_status(CONNECTING_SERVER); + return false; + } + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + else { + __exit_handler_again___status_CONNECTING_SERVER_with_err: + bool is_error_present = myconn->is_error_present(); + if (is_error_present) { + client_myds->myprot.generate_error_packet(true, true, myconn->error_info.message.c_str(), + myconn->error_info.code, false, true); + } else { + char buf[256]; + sprintf(buf, "Max connect failure while reaching hostgroup %d", current_hostgroup); + client_myds->myprot.generate_error_packet(true, true, buf, PGSQL_ERROR_CODES::ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION, + false, true); + if (thread) { + thread->status_variables.stvar[st_var_max_connect_timeout_err]++; + } + } + if (session_fast_forward == false) { + // see bug #979 + RequestEnd(myds); + } + while (previous_status.size()) { + st = previous_status.top(); + previous_status.pop(); + } + if (mirror) { + PROXY_TRACE(); + } + myds->destroy_MySQL_Connection_From_Pool(is_error_present); + myds->max_connect_time = 0; + NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); + } + break; + case 1: // continue on next loop + default: + break; + } + } + return false; +} +bool PgSQL_Session::handler_again___status_RESETTING_CONNECTION(int* _rc) { + assert(mybe->server_myds->myconn); + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + myds->DSS = STATE_MARIADB_QUERY; + enum session_status st = status; + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + // we recreate local_stmts : see issue #752 + delete myconn->local_stmts; + myconn->local_stmts = new MySQL_STMTs_local_v14(false); // false by default, it is a backend + if (pgsql_thread___connect_timeout_server_max) { + if (mybe->server_myds->max_connect_time == 0) { + mybe->server_myds->max_connect_time = thread->curtime + pgsql_thread___connect_timeout_server_max * 1000; + } + } + int rc = myconn->async_reset_session(myds->revents); + if (rc == 0) { + __sync_fetch_and_add(&PgHGM->status.backend_reset_connection, 1); + //myds->myconn->userinfo->set(client_myds->myconn->userinfo); + myds->myconn->reset(); + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + NEXT_IMMEDIATE_NEW(st); + } else { + if (rc == -1) { + // the command failed + const bool error_present = myconn->is_error_present(); + PgHGM->p_update_pgsql_error_counter( + p_pgsql_error_type::pgsql, + myconn->parent->myhgc->hid, + myconn->parent->address, + myconn->parent->port, + (error_present ? 9999 : ER_PROXYSQL_OFFLINE_SRV) // TOFIX: 9999 is a placeholder for the actual error code + ); + if (error_present == false || (error_present == true && myconn->is_connection_in_reusable_state() == false)) { + bool retry_conn = false; + // client error, serious + detected_broken_connection(__FILE__, __LINE__, __func__, "during Resetting Connection", myconn); + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; + return false; + } else { + proxy_warning("Error during Resetting Connection: %s\n", myconn->get_error_code_with_message().c_str()); + // we won't go back to PROCESSING_QUERY + st = previous_status.top(); + previous_status.pop(); + client_myds->myprot.generate_error_packet(true, true, myconn->get_error_message().c_str(), myconn->get_error_code(), false); + myds->destroy_MySQL_Connection_From_Pool(true); + myds->fd = 0; + RequestEnd(myds); //fix bug #682 + } + } else { + if (rc == -2) { + bool retry_conn = false; + proxy_error("Timeout during Resetting Connection on %s , %d\n", myconn->parent->address, myconn->parent->port); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::pgsql, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, ER_PROXYSQL_CHANGE_USER_TIMEOUT); + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + retry_conn = true; + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); + } + *_rc = -1; + return false; + } else { + // rc==1 , nothing to do for now + } + } + } + return false; +} + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// where: +// status = WAITING_CLIENT_DATA +// client_myds->DSS = STATE_SLEEP +// enum_mysql_command = _MYSQL_COM_STMT_PREPARE +// +// all break were replaced with a return +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_PREPARE(PtrSize_t& pkt) { + if (session_type != PROXYSQL_SESSION_PGSQL) { // only MySQL module supports prepared statement!! + l_free(pkt.size, pkt.ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, 1045, (char*)"28000", (char*)"Command not supported"); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + return; + } + else { + thread->status_variables.stvar[st_var_frontend_stmt_prepare]++; + thread->status_variables.stvar[st_var_queries]++; + // if we reach here, we are not on MySQL module + bool rc_break = false; + bool lock_hostgroup = false; + + // Note: CurrentQuery sees the query as sent by the client. + // shortly after, the packets it used to contain the query will be deallocated + // Note2 : we call the next function as if it was _MYSQL_COM_QUERY + // because the offset will be identical + CurrentQuery.begin((unsigned char*)pkt.ptr, pkt.size, true); + + timespec begint; + timespec endt; + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &begint); + } + qpo = GloPgQPro->process_query(this, pkt.ptr, pkt.size, &CurrentQuery); + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &endt); + thread->status_variables.stvar[st_var_query_processor_time] = thread->status_variables.stvar[st_var_query_processor_time] + + (endt.tv_sec * 1000000000 + endt.tv_nsec) - + (begint.tv_sec * 1000000000 + begint.tv_nsec); + } + assert(qpo); // GloPgQPro->process_mysql_query() should always return a qpo + // setting 'prepared' to prevent fetching results from the cache if the digest matches + rc_break = handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(&pkt, &lock_hostgroup, PgSQL_ps_type_prepare_stmt); + if (rc_break == true) { + return; + } + if (pgsql_thread___set_query_lock_on_hostgroup == 1) { // algorithm introduced in 2.0.6 + if (locked_on_hostgroup < 0) { + if (lock_hostgroup) { + // we are locking on hostgroup now + locked_on_hostgroup = current_hostgroup; + } + } + if (locked_on_hostgroup >= 0) { + if (current_hostgroup != locked_on_hostgroup) { + client_myds->DSS = STATE_QUERY_SENT_NET; + int l = CurrentQuery.QueryLength; + char* end = (char*)""; + if (l > 256) { + l = 253; + end = (char*)"..."; + } + string nqn = string((char*)CurrentQuery.QueryPointer, l); + char* err_msg = (char*)"Session trying to reach HG %d while locked on HG %d . Rejecting query: %s"; + char* buf = (char*)malloc(strlen(err_msg) + strlen(nqn.c_str()) + strlen(end) + 64); + sprintf(buf, err_msg, current_hostgroup, locked_on_hostgroup, nqn.c_str(), end); + client_myds->myprot.generate_error_packet(true, true, buf, PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION, + false, true); + thread->status_variables.stvar[st_var_hostgroup_locked_queries]++; + RequestEnd(NULL); + free(buf); + l_free(pkt.size, pkt.ptr); + return; + } + } + } + mybe = find_or_create_backend(current_hostgroup); + if (client_myds->myconn->local_stmts == NULL) { + client_myds->myconn->local_stmts = new MySQL_STMTs_local_v14(true); + } + uint64_t hash = client_myds->myconn->local_stmts->compute_hash( + (char*)client_myds->myconn->userinfo->username, + (char*)client_myds->myconn->userinfo->dbname, + (char*)CurrentQuery.QueryPointer, + CurrentQuery.QueryLength + ); + MySQL_STMT_Global_info* stmt_info = NULL; + // we first lock GloStmt + GloMyStmt->wrlock(); + stmt_info = GloMyStmt->find_prepared_statement_by_hash(hash); + if (stmt_info) { + // the prepared statement exists in GloMyStmt + // for this reason, we do not need to prepare it again, and we can already reply to the client + // we will now generate a unique stmt and send it to the client + uint32_t new_stmt_id = client_myds->myconn->local_stmts->generate_new_client_stmt_id(stmt_info->statement_id); + CurrentQuery.stmt_client_id = new_stmt_id; + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_STMT_PREPARE_RESPONSE(client_myds->pkt_sid + 1, stmt_info, new_stmt_id); + LogQuery(NULL); + l_free(pkt.size, pkt.ptr); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + CurrentQuery.end_time = thread->curtime; + CurrentQuery.end(); + } + else { + mybe = find_or_create_backend(current_hostgroup); + status = PROCESSING_STMT_PREPARE; + mybe->server_myds->connect_retries_on_failure = pgsql_thread___connect_retries_on_failure; + mybe->server_myds->wait_until = 0; + pause_until = 0; + mybe->server_myds->killed_at = 0; + mybe->server_myds->kill_type = 0; + mybe->server_myds->mysql_real_query.init(&pkt); // fix memory leak for PREPARE in prepared statements #796 + mybe->server_myds->statuses.questions++; + client_myds->setDSS_STATE_QUERY_SENT_NET(); + } + GloMyStmt->unlock(); + return; // make sure to not return before unlocking GloMyStmt + } +} + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// where: +// status = WAITING_CLIENT_DATA +// client_myds->DSS = STATE_SLEEP +// enum_mysql_command = _MYSQL_COM_STMT_EXECUTE +// +// all break were replaced with a return +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_EXECUTE(PtrSize_t& pkt) { + if (session_type != PROXYSQL_SESSION_PGSQL) { // only MySQL module supports prepared statement!! + l_free(pkt.size, pkt.ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, 1045, (char*)"28000", (char*)"Command not supported"); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + return; + } + else { + // if we reach here, we are on MySQL module + bool rc_break = false; + bool lock_hostgroup = false; + thread->status_variables.stvar[st_var_frontend_stmt_execute]++; + thread->status_variables.stvar[st_var_queries]++; + uint32_t client_stmt_id = 0; + uint64_t stmt_global_id = 0; + memcpy(&client_stmt_id, (char*)pkt.ptr + 5, sizeof(uint32_t)); + CurrentQuery.stmt_client_id = client_stmt_id; + stmt_global_id = client_myds->myconn->local_stmts->find_global_stmt_id_from_client(client_stmt_id); + if (stmt_global_id == 0) { + // FIXME: add error handling + // LCOV_EXCL_START + assert(0); + // LCOV_EXCL_STOP + } + CurrentQuery.stmt_global_id = stmt_global_id; + // now we get the statement information + MySQL_STMT_Global_info* stmt_info = NULL; + stmt_info = GloMyStmt->find_prepared_statement_by_stmt_id(stmt_global_id); + if (stmt_info == NULL) { + // we couldn't find it + l_free(pkt.size, pkt.ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, 1045, (char*)"28000", (char*)"Prepared statement doesn't exist", true); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + return; + } + CurrentQuery.stmt_info = stmt_info; + CurrentQuery.start_time = thread->curtime; + + timespec begint; + timespec endt; + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &begint); + } + qpo = GloPgQPro->process_query(this, NULL, 0, &CurrentQuery); + if (qpo->max_lag_ms >= 0) { + thread->status_variables.stvar[st_var_queries_with_max_lag_ms]++; + } + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &endt); + thread->status_variables.stvar[st_var_query_processor_time] = thread->status_variables.stvar[st_var_query_processor_time] + + (endt.tv_sec * 1000000000 + endt.tv_nsec) - + (begint.tv_sec * 1000000000 + begint.tv_nsec); + } + assert(qpo); // GloPgQPro->process_mysql_query() should always return a qpo + // we now take the metadata associated with STMT_EXECUTE from MySQL_STMTs_meta + bool stmt_meta_found = true; // let's be optimistic and we assume we will found it + stmt_execute_metadata_t* stmt_meta = sess_STMTs_meta->find(stmt_global_id); + if (stmt_meta == NULL) { // we couldn't find any metadata + stmt_meta_found = false; + } + stmt_meta = client_myds->myprot.get_binds_from_pkt(pkt.ptr, pkt.size, stmt_info, &stmt_meta); + if (stmt_meta == NULL) { + l_free(pkt.size, pkt.ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, 1045, (char*)"28000", (char*)"Error in prepared statement execution", true); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + //__sync_fetch_and_sub(&stmt_info->ref_count,1); // decrease reference count + stmt_info = NULL; + return; + } + if (stmt_meta_found == false) { + // previously we didn't find any metadata + // but as we reached here, stmt_meta is not null and we save the metadata + sess_STMTs_meta->insert(stmt_global_id, stmt_meta); + } + // else + + CurrentQuery.stmt_meta = stmt_meta; + //current_hostgroup=qpo->destination_hostgroup; + rc_break = handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(&pkt, &lock_hostgroup, PgSQL_ps_type_execute_stmt); + if (rc_break == true) { + return; + } + if (pgsql_thread___set_query_lock_on_hostgroup == 1) { // algorithm introduced in 2.0.6 + if (locked_on_hostgroup < 0) { + if (lock_hostgroup) { + // we are locking on hostgroup now + locked_on_hostgroup = current_hostgroup; + } + } + if (locked_on_hostgroup >= 0) { + if (current_hostgroup != locked_on_hostgroup) { + client_myds->DSS = STATE_QUERY_SENT_NET; + //int l = CurrentQuery.QueryLength; + int l = CurrentQuery.stmt_info->query_length; + char* end = (char*)""; + if (l > 256) { + l = 253; + end = (char*)"..."; + } + string nqn = string((char*)CurrentQuery.stmt_info->query, l); + char* err_msg = (char*)"Session trying to reach HG %d while locked on HG %d . Rejecting query: %s"; + char* buf = (char*)malloc(strlen(err_msg) + strlen(nqn.c_str()) + strlen(end) + 64); + sprintf(buf, err_msg, current_hostgroup, locked_on_hostgroup, nqn.c_str(), end); + client_myds->myprot.generate_error_packet(true, true, buf, PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION, + false, true); + thread->status_variables.stvar[st_var_hostgroup_locked_queries]++; + RequestEnd(NULL); + free(buf); + l_free(pkt.size, pkt.ptr); + return; + } + } + } + mybe = find_or_create_backend(current_hostgroup); + status = PROCESSING_STMT_EXECUTE; + mybe->server_myds->connect_retries_on_failure = pgsql_thread___connect_retries_on_failure; + mybe->server_myds->wait_until = 0; + mybe->server_myds->killed_at = 0; + mybe->server_myds->kill_type = 0; + client_myds->setDSS_STATE_QUERY_SENT_NET(); + } +} + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// ClickHouse doesn't support COM_INIT_DB , so we replace it +// with a COM_QUERY running USE +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_INIT_DB_replace_CLICKHOUSE(PtrSize_t& pkt) { + PtrSize_t _new_pkt; + _new_pkt.ptr = malloc(pkt.size + 4); // USE + space + memcpy(_new_pkt.ptr, pkt.ptr, 4); + unsigned char* _c = (unsigned char*)_new_pkt.ptr; + _c += 4; *_c = 0x03; + _c += 1; *_c = 'U'; + _c += 1; *_c = 'S'; + _c += 1; *_c = 'E'; + _c += 1; *_c = ' '; + memcpy((char*)_new_pkt.ptr + 9, (char*)pkt.ptr + 5, pkt.size - 5); + l_free(pkt.size, pkt.ptr); + pkt.size += 4; + pkt.ptr = _new_pkt.ptr; +} + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// where: +// status = WAITING_CLIENT_DATA +// client_myds->DSS = STATE_SLEEP +// enum_mysql_command = _MYSQL_COM_QUERY +// it processes the session not MYSQL_SESSION +// Make sure that handler_function() doesn't free the packet +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___not_mysql(PtrSize_t& pkt) { + switch (session_type) { + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + // this is processed by the admin module + handler_function(this, (void*)GloAdmin, &pkt); + l_free(pkt.size, pkt.ptr); + break; + case PROXYSQL_SESSION_SQLITE: + handler_function(this, (void*)GloSQLite3Server, &pkt); + l_free(pkt.size, pkt.ptr); + break; +#ifdef PROXYSQLCLICKHOUSE + case PROXYSQL_SESSION_CLICKHOUSE: + handler_function(this, (void*)GloClickHouseServer, &pkt); + l_free(pkt.size, pkt.ptr); + break; +#endif /* PROXYSQLCLICKHOUSE */ + default: + // LCOV_EXCL_START + assert(0); + // LCOV_EXCL_STOP + } +} + + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// where: +// status = WAITING_CLIENT_DATA +// client_myds->DSS = STATE_SLEEP +// enum_mysql_command = _MYSQL_COM_QUERY +// it searches for SQL injection +// it returns true if it detected an SQL injection +bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_detect_SQLi() { + if (client_myds->com_field_list == false) { + if (qpo->firewall_whitelist_mode != WUS_OFF) { + struct libinjection_sqli_state state; + int issqli; + const char* input = (char*)CurrentQuery.QueryPointer; + size_t slen = CurrentQuery.QueryLength; + libinjection_sqli_init(&state, input, slen, FLAG_SQL_MYSQL); + issqli = libinjection_is_sqli(&state); + if (issqli) { + bool allow_sqli = false; + allow_sqli = GloPgQPro->whitelisted_sqli_fingerprint(state.fingerprint); + if (allow_sqli) { + thread->status_variables.stvar[st_var_mysql_whitelisted_sqli_fingerprint]++; + } + else { + thread->status_variables.stvar[st_var_automatic_detected_sqli]++; + char* username = client_myds->myconn->userinfo->username; + char* client_address = client_myds->addr.addr; + proxy_error("SQLinjection detected with fingerprint of '%s' from client %s@%s . Query listed below:\n", state.fingerprint, username, client_address); + fwrite(CurrentQuery.QueryPointer, CurrentQuery.QueryLength, 1, stderr); + fprintf(stderr, "\n"); + RequestEnd(NULL); + return true; + } + } + } + } + return false; +} + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// where: +// status = WAITING_CLIENT_DATA +// client_myds->DSS = STATE_SLEEP_MULTI_PACKET +// +// replacing the single goto with return true +bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP_MULTI_PACKET(PtrSize_t& pkt) { + if (client_myds->multi_pkt.ptr == NULL) { + // not initialized yet + client_myds->multi_pkt.ptr = pkt.ptr; + client_myds->multi_pkt.size = pkt.size; + } + else { + PtrSize_t tmp_pkt; + tmp_pkt.ptr = client_myds->multi_pkt.ptr; + tmp_pkt.size = client_myds->multi_pkt.size; + client_myds->multi_pkt.size = pkt.size + tmp_pkt.size - sizeof(mysql_hdr); + client_myds->multi_pkt.ptr = l_alloc(client_myds->multi_pkt.size); + memcpy(client_myds->multi_pkt.ptr, tmp_pkt.ptr, tmp_pkt.size); + memcpy((char*)client_myds->multi_pkt.ptr + tmp_pkt.size, (char*)pkt.ptr + sizeof(mysql_hdr), pkt.size - sizeof(mysql_hdr)); // the header is not copied + l_free(tmp_pkt.size, tmp_pkt.ptr); + l_free(pkt.size, pkt.ptr); + } + if (pkt.size == (0xFFFFFF + sizeof(mysql_hdr))) { // there are more packets + //goto __get_pkts_from_client; + return true; + } + else { + // no more packets, move everything back to pkt and proceed + pkt.ptr = client_myds->multi_pkt.ptr; + pkt.size = client_myds->multi_pkt.size; + client_myds->multi_pkt.size = 0; + client_myds->multi_pkt.ptr = NULL; + client_myds->DSS = STATE_SLEEP; + } + return false; +} + + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// where: +// status = WAITING_CLIENT_DATA +// client_myds->DSS = STATE_SLEEP +// enum_mysql_command in a large list of possible values +// the most common values for enum_mysql_command are handled from the calling function +// here we only process the not so common ones +// we return false if the enum_mysql_command is not found +bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM__various(PtrSize_t* pkt, bool* wrong_pass) { + unsigned char c; + c = *((unsigned char*)pkt->ptr + sizeof(mysql_hdr)); + switch ((enum_mysql_command)c) { + case _MYSQL_COM_CHANGE_USER: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_CHANGE_USER(pkt, wrong_pass); + break; + case _MYSQL_COM_PING: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_PING(pkt); + break; + case _MYSQL_COM_SET_OPTION: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_SET_OPTION(pkt); + break; + case _MYSQL_COM_STATISTICS: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STATISTICS(pkt); + break; + case _MYSQL_COM_INIT_DB: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_INIT_DB(pkt); + break; + case _MYSQL_COM_FIELD_LIST: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_FIELD_LIST(pkt); + break; + case _MYSQL_COM_PROCESS_KILL: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_PROCESS_KILL(pkt); + break; + case _MYSQL_COM_RESET_CONNECTION: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_RESET_CONNECTION(pkt); + break; + default: + return false; + break; + } + return true; +} + + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// where: +// status = NONE or default +// +// this is triggered when proxysql receives a packet when doesn't expect any +// for example while it is supposed to be sending resultset to client +void PgSQL_Session::handler___status_NONE_or_default(PtrSize_t& pkt) { + char buf[INET6_ADDRSTRLEN]; + switch (client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + break; + } + default: + sprintf(buf, "localhost"); + break; + } + if (pkt.size == 5) { + unsigned char c = *((unsigned char*)pkt.ptr + sizeof(mysql_hdr)); + if (c == _MYSQL_COM_QUIT) { + proxy_error("Unexpected COM_QUIT from client %s . Session_status: %d , client_status: %d Disconnecting it\n", buf, status, client_myds->status); + if (GloPgSQL_Logger) { GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_QUIT, this, NULL); } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_QUIT packet\n"); + l_free(pkt.size, pkt.ptr); + if (thread) { + thread->status_variables.stvar[st_var_unexpected_com_quit]++; + } + return; + } + } + proxy_error2(10001, "Unexpected packet from client %s . Session_status: %d , client_status: %d Disconnecting it\n", buf, status, client_myds->status); + if (thread) { + thread->status_variables.stvar[st_var_unexpected_packet]++; + } + return; +} + +// this function was inline inside PgSQL_Session::get_pkts_from_client +// where: +// status = WAITING_CLIENT_DATA +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___default() { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Statuses: WAITING_CLIENT_DATA - STATE_UNKNOWN\n"); + if (mirror == false) { + char buf[INET6_ADDRSTRLEN]; + switch (client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + break; + } + default: + sprintf(buf, "localhost"); + break; + } + // PMC-10001: A unexpected packet has been received from client. This error has two potential causes: + // * Bug: ProxySQL state machine wasn't in the correct state when a legitimate client packet was received. + // * Client error: The client incorrectly sent a packet breaking MySQL protocol. + proxy_error2(10001, "Unexpected packet from client %s . Session_status: %d , client_status: %d Disconnecting it\n", buf, status, client_myds->status); + } +} + +int PgSQL_Session::get_pkts_from_client(bool& wrong_pass, PtrSize_t& pkt) { + int handler_ret = 0; + unsigned char c; + +__get_pkts_from_client: + + // implement a more complex logic to run even in case of mirror + // if client_myds , this is a regular client + // if client_myds == NULL , it is a mirror + // process mirror only status==WAITING_CLIENT_DATA + for (unsigned int j = 0; j < (client_myds->PSarrayIN ? client_myds->PSarrayIN->len : 0) || (mirror == true && status == WAITING_CLIENT_DATA);) { + if (mirror == false) { + client_myds->PSarrayIN->remove_index(0, &pkt); + } + switch (status) { + + case CONNECTING_CLIENT: + switch (client_myds->DSS) { + case STATE_SSL_INIT: + case STATE_SERVER_HANDSHAKE: + handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(&pkt, &wrong_pass); + break; + default: + proxy_error("Detected not valid state client state: %d\n", client_myds->DSS); + handler_ret = -1; //close connection + return handler_ret; + break; + } + break; + + case WAITING_CLIENT_DATA: + // this is handled only for real traffic, not mirror + if (pkt.size == (0xFFFFFF + sizeof(mysql_hdr))) { + // we are handling a multi-packet + switch (client_myds->DSS) { // real traffic only + case STATE_SLEEP: + client_myds->DSS = STATE_SLEEP_MULTI_PACKET; + break; + case STATE_SLEEP_MULTI_PACKET: + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + } + switch (client_myds->DSS) { + case STATE_SLEEP_MULTI_PACKET: + if (handler___status_WAITING_CLIENT_DATA___STATE_SLEEP_MULTI_PACKET(pkt)) { + // if handler___status_WAITING_CLIENT_DATA___STATE_SLEEP_MULTI_PACKET + // returns true it meansa we need to reiterate + goto __get_pkts_from_client; + } + // Note: the above function can change DSS to STATE_SLEEP + // in that case we don't break from the witch but continue + if (client_myds->DSS != STATE_SLEEP) // if DSS==STATE_SLEEP , we continue + break; + case STATE_SLEEP: // only this section can be executed ALSO by mirror + command_counters->incr(thread->curtime / 1000000); + if (transaction_persistent_hostgroup == -1) { + if (pgsql_thread___set_query_lock_on_hostgroup == 0) { // behavior before 2.0.6 + current_hostgroup = default_hostgroup; + } + else { + if (locked_on_hostgroup == -1) { + current_hostgroup = default_hostgroup; + } + else { + current_hostgroup = locked_on_hostgroup; + } + } + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , client_myds=%p . Statuses: WAITING_CLIENT_DATA - STATE_SLEEP\n", this, client_myds); + if (session_fast_forward == true) { // if it is fast forward + // If this is a 'fast_forward' session that hasn't yet received a backend connection, we don't + // forward 'COM_QUIT' packets, since this will make the act of obtaining a connection pointless. + // Instead, we intercept the 'COM_QUIT' packet and end the 'PgSQL_Session'. + unsigned char command = *(static_cast(pkt.ptr) + sizeof(mysql_hdr)); + if (command == _MYSQL_COM_QUIT) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_QUIT packet\n"); + if (GloPgSQL_Logger) { GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_QUIT, this, NULL); } + l_free(pkt.size, pkt.ptr); + handler_ret = -1; + return handler_ret; + } + + mybe = find_or_create_backend(current_hostgroup); // set a backend + mybe->server_myds->reinit_queues(); // reinitialize the queues in the myds . By default, they are not active + mybe->server_myds->PSarrayOUT->add(pkt.ptr, pkt.size); // move the first packet + previous_status.push(FAST_FORWARD); // next status will be FAST_FORWARD . Now we need a connection + + // If this is a 'fast_forward' session, we impose the 'connect_timeout' prior to actually getting the + // connection from the 'connection_pool'. This is used to ensure that we kill the session if + // 'CONNECTING_SERVER' isn't completed before this timeout expiring. For example, if 'max_connections' + // is reached for the target hostgroup. + if (mybe->server_myds->max_connect_time == 0) { + uint64_t connect_timeout = + pgsql_thread___connect_timeout_server < pgsql_thread___connect_timeout_server_max ? + pgsql_thread___connect_timeout_server_max : pgsql_thread___connect_timeout_server; + mybe->server_myds->max_connect_time = thread->curtime + connect_timeout * 1000; + } + // Impose the same connection retrying policy as done for regular connections during + // 'MYSQL_CON_QUERY'. + mybe->server_myds->connect_retries_on_failure = pgsql_thread___connect_retries_on_failure; + // 'CurrentQuery' isn't used for 'FAST_FORWARD' but we update it for using it as a session + // startup time for when a fast_forward session has attempted to obtain a connection. + CurrentQuery.start_time = thread->curtime; + + { + //NEXT_IMMEDIATE(CONNECTING_SERVER); // we create a connection . next status will be FAST_FORWARD + // we can't use NEXT_IMMEDIATE() inside get_pkts_from_client() + // instead we set status to CONNECTING_SERVER and return 0 + // when we exit from get_pkts_from_client() we expect the label "handler_again" + set_status(CONNECTING_SERVER); + return 0; + } + } + c = *((unsigned char*)pkt.ptr + sizeof(mysql_hdr)); + if (client_myds != NULL) { + if (session_type == PROXYSQL_SESSION_ADMIN || session_type == PROXYSQL_SESSION_STATS) { + c = *((unsigned char*)pkt.ptr + 0); + if (c == 'Q') { + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___not_mysql(pkt); + } + else if (c == 'X') { + //proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_QUIT packet\n"); + //if (GloPgSQL_Logger) { GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_QUIT, this, NULL); } + l_free(pkt.size, pkt.ptr); + handler_ret = -1; + return handler_ret; + } + else { + proxy_error("Not implemented yet"); + assert(0); + } + } + else { + char command = c = *((unsigned char*)pkt.ptr + 0); + switch (command) { + case 'Q': + { + __sync_add_and_fetch(&thread->status_variables.stvar[st_var_queries], 1); + if (session_type == PROXYSQL_SESSION_PGSQL) { + bool rc_break = false; + bool lock_hostgroup = false; + if (session_fast_forward == false) { + // Note: CurrentQuery sees the query as sent by the client. + // shortly after, the packets it used to contain the query will be deallocated + CurrentQuery.begin((unsigned char*)pkt.ptr, pkt.size, true); + } + rc_break = handler_special_queries(&pkt); + if (rc_break == true) { + if (mirror == false) { + // track also special queries + //RequestEnd(NULL); + // we moved this inside handler_special_queries() + // because a pointer was becoming invalid + break; + } + else { + handler_ret = -1; + return handler_ret; + } + } + timespec begint; + timespec endt; + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &begint); + } + qpo = GloPgQPro->process_query(this, pkt.ptr, pkt.size, &CurrentQuery); + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &endt); + thread->status_variables.stvar[st_var_query_processor_time] = thread->status_variables.stvar[st_var_query_processor_time] + + (endt.tv_sec * 1000000000 + endt.tv_nsec) - + (begint.tv_sec * 1000000000 + begint.tv_nsec); + } + assert(qpo); // GloPgQPro->process_mysql_query() should always return a qpo + // This block was moved from 'handler_special_queries' to support + // handling of 'USE' statements which are preceded by a comment. + // For more context check issue: #3493. + // =================================================== + if (session_type != PROXYSQL_SESSION_CLICKHOUSE) { + const char* qd = CurrentQuery.get_digest_text(); + bool use_db_query = false; + + if (qd != NULL) { + if ( + (strncasecmp((char*)"USE", qd, 3) == 0) + && + ( + (strncasecmp((char*)"USE ", qd, 4) == 0) + || + (strncasecmp((char*)"USE`", qd, 4) == 0) + ) + ) { + use_db_query = true; + } + } + else { + if (pkt.size > (5 + 4) && strncasecmp((char*)"USE ", (char*)pkt.ptr + 5, 4) == 0) { + use_db_query = true; + } + } + + if (use_db_query) { + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_USE_DB(&pkt); + + if (mirror == false) { + break; + } + else { + handler_ret = -1; + return handler_ret; + } + } + } + // =================================================== + if (qpo->max_lag_ms >= 0) { + thread->status_variables.stvar[st_var_queries_with_max_lag_ms]++; + } + rc_break = handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(&pkt, &lock_hostgroup); + if (mirror == false && rc_break == false) { + if (pgsql_thread___automatic_detect_sqli) { + if (handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_detect_SQLi()) { + handler_ret = -1; + return handler_ret; + } + } + } + if (rc_break == true) { + if (mirror == false) { + break; + } + else { + handler_ret = -1; + return handler_ret; + } + } + if (mirror == false) { + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___create_mirror_session(); + } + + if (autocommit_on_hostgroup >= 0) { + } + if (pgsql_thread___set_query_lock_on_hostgroup == 1) { // algorithm introduced in 2.0.6 + if (locked_on_hostgroup < 0) { + if (lock_hostgroup) { + // we are locking on hostgroup now + if (qpo->destination_hostgroup >= 0) { + if (transaction_persistent_hostgroup == -1) { + current_hostgroup = qpo->destination_hostgroup; + } + } + locked_on_hostgroup = current_hostgroup; + thread->status_variables.stvar[st_var_hostgroup_locked]++; + thread->status_variables.stvar[st_var_hostgroup_locked_set_cmds]++; + } + } + if (locked_on_hostgroup >= 0) { + if (current_hostgroup != locked_on_hostgroup) { + client_myds->DSS = STATE_QUERY_SENT_NET; + int l = CurrentQuery.QueryLength; + char* end = (char*)""; + if (l > 256) { + l = 253; + end = (char*)"..."; + } + string nqn = string((char*)CurrentQuery.QueryPointer, l); + char* err_msg = (char*)"Session trying to reach HG %d while locked on HG %d . Rejecting query: %s"; + char* buf = (char*)malloc(strlen(err_msg) + strlen(nqn.c_str()) + strlen(end) + 64); + sprintf(buf, err_msg, current_hostgroup, locked_on_hostgroup, nqn.c_str(), end); + client_myds->myprot.generate_error_packet(true, true, buf, PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION, + false, true); + thread->status_variables.stvar[st_var_hostgroup_locked_queries]++; + RequestEnd(NULL); + free(buf); + l_free(pkt.size, pkt.ptr); + break; + } + } + } + mybe = find_or_create_backend(current_hostgroup); + status = PROCESSING_QUERY; + // set query retries + mybe->server_myds->query_retries_on_failure = pgsql_thread___query_retries_on_failure; + // if a number of retries is set in mysql_query_rules, that takes priority + if (qpo) { + if (qpo->retries >= 0) { + mybe->server_myds->query_retries_on_failure = qpo->retries; + } + } + mybe->server_myds->connect_retries_on_failure = pgsql_thread___connect_retries_on_failure; + mybe->server_myds->wait_until = 0; + pause_until = 0; + if (pgsql_thread___default_query_delay) { + pause_until = thread->curtime + pgsql_thread___default_query_delay * 1000; + } + if (qpo) { + if (qpo->delay > 0) { + if (pause_until == 0) + pause_until = thread->curtime; + pause_until += qpo->delay * 1000; + } + } + + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Received query to be processed with MariaDB Client library\n"); + mybe->server_myds->killed_at = 0; + mybe->server_myds->kill_type = 0; +#if 0 + if (GloMyLdapAuth) { + if (session_type == PROXYSQL_SESSION_PGSQL) { + if (mysql_thread___add_ldap_user_comment && strlen(mysql_thread___add_ldap_user_comment)) { + add_ldap_comment_to_pkt(&pkt); + } + } + } +#endif // 0 + mybe->server_myds->mysql_real_query.init(&pkt); + mybe->server_myds->statuses.questions++; + client_myds->setDSS_STATE_QUERY_SENT_NET(); + } + } + break; + case 'X': + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got QUIT packet\n"); + if (GloPgSQL_Logger) { GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_QUIT, this, NULL); } + l_free(pkt.size, pkt.ptr); + handler_ret = -1; + return handler_ret; + break; + default: + proxy_error("Not implemented yet"); + assert(0); + } + } + break; + } + if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { + if ((enum_mysql_command)c == _MYSQL_COM_INIT_DB) { + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_INIT_DB_replace_CLICKHOUSE(pkt); + c = *((unsigned char*)pkt.ptr + sizeof(mysql_hdr)); + } + } + client_myds->com_field_list = false; // default + if (c == _MYSQL_COM_FIELD_LIST) { + if (session_type == PROXYSQL_SESSION_PGSQL) { + MySQL_Protocol* myprot = &client_myds->myprot; + bool rcp = myprot->generate_COM_QUERY_from_COM_FIELD_LIST(&pkt); + if (rcp) { + // all went well + c = *((unsigned char*)pkt.ptr + sizeof(mysql_hdr)); + client_myds->com_field_list = true; + } + else { + // parsing failed, proxysql will return not suppported command + } + } + } + switch ((enum_mysql_command)c) { + case _MYSQL_COM_QUERY: + __sync_add_and_fetch(&thread->status_variables.stvar[st_var_queries], 1); + if (session_type == PROXYSQL_SESSION_PGSQL) { + bool rc_break = false; + bool lock_hostgroup = false; + if (session_fast_forward == false) { + // Note: CurrentQuery sees the query as sent by the client. + // shortly after, the packets it used to contain the query will be deallocated + CurrentQuery.begin((unsigned char*)pkt.ptr, pkt.size, true); + } + rc_break = handler_special_queries(&pkt); + if (rc_break == true) { + if (mirror == false) { + // track also special queries + //RequestEnd(NULL); + // we moved this inside handler_special_queries() + // because a pointer was becoming invalid + break; + } + else { + handler_ret = -1; + return handler_ret; + } + } + timespec begint; + timespec endt; + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &begint); + } + qpo = GloPgQPro->process_query(this, pkt.ptr, pkt.size, &CurrentQuery); + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &endt); + thread->status_variables.stvar[st_var_query_processor_time] = thread->status_variables.stvar[st_var_query_processor_time] + + (endt.tv_sec * 1000000000 + endt.tv_nsec) - + (begint.tv_sec * 1000000000 + begint.tv_nsec); + } + assert(qpo); // GloPgQPro->process_mysql_query() should always return a qpo + // This block was moved from 'handler_special_queries' to support + // handling of 'USE' statements which are preceded by a comment. + // For more context check issue: #3493. + // =================================================== + if (session_type != PROXYSQL_SESSION_CLICKHOUSE) { + const char* qd = CurrentQuery.get_digest_text(); + bool use_db_query = false; + + if (qd != NULL) { + if ( + (strncasecmp((char*)"USE", qd, 3) == 0) + && + ( + (strncasecmp((char*)"USE ", qd, 4) == 0) + || + (strncasecmp((char*)"USE`", qd, 4) == 0) + ) + ) { + use_db_query = true; + } + } + else { + if (pkt.size > (5 + 4) && strncasecmp((char*)"USE ", (char*)pkt.ptr + 5, 4) == 0) { + use_db_query = true; + } + } + + if (use_db_query) { + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_USE_DB(&pkt); + + if (mirror == false) { + break; + } + else { + handler_ret = -1; + return handler_ret; + } + } + } + // =================================================== + if (qpo->max_lag_ms >= 0) { + thread->status_variables.stvar[st_var_queries_with_max_lag_ms]++; + } + rc_break = handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(&pkt, &lock_hostgroup); + if (mirror == false && rc_break == false) { + if (pgsql_thread___automatic_detect_sqli) { + if (handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_detect_SQLi()) { + handler_ret = -1; + return handler_ret; + } + } + } + if (rc_break == true) { + if (mirror == false) { + break; + } + else { + handler_ret = -1; + return handler_ret; + } + } + if (mirror == false) { + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___create_mirror_session(); + } + + if (autocommit_on_hostgroup >= 0) { + } + if (pgsql_thread___set_query_lock_on_hostgroup == 1) { // algorithm introduced in 2.0.6 + if (locked_on_hostgroup < 0) { + if (lock_hostgroup) { + // we are locking on hostgroup now + if (qpo->destination_hostgroup >= 0) { + if (transaction_persistent_hostgroup == -1) { + current_hostgroup = qpo->destination_hostgroup; + } + } + locked_on_hostgroup = current_hostgroup; + thread->status_variables.stvar[st_var_hostgroup_locked]++; + thread->status_variables.stvar[st_var_hostgroup_locked_set_cmds]++; + } + } + if (locked_on_hostgroup >= 0) { + if (current_hostgroup != locked_on_hostgroup) { + client_myds->DSS = STATE_QUERY_SENT_NET; + int l = CurrentQuery.QueryLength; + char* end = (char*)""; + if (l > 256) { + l = 253; + end = (char*)"..."; + } + string nqn = string((char*)CurrentQuery.QueryPointer, l); + char* err_msg = (char*)"Session trying to reach HG %d while locked on HG %d . Rejecting query: %s"; + char* buf = (char*)malloc(strlen(err_msg) + strlen(nqn.c_str()) + strlen(end) + 64); + sprintf(buf, err_msg, current_hostgroup, locked_on_hostgroup, nqn.c_str(), end); + client_myds->myprot.generate_error_packet(true, true, buf, PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION, + false, true); + thread->status_variables.stvar[st_var_hostgroup_locked_queries]++; + RequestEnd(NULL); + free(buf); + l_free(pkt.size, pkt.ptr); + break; + } + } + } + mybe = find_or_create_backend(current_hostgroup); + status = PROCESSING_QUERY; + // set query retries + mybe->server_myds->query_retries_on_failure = pgsql_thread___query_retries_on_failure; + // if a number of retries is set in mysql_query_rules, that takes priority + if (qpo) { + if (qpo->retries >= 0) { + mybe->server_myds->query_retries_on_failure = qpo->retries; + } + } + mybe->server_myds->connect_retries_on_failure = pgsql_thread___connect_retries_on_failure; + mybe->server_myds->wait_until = 0; + pause_until = 0; + if (pgsql_thread___default_query_delay) { + pause_until = thread->curtime + pgsql_thread___default_query_delay * 1000; + } + if (qpo) { + if (qpo->delay > 0) { + if (pause_until == 0) + pause_until = thread->curtime; + pause_until += qpo->delay * 1000; + } + } + + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Received query to be processed with MariaDB Client library\n"); + mybe->server_myds->killed_at = 0; + mybe->server_myds->kill_type = 0; +#if 0 + if (GloMyLdapAuth) { + if (session_type == PROXYSQL_SESSION_PGSQL) { + if (mysql_thread___add_ldap_user_comment && strlen(mysql_thread___add_ldap_user_comment)) { + add_ldap_comment_to_pkt(&pkt); + } + } + } +#endif // 0 + mybe->server_myds->mysql_real_query.init(&pkt); + mybe->server_myds->statuses.questions++; + client_myds->setDSS_STATE_QUERY_SENT_NET(); + } + else { + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY___not_mysql(pkt); + } + break; + case _MYSQL_COM_STMT_PREPARE: +#if 0 + if (GloMyLdapAuth) { + if (session_type == PROXYSQL_SESSION_PGSQL) { + if (mysql_thread___add_ldap_user_comment && strlen(mysql_thread___add_ldap_user_comment)) { + add_ldap_comment_to_pkt(&pkt); + } + } + } +#endif // 0 + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_PREPARE(pkt); + break; + case _MYSQL_COM_STMT_EXECUTE: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_EXECUTE(pkt); + break; +#if 0 + case _MYSQL_COM_STMT_RESET: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_RESET(pkt); + break; + case _MYSQL_COM_STMT_CLOSE: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_CLOSE(pkt); + break; + case _MYSQL_COM_STMT_SEND_LONG_DATA: + handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_SEND_LONG_DATA(pkt); + break; + case _MYSQL_COM_BINLOG_DUMP: + case _MYSQL_COM_BINLOG_DUMP_GTID: + case _MYSQL_COM_REGISTER_SLAVE: + // In this switch we handle commands that download binlog events from MySQL + // servers. For these commands a lot of the features provided by ProxySQL + // aren't useful, like multiplexing, query parsing, etc. For this reason, + // ProxySQL enables fast_forward when it receives these commands.  + { + // we use a switch to write the command in the info message + std::string q = "Received command "; + switch ((enum_mysql_command)c) { + case _MYSQL_COM_BINLOG_DUMP: + q += "MYSQL_COM_BINLOG_DUMP"; + break; + case _MYSQL_COM_BINLOG_DUMP_GTID: + q += "MYSQL_COM_BINLOG_DUMP_GTID"; + break; + case _MYSQL_COM_REGISTER_SLAVE: + q += "MYSQL_COM_REGISTER_SLAVE"; + break; + default: + assert(0); + break; + }; + // we add the client details in the info message + if (client_myds && client_myds->addr.addr) { + q += " from client " + std::string(client_myds->addr.addr) + ":" + std::to_string(client_myds->addr.port); + } + q += " . Changing session fast_forward to true"; + proxy_info("%s\n", q.c_str()); + } + session_fast_forward = true; + + if (client_myds->PSarrayIN->len) { + proxy_error("UNEXPECTED PACKET FROM CLIENT -- PLEASE REPORT A BUG\n"); + assert(0); + } + client_myds->PSarrayIN->add(pkt.ptr, pkt.size); + + // The following code prepares the session as if it was configured with fast + // forward before receiving the command. This way the state machine will + // handle the command automatically. + current_hostgroup = previous_hostgroup; + mybe = find_or_create_backend(current_hostgroup); // set a backend + mybe->server_myds->reinit_queues(); // reinitialize the queues in the myds . By default, they are not active + // We reinitialize the 'wait_until' since this session shouldn't wait for processing as + // we are now transitioning to 'FAST_FORWARD'. + mybe->server_myds->wait_until = 0; + if (mybe->server_myds->DSS == STATE_NOT_INITIALIZED) { + // NOTE: This section is entirely borrowed from 'STATE_SLEEP' for 'session_fast_forward'. + // Check comments there for extra information. + // ============================================================================= + if (mybe->server_myds->max_connect_time == 0) { + uint64_t connect_timeout = + pgsql_thread___connect_timeout_server < pgsql_thread___connect_timeout_server_max ? + pgsql_thread___connect_timeout_server_max : pgsql_thread___connect_timeout_server; + mybe->server_myds->max_connect_time = thread->curtime + connect_timeout * 1000; + } + mybe->server_myds->connect_retries_on_failure = pgsql_thread___connect_retries_on_failure; + CurrentQuery.start_time = thread->curtime; + // ============================================================================= + + // we don't have a connection + previous_status.push(FAST_FORWARD); // next status will be FAST_FORWARD + set_status(CONNECTING_SERVER); // now we need a connection + } + else { + // In case of having a connection, we need to make user to reset the state machine + // for current server 'PgSQL_Data_Stream', setting it outside of any state handled + // by 'mariadb' library. Otherwise 'MySQL_Thread' will threat this + // 'PgSQL_Data_Stream' as library handled. + mybe->server_myds->DSS = STATE_READY; + // myds needs to have encrypted value set correctly + { + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + assert(myconn != NULL); + // PMC-10005 + // if backend connection uses SSL we will set + // encrypted = true and we will start using the SSL structure + // directly from P_MARIADB_TLS structure. + MYSQL* pgsql = myconn->pgsql; + if (pgsql && myconn->ret_mysql) { + if (pgsql->options.use_ssl == 1) { + P_MARIADB_TLS* matls = (P_MARIADB_TLS*)pgsql->net.pvio->ctls; + if (matls != NULL) { + myds->encrypted = true; + myds->ssl = (SSL*)matls->ssl; + myds->rbio_ssl = BIO_new(BIO_s_mem()); + myds->wbio_ssl = BIO_new(BIO_s_mem()); + SSL_set_bio(myds->ssl, myds->rbio_ssl, myds->wbio_ssl); + } + else { + // if pgsql->options.use_ssl == 1 but matls == NULL + // it means that ProxySQL tried to use SSL to connect to the backend + // but the backend didn't support SSL + } + } + } + } + set_status(FAST_FORWARD); // we can set status to FAST_FORWARD + } + + break; + case _MYSQL_COM_QUIT: + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_QUIT packet\n"); + if (GloPgSQL_Logger) { GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_QUIT, this, NULL); } + l_free(pkt.size, pkt.ptr); + handler_ret = -1; + return handler_ret; + break; +#endif // 0 + default: + // in this switch we only handle the most common commands. + // The not common commands are handled by "default" , that + // calls the following function + // handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM__various + if (handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM__various(&pkt, &wrong_pass) == false) { + // If even this cannot find the command, we return an error to the client + proxy_error("RECEIVED AN UNKNOWN COMMAND: %d -- PLEASE REPORT A BUG\n", c); + l_free(pkt.size, pkt.ptr); + handler_ret = -1; // immediately drop the connection + return handler_ret; + } + break; + } + break; + default: + handler___status_WAITING_CLIENT_DATA___default(); + handler_ret = -1; + return handler_ret; + break; + } + break; + case FAST_FORWARD: + mybe->server_myds->PSarrayOUT->add(pkt.ptr, pkt.size); + break; + // This state is required because it covers the following situation: + // 1. A new connection is created by a client and the 'FAST_FORWARD' mode is enabled. + // 2. The first packet received for this connection isn't a whole packet, i.e, it's either + // split into multiple packets, or it doesn't fit 'queueIN' size (typically + // QUEUE_T_DEFAULT_SIZE). + // 3. Session is still in 'CONNECTING_SERVER' state, BUT further packets remain to be received + // from the initial split packet. + // + // Because of this, packets received during 'CONNECTING_SERVER' when the previous state is + // 'FAST_FORWARD' should be pushed to 'PSarrayOUT'. + case CONNECTING_SERVER: + if (previous_status.empty() == false && previous_status.top() == FAST_FORWARD) { + mybe->server_myds->PSarrayOUT->add(pkt.ptr, pkt.size); + break; + } + case session_status___NONE: + default: + handler___status_NONE_or_default(pkt); + handler_ret = -1; + return handler_ret; + break; + } + } + return handler_ret; +} +// end of PgSQL_Session::get_pkts_from_client() + + +// this function returns: +// 0 : no action +// -1 : the calling function will return +// 1 : call to NEXT_IMMEDIATE +int PgSQL_Session::handler_ProcessingQueryError_CheckBackendConnectionStatus(PgSQL_Data_Stream* myds) { + PgSQL_Connection* myconn = myds->myconn; + // the query failed + if (myconn->IsServerOffline()) { + // Set maximum connect time if connect timeout is configured + if (pgsql_thread___connect_timeout_server_max) { + myds->max_connect_time = thread->curtime + pgsql_thread___connect_timeout_server_max * 1000; + } + + // Variables to track retry and error conditions + bool retry_conn = false; + if (myconn->server_status == MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) { + thread->status_variables.stvar[st_var_backend_lagging_during_query]++; + proxy_error("Detected a lagging server during query: %s, %d\n", myconn->parent->address, myconn->parent->port); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, ER_PROXYSQL_LAGGING_SRV); + } else { + thread->status_variables.stvar[st_var_backend_offline_during_query]++; + proxy_error("Detected an offline server during query: %s, %d\n", myconn->parent->address, myconn->parent->port); + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, ER_PROXYSQL_OFFLINE_SRV); + } + + // Retry the query if retries are allowed and conditions permit + if (myds->query_retries_on_failure > 0) { + myds->query_retries_on_failure--; + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + if (myds->myconn->query_result && myds->myconn->query_result->is_transfer_started()) { + // transfer to frontend has started, we cannot retry + } else { + retry_conn = true; + proxy_warning("Retrying query.\n"); + } + } + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + // Sets the previous status of the PgSQL session according to the current status. + set_previous_status_mode3(); + return 1; + } + return -1; + } + return 0; +} + +void PgSQL_Session::SetQueryTimeout() { + mybe->server_myds->wait_until = 0; + if (qpo) { + if (qpo->timeout > 0) { + unsigned long long qr_timeout = qpo->timeout; + mybe->server_myds->wait_until = thread->curtime; + mybe->server_myds->wait_until += qr_timeout * 1000; + } + } + if (pgsql_thread___default_query_timeout) { + if (mybe->server_myds->wait_until == 0) { + mybe->server_myds->wait_until = thread->curtime; + unsigned long long def_query_timeout = pgsql_thread___default_query_timeout; + mybe->server_myds->wait_until += def_query_timeout * 1000; + } + } +} + +// this function used to be inline. +// now it returns: +// true: NEXT_IMMEDIATE(st) needs to be called +// false: continue +bool PgSQL_Session::handler_rc0_PROCESSING_STMT_PREPARE(enum session_status& st, PgSQL_Data_Stream* myds, bool& prepared_stmt_with_no_params) { + thread->status_variables.stvar[st_var_backend_stmt_prepare]++; + GloMyStmt->wrlock(); + uint32_t client_stmtid = 0; + uint64_t global_stmtid; + //bool is_new; + MySQL_STMT_Global_info* stmt_info = NULL; + stmt_info = GloMyStmt->add_prepared_statement( + (char*)client_myds->myconn->userinfo->username, + (char*)client_myds->myconn->userinfo->dbname, + (char*)CurrentQuery.QueryPointer, + CurrentQuery.QueryLength, + CurrentQuery.QueryParserArgs.first_comment, + CurrentQuery.mysql_stmt, + false); + if (CurrentQuery.QueryParserArgs.digest_text) { + if (stmt_info->digest_text == NULL) { + stmt_info->digest_text = strdup(CurrentQuery.QueryParserArgs.digest_text); + stmt_info->digest = CurrentQuery.QueryParserArgs.digest; // copy digest + //stmt_info->MyComQueryCmd = CurrentQuery.PgQueryCmd; // copy MyComQueryCmd + stmt_info->calculate_mem_usage(); + } + } + global_stmtid = stmt_info->statement_id; + myds->myconn->local_stmts->backend_insert(global_stmtid, CurrentQuery.mysql_stmt); + // We only perform the generation for a new 'client_stmt_id' when there is no previous status, this + // is, when 'PROCESSING_STMT_PREPARE' is reached directly without transitioning from a previous status + // like 'PROCESSING_STMT_EXECUTE'. The same condition needs to hold for setting 'stmt_client_id', + // otherwise we could be resetting it's current value from the previous state. + if (previous_status.size() == 0) { + client_stmtid = client_myds->myconn->local_stmts->generate_new_client_stmt_id(global_stmtid); + CurrentQuery.stmt_client_id = client_stmtid; + } + CurrentQuery.mysql_stmt = NULL; + st = status; + size_t sts = previous_status.size(); + if (sts) { + myds->myconn->async_state_machine = ASYNC_IDLE; + myds->DSS = STATE_MARIADB_GENERIC; + st = previous_status.top(); + previous_status.pop(); + GloMyStmt->unlock(); + return true; + //NEXT_IMMEDIATE(st); + } + else { + client_myds->myprot.generate_STMT_PREPARE_RESPONSE(client_myds->pkt_sid + 1, stmt_info, client_stmtid); + if (stmt_info->num_params == 0) { + prepared_stmt_with_no_params = true; + } + LogQuery(myds); + GloMyStmt->unlock(); + } + return false; +} + + +// this function used to be inline +void PgSQL_Session::handler_rc0_PROCESSING_STMT_EXECUTE(PgSQL_Data_Stream* myds) { + thread->status_variables.stvar[st_var_backend_stmt_execute]++; + PROXY_TRACE2(); + if (CurrentQuery.mysql_stmt) { + // See issue #1574. Metadata needs to be updated in case of need also + // during STMT_EXECUTE, so a failure in the prepared statement + // metadata cache is only hit once. This way we ensure that the next + // 'PREPARE' will be answered with the properly updated metadata. + /********************************************************************/ + // Lock the global statement manager + GloMyStmt->wrlock(); + // Update the global prepared statement metadata + MySQL_STMT_Global_info* stmt_info = GloMyStmt->find_prepared_statement_by_stmt_id(CurrentQuery.stmt_global_id, false); + stmt_info->update_metadata(CurrentQuery.mysql_stmt); + // Unlock the global statement manager + GloMyStmt->unlock(); + /********************************************************************/ + } + MySQL_Stmt_Result_to_MySQL_wire(CurrentQuery.mysql_stmt, myds->myconn); + LogQuery(myds); + if (CurrentQuery.stmt_meta) { + if (CurrentQuery.stmt_meta->pkt) { + uint32_t stmt_global_id = 0; + memcpy(&stmt_global_id, (char*)(CurrentQuery.stmt_meta->pkt) + 5, sizeof(uint32_t)); + SLDH->reset(stmt_global_id); + free(CurrentQuery.stmt_meta->pkt); + CurrentQuery.stmt_meta->pkt = NULL; + } + + // free for all the buffer types in which we allocate + for (int i = 0; i < CurrentQuery.stmt_meta->num_params; i++) { + enum enum_field_types buffer_type = + CurrentQuery.stmt_meta->binds[i].buffer_type; + + if ( + (buffer_type == MYSQL_TYPE_TIME) || + (buffer_type == MYSQL_TYPE_DATE) || + (buffer_type == MYSQL_TYPE_TIMESTAMP) || + (buffer_type == MYSQL_TYPE_DATETIME) + ) { + free(CurrentQuery.stmt_meta->binds[i].buffer); + // NOTE: This memory should be zeroed during initialization, + // but we also nullify it here for extra safety. See #3546. + CurrentQuery.stmt_meta->binds[i].buffer = NULL; + } + } + } + CurrentQuery.mysql_stmt = NULL; +} + +// this function used to be inline. +// now it returns: +// true: NEXT_IMMEDIATE(CONNECTING_SERVER) needs to be called +// false: continue +bool PgSQL_Session::handler_minus1_ClientLibraryError(PgSQL_Data_Stream* myds) { + PgSQL_Connection* myconn = myds->myconn; + bool retry_conn = false; + // client error, serious + detected_broken_connection(__FILE__, __LINE__, __func__, "running query", myconn, true); + if (myds->query_retries_on_failure > 0) { + myds->query_retries_on_failure--; + if ((myconn->reusable == true) && myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false) { + if (myconn->query_result && myconn->query_result->is_transfer_started()) { + // transfer to frontend has started, we cannot retry + } else { + // This should never occur. + if (myconn->processing_multi_statement == true) { + // we are in the process of retriving results from a multi-statement query + proxy_warning("Disabling query retry because we were in middle of processing results\n"); + } else { + retry_conn = true; + proxy_warning("Retrying query.\n"); + } + } + } + } + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + // Sets the previous status of the PgSQL session according to the current status. + set_previous_status_mode3(); + return true; + } + return false; +} + + +// this function was inline +void PgSQL_Session::handler_minus1_LogErrorDuringQuery(PgSQL_Connection* myconn) { + if (pgsql_thread___verbose_query_error) { + proxy_warning("Error during query on (%d,%s,%d,%lu) , user \"%s@%s\" , dbname \"%s\" , %s . digest_text = \"%s\"\n", myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, myconn->get_mysql_thread_id(), client_myds->myconn->userinfo->username, (client_myds->addr.addr ? client_myds->addr.addr : (char*)"unknown"), client_myds->myconn->userinfo->dbname, myconn->get_error_code_with_message().c_str(), CurrentQuery.QueryParserArgs.digest_text); + } else { + proxy_warning("Error during query on (%d,%s,%d,%lu): %s\n", myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, myconn->get_mysql_thread_id(), myconn->get_error_code_with_message().c_str()); + } + PgHGM->add_pgsql_errors(myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, client_myds->myconn->userinfo->username, + (client_myds->addr.addr ? client_myds->addr.addr : "unknown"), client_myds->myconn->userinfo->dbname, + myconn->get_error_code_str(), myconn->get_error_message().c_str()); +} + + +// this function used to be inline. +// now it returns: +// true: +// if handler_ret == -1 : return +// if handler_ret == 0 : NEXT_IMMEDIATE(CONNECTING_SERVER) needs to be called +// false: continue +bool PgSQL_Session::handler_minus1_HandleErrorCodes(PgSQL_Data_Stream* myds, int& handler_ret) { + bool retry_conn = false; + PgSQL_Connection* myconn = myds->myconn; + handler_ret = 0; // default + switch (myconn->get_error_code()) { + case PGSQL_ERROR_CODES::ERRCODE_QUERY_CANCELED: // Query execution was interrupted + if (killed == true) { // this session is being kiled + handler_ret = -1; + return true; + } + if (myds->killed_at) { + // we intentionally killed the query + break; + } + break; + case PGSQL_ERROR_CODES::ERRCODE_ADMIN_SHUTDOWN: // Server shutdown in progress. Requested by Admin + case PGSQL_ERROR_CODES::ERRCODE_CRASH_SHUTDOWN: // Server shutdown in progress + case PGSQL_ERROR_CODES::ERRCODE_CANNOT_CONNECT_NOW: // Server in initialization mode and not ready to handle new connections + myconn->parent->connect_error(9999); + if (myds->query_retries_on_failure > 0) { + myds->query_retries_on_failure--; + if ((myconn->reusable == true) && myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false) { + retry_conn = true; + proxy_warning("Retrying query.\n"); + } + } + myds->destroy_MySQL_Connection_From_Pool(false); + myconn = myds->myconn; + myds->fd = 0; + if (retry_conn) { + myds->DSS = STATE_NOT_INITIALIZED; + //previous_status.push(PROCESSING_QUERY); + set_previous_status_mode3(false); + return true; // it will call NEXT_IMMEDIATE(CONNECTING_SERVER); + //NEXT_IMMEDIATE(CONNECTING_SERVER); + } + //handler_ret = -1; + //return handler_ret; + break; + case PGSQL_ERROR_CODES::ERRCODE_OUT_OF_MEMORY: + proxy_warning("Error OUT_OF_MEMORY during query on (%d,%s,%d,%lu): %s\n", myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, myconn->get_mysql_thread_id(), myconn->get_error_code_with_message().c_str()); + break; + default: + break; // continue normally + } + return false; +} + +// this function used to be inline. +void PgSQL_Session::handler_minus1_GenerateErrorMessage(PgSQL_Data_Stream* myds, bool& wrong_pass) { + PgSQL_Connection* myconn = myds->myconn; + switch (status) { + case PROCESSING_QUERY: + if (myconn) { + PgSQL_Result_to_PgSQL_wire(myconn, myds); + } + else { + PgSQL_Result_to_PgSQL_wire(NULL, myds); + } + break; + case PROCESSING_STMT_PREPARE: + { + char sqlstate[10]; + if (myconn && myconn->pgsql) { + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, mysql_errno(myconn->pgsql), sqlstate, (char*)mysql_stmt_error(myconn->query.stmt)); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_CLOSE, this, NULL); + } + else { + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, 2013, (char*)"HY000", (char*)"Lost connection to MySQL server during query"); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_CLOSE, this, NULL); + } + client_myds->pkt_sid++; + if (previous_status.size()) { + // an STMT_PREPARE failed + // we have a previous status, probably STMT_EXECUTE, + // but returning to that status is not safe after STMT_PREPARE failed + // for this reason we exit immediately + wrong_pass = true; + } + } + break; + case PROCESSING_STMT_EXECUTE: + { + char sqlstate[10]; + if (myconn && myconn->pgsql) { + if (myconn->query_result) { + PROXY_TRACE2(); + myds->sess->handler_rc0_PROCESSING_STMT_EXECUTE(myds); + } + else { + sprintf(sqlstate, "%s", mysql_sqlstate(myconn->pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, mysql_errno(myconn->pgsql), sqlstate, (char*)mysql_stmt_error(myconn->query.stmt)); + } + } + else { + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, 2013, (char*)"HY000", (char*)"Lost connection to MySQL server during query"); + } + client_myds->pkt_sid++; + } + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } +} + +// this function was inline +void PgSQL_Session::handler_minus1_HandleBackendConnection(PgSQL_Data_Stream* myds) { + PgSQL_Connection* myconn = myds->myconn; + if (myconn) { + myconn->reduce_auto_increment_delay_token(); + if (pgsql_thread___multiplexing && (myconn->reusable == true) && myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false) { + myds->DSS = STATE_NOT_INITIALIZED; + if (mysql_thread___autocommit_false_not_reusable && myconn->IsAutoCommit() == false) { + create_new_session_and_reset_connection(myds); + } else { + myds->return_MySQL_Connection_To_Pool(); + } + } else { + myconn->async_state_machine = ASYNC_IDLE; + myds->DSS = STATE_MARIADB_GENERIC; + } + } +} + +// this function was inline +int PgSQL_Session::RunQuery(PgSQL_Data_Stream* myds, PgSQL_Connection* myconn) { + PROXY_TRACE2(); + int rc = 0; + switch (status) { + case PROCESSING_QUERY: + rc = myconn->async_query(myds->revents, myds->mysql_real_query.QueryPtr, myds->mysql_real_query.QuerySize); + break; + case PROCESSING_STMT_PREPARE: + rc = myconn->async_query(myds->revents, (char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength, &CurrentQuery.mysql_stmt); + break; + case PROCESSING_STMT_EXECUTE: + PROXY_TRACE2(); + rc = myconn->async_query(myds->revents, (char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength, &CurrentQuery.mysql_stmt, CurrentQuery.stmt_meta); + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + return rc; +} + +// this function was inline +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA() { + // NOTE: Maintenance of 'multiplex_delayed' has been moved to 'housekeeping_before_pkts'. The previous impl + // is left below as an example of how to perform a more passive maintenance over session connections. +} + +// this function was inline +void PgSQL_Session::handler_rc0_Process_GTID(PgSQL_Connection* myconn) { + if (myconn->get_gtid(mybe->gtid_uuid, &mybe->gtid_trxid)) { + + } +} + +int PgSQL_Session::handler() { +#if ENABLE_TIMER + Timer timer(thread->Timers.Sessions_Handlers); +#endif // ENABLE_TIMER + int handler_ret = 0; + bool prepared_stmt_with_no_params = false; + bool wrong_pass = false; + if (to_process == 0) return 0; // this should be redundant if the called does the same check + proxy_debug(PROXY_DEBUG_NET, 1, "Thread=%p, Session=%p -- Processing session %p\n", this->thread, this, this); + //unsigned int j; + //unsigned char c; + +// FIXME: Sessions without frontend are an ugly hack + if (session_fast_forward == false) { + if (client_myds == NULL) { + // if we are here, probably we are trying to ping backends + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Processing session %p without client_myds\n", this); + assert(mybe); + assert(mybe->server_myds); + goto handler_again; + } + else { + if (mirror == true) { + if (mirrorPkt.ptr) { // this is the first time we call handler() + pkt.ptr = mirrorPkt.ptr; + pkt.size = mirrorPkt.size; + mirrorPkt.ptr = NULL; // this will prevent the copy to happen again + } + else { + if (status == WAITING_CLIENT_DATA) { + // we are being called a second time with WAITING_CLIENT_DATA + handler_ret = 0; + return handler_ret; + } + } + } + } + } + + housekeeping_before_pkts(); + handler_ret = get_pkts_from_client(wrong_pass, pkt); + if (handler_ret != 0) { + return handler_ret; + } + +handler_again: + + switch (status) { + case WAITING_CLIENT_DATA: + // housekeeping + handler___status_WAITING_CLIENT_DATA(); + break; + case FAST_FORWARD: + if (mybe->server_myds->mypolls == NULL) { + // register the PgSQL_Data_Stream + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + client_myds->PSarrayOUT->copy_add(mybe->server_myds->PSarrayIN, 0, mybe->server_myds->PSarrayIN->len); + while (mybe->server_myds->PSarrayIN->len) mybe->server_myds->PSarrayIN->remove_index(mybe->server_myds->PSarrayIN->len - 1, NULL); + break; + case CONNECTING_CLIENT: + //fprintf(stderr,"CONNECTING_CLIENT\n"); + // FIXME: to implement + break; + case PINGING_SERVER: + { + int rc = handler_again___status_PINGING_SERVER(); + if (rc == -1) { // if the ping fails, we destroy the session + handler_ret = -1; + return handler_ret; + } + } + break; + + case RESETTING_CONNECTION: + { + int rc = handler_again___status_RESETTING_CONNECTION(); + if (rc == -1) { // we always destroy the session + handler_ret = -1; + return handler_ret; + } + } + break; + + case PROCESSING_STMT_PREPARE: + case PROCESSING_STMT_EXECUTE: + case PROCESSING_QUERY: + //fprintf(stderr,"PROCESSING_QUERY\n"); + if (pause_until > thread->curtime) { + handler_ret = 0; + return handler_ret; + } + if (pgsql_thread___connect_timeout_server_max) { + if (mybe->server_myds->max_connect_time == 0) + mybe->server_myds->max_connect_time = thread->curtime + (long long)pgsql_thread___connect_timeout_server_max * 1000; + } + else { + mybe->server_myds->max_connect_time = 0; + } + if ( + (mybe->server_myds->myconn && mybe->server_myds->myconn->async_state_machine != ASYNC_IDLE && mybe->server_myds->wait_until && thread->curtime >= mybe->server_myds->wait_until) + // query timed out + || + (killed == true) // session was killed by admin + ) { + // we only log in case on timing out here. Logging for 'killed' is done in the places that hold that contextual information. + if (mybe->server_myds->myconn && (mybe->server_myds->myconn->async_state_machine != ASYNC_IDLE) && mybe->server_myds->wait_until && (thread->curtime >= mybe->server_myds->wait_until)) { + std::string query{}; + + if (CurrentQuery.stmt_info == NULL) { // text protocol + query = std::string{ mybe->server_myds->myconn->query.ptr, mybe->server_myds->myconn->query.length }; + } + else { // prepared statement + query = std::string{ CurrentQuery.stmt_info->query, CurrentQuery.stmt_info->query_length }; + } + + std::string client_addr{ "" }; + int client_port = 0; + + if (client_myds) { + client_addr = client_myds->addr.addr ? client_myds->addr.addr : ""; + client_port = client_myds->addr.port; + } + + proxy_warning( + "Killing connection %s:%d because query '%s' from client '%s':%d timed out.\n", + mybe->server_myds->myconn->parent->address, + mybe->server_myds->myconn->parent->port, + query.c_str(), + client_addr.c_str(), + client_port + ); + } + handler_again___new_thread_to_kill_connection(); + } + if (mybe->server_myds->DSS == STATE_NOT_INITIALIZED) { + // we don't have a backend yet + // It saves the current processing status of the session (status) onto the previous_status stack + // Sets the previous status of the PgSQL session according to the current status. + set_previous_status_mode3(); + // It transitions the session to the CONNECTING_SERVER state immediately. + NEXT_IMMEDIATE(CONNECTING_SERVER); + } else { + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + mybe->server_myds->max_connect_time = 0; + // we insert it in mypolls only if not already there + if (myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, mybe->server_myds->fd, mybe->server_myds, thread->curtime); + } + if (default_hostgroup >= 0) { + if (handler_again___verify_backend_user_db()) { + goto handler_again; + } + if (mirror == false) { // do not care about autocommit and charset if mirror + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session %p , default_HG=%d server_myds DSS=%d , locked_on_HG=%d\n", this, default_hostgroup, mybe->server_myds->DSS, locked_on_hostgroup); + if (mybe->server_myds->DSS == STATE_READY || mybe->server_myds->DSS == STATE_MARIADB_GENERIC) { + if (handler_again___verify_init_connect()) { + goto handler_again; + } +#if 0 + if (use_ldap_auth) { + if (handler_again___verify_ldap_user_variable()) { + goto handler_again; + } + } + if (handler_again___verify_backend_autocommit()) { + goto handler_again; + } +#endif // 0 + if (locked_on_hostgroup == -1 || locked_on_hostgroup_and_all_variables_set == false) { + +#if 0 + if (handler_again___verify_backend_multi_statement()) { + goto handler_again; + } + + if (handler_again___verify_backend_session_track_gtids()) { + goto handler_again; + } +#endif // 0 + // Optimize network traffic when we can use 'SET NAMES' + //if (verify_set_names(this)) { + // goto handler_again; + //} + + for (auto i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + auto client_hash = client_myds->myconn->var_hash[i]; +#ifdef DEBUG + if (GloVars.global.gdbg) { + switch (i) { + case SQL_CHARACTER_SET: + case SQL_SET_NAMES: + case SQL_CHARACTER_SET_RESULTS: + case SQL_CHARACTER_SET_CONNECTION: + case SQL_CHARACTER_SET_CLIENT: + case SQL_COLLATION_CONNECTION: + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Session %p , variable %s has value %s\n", this, mysql_tracked_variables[i].set_variable_name, client_myds->myconn->variables[i].value); + default: + break; + } + } +#endif // DEBUG + if (client_hash) { + auto server_hash = myconn->var_hash[i]; + if (client_hash != server_hash) { + if (!myconn->var_absent[i] && pgsql_variables.verify_variable(this, i)) { + goto handler_again; + } + } + } + } + PgSQL_Connection* c_con = client_myds->myconn; + vector::const_iterator it_c = c_con->dynamic_variables_idx.begin(); // client connection iterator + for (; it_c != c_con->dynamic_variables_idx.end(); it_c++) { + auto i = *it_c; + auto client_hash = c_con->var_hash[i]; + auto server_hash = myconn->var_hash[i]; + if (client_hash != server_hash) { + if ( + !myconn->var_absent[i] + && + pgsql_variables.verify_variable(this, i) + ) { + goto handler_again; + } + } + } + + if (locked_on_hostgroup != -1) { + locked_on_hostgroup_and_all_variables_set = true; + } + } + } + if (status == PROCESSING_STMT_EXECUTE) { + CurrentQuery.mysql_stmt = myconn->local_stmts->find_backend_stmt_by_global_id(CurrentQuery.stmt_global_id); + if (CurrentQuery.mysql_stmt == NULL) { + MySQL_STMT_Global_info* stmt_info = NULL; + // the connection we too doesn't have the prepared statements prepared + // we try to create it now + stmt_info = GloMyStmt->find_prepared_statement_by_stmt_id(CurrentQuery.stmt_global_id); + CurrentQuery.QueryLength = stmt_info->query_length; + CurrentQuery.QueryPointer = (unsigned char*)stmt_info->query; + // NOTE: Update 'first_comment' with the 'first_comment' from the retrieved + // 'stmt_info' from the found prepared statement. 'CurrentQuery' requires its + // own copy of 'first_comment' because it will later be free by 'QueryInfo::end'. + if (stmt_info->first_comment) { + CurrentQuery.QueryParserArgs.first_comment = strdup(stmt_info->first_comment); + } + previous_status.push(PROCESSING_STMT_EXECUTE); + NEXT_IMMEDIATE(PROCESSING_STMT_PREPARE); + if (CurrentQuery.stmt_global_id != stmt_info->statement_id) { + PROXY_TRACE(); + } + } + } + } + } + + if (myconn->async_state_machine == ASYNC_IDLE) { + SetQueryTimeout(); + } + int rc; + timespec begint; + if (thread->variables.stats_time_backend_query) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &begint); + } + rc = RunQuery(myds, myconn); + timespec endt; + if (thread->variables.stats_time_backend_query) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &endt); + thread->status_variables.stvar[st_var_backend_query_time] = thread->status_variables.stvar[st_var_backend_query_time] + + (endt.tv_sec * 1000000000 + endt.tv_nsec) - + (begint.tv_sec * 1000000000 + begint.tv_nsec); + } + + if (rc == 0) { + + if (active_transactions != 0) { // run this only if currently we think there is a transaction + if (myconn->IsKnownActiveTransaction() == false) { // there is no transaction on the backend connection + active_transactions = NumActiveTransactions(); // we check all the hostgroups/backends + if (active_transactions == 0) + transaction_started_at = 0; // reset it + } + } + + //handler_rc0_Process_GTID(myconn); + + // if we are locked on hostgroup, the value of autocommit is copied from the backend connection + // see bug #3549 + if (locked_on_hostgroup >= 0) { + assert(myconn != NULL); + assert(myconn->pgsql_conn != NULL); + //autocommit = myconn->pgsql->server_status & SERVER_STATUS_AUTOCOMMIT; + } + + /*if (mirror == false && myconn->pgsql) { + // Support for LAST_INSERT_ID() + if (myconn->pgsql->insert_id) { + last_insert_id = myconn->pgsql->insert_id; + } + if (myconn->pgsql->affected_rows) { + if (myconn->pgsql->affected_rows != ULLONG_MAX) { + last_HG_affected_rows = current_hostgroup; + if (pgsql_thread___auto_increment_delay_multiplex && myconn->pgsql->insert_id) { + myconn->auto_increment_delay_token = pgsql_thread___auto_increment_delay_multiplex + 1; + __sync_fetch_and_add(&PgHGM->status.auto_increment_delay_multiplex, 1); + } + } + } + }*/ + + switch (status) { + case PROCESSING_QUERY: + PgSQL_Result_to_PgSQL_wire(myconn, myconn->myds); + break; + case PROCESSING_STMT_PREPARE: + { + enum session_status st; + if (handler_rc0_PROCESSING_STMT_PREPARE(st, myds, prepared_stmt_with_no_params)) { + NEXT_IMMEDIATE(st); + } + } + break; + case PROCESSING_STMT_EXECUTE: + handler_rc0_PROCESSING_STMT_EXECUTE(myds); + break; + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + + if (mysql_thread___log_mysql_warnings_enabled) { + auto warn_no = mysql_warning_count(myconn->pgsql); + if (warn_no > 0) { + // Backup actual digest causing the warning before it's destroyed by finishing the request + const char* digest_text = CurrentQuery.get_digest_text(); + CurrentQuery.show_warnings_prev_query_digest = digest_text == NULL ? "" : digest_text; + + RequestEnd(myds); + writeout(); + + myconn->async_state_machine = ASYNC_IDLE; + myds->DSS = STATE_MARIADB_GENERIC; + + NEXT_IMMEDIATE(SHOW_WARNINGS); + } + } + + RequestEnd(myds); + finishQuery(myds, myconn, prepared_stmt_with_no_params); + } + else { + if (rc == -1) { + // the query failed + const bool is_error_present = myconn->is_error_present(); // false means failure is due to server being in OFFLINE state + if (is_error_present == false) { + + /*if (CurrentQuery.mysql_stmt) { + myerr = mysql_stmt_errno(CurrentQuery.mysql_stmt); + errmsg = strdup(mysql_stmt_error(CurrentQuery.mysql_stmt)); + }*/ + } + PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::pgsql, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, 9999); // TOFIX + //CurrentQuery.mysql_stmt = NULL; // immediately reset mysql_stmt + int rc1 = handler_ProcessingQueryError_CheckBackendConnectionStatus(myds); + if (rc1 == -1) { + handler_ret = -1; + return handler_ret; + } + else { + if (rc1 == 1) + NEXT_IMMEDIATE(CONNECTING_SERVER); + } + if (myconn->is_connection_in_reusable_state() == false) { + if (handler_minus1_ClientLibraryError(myds)) { + NEXT_IMMEDIATE(CONNECTING_SERVER); + } else { + handler_ret = -1; + return handler_ret; + } + } else { + handler_minus1_LogErrorDuringQuery(myconn); + if (handler_minus1_HandleErrorCodes(myds, handler_ret)) { + if (handler_ret == 0) + NEXT_IMMEDIATE(CONNECTING_SERVER); + return handler_ret; + } + handler_minus1_GenerateErrorMessage(myds, wrong_pass); + RequestEnd(myds); + handler_minus1_HandleBackendConnection(myds); + } + } else { + switch (rc) { + // rc==1 , query is still running + // start sending to frontend if pgsql_thread___threshold_resultset_size is reached + case 1: + if (myconn->query_result && myconn->query_result->get_resultset_size() > (unsigned int)pgsql_thread___threshold_resultset_size) { + myconn->query_result->get_resultset(client_myds->PSarrayOUT); + } + break; + // rc==2 : a multi-resultset (or multi statement) was detected, and the current statement is completed + case 2: + PgSQL_Result_to_PgSQL_wire(myconn, myconn->myds); + if (myconn->query_result) { // we also need to clear query_result, so that the next statement will recreate it if needed + if (myconn->query_result_reuse) { + delete myconn->query_result_reuse; + } + myconn->query_result_reuse = myconn->query_result; + myconn->query_result = NULL; + } + NEXT_IMMEDIATE(PROCESSING_QUERY); + break; + // rc==3 , a multi statement query is still running + // start sending to frontend if pgsql_thread___threshold_resultset_size is reached + case 3: + if (myconn->query_result && myconn->query_result->get_resultset_size() > (unsigned int)pgsql_thread___threshold_resultset_size) { + myconn->query_result->get_resultset(client_myds->PSarrayOUT); + } + break; + default: + break; + } + } + } + goto __exit_DSS__STATE_NOT_INITIALIZED; + } + break; + + case SETTING_ISOLATION_LEVEL: + case SETTING_TRANSACTION_READ: + case SETTING_CHARSET: + case SETTING_VARIABLE: + case SETTING_NEXT_ISOLATION_LEVEL: + case SETTING_NEXT_TRANSACTION_READ: + { + int rc = 0; + if (pgsql_variables.update_variable(this, status, rc)) { + goto handler_again; + } + if (rc == -1) { + handler_ret = -1; + return handler_ret; + } + } + break; + + case SHOW_WARNINGS: + // Performs a 'SHOW WARNINGS' query over the current backend connection and returns the connection back + // to the connection pool when finished. Actual logging of received warnings is performed in + // 'PgSQL_Connection' while processing 'ASYNC_USE_RESULT_CONT'. + { + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = myds->myconn; + + // Setting POLLOUT is required just in case this state has been reached when 'RunQuery' from + // 'PROCESSING_QUERY' state has immediately return. This is because in case 'mysql_real_query_start' + // immediately returns with '0' the session is never processed again by 'MySQL_Thread', and 'revents' is + // never updated with the result of polling through the 'MySQL_Thread::mypolls'. + myds->revents |= POLLOUT; + + int rc = myconn->async_query( + mybe->server_myds->revents, (char*)"SHOW WARNINGS", strlen((char*)"SHOW WARNINGS") + ); + if (rc == 0 || rc == -1) { + // Cleanup the connection resulset from 'SHOW WARNINGS' for the next query. + if (myconn->query_result != NULL) { + delete myconn->query_result; + myconn->query_result = NULL; + } + + if (rc == -1) { + int myerr = mysql_errno(myconn->pgsql); + proxy_error( + "'SHOW WARNINGS' failed to be executed over backend connection with error: '%d'\n", myerr + ); + } + + RequestEnd(myds); + finishQuery(myds, myconn, prepared_stmt_with_no_params); + + handler_ret = 0; + return handler_ret; + } + else { + goto handler_again; + } + } + break; + + case CONNECTING_SERVER: + { + int rc = 0; + if (handler_again___status_CONNECTING_SERVER(&rc)) + goto handler_again; // we changed status + if (rc == 1) //handler_again___status_CONNECTING_SERVER returns 1 + goto __exit_DSS__STATE_NOT_INITIALIZED; + } + break; + case session_status___NONE: + fprintf(stderr, "NONE\n"); + default: + { + int rc = 0; + if (handler_again___multiple_statuses(&rc)) // a sort of catch all + goto handler_again; // we changed status + if (rc == -1) { // we have an error we can't handle + handler_ret = -1; + return handler_ret; + } + } + break; + } + + +__exit_DSS__STATE_NOT_INITIALIZED: + + + if (mybe && mybe->server_myds) { + if (mybe->server_myds->DSS > STATE_MARIADB_BEGIN && mybe->server_myds->DSS < STATE_MARIADB_END) { +#ifdef DEBUG + PgSQL_Data_Stream* myds = mybe->server_myds; + PgSQL_Connection* myconn = mybe->server_myds->myconn; +#endif /* DEBUG */ + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p, status=%d, server_myds->DSS==%d , revents==%d , async_state_machine=%d\n", this, status, mybe->server_myds->DSS, myds->revents, myconn->async_state_machine); + } + } + + writeout(); + + if (wrong_pass == true) { + client_myds->array2buffer_full(); + client_myds->write_to_net(); + handler_ret = -1; + return handler_ret; + } + handler_ret = 0; + return handler_ret; +} +// end ::handler() + + +bool PgSQL_Session::handler_again___multiple_statuses(int* rc) { + bool ret = false; + switch (status) { + case RESETTING_CONNECTION_V2: + ret = handler_again___status_RESETTING_CONNECTION(rc); + break; +#if 0 + case CHANGING_AUTOCOMMIT: + ret = handler_again___status_CHANGING_AUTOCOMMIT(rc); + break; + case CHANGING_SCHEMA: + ret = handler_again___status_CHANGING_SCHEMA(rc); + break; + case SETTING_LDAP_USER_VARIABLE: + ret = handler_again___status_SETTING_LDAP_USER_VARIABLE(rc); + break; +#endif // 0 + case SETTING_INIT_CONNECT: + ret = handler_again___status_SETTING_INIT_CONNECT(rc); + break; +#if 0 + case SETTING_MULTI_STMT: + ret = handler_again___status_SETTING_MULTI_STMT(rc); + break; + case SETTING_SESSION_TRACK_GTIDS: + ret = handler_again___status_SETTING_SESSION_TRACK_GTIDS(rc); + break; +#endif // 0 + case SETTING_SET_NAMES: + ret = handler_again___status_CHANGING_CHARSET(rc); + break; + default: + break; + } + return ret; +} + +/* +void PgSQL_Session::handler___status_CHANGING_USER_CLIENT___STATE_CLIENT_HANDSHAKE(PtrSize_t *pkt, bool *wrong_pass) { + // FIXME: no support for SSL yet + if ( + client_myds->myprot.process_pkt_auth_swich_response((unsigned char *)pkt->ptr,pkt->size)==true + ) { + l_free(pkt->size,pkt->ptr); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , DS=%p . Successful connection\n", this, client_myds); + client_myds->myprot.generate_pkt_OK(true,NULL,NULL,2,0,0,0,0,NULL); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_CHANGE_USER_OK, this, NULL); + status=WAITING_CLIENT_DATA; + client_myds->DSS=STATE_SLEEP; + } else { + l_free(pkt->size,pkt->ptr); + *wrong_pass=true; + // FIXME: this should become close connection + client_myds->setDSS_STATE_QUERY_SENT_NET(); + char *client_addr=NULL; + if (client_myds->client_addr) { + char buf[512]; + switch (client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)client_myds->client_addr; + if (ipv4->sin_port) { + inet_ntop(client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + client_addr = strdup(buf); + } else { + client_addr = strdup((char *)"localhost"); + } + break; + } + case AF_INET6: { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + default: + client_addr = strdup((char *)"localhost"); + break; + } + } else { + client_addr = strdup((char *)""); + } + char *_s=(char *)malloc(strlen(client_myds->myconn->userinfo->username)+100+strlen(client_addr)); + sprintf(_s,"ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + proxy_error("ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,2,1045,(char *)"28000", _s, true); +#ifdef DEBUG + if (client_myds->myconn->userinfo->password) { + char *tmp_pass=strdup(client_myds->myconn->userinfo->password); + int lpass = strlen(tmp_pass); + for (int i=2; imyconn->userinfo->username, client_addr, tmp_pass); + free(tmp_pass); + } else { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , DS=%p . Wrong credentials for frontend: %s:%s . No password. Disconnecting\n", this, client_myds, client_myds->myconn->userinfo->username, client_addr); + } +#endif //DEBUG + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_CHANGE_USER_ERR, this, NULL); + free(_s); + __sync_fetch_and_add(&PgHGM->status.access_denied_wrong_password, 1); + } +} +*/ + +void PgSQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(PtrSize_t* pkt, bool* wrong_pass) { + bool is_encrypted = client_myds->encrypted; + bool handshake_response_return = false; + bool ssl_request = false; + + if (client_myds->auth_received_startup == false) { + if (client_myds->myprot.process_startup_packet((unsigned char*)pkt->ptr, pkt->size, ssl_request) == true ) { + if (ssl_request) { + if (is_encrypted == false && client_myds->encrypted == true) { + // switch to SSL... + } else { + // if sslmode is prefer, same connection will be used for plain text + l_free(pkt->size, pkt->ptr); + return; + } + } else if (client_myds->myprot.generate_pkt_initial_handshake(true, NULL, NULL, &thread_session_id, true) == true) { + client_myds->auth_received_startup = true; + l_free(pkt->size, pkt->ptr); + return; + } else { + assert(0); // this should never happen + } + } else { + *wrong_pass = true; //to forcefully close the connection. Is there a better way to do it? + client_myds->setDSS_STATE_QUERY_SENT_NET(); + l_free(pkt->size, pkt->ptr); + return; + } + } + + bool handshake_err = true; + + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 8, "Session=%p , DS=%p , handshake_response=%d , switching_auth_stage=%d , is_encrypted=%d , client_encrypted=%d\n", this, client_myds, handshake_response_return, client_myds->switching_auth_stage, is_encrypted, client_myds->encrypted); + + if (client_myds->auth_received_startup) { + EXECUTION_STATE state = client_myds->myprot.process_handshake_response_packet((unsigned char*)pkt->ptr, pkt->size); + + if (state == EXECUTION_STATE::PENDING) { + l_free(pkt->size, pkt->ptr); + return; + } + + handshake_response_return = (state == EXECUTION_STATE::SUCCESSFUL) ? true : false; + } + + if ( + (handshake_response_return == false) && (client_myds->switching_auth_stage == 1) + ) { + l_free(pkt->size, pkt->ptr); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 8, "Session=%p , DS=%p . Returning\n", this, client_myds); + return; + } + + if ( + (is_encrypted == false) && // the connection was encrypted + (handshake_response_return == false) && // the authentication didn't complete + (client_myds->encrypted == true) // client is asking for encryption + ) { + // use SSL + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 8, "Session=%p , DS=%p . SSL_INIT\n", this, client_myds); + client_myds->DSS = STATE_SSL_INIT; + client_myds->rbio_ssl = BIO_new(BIO_s_mem()); + client_myds->wbio_ssl = BIO_new(BIO_s_mem()); + client_myds->ssl = GloVars.get_SSL_new(); + SSL_set_fd(client_myds->ssl, client_myds->fd); + SSL_set_accept_state(client_myds->ssl); + SSL_set_bio(client_myds->ssl, client_myds->rbio_ssl, client_myds->wbio_ssl); + l_free(pkt->size, pkt->ptr); + proxysql_keylog_attach_callback(GloVars.get_SSL_ctx()); + return; + } + + if ( + //(client_myds->myprot.process_pkt_handshake_response((unsigned char *)pkt->ptr,pkt->size)==true) + (handshake_response_return == true) + && + ( +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) + (default_hostgroup < 0 && (session_type == PROXYSQL_SESSION_ADMIN || session_type == PROXYSQL_SESSION_STATS || session_type == PROXYSQL_SESSION_SQLITE)) +#else + (default_hostgroup < 0 && (session_type == PROXYSQL_SESSION_ADMIN || session_type == PROXYSQL_SESSION_STATS)) +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP + || + (default_hostgroup == 0 && session_type == PROXYSQL_SESSION_CLICKHOUSE) + || + //(default_hostgroup>=0 && session_type == PROXYSQL_SESSION_PGSQL) + (default_hostgroup >= 0 && (session_type == PROXYSQL_SESSION_PGSQL || session_type == PROXYSQL_SESSION_SQLITE)) + || + ( + client_myds->encrypted == false + && + strncmp(client_myds->myconn->userinfo->username, mysql_thread___monitor_username, strlen(mysql_thread___monitor_username)) == 0 + ) + ) // Do not delete this line. See bug #492 + ) { + if (session_type == PROXYSQL_SESSION_ADMIN) { + if ((default_hostgroup < 0) || (strncmp(client_myds->myconn->userinfo->username, mysql_thread___monitor_username, strlen(mysql_thread___monitor_username)) == 0)) { + if (default_hostgroup == STATS_HOSTGROUP) { + session_type = PROXYSQL_SESSION_STATS; + } + } + } + l_free(pkt->size, pkt->ptr); + //if (client_myds->encrypted==false) { + if (client_myds->myconn->userinfo->dbname == NULL) { +#if 0 +#ifdef PROXYSQLCLICKHOUSE + if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { + if (strlen(default_schema) == 0) { + free(default_schema); + default_schema = strdup((char*)"default"); + } + } +#endif /* PROXYSQLCLICKHOUSE */ +#endif + client_myds->myconn->userinfo->set_dbname(default_schema); + } + int free_users = 0; + int used_users = 0; + if ( + (max_connections_reached == false) + && + (session_type == PROXYSQL_SESSION_PGSQL || session_type == PROXYSQL_SESSION_CLICKHOUSE || session_type == PROXYSQL_SESSION_SQLITE) + ) { + //if (session_type == PROXYSQL_SESSION_PGSQL || session_type == PROXYSQL_SESSION_CLICKHOUSE) { + client_authenticated = true; + switch (session_type) { + case PROXYSQL_SESSION_SQLITE: + //#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) + free_users = 1; + break; + //#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP + case PROXYSQL_SESSION_PGSQL: + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 8, "Session=%p , DS=%p , session_type=PROXYSQL_SESSION_PGSQL\n", this, client_myds); + if (use_ldap_auth == false) { + free_users = GloPgAuth->increase_frontend_user_connections(client_myds->myconn->userinfo->username, &used_users); + } + else { + free_users = GloMyLdapAuth->increase_frontend_user_connections(client_myds->myconn->userinfo->fe_username, &used_users); + } + break; +#ifdef PROXYSQLCLICKHOUSE + case PROXYSQL_SESSION_CLICKHOUSE: + free_users = GloClickHouseAuth->increase_frontend_user_connections(client_myds->myconn->userinfo->username, &used_users); + break; +#endif /* PROXYSQLCLICKHOUSE */ + default: + // LCOV_EXCL_START + assert(0); + break; + // LCOV_EXCL_STOP + } + } + else { + free_users = 1; + } + if (max_connections_reached == true || free_users <= 0) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 8, "Session=%p , DS=%p , max_connections_reached=%d , free_users=%d\n", this, client_myds, max_connections_reached, free_users); + client_authenticated = false; + *wrong_pass = true; + client_myds->setDSS_STATE_QUERY_SENT_NET(); + uint8_t _pid = 2; + if (client_myds->switching_auth_stage) _pid += 2; + if (max_connections_reached == true) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , DS=%p , Too many connections\n", this, client_myds); + client_myds->myprot.generate_error_packet(true, false, "Too many connections", PGSQL_ERROR_CODES::ERRCODE_TOO_MANY_CONNECTIONS, + true, true); + proxy_warning("pgsql-max_connections reached. Returning 'Too many connections'\n"); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_ERR, this, NULL, (char*)"pgsql-max_connections reached"); + __sync_fetch_and_add(&PgHGM->status.access_denied_max_connections, 1); + } + else { // see issue #794 + __sync_fetch_and_add(&PgHGM->status.access_denied_max_user_connections, 1); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , DS=%p . User '%s' has exceeded the 'max_user_connections' resource (current value: %d)\n", this, client_myds, client_myds->myconn->userinfo->username, used_users); + char* a = (char*)"User '%s' has exceeded the 'max_user_connections' resource (current value: %d)"; + char* b = (char*)malloc(strlen(a) + strlen(client_myds->myconn->userinfo->username) + 16); + sprintf(b, a, client_myds->myconn->userinfo->username, used_users); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_ERR, this, NULL, b); + client_myds->myprot.generate_error_packet(true, false, b, PGSQL_ERROR_CODES::ERRCODE_TOO_MANY_CONNECTIONS, + true, true); + proxy_warning("User '%s' has exceeded the 'max_user_connections' resource (current value: %d)\n", client_myds->myconn->userinfo->username, used_users); + free(b); + } + __sync_add_and_fetch(&PgHGM->status.client_connections_aborted, 1); + client_myds->DSS = STATE_SLEEP; + } + else { + if ( + (default_hostgroup == ADMIN_HOSTGROUP && strcmp(client_myds->myconn->userinfo->username, (char*)"admin") == 0) + || + (default_hostgroup == STATS_HOSTGROUP && strcmp(client_myds->myconn->userinfo->username, (char*)"stats") == 0) + || + (default_hostgroup < 0 && strcmp(client_myds->myconn->userinfo->username, (char*)"monitor") == 0) + ) { + char* client_addr = NULL; + union { + struct sockaddr_in in; + struct sockaddr_in6 in6; + } custom_sockaddr; + struct sockaddr* addr = (struct sockaddr*)malloc(sizeof(custom_sockaddr)); + socklen_t addrlen = sizeof(custom_sockaddr); + memset(addr, 0, sizeof(custom_sockaddr)); + int rc = 0; + rc = getpeername(client_myds->fd, addr, &addrlen); + if (rc == 0) { + char buf[512]; + switch (addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)addr; + inet_ntop(addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)addr; + inet_ntop(addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + default: + client_addr = strdup((char*)"localhost"); + break; + } + } + else { + client_addr = strdup((char*)""); + } + uint8_t _pid = 2; + if (client_myds->switching_auth_stage) _pid += 2; + if (is_encrypted) _pid++; + if ( + (strcmp(client_addr, (char*)"127.0.0.1") == 0) + || + (strcmp(client_addr, (char*)"localhost") == 0) + || + (strcmp(client_addr, (char*)"::1") == 0) + ) { + // we are good! + client_myds->myprot.welcome_client(); + handshake_err = false; + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_OK, this, NULL); + status = WAITING_CLIENT_DATA; + client_myds->DSS = STATE_CLIENT_AUTH_OK; + } + else { + char* a = (char*)"User '%s' can only connect locally"; + char* b = (char*)malloc(strlen(a) + strlen(client_myds->myconn->userinfo->username)); + sprintf(b, a, client_myds->myconn->userinfo->username); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_ERR, this, NULL, b); + client_myds->myprot.generate_error_packet(true, false, b, PGSQL_ERROR_CODES::ERRCODE_SQLSERVER_REJECTED_ESTABLISHMENT_OF_SQLCONNECTION, + true, true); + free(b); + } + free(addr); + free(client_addr); + } + else { + uint8_t _pid = 2; + if (client_myds->switching_auth_stage) _pid += 2; + if (is_encrypted) _pid++; + // If this condition is met, it means that the + // 'STATE_SERVER_HANDSHAKE' being performed isn't from the start of a + // connection, but as a consequence of a 'COM_USER_CHANGE' which + // requires an 'Auth Switch'. Thus, we impose a 'pid' of '3' for the + // response 'OK' packet. See #3504 for more context. + if (change_user_auth_switch) { + _pid = 3; + change_user_auth_switch = 0; + } + if (use_ssl == true && is_encrypted == false) { + *wrong_pass = true; + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_ERR, this, NULL); + + char* _a = (char*)"ProxySQL Error: Access denied for user '%s' (using password: %s). SSL is required"; + char* _s = (char*)malloc(strlen(_a) + strlen(client_myds->myconn->userinfo->username) + 32); + sprintf(_s, _a, client_myds->myconn->userinfo->username, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + client_myds->myprot.generate_error_packet(true, false, _s, PGSQL_ERROR_CODES::ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION, + true, true); + proxy_error("ProxySQL Error: Access denied for user '%s' (using password: %s). SSL is required\n", client_myds->myconn->userinfo->username, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 8, "Session=%p , DS=%p . Access denied for user '%s' (using password: %s). SSL is required\n", this, client_myds, client_myds->myconn->userinfo->username, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + __sync_add_and_fetch(&PgHGM->status.client_connections_aborted, 1); + free(_s); + __sync_fetch_and_add(&PgHGM->status.access_denied_wrong_password, 1); + } + else { + // we are good! + //client_myds->myprot.generate_pkt_OK(true,NULL,NULL, (is_encrypted ? 3 : 2), 0,0,0,0,NULL,false); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 8, "Session=%p , DS=%p . STATE_CLIENT_AUTH_OK\n", this, client_myds); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_OK, this, NULL); + client_myds->myprot.welcome_client(); + handshake_err = false; + status = WAITING_CLIENT_DATA; + client_myds->DSS = STATE_CLIENT_AUTH_OK; + } + } + } + } + else { + l_free(pkt->size, pkt->ptr); + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , DS=%p . Wrong credentials for frontend: disconnecting\n", this, client_myds); + *wrong_pass = true; + // FIXME: this should become close connection + client_myds->setDSS_STATE_QUERY_SENT_NET(); + char* client_addr = NULL; + if (client_myds->client_addr && client_myds->myconn->userinfo->username) { + char buf[512]; + switch (client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)client_myds->client_addr; + if (ipv4->sin_port) { + inet_ntop(client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + client_addr = strdup(buf); + } + else { + client_addr = strdup((char*)"localhost"); + } + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + default: + client_addr = strdup((char*)"localhost"); + break; + } + } + else { + client_addr = strdup((char*)""); + } + if (client_myds->myconn->userinfo->username) { + char* _s = (char*)malloc(strlen(client_myds->myconn->userinfo->username) + 100 + strlen(client_addr)); + uint8_t _pid = 2; + if (client_myds->switching_auth_stage) _pid += 2; + if (is_encrypted) _pid++; +#ifdef DEBUG + if (client_myds->myconn->userinfo->password) { + char* tmp_pass = strdup(client_myds->myconn->userinfo->password); + int lpass = strlen(tmp_pass); + for (int i = 2; i < lpass - 1; i++) { + tmp_pass[i] = '*'; + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , DS=%p . Error: Access denied for user '%s'@'%s' , Password='%s'. Disconnecting\n", this, client_myds, client_myds->myconn->userinfo->username, client_addr, tmp_pass); + free(tmp_pass); + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , DS=%p . Error: Access denied for user '%s'@'%s' . No password. Disconnecting\n", this, client_myds, client_myds->myconn->userinfo->username, client_addr); + } +#endif // DEBUG + sprintf(_s, "ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + client_myds->myprot.generate_error_packet(true, false, _s, PGSQL_ERROR_CODES::ERRCODE_INVALID_PASSWORD, true, true); + proxy_error("%s\n", _s); + free(_s); + __sync_fetch_and_add(&PgHGM->status.access_denied_wrong_password, 1); + } + if (client_addr) { + free(client_addr); + } + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_ERR, this, NULL); + __sync_add_and_fetch(&PgHGM->status.client_connections_aborted, 1); + client_myds->DSS = STATE_SLEEP; + } + + if (pgsql_thread___client_host_cache_size) { + GloPTH->update_client_host_cache(client_myds->client_addr, handshake_err); + } +} + +// Note: as commented in issue #546 and #547 , some clients ignore the status of CLIENT_MULTI_STATEMENTS +// therefore tracking it is not needed, unless in future this should become a security enhancement, +// returning errors to all clients trying to send multi-statements . +// see also #1140 +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_SET_OPTION(PtrSize_t* pkt) { + + char v; + v = *((char*)pkt->ptr + 3); + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_SET_OPTION packet , value %d\n", v); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + + bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + if (deprecate_eof_active) + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL, true); + else + client_myds->myprot.generate_pkt_EOF(true, NULL, NULL, 1, 0, setStatus); + + if (v == 1) { // disabled. MYSQL_OPTION_MULTI_STATEMENTS_OFF == 1 + client_myds->myconn->options.client_flag &= ~CLIENT_MULTI_STATEMENTS; + } + else { // enabled, MYSQL_OPTION_MULTI_STATEMENTS_ON == 0 + client_myds->myconn->options.client_flag |= CLIENT_MULTI_STATEMENTS; + } + client_myds->DSS = STATE_SLEEP; + l_free(pkt->size, pkt->ptr); +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_PING(PtrSize_t* pkt) { + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_PING packet\n"); + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + client_myds->DSS = STATE_SLEEP; +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_FIELD_LIST(PtrSize_t* pkt) { + if (session_type == PROXYSQL_SESSION_PGSQL) { + /* FIXME: temporary */ + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_error_packet(true, true, "Command not supported", PGSQL_ERROR_CODES::ERRCODE_FEATURE_NOT_SUPPORTED, + false, true); + client_myds->DSS = STATE_SLEEP; + } + else { + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_error_packet(true, true, "Command not supported", PGSQL_ERROR_CODES::ERRCODE_FEATURE_NOT_SUPPORTED, + false, true); + client_myds->DSS = STATE_SLEEP; + } +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_PROCESS_KILL(PtrSize_t* pkt) { + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_error_packet(true, true, "Command not supported", PGSQL_ERROR_CODES::ERRCODE_FEATURE_NOT_SUPPORTED, false); + client_myds->DSS = STATE_SLEEP; +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_INIT_DB(PtrSize_t* pkt) { + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_INIT_DB packet\n"); + if (session_type == PROXYSQL_SESSION_PGSQL) { + //__sync_fetch_and_add(&PgHGM->status.frontend_init_db, 1); + //client_myds->myconn->userinfo->set_dbname((char*)pkt->ptr + sizeof(mysql_hdr) + 1, pkt->size - sizeof(mysql_hdr) - 1); + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_INITDB, this, NULL); + client_myds->DSS = STATE_SLEEP; + } + else { + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + client_myds->DSS = STATE_SLEEP; + } +} + +// this function was introduced due to isseu #718 +// some application (like the one written in Perl) do not use COM_INIT_DB , but COM_QUERY with USE dbname +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_USE_DB(PtrSize_t* pkt) { + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_QUERY with USE dbname\n"); + if (session_type == PROXYSQL_SESSION_PGSQL) { + //__sync_fetch_and_add(&PgHGM->status.frontend_use_db, 1); + string nq = string((char*)pkt->ptr + sizeof(mysql_hdr) + 1, pkt->size - sizeof(mysql_hdr) - 1); + RE2::GlobalReplace(&nq, (char*)"(?U)/\\*.*\\*/", (char*)" "); + char* sn_tmp = (char*)nq.c_str(); + while (sn_tmp < (nq.c_str() + nq.length() - 4) && *sn_tmp == ' ') + sn_tmp++; + //char *schemaname=strdup(nq.c_str()+4); + char* schemaname = strdup(sn_tmp + 3); + char* schemanameptr = trim_spaces_and_quotes_in_place(schemaname); + // handle cases like "USE `schemaname` + if (schemanameptr[0] == '`' && schemanameptr[strlen(schemanameptr) - 1] == '`') { + schemanameptr[strlen(schemanameptr) - 1] = '\0'; + schemanameptr++; + } + //client_myds->myconn->userinfo->set_dbname(schemanameptr); + free(schemaname); + if (mirror == false) { + RequestEnd(NULL); + } + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_INITDB, this, NULL); + client_myds->DSS = STATE_SLEEP; + } + else { + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + client_myds->DSS = STATE_SLEEP; + } +} + + +// this function as inline in handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo +void PgSQL_Session::handler_WCD_SS_MCQ_qpo_QueryRewrite(PtrSize_t* pkt) { + // the query was rewritten + l_free(pkt->size, pkt->ptr); // free old pkt + // allocate new pkt + timespec begint; + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &begint); + } + pkt->size = sizeof(mysql_hdr) + 1 + qpo->new_query->length(); + pkt->ptr = l_alloc(pkt->size); + mysql_hdr hdr; + hdr.pkt_id = 0; + hdr.pkt_length = pkt->size - sizeof(mysql_hdr); + memcpy((unsigned char*)pkt->ptr, &hdr, sizeof(mysql_hdr)); // copy header + unsigned char* c = (unsigned char*)pkt->ptr + sizeof(mysql_hdr); + *c = (unsigned char)_MYSQL_COM_QUERY; // set command type + memcpy((unsigned char*)pkt->ptr + sizeof(mysql_hdr) + 1, qpo->new_query->data(), qpo->new_query->length()); // copy query + CurrentQuery.query_parser_free(); + CurrentQuery.begin((unsigned char*)pkt->ptr, pkt->size, true); + delete qpo->new_query; + timespec endt; + if (thread->variables.stats_time_query_processor) { + clock_gettime(CLOCK_THREAD_CPUTIME_ID, &endt); + thread->status_variables.stvar[st_var_query_processor_time] = thread->status_variables.stvar[st_var_query_processor_time] + + (endt.tv_sec * 1000000000 + endt.tv_nsec) - + (begint.tv_sec * 1000000000 + begint.tv_nsec); + } +} + +// this function as inline in handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo +void PgSQL_Session::handler_WCD_SS_MCQ_qpo_OK_msg(PtrSize_t* pkt) { + + client_myds->DSS = STATE_QUERY_SENT_NET; + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, client_myds->pkt_sid + 1, 0, 0, setStatus, 0, qpo->OK_msg); + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); +} + +// this function as inline in handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo +void PgSQL_Session::handler_WCD_SS_MCQ_qpo_error_msg(PtrSize_t* pkt) { + client_myds->DSS = STATE_QUERY_SENT_NET; + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, 1148, (char*)"42000", qpo->error_msg); + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); +} + +// this function as inline in handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo +void PgSQL_Session::handler_WCD_SS_MCQ_qpo_LargePacket(PtrSize_t* pkt) { + // ER_NET_PACKET_TOO_LARGE + client_myds->DSS = STATE_QUERY_SENT_NET; + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, 1153, (char*)"08S01", (char*)"Got a packet bigger than 'max_allowed_packet' bytes", true); + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); +} + +bool PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(PtrSize_t* pkt, bool* lock_hostgroup, PgSQL_ps_type prepare_stmt_type) { + /* + lock_hostgroup: + If this variable is set to true, this session will get lock to a + specific hostgroup, and also have multiplexing disabled. + It means that parsing the query wasn't completely possible (mostly + a SET statement) and proxysql won't be able to set the same variable + in another connection. + This algorithm will be become obsolete once we implement session + tracking for MySQL 5.7+ + */ + bool exit_after_SetParse = true; + unsigned char command_type = *((unsigned char*)pkt->ptr + sizeof(mysql_hdr)); + if (qpo->new_query) { + handler_WCD_SS_MCQ_qpo_QueryRewrite(pkt); + } + + if (pkt->size > (unsigned int)pgsql_thread___max_allowed_packet) { + handler_WCD_SS_MCQ_qpo_LargePacket(pkt); + reset_warning_hostgroup_flag_and_release_connection(); + return true; + } + + if (qpo->OK_msg) { + handler_WCD_SS_MCQ_qpo_OK_msg(pkt); + reset_warning_hostgroup_flag_and_release_connection(); + return true; + } + + if (qpo->error_msg) { + handler_WCD_SS_MCQ_qpo_error_msg(pkt); + reset_warning_hostgroup_flag_and_release_connection(); + return true; + } + + if (prepare_stmt_type & PgSQL_ps_type_execute_stmt) { // for prepared statement execute we exit here + reset_warning_hostgroup_flag_and_release_connection(); + goto __exit_set_destination_hostgroup; + } + + // handle warnings + if (CurrentQuery.QueryParserArgs.digest_text) { + const char* dig_text = CurrentQuery.QueryParserArgs.digest_text; + const size_t dig_len = strlen(dig_text); + + if (dig_len > 0) { + if ((dig_len == 13) && (strncasecmp(dig_text, "SHOW WARNINGS", 13) == 0)) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Intercepted '%s'\n", dig_text); + if (warning_in_hg > -1) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Changing current_hostgroup to '%d'\n", warning_in_hg); + current_hostgroup = warning_in_hg; + return false; + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "No warnings were detected in the previous query. Sending an empty response.\n"); + std::unique_ptr resultset(new SQLite3_result(3)); + resultset->add_column_definition(SQLITE_TEXT, "Level"); + resultset->add_column_definition(SQLITE_TEXT, "Code"); + resultset->add_column_definition(SQLITE_TEXT, "Message"); + SQLite3_to_MySQL(resultset.get(), NULL, 0, &client_myds->myprot, false, (client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF)); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + if (mirror == false) { + RequestEnd(NULL); + } + l_free(pkt->size, pkt->ptr); + return true; + } + } + + if ((dig_len == 22) && (strncasecmp(dig_text, "SHOW COUNT(*) WARNINGS", 22) == 0)) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Intercepted '%s'\n", dig_text); + std::string warning_count = "0"; + if (warning_in_hg > -1) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Changing current_hostgroup to '%d'\n", warning_in_hg); + current_hostgroup = warning_in_hg; + assert(mybe && mybe->server_myds && mybe->server_myds->myconn && mybe->server_myds->myconn->pgsql); + warning_count = std::to_string(mybe->server_myds->myconn->warning_count); + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "No warnings were detected in the previous query. Sending an empty response.\n"); + } + std::unique_ptr resultset(new SQLite3_result(1)); + resultset->add_column_definition(SQLITE_TEXT, "@@session.warning_count"); + char* pta[1]; + pta[0] = (char*)warning_count.c_str(); + resultset->add_row(pta); + SQLite3_to_MySQL(resultset.get(), NULL, 0, &client_myds->myprot, false, (client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF)); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + if (mirror == false) { + RequestEnd(NULL); + } + l_free(pkt->size, pkt->ptr); + return true; + } + } + } + + reset_warning_hostgroup_flag_and_release_connection(); + + // handle here #509, #815 and #816 + if (CurrentQuery.QueryParserArgs.digest_text) { + char* dig = CurrentQuery.QueryParserArgs.digest_text; + unsigned int nTrx = NumActiveTransactions(); + if ((locked_on_hostgroup == -1) && (strncasecmp(dig, (char*)"SET ", 4) == 0)) { + // this code is executed only if locked_on_hostgroup is not set yet + // if locked_on_hostgroup is set, we do not try to parse the SET statement +#ifdef DEBUG + { + string nqn = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Parsing SET command = %s\n", nqn.c_str()); + } +#endif + if (index(dig, ';') && (index(dig, ';') != dig + strlen(dig) - 1)) { + string nqn; + if (pgsql_thread___parse_failure_logs_digest) + nqn = string(CurrentQuery.get_digest_text()); + else + nqn = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); + proxy_warning( + "Unable to parse multi-statements command with SET statement from client" + " %s:%d: setting lock hostgroup. Command: %s\n", client_myds->addr.addr, + client_myds->addr.port, nqn.c_str() + ); + *lock_hostgroup = true; + return false; + } + int rc; + string nq = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); + RE2::GlobalReplace(&nq, (char*)"^/\\*!\\d\\d\\d\\d\\d SET(.*)\\*/", (char*)"SET\\1"); + RE2::GlobalReplace(&nq, (char*)"(?U)/\\*.*\\*/", (char*)""); + // remove trailing space and semicolon if present. See issue#4380 + nq.erase(nq.find_last_not_of(" ;") + 1); + if ( + ( + match_regexes && (match_regexes[1]->match(dig)) + ) + || + (strncasecmp(dig, (char*)"SET NAMES", strlen((char*)"SET NAMES")) == 0) + || + (strcasestr(dig, (char*)"autocommit")) + ) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Parsing SET command %s\n", nq.c_str()); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Parsing SET command = %s\n", nq.c_str()); + SetParser parser(nq); + std::map> set = {}; + if (pgsql_thread___set_parser_algorithm == 1) { // legacy behavior + set = parser.parse1(); + } else if (pgsql_thread___set_parser_algorithm == 2) { // we use a single SetParser per thread + thread->thr_SetParser->set_query(nq); // replace the query + set = thread->thr_SetParser->parse1v2(); // use algorithm v2 + } else { + assert(0); + } + // Flag to be set if any variable within the 'SET' statement fails to be tracked, + // due to being unknown or because it's an user defined variable. + bool failed_to_parse_var = false; + for (auto it = std::begin(set); it != std::end(set); ++it) { + std::string var = it->first; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET variable %s\n", var.c_str()); + if (it->second.size() < 1 || it->second.size() > 2) { + // error not enough arguments + string query_str = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); + string digest_str = string(CurrentQuery.get_digest_text()); + string nqn; + if (pgsql_thread___parse_failure_logs_digest) + nqn = digest_str; + else + nqn = query_str; + // PMC-10002: A query has failed to be parsed. This can be due a incorrect query or + // due to ProxySQL not being able to properly parse it. In case the query is correct a + // bug report should be filed including the offending query. + proxy_error2(10002, "Unable to parse query. If correct, report it as a bug: %s\n", nqn.c_str()); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Locking hostgroup for query %s\n", + query_str.c_str()); + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + auto values = std::begin(it->second); + if (var == "sql_mode") { + std::string value1 = *values; + if (strcasestr(value1.c_str(), "NO_BACKSLASH_ESCAPES") != NULL) { + // client is setting NO_BACKSLASH_ESCAPES in sql_mode + // Because we will reply with an OK packet without + // first setting sql_mode to the backend (this is + // by design) we need to set no_backslash_escapes + // in the client connection + if (client_myds && client_myds->myconn) { // some extra sanity check + client_myds->myconn->set_no_backslash_escapes(true); + } + } + if ( + (strcasecmp(value1.c_str(), (char*)"CONCAT") == 0) + || + (strcasecmp(value1.c_str(), (char*)"REPLACE") == 0) + || + (strcasecmp(value1.c_str(), (char*)"IFNULL") == 0) + ) { + string query_str = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); + string digest_str = string(CurrentQuery.get_digest_text()); + string nqn; + if (pgsql_thread___parse_failure_logs_digest) + nqn = digest_str; + else + nqn = query_str; + proxy_error2(10002, "Unable to parse query. If correct, report it as a bug: %s\n", nqn.c_str()); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, + "Locking hostgroup for query %s\n", query_str.c_str()); + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + std::size_t found_at = value1.find("@"); + if (found_at != std::string::npos) { + char* v1 = strdup(value1.c_str()); + char* v1t = v1; + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Found @ in SQL_MODE . v1 = %s\n", v1); + char* v2 = NULL; + while (v1 && (v2 = strstr(v1, (const char*)"@"))) { + // we found a @ . Maybe we need to lock hostgroup + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Found @ in SQL_MODE . v2 = %s\n", v2); + if (strncasecmp(v2, (const char*)"@@sql_mode", strlen((const char*)"@@sql_mode"))) { + unable_to_parse_set_statement(lock_hostgroup); + free(v1); + return false; + } + else { + v2++; + } + if (strlen(v2) > 1) { + v1 = v2 + 1; + } + } + free(v1t); + } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET SQL Mode value %s\n", value1.c_str()); + uint32_t sql_mode_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + if (pgsql_variables.client_get_hash(this, SQL_SQL_MODE) != sql_mode_int) { + if (!pgsql_variables.client_set_value(this, SQL_SQL_MODE, value1.c_str())) { + return false; + } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection SQL Mode to %s\n", value1.c_str()); + } + } + else if (pgsql_variables_strings.find(var) != pgsql_variables_strings.end()) { + std::string value1 = *values; + std::size_t found_at = value1.find("@"); + if (found_at != std::string::npos) { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + int idx = SQL_NAME_LAST_HIGH_WM; + for (int i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + if (mysql_tracked_variables[i].is_number == false && mysql_tracked_variables[i].is_bool == false) { + if (!strcasecmp(var.c_str(), mysql_tracked_variables[i].set_variable_name)) { + idx = mysql_tracked_variables[i].idx; + break; + } + } + } + if (idx != SQL_NAME_LAST_HIGH_WM) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection %s to %s\n", var.c_str(), value1.c_str()); + uint32_t var_hash_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + if (pgsql_variables.client_get_hash(this, mysql_tracked_variables[idx].idx) != var_hash_int) { + if (!pgsql_variables.client_set_value(this, mysql_tracked_variables[idx].idx, value1.c_str())) { + return false; + } + } + } + } + else if (pgsql_variables_boolean.find(var) != pgsql_variables_boolean.end()) { + int idx = SQL_NAME_LAST_HIGH_WM; + for (int i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + if (mysql_tracked_variables[i].is_bool) { + if (!strcasecmp(var.c_str(), mysql_tracked_variables[i].set_variable_name)) { + idx = mysql_tracked_variables[i].idx; + break; + } + } + } + if (idx != SQL_NAME_LAST_HIGH_WM) { + if (pgsql_variables.parse_variable_boolean(this, idx, *values, lock_hostgroup) == false) { + return false; + } + } + } + else if (pgsql_variables_numeric.find(var) != pgsql_variables_numeric.end()) { + int idx = SQL_NAME_LAST_HIGH_WM; + for (int i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + if (mysql_tracked_variables[i].is_number) { + if (!strcasecmp(var.c_str(), mysql_tracked_variables[i].set_variable_name)) { + idx = mysql_tracked_variables[i].idx; + break; + } + } + } + if (idx != SQL_NAME_LAST_HIGH_WM) { + if (var == "query_cache_type") { + // note that query_cache_type variable can act both as boolean AND a number , but also accept "DEMAND" + // See https://dev.pgsql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_query_cache_type + std::string value1 = *values; + if (strcasecmp(value1.c_str(), "off") == 0 || strcasecmp(value1.c_str(), "false") == 0) { + value1 = "0"; + } + else if (strcasecmp(value1.c_str(), "on") == 0 || strcasecmp(value1.c_str(), "true") == 0) { + value1 = "1"; + } + else if (strcasecmp(value1.c_str(), "demand") == 0 || strcasecmp(value1.c_str(), "true") == 0) { + value1 = "2"; + } + if (pgsql_variables.parse_variable_number(this, idx, value1, lock_hostgroup) == false) { + return false; + } + } + else { + if (pgsql_variables.parse_variable_number(this, idx, *values, lock_hostgroup) == false) { + return false; + } + } + } + } + else if (var == "autocommit") { + std::string value1 = *values; + std::size_t found_at = value1.find("@"); + if (found_at != std::string::npos) { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET autocommit value %s\n", value1.c_str()); + int __tmp_autocommit = -1; + if ( + (strcasecmp(value1.c_str(), (char*)"0") == 0) || + (strcasecmp(value1.c_str(), (char*)"false") == 0) || + (strcasecmp(value1.c_str(), (char*)"off") == 0) + ) { + __tmp_autocommit = 0; + } + else { + if ( + (strcasecmp(value1.c_str(), (char*)"1") == 0) || + (strcasecmp(value1.c_str(), (char*)"true") == 0) || + (strcasecmp(value1.c_str(), (char*)"on") == 0) + ) { + __tmp_autocommit = 1; + } + } + if (__tmp_autocommit >= 0 && autocommit_handled == false) { + int fd = __tmp_autocommit; + __sync_fetch_and_add(&PgHGM->status.autocommit_cnt, 1); + // we immediately process the number of transactions + unsigned int nTrx = NumActiveTransactions(); + if (fd == 1 && autocommit == true) { + // nothing to do, return OK + } + if (fd == 1 && autocommit == false) { + if (nTrx) { + // there is an active transaction, we need to forward it + // because this can potentially close the transaction + autocommit = true; + client_myds->myconn->set_autocommit(autocommit); + autocommit_on_hostgroup = FindOneActiveTransaction(); + exit_after_SetParse = false; + sending_set_autocommit = true; + } + else { + // as there is no active transaction, we do no need to forward it + // just change internal state + autocommit = true; + client_myds->myconn->set_autocommit(autocommit); + } + } + + if (fd == 0) { + autocommit = false; // we set it, no matter if already set or not + client_myds->myconn->set_autocommit(autocommit); + } + } + else { + if (autocommit_handled == true) { + exit_after_SetParse = false; + } + } + } + else if (var == "time_zone") { + std::string value1 = *values; + std::size_t found_at = value1.find("@"); + if (found_at != std::string::npos) { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET Time Zone value %s\n", value1.c_str()); + { + // reformat +1:23 to +01:23 + if (value1.length() == 5) { + if (value1[0] == '+' || value1[0] == '-') { + if (value1[2] == ':') { + std::string s = std::string(value1, 0, 1); + s += "0"; + s += std::string(value1, 1, 4); + value1 = s; + } + } + } + } + uint32_t time_zone_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + if (pgsql_variables.client_get_hash(this, SQL_TIME_ZONE) != time_zone_int) { + if (!pgsql_variables.client_set_value(this, SQL_TIME_ZONE, value1.c_str())) + return false; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection Time zone to %s\n", value1.c_str()); + } + } + else if (var == "session_track_gtids") { + std::string value1 = *values; + if ((strcasecmp(value1.c_str(), "OWN_GTID") == 0) || (strcasecmp(value1.c_str(), "OFF") == 0) || (strcasecmp(value1.c_str(), "ALL_GTIDS") == 0)) { + if (strcasecmp(value1.c_str(), "ALL_GTIDS") == 0) { + // we convert session_track_gtids=ALL_GTIDS to session_track_gtids=OWN_GTID + std::string a = ""; + if (client_myds && client_myds->addr.addr) { + a = " . Client "; + a += client_myds->addr.addr; + a += ":" + std::to_string(client_myds->addr.port); + } + proxy_warning("SET session_track_gtids=ALL_GTIDS is not allowed. Switching to session_track_gtids=OWN_GTID%s\n", a.c_str()); + value1 = "OWN_GTID"; + } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 7, "Processing SET session_track_gtids value %s\n", value1.c_str()); + uint32_t session_track_gtids_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + if (client_myds->myconn->options.session_track_gtids_int != session_track_gtids_int) { + client_myds->myconn->options.session_track_gtids_int = session_track_gtids_int; + if (client_myds->myconn->options.session_track_gtids) { + free(client_myds->myconn->options.session_track_gtids); + } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Changing connection session_track_gtids to %s\n", value1.c_str()); + client_myds->myconn->options.session_track_gtids = strdup(value1.c_str()); + } + } + else { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + } + else if ((var == "character_set_results") || (var == "collation_connection") || + (var == "character_set_connection") || (var == "character_set_client") || + (var == "character_set_database")) { + std::string value1 = *values; + int vl = strlen(value1.c_str()); + const char* v = value1.c_str(); + bool only_normal_chars = true; + for (int i = 0; i < vl && only_normal_chars == true; i++) { + if (is_normal_char(v[i]) == 0) { + only_normal_chars = false; + } + } + if (only_normal_chars) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 7, "Processing SET %s value %s\n", var.c_str(), value1.c_str()); + uint32_t var_value_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + int idx = SQL_NAME_LAST_HIGH_WM; + for (int i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + if (!strcasecmp(var.c_str(), mysql_tracked_variables[i].set_variable_name)) { + idx = mysql_tracked_variables[i].idx; + break; + } + } + if (idx == SQL_NAME_LAST_HIGH_WM) { + proxy_error("Variable %s not found in mysql_tracked_variables[]\n", var.c_str()); + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + if (pgsql_variables.client_get_hash(this, idx) != var_value_int) { + const MARIADB_CHARSET_INFO* ci = NULL; + if (var == "character_set_results" || var == "character_set_connection" || + var == "character_set_client" || var == "character_set_database") { + ci = proxysql_find_charset_name(value1.c_str()); + } + else if (var == "collation_connection") + ci = proxysql_find_charset_collate(value1.c_str()); + + if (!ci) { + if (var == "character_set_results") { + if (!strcasecmp("NULL", value1.c_str())) { + if (!pgsql_variables.client_set_value(this, idx, "NULL")) { + return false; + } + } + else if (!strcasecmp("binary", value1.c_str())) { + if (!pgsql_variables.client_set_value(this, idx, "binary")) { + return false; + } + } + else { + // LCOV_EXCL_START + proxy_error("Cannot find charset/collation [%s]\n", value1.c_str()); + assert(0); + // LCOV_EXCL_STOP + } + } + } + else { + std::stringstream ss; + ss << ci->nr; + /* changing collation_connection the character_set_connection will be changed as well + * and vice versa + */ + if (var == "collation_connection") { + if (!pgsql_variables.client_set_value(this, SQL_CHARACTER_SET_CONNECTION, ss.str().c_str())) + return false; + } + if (var == "character_set_connection") { + if (!pgsql_variables.client_set_value(this, SQL_COLLATION_CONNECTION, ss.str().c_str())) + return false; + } + + /* this is explicit statement from client. we do not multiplex, therefor we must + * remember client's choice in the client's variable for future use in verifications, multiplexing etc. + */ + if (!pgsql_variables.client_set_value(this, idx, ss.str().c_str())) + return false; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Changing connection %s to %s\n", var.c_str(), value1.c_str()); + } + } + } + else { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + } + else if (var == "names") { + std::string value1 = *values++; + std::size_t found_at = value1.find("@"); + if (found_at != std::string::npos) { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET NAMES %s\n", value1.c_str()); + const MARIADB_CHARSET_INFO* c; + std::string value2; + if (values != std::end(it->second)) { + value2 = *values; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET NAMES With COLLATE %s\n", value2.c_str()); + c = proxysql_find_charset_collate_names(value1.c_str(), value2.c_str()); + } + else { + c = proxysql_find_charset_name(value1.c_str()); + } + if (!c) { + char* m = NULL; + char* errmsg = NULL; + if (value2.length()) { + m = (char*)"Unknown character set '%s' or collation '%s'"; + errmsg = (char*)malloc(value1.length() + value2.length() + strlen(m)); + sprintf(errmsg, m, value1.c_str(), value2.c_str()); + } + else { + m = (char*)"Unknown character set: '%s'"; + errmsg = (char*)malloc(value1.length() + strlen(m)); + sprintf(errmsg, m, value1.c_str()); + } + client_myds->DSS = STATE_QUERY_SENT_NET; + client_myds->myprot.generate_error_packet(true, true, errmsg, + PGSQL_ERROR_CODES::ERRCODE_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION, false, true); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + free(errmsg); + return true; + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection charset to %d\n", c->nr); + //-- client_myds->myconn->set_charset(c->nr, NAMES); + } + } + else if (var == "tx_isolation") { + std::string value1 = *values; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET tx_isolation value %s\n", value1.c_str()); + auto pos = value1.find('-'); + if (pos != std::string::npos) + value1[pos] = ' '; + uint32_t isolation_level_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + if (pgsql_variables.client_get_hash(this, SQL_ISOLATION_LEVEL) != isolation_level_int) { + if (!pgsql_variables.client_set_value(this, SQL_ISOLATION_LEVEL, value1.c_str())) + return false; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection TX ISOLATION to %s\n", value1.c_str()); + } + } + else if (var == "tx_read_only") { + std::string value1 = *values; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET tx_read_only value %s\n", value1.c_str()); + + if ( + (value1 == "0") || + (strcasecmp(value1.c_str(), "false") == 0) || + (strcasecmp(value1.c_str(), "off") == 0) + ) { + value1 = "WRITE"; + } + else if ( + (value1 == "1") || + (strcasecmp(value1.c_str(), "true") == 0) || + (strcasecmp(value1.c_str(), "on") == 0) + ) { + value1 = "ONLY"; + } + else { + //proxy_warning("Unknown tx_read_only value \"%s\"\n", value1.c_str()); + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + uint32_t read_only_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + if (pgsql_variables.client_get_hash(this, SQL_TRANSACTION_READ) != read_only_int) { + if (!pgsql_variables.client_set_value(this, SQL_TRANSACTION_READ, value1.c_str())) + return false; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection TX ACCESS MODE to READ %s\n", value1.c_str()); + } + } + else if (std::find(pgsql_variables.ignore_vars.begin(), pgsql_variables.ignore_vars.end(), var) != pgsql_variables.ignore_vars.end()) { + // this is a variable we parse but ignore + // see MySQL_Variables::MySQL_Variables() for a list of ignored variables +#ifdef DEBUG + std::string value1 = *values; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET %s value %s\n", var.c_str(), value1.c_str()); +#endif // DEBUG + } + else { + // At this point the variable is unknown to us, or it's a user variable + // prefixed by '@', in both cases, we should fail to parse. We don't + // fail inmediately so we can anyway keep track of the other variables + // supplied within the 'SET' statement being parsed. + failed_to_parse_var = true; + } + } + + if (failed_to_parse_var) { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + /* + if (exit_after_SetParse) { + goto __exit_set_destination_hostgroup; + } + */ + // parseSetCommand wasn't able to parse anything... + if (set.size() == 0) { + // try case listed in #1373 + // SET @@SESSION.sql_mode = CONCAT(CONCAT(@@sql_mode, ',STRICT_ALL_TABLES'), ',NO_AUTO_VALUE_ON_ZERO'), @@SESSION.sql_auto_is_null = 0, @@SESSION.wait_timeout = 2147483 + // this is not a complete solution. A right solution involves true parsing + size_t query_no_space_length = nq.length(); + char* query_no_space = (char*)malloc(query_no_space_length + 1); + memcpy(query_no_space, nq.c_str(), query_no_space_length); + query_no_space[query_no_space_length] = '\0'; + query_no_space_length = remove_spaces(query_no_space); + + string nq1 = string(query_no_space); + free(query_no_space); + RE2::GlobalReplace(&nq1, (char*)"SESSION.", (char*)""); + RE2::GlobalReplace(&nq1, (char*)"SESSION ", (char*)""); + RE2::GlobalReplace(&nq1, (char*)"session.", (char*)""); + RE2::GlobalReplace(&nq1, (char*)"session ", (char*)""); + //fprintf(stderr,"%s\n",nq1.c_str()); + re2::RE2::Options* opt2 = new re2::RE2::Options(RE2::Quiet); + opt2->set_case_sensitive(false); + char* pattern = (char*)"^SET @@SQL_MODE *(?:|:)= *(?:'||\")(.*)(?:'||\") *, *@@sql_auto_is_null *(?:|:)= *(?:(?:\\w|\\d)*) *, @@wait_timeout *(?:|:)= *(?:\\d*)$"; + re2::RE2* re = new RE2(pattern, *opt2); + string s1; + rc = RE2::FullMatch(nq1, *re, &s1); + delete re; + delete opt2; + if (rc) { + uint32_t sql_mode_int = SpookyHash::Hash32(s1.c_str(), s1.length(), 10); + if (pgsql_variables.client_get_hash(this, SQL_SQL_MODE) != sql_mode_int) { + if (!pgsql_variables.client_set_value(this, SQL_SQL_MODE, s1.c_str())) + return false; + std::size_t found_at = s1.find("@"); + if (found_at != std::string::npos) { + char* v1 = strdup(s1.c_str()); + char* v2 = NULL; + while (v1 && (v2 = strstr(v1, (const char*)"@"))) { + // we found a @ . Maybe we need to lock hostgroup + if (strncasecmp(v2, (const char*)"@@sql_mode", strlen((const char*)"@@sql_mode"))) { +#ifdef DEBUG + string nqn = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Locking hostgroup for query %s\n", nqn.c_str()); +#endif + * lock_hostgroup = true; + } + if (strlen(v2) > 1) { + v1 = v2 + 1; + } + } + free(v1); + if (*lock_hostgroup) { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + } + } + } + else { + if (memchr((const char*)CurrentQuery.QueryPointer, '@', CurrentQuery.QueryLength)) { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + int kq = 0; + kq = strncmp((const char*)CurrentQuery.QueryPointer, (const char*)"/*!40101 SET SQL_MODE=@OLD_SQL_MODE */", CurrentQuery.QueryLength); + if (kq != 0) { + kq = strncmp((const char*)CurrentQuery.QueryPointer, (const char*)"/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */", CurrentQuery.QueryLength); + if (kq != 0) { + string nqn; + if (pgsql_thread___parse_failure_logs_digest) + nqn = string(CurrentQuery.get_digest_text()); + else + nqn = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); + proxy_error2(10002, "Unable to parse query. If correct, report it as a bug: %s\n", nqn.c_str()); + return false; + } + } + } + } + + if (exit_after_SetParse) { + if (command_type == _MYSQL_COM_QUERY) { + client_myds->DSS = STATE_QUERY_SENT_NET; + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); + return true; + } + } + } + else if (match_regexes && match_regexes[2]->match(dig)) { + SetParser parser(nq); + std::map> set = parser.parse2(); + + for (auto it = std::begin(set); it != std::end(set); ++it) { + + const std::vector& val = split_string(it->first, ':'); + + if (val.size() == 2) { + + const auto values = std::begin(it->second); + const std::string& var = val[1]; + + enum mysql_variable_name isolation_level_val; + enum mysql_variable_name transaction_read_val; + + if (val[0] == "session") { + isolation_level_val = SQL_ISOLATION_LEVEL; + transaction_read_val = SQL_TRANSACTION_READ; + } + else { + isolation_level_val = SQL_NEXT_ISOLATION_LEVEL; + transaction_read_val = SQL_NEXT_TRANSACTION_READ; + } + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET variable %s\n", var.c_str()); + if (var == "isolation level") { + const std::string& value1 = *values; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET %s TRANSACTION ISOLATION LEVEL value %s\n", val[0].c_str(), value1.c_str()); + const uint32_t isolation_level_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + if (pgsql_variables.client_get_hash(this, isolation_level_val) != isolation_level_int) { + if (!pgsql_variables.client_set_value(this, isolation_level_val, value1.c_str())) + return false; + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection TRANSACTION ISOLATION LEVEL to %s\n", value1.c_str()); + } + } + else if (var == "read") { + const std::string& value1 = *values; + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET %s TRANSACTION READ value %s\n", val[0].c_str(), value1.c_str()); + const uint32_t transaction_read_int = SpookyHash::Hash32(value1.c_str(), value1.length(), 10); + if (pgsql_variables.client_get_hash(this, transaction_read_val) != transaction_read_int) { + if (!pgsql_variables.client_set_value(this, transaction_read_val, value1.c_str())) + return false; + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection TRANSACTION READ to %s\n", value1.c_str()); + } + } + else { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + } + else { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + } + if (exit_after_SetParse) { + if (command_type == _MYSQL_COM_QUERY) { + client_myds->DSS = STATE_QUERY_SENT_NET; + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); + return true; + } + } + } + else if (match_regexes && match_regexes[3]->match(dig)) { + SetParser parser(nq); + std::string charset = parser.parse_character_set(); + const MARIADB_CHARSET_INFO* c; + if (!charset.empty()) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET CHARACTER SET %s\n", charset.c_str()); + c = proxysql_find_charset_name(charset.c_str()); + } + else { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + if (!c) { + char* m = NULL; + char* errmsg = NULL; + m = (char*)"Unknown character set: '%s'"; + errmsg = (char*)malloc(charset.length() + strlen(m)); + sprintf(errmsg, m, charset.c_str()); + client_myds->DSS = STATE_QUERY_SENT_NET; + client_myds->myprot.generate_error_packet(true, true, errmsg, + PGSQL_ERROR_CODES::ERRCODE_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION, false, true); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + free(errmsg); + return true; + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 8, "Changing connection charset to %d\n", c->nr); + //-- client_myds->myconn->set_charset(c->nr, CHARSET); + } + if (exit_after_SetParse) { + if (command_type == _MYSQL_COM_QUERY) { + client_myds->DSS = STATE_QUERY_SENT_NET; + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); + return true; + } + } + } + else { + unable_to_parse_set_statement(lock_hostgroup); + return false; + } + } + } + + if (mirror == true) { // for mirror session we exit here + current_hostgroup = qpo->destination_hostgroup; + return false; + } + + // handle case #1797 + // handle case #2564 + if ((pkt->size == SELECT_CONNECTION_ID_LEN + 5 && *((char*)(pkt->ptr) + 4) == (char)0x03 && strncasecmp((char*)SELECT_CONNECTION_ID, (char*)pkt->ptr + 5, pkt->size - 5) == 0)) { + char buf[32]; + char buf2[32]; + sprintf(buf, "%u", thread_session_id); + int l0 = strlen("CONNECTION_ID()"); + memcpy(buf2, (char*)pkt->ptr + 5 + SELECT_CONNECTION_ID_LEN - l0, l0); + buf2[l0] = 0; + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + PgSQL_Data_Stream* myds = client_myds; + MySQL_Protocol* myprot = &client_myds->myprot; + myds->DSS = STATE_QUERY_SENT_DS; + int sid = 1; + myprot->generate_pkt_column_count(true, NULL, NULL, sid, 1); sid++; + myprot->generate_pkt_field(true, NULL, NULL, sid, (char*)"", (char*)"", (char*)"", buf2, (char*)"", 63, 31, MYSQL_TYPE_LONGLONG, 161, 0, false, 0, NULL); sid++; + myds->DSS = STATE_COLUMN_DEFINITION; + + bool deprecate_eof_active = myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + if (!deprecate_eof_active) { + myprot->generate_pkt_EOF(true, NULL, NULL, sid, 0, setStatus); sid++; + } + + char** p = (char**)malloc(sizeof(char*) * 1); + unsigned long* l = (unsigned long*)malloc(sizeof(unsigned long*) * 1); + l[0] = strlen(buf); + p[0] = buf; + myprot->generate_pkt_row(true, NULL, NULL, sid, 1, l, p); sid++; + myds->DSS = STATE_ROW; + + if (deprecate_eof_active) { + myprot->generate_pkt_OK(true, NULL, NULL, sid, 0, 0, setStatus, 0, NULL, true); sid++; + } + else { + myprot->generate_pkt_EOF(true, NULL, NULL, sid, 0, setStatus); sid++; + } + myds->DSS = STATE_SLEEP; + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); + free(p); + free(l); + return true; + } + + // handle case #1421 , about LAST_INSERT_ID + if (CurrentQuery.QueryParserArgs.digest_text) { + char* dig = CurrentQuery.QueryParserArgs.digest_text; + if (strcasestr(dig, "LAST_INSERT_ID") || strcasestr(dig, "@@IDENTITY")) { + // we need to try to execute it where the last write was successful + if (last_HG_affected_rows >= 0) { + PgSQL_Backend* _mybe = NULL; + _mybe = find_backend(last_HG_affected_rows); + if (_mybe) { + if (_mybe->server_myds) { + if (_mybe->server_myds->myconn) { + if (_mybe->server_myds->myconn->pgsql) { // we have an established connection + // this seems to be the right backend + qpo->destination_hostgroup = last_HG_affected_rows; + current_hostgroup = qpo->destination_hostgroup; + return false; // execute it on backend! + } + } + } + } + } + // if we reached here, we don't know the right backend + // we try to determine if it is a simple "SELECT LAST_INSERT_ID()" or "SELECT @@IDENTITY" and we return pgsql->last_insert_id + + //handle 2564 + if ( + (pkt->size == SELECT_LAST_INSERT_ID_LEN + 5 && *((char*)(pkt->ptr) + 4) == (char)0x03 && strncasecmp((char*)SELECT_LAST_INSERT_ID, (char*)pkt->ptr + 5, pkt->size - 5) == 0) + || + (pkt->size == SELECT_LAST_INSERT_ID_LIMIT1_LEN + 5 && *((char*)(pkt->ptr) + 4) == (char)0x03 && strncasecmp((char*)SELECT_LAST_INSERT_ID_LIMIT1, (char*)pkt->ptr + 5, pkt->size - 5) == 0) + || + (pkt->size == SELECT_VARIABLE_IDENTITY_LEN + 5 && *((char*)(pkt->ptr) + 4) == (char)0x03 && strncasecmp((char*)SELECT_VARIABLE_IDENTITY, (char*)pkt->ptr + 5, pkt->size - 5) == 0) + || + (pkt->size == SELECT_VARIABLE_IDENTITY_LIMIT1_LEN + 5 && *((char*)(pkt->ptr) + 4) == (char)0x03 && strncasecmp((char*)SELECT_VARIABLE_IDENTITY_LIMIT1, (char*)pkt->ptr + 5, pkt->size - 5) == 0) + ) { + char buf[32]; + sprintf(buf, "%llu", last_insert_id); + char buf2[32]; + int l0 = 0; + if (strcasestr(dig, "LAST_INSERT_ID")) { + l0 = strlen("LAST_INSERT_ID()"); + memcpy(buf2, (char*)pkt->ptr + 5 + SELECT_LAST_INSERT_ID_LEN - l0, l0); + } + else if (strcasestr(dig, "@@IDENTITY")) { + l0 = strlen("@@IDENTITY"); + memcpy(buf2, (char*)pkt->ptr + 5 + SELECT_VARIABLE_IDENTITY_LEN - l0, l0); + } + buf2[l0] = 0; + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + PgSQL_Data_Stream* myds = client_myds; + MySQL_Protocol* myprot = &client_myds->myprot; + myds->DSS = STATE_QUERY_SENT_DS; + int sid = 1; + myprot->generate_pkt_column_count(true, NULL, NULL, sid, 1); sid++; + myprot->generate_pkt_field(true, NULL, NULL, sid, (char*)"", (char*)"", (char*)"", buf2, (char*)"", 63, 31, MYSQL_TYPE_LONGLONG, 161, 0, false, 0, NULL); sid++; + myds->DSS = STATE_COLUMN_DEFINITION; + + bool deprecate_eof_active = myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + if (!deprecate_eof_active) { + myprot->generate_pkt_EOF(true, NULL, NULL, sid, 0, setStatus); sid++; + } + char** p = (char**)malloc(sizeof(char*) * 1); + unsigned long* l = (unsigned long*)malloc(sizeof(unsigned long*) * 1); + l[0] = strlen(buf); + p[0] = buf; + myprot->generate_pkt_row(true, NULL, NULL, sid, 1, l, p); sid++; + myds->DSS = STATE_ROW; + if (deprecate_eof_active) { + myprot->generate_pkt_OK(true, NULL, NULL, sid, 0, 0, setStatus, 0, NULL, true); sid++; + } + else { + myprot->generate_pkt_EOF(true, NULL, NULL, sid, 0, setStatus); sid++; + } + myds->DSS = STATE_SLEEP; + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); + free(p); + free(l); + return true; + } + + // if we reached here, we don't know the right backend and we cannot answer the query directly + // We continue the normal way + + // as a precaution, we reset cache_ttl + qpo->cache_ttl = 0; + } + } + + // handle command KILL #860 + //if (prepared == false) { + if (handle_command_query_kill(pkt)) { + return true; + } + //} + if (qpo->cache_ttl > 0 && ((prepare_stmt_type & PgSQL_ps_type_prepare_stmt) == 0)) { + bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + uint32_t resbuf = 0; + unsigned char* aa = GloQC->get( + client_myds->myconn->userinfo->hash, + (const unsigned char*)CurrentQuery.QueryPointer, + CurrentQuery.QueryLength, + &resbuf, + thread->curtime / 1000, + qpo->cache_ttl, + deprecate_eof_active + ); + if (aa) { + client_myds->buffer2resultset(aa, resbuf); + free(aa); + client_myds->PSarrayOUT->copy_add(client_myds->resultset, 0, client_myds->resultset->len); + while (client_myds->resultset->len) client_myds->resultset->remove_index(client_myds->resultset->len - 1, NULL); + if (transaction_persistent_hostgroup == -1) { + // not active, we can change it + current_hostgroup = -1; + } + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); + return true; + } + } + +__exit_set_destination_hostgroup: + + if (qpo->next_query_flagIN >= 0) { + next_query_flagIN = qpo->next_query_flagIN; + } + if (qpo->destination_hostgroup >= 0) { + if (transaction_persistent_hostgroup == -1) { + current_hostgroup = qpo->destination_hostgroup; + } + } + + if (pgsql_thread___set_query_lock_on_hostgroup == 1) { // algorithm introduced in 2.0.6 + if (locked_on_hostgroup >= 0) { + if (current_hostgroup != locked_on_hostgroup) { + client_myds->DSS = STATE_QUERY_SENT_NET; + char buf[140]; + sprintf(buf, "ProxySQL Error: connection is locked to hostgroup %d but trying to reach hostgroup %d", locked_on_hostgroup, current_hostgroup); + client_myds->myprot.generate_error_packet(true, true, buf, + PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION, false); + thread->status_variables.stvar[st_var_hostgroup_locked_queries]++; + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); + return true; + } + } + } + return false; +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STATISTICS(PtrSize_t* pkt) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_STATISTICS packet\n"); + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_statistics_response(true, NULL, NULL); + client_myds->DSS = STATE_SLEEP; +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_CHANGE_USER(PtrSize_t* pkt, bool* wrong_pass) { + + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_CHANGE_USER packet\n"); + //if (session_type == PROXYSQL_SESSION_PGSQL) { + if (session_type == PROXYSQL_SESSION_PGSQL || session_type == PROXYSQL_SESSION_SQLITE) { + reset(); + init(); + if (client_authenticated) { + if (use_ldap_auth == false) { + GloPgAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->username); + } + else { + GloMyLdapAuth->decrease_frontend_user_connections(client_myds->myconn->userinfo->fe_username); + } + } + client_authenticated = false; + if (client_myds->myprot.process_pkt_COM_CHANGE_USER((unsigned char*)pkt->ptr, pkt->size) == true) { + l_free(pkt->size, pkt->ptr); + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, 2, 0, NULL); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + *wrong_pass = false; + client_authenticated = true; + //int free_users=0; + int used_users = 0; + /*free_users */GloPgAuth->increase_frontend_user_connections(client_myds->myconn->userinfo->username, &used_users); + // FIXME: max_connections is not handled for CHANGE_USER + } + else { + l_free(pkt->size, pkt->ptr); + // 'COM_CHANGE_USER' didn't supply a password, and an 'Auth Switch Response' is + // required, going back to 'STATE_SERVER_HANDSHAKE' to perform the regular + // 'Auth Switch Response' for a connection is required. See #3504 for more context. + if (change_user_auth_switch) { + client_myds->DSS = STATE_SERVER_HANDSHAKE; + status = CONNECTING_CLIENT; + return; + } + + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Wrong credentials for frontend: disconnecting\n"); + *wrong_pass = true; + // FIXME: this should become close connection + client_myds->setDSS_STATE_QUERY_SENT_NET(); + char* client_addr = NULL; + if (client_myds->client_addr) { + char buf[512]; + switch (client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + default: + client_addr = strdup((char*)"localhost"); + break; + } + } + else { + client_addr = strdup((char*)""); + } + char* _s = (char*)malloc(strlen(client_myds->myconn->userinfo->username) + 100 + strlen(client_addr)); + sprintf(_s, "ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + proxy_error("ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)\n", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 2, 1045, (char*)"28000", _s, true); + free(_s); + __sync_fetch_and_add(&PgHGM->status.access_denied_wrong_password, 1); + } + } + else { + //FIXME: send an error message saying "not supported" or disconnect + l_free(pkt->size, pkt->ptr); + } +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_RESET_CONNECTION(PtrSize_t* pkt) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got MYSQL_COM_RESET_CONNECTION packet\n"); + + if (session_type == PROXYSQL_SESSION_PGSQL || session_type == PROXYSQL_SESSION_SQLITE) { + // Backup the current relevant session values + int default_hostgroup = this->default_hostgroup; + bool transaction_persistent = this->transaction_persistent; + + // Re-initialize the session + reset(); + init(); + + // Recover the relevant session values + this->default_hostgroup = default_hostgroup; + this->transaction_persistent = transaction_persistent; + //-- client_myds->myconn->set_charset(default_charset, NAMES); + + if (user_attributes != NULL && strlen(user_attributes)) { + nlohmann::json j_user_attributes = nlohmann::json::parse(user_attributes); + auto default_transaction_isolation = j_user_attributes.find("default-transaction_isolation"); + + if (default_transaction_isolation != j_user_attributes.end()) { + std::string def_trx_isolation_val = + j_user_attributes["default-transaction_isolation"].get(); + pgsql_variables.client_set_value(this, SQL_ISOLATION_LEVEL, def_trx_isolation_val.c_str()); + } + } + + l_free(pkt->size, pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, 2, 0, NULL); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + } + else { + l_free(pkt->size, pkt->ptr); + + std::string t_sql_error_msg{ "Received unsupported 'COM_RESET_CONNECTION' for session type '%s'" }; + std::string sql_error_msg{}; + string_format(t_sql_error_msg, sql_error_msg, proxysql_session_type_str(session_type).c_str()); + + client_myds->setDSS_STATE_QUERY_SENT_NET(); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, 2, 1047, (char*)"28000", sql_error_msg.c_str(), true); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + } +} + +void PgSQL_Session::handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED__get_connection() { + // Get a MySQL Connection + + PgSQL_Connection* mc = NULL; + char uuid[64]; + uint64_t trxid = 0; + unsigned long long now_us = 0; + if (qpo->max_lag_ms >= 0) { + if (qpo->max_lag_ms > 360000) { // this is an absolute time, we convert it to relative + if (now_us == 0) { + now_us = realtime_time(); + } + long long now_ms = now_us / 1000; + qpo->max_lag_ms = now_ms - qpo->max_lag_ms; + if (qpo->max_lag_ms < 0) { + qpo->max_lag_ms = -1; // time expired + } + } + } + if (session_fast_forward == false && qpo->create_new_conn == false) { +#ifndef STRESSTEST_POOL + mc = thread->get_MyConn_local(mybe->hostgroup_id, this, NULL, 0, (int)qpo->max_lag_ms); +#endif // STRESSTEST_POOL + } +#ifdef STRESSTEST_POOL + // Check STRESSTEST_POOL in MySQL_HostGroups_Manager.h + // Note: this works only if session_fast_forward==false and create_new_conn is false too +#define NUM_SLOW_LOOPS 1000 + // if STRESSTESTPOOL_MEASURE is define, time is measured in Query_Processor_time_nsec + // even if not the right variable +//#define STRESSTESTPOOL_MEASURE +#ifdef STRESSTESTPOOL_MEASURE + timespec begint; + timespec endt; + clock_gettime(CLOCK_MONOTONIC, &begint); +#endif // STRESSTESTPOOL_MEASURE + for (unsigned int loops = 0; loops < NUM_SLOW_LOOPS; loops++) { +#endif // STRESSTEST_POOL + + if (mc == NULL) { + if (trxid) { + mc = PgHGM->get_MyConn_from_pool(mybe->hostgroup_id, this, (session_fast_forward || qpo->create_new_conn), uuid, trxid, -1); + } + else { + mc = PgHGM->get_MyConn_from_pool(mybe->hostgroup_id, this, (session_fast_forward || qpo->create_new_conn), NULL, 0, (int)qpo->max_lag_ms); + } +#ifdef STRESSTEST_POOL + if (mc && (loops < NUM_SLOW_LOOPS - 1)) { + if (mc->pgsql) { + mybe->server_myds->attach_connection(mc); + mybe->server_myds->DSS = STATE_NOT_INITIALIZED; + mybe->server_myds->return_MySQL_Connection_To_Pool(); + mc = NULL; + } + } +#endif // STRESSTEST_POOL + } + else { + thread->status_variables.stvar[st_var_ConnPool_get_conn_immediate]++; + } +#ifdef STRESSTEST_POOL +#ifdef STRESSTESTPOOL_MEASURE + clock_gettime(CLOCK_MONOTONIC, &endt); + thread->status_variables.query_processor_time = thread->status_variables.query_processor_time + + (endt.tv_sec * 1000000000 + endt.tv_nsec) - + (begint.tv_sec * 1000000000 + begint.tv_nsec); +#endif // STRESSTESTPOOL_MEASURE + } +#endif // STRESSTEST_POOL + if (mc) { + mybe->server_myds->attach_connection(mc); + thread->status_variables.stvar[st_var_ConnPool_get_conn_success]++; + } + else { + thread->status_variables.stvar[st_var_ConnPool_get_conn_failure]++; + } + if (qpo->max_lag_ms >= 0) { + if (qpo->max_lag_ms <= 360000) { // this is a relative time , we convert it to absolute + if (mc == NULL) { + if (CurrentQuery.waiting_since == 0) { + CurrentQuery.waiting_since = thread->curtime; + thread->status_variables.stvar[st_var_queries_with_max_lag_ms__delayed]++; + } + } + if (now_us == 0) { + now_us = realtime_time(); + } + long long now_ms = now_us / 1000; + qpo->max_lag_ms = now_ms - qpo->max_lag_ms; + } + } + if (mc) { + if (CurrentQuery.waiting_since) { + unsigned long long waited = thread->curtime - CurrentQuery.waiting_since; + thread->status_variables.stvar[st_var_queries_with_max_lag_ms__total_wait_time_us] += waited; + CurrentQuery.waiting_since = 0; + } + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p -- server_myds=%p -- PgSQL_Connection %p\n", this, mybe->server_myds, mybe->server_myds->myconn); + if (mybe->server_myds->myconn == NULL) { + // we couldn't get a connection for whatever reason, ex: no backends, or too busy + if (thread->mypolls.poll_timeout == 0) { // tune poll timeout + thread->mypolls.poll_timeout = pgsql_thread___poll_timeout_on_failure * 1000; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Session=%p , DS=%p , poll_timeout=%u\n", mybe->server_myds->sess, mybe->server_myds, thread->mypolls.poll_timeout); + } + else { + if (thread->mypolls.poll_timeout > (unsigned int)pgsql_thread___poll_timeout_on_failure * 1000) { + thread->mypolls.poll_timeout = pgsql_thread___poll_timeout_on_failure * 1000; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 7, "Session=%p , DS=%p , poll_timeout=%u\n", mybe->server_myds->sess, mybe->server_myds, thread->mypolls.poll_timeout); + } + } + return; + } + if (mybe->server_myds->myconn->fd == -1) { + // we didn't get a valid connection, we need to create one + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p -- MySQL Connection has no FD\n", this); + PgSQL_Connection* myconn = mybe->server_myds->myconn; + myconn->userinfo->set(client_myds->myconn->userinfo); + + myconn->handler(0); + mybe->server_myds->fd = myconn->fd; + mybe->server_myds->DSS = STATE_MARIADB_CONNECTING; + status = CONNECTING_SERVER; + mybe->server_myds->myconn->reusable = true; + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Sess=%p -- MySQL Connection found = %p\n", this, mybe->server_myds->myconn); + mybe->server_myds->assign_fd_from_mysql_conn(); + mybe->server_myds->myds_type = MYDS_BACKEND; + mybe->server_myds->DSS = STATE_READY; + + if (session_fast_forward == true) { + status = FAST_FORWARD; + mybe->server_myds->myconn->reusable = false; // the connection cannot be usable anymore + } + } +} + +void PgSQL_Session::MySQL_Stmt_Result_to_MySQL_wire(MYSQL_STMT* stmt, PgSQL_Connection* myconn) { + PgSQL_Query_Result* query_result = NULL; + if (myconn) { + if (myconn->query_result) { + query_result = myconn->query_result; + } + } + /* + MYSQL_RES *stmt_result=myconn->query.stmt_result; + if (stmt_result) { + MySQL_ResultSet *query_result=new MySQL_ResultSet(); + query_result->init(&client_myds->myprot, stmt_result, stmt->pgsql, stmt); + query_result->get_resultset(client_myds->PSarrayOUT); + CurrentQuery.rows_sent = query_result->num_rows; + //removed bool resultset_completed=query_result->get_resultset(client_myds->PSarrayOUT); + delete query_result; + */ + if (query_result) { + //assert(query_result->result); + //query_result->init_with_stmt(myconn); + CurrentQuery.rows_sent = query_result->get_num_rows(); + const auto _affected_rows = query_result->get_affected_rows(); + if (_affected_rows != -1) { + CurrentQuery.affected_rows = _affected_rows; + CurrentQuery.have_affected_rows = true; + } + bool resultset_completed = query_result->get_resultset(client_myds->PSarrayOUT); + assert(resultset_completed); // the resultset should always be completed if MySQL_Result_to_MySQL_wire is called + } + else { + MYSQL* pgsql = stmt->mysql; + // no result set + int myerrno = mysql_stmt_errno(stmt); + if (myerrno == 0) { + unsigned int num_rows = mysql_affected_rows(stmt->mysql); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + if (pgsql->server_status & SERVER_MORE_RESULTS_EXIST) + setStatus |= SERVER_MORE_RESULTS_EXIST; + setStatus |= (pgsql->server_status & ~SERVER_STATUS_AUTOCOMMIT); // get flags from server_status but ignore autocommit + setStatus = setStatus & ~SERVER_STATUS_CURSOR_EXISTS; // Do not send cursor #1128 + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, client_myds->pkt_sid + 1, num_rows, pgsql->insert_id, setStatus, myconn ? myconn->warning_count : 0, pgsql->info); + client_myds->pkt_sid++; + } + else { + // error + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(pgsql)); + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, mysql_errno(pgsql), sqlstate, mysql_error(pgsql)); + client_myds->pkt_sid++; + } + } +} + +void PgSQL_Session::PgSQL_Result_to_PgSQL_wire(PgSQL_Connection* _conn, PgSQL_Data_Stream* _myds) { + if (_conn == NULL) { + // error + client_myds->myprot.generate_error_packet(true, true, "Lost connection to PostgreSQL server during query", + PGSQL_ERROR_CODES::ERRCODE_CONNECTION_FAILURE, false); + return; + } + + PgSQL_Query_Result* query_result = _conn->query_result; + + if (query_result && query_result->get_result_packet_type() != PGSQL_QUERY_RESULT_NO_DATA) { + bool transfer_started = query_result->is_transfer_started(); + // if there is an error, it will be false so results are not cached + bool is_tuple = query_result->get_result_packet_type() == (PGSQL_QUERY_RESULT_TUPLE | PGSQL_QUERY_RESULT_COMMAND | PGSQL_QUERY_RESULT_READY); + CurrentQuery.rows_sent = query_result->get_num_rows(); + const auto _affected_rows = query_result->get_affected_rows(); + if (_affected_rows != -1) { + CurrentQuery.affected_rows = _affected_rows; + CurrentQuery.have_affected_rows = true; + } + bool resultset_completed = query_result->get_resultset(client_myds->PSarrayOUT); + if (_conn->processing_multi_statement == false) + assert(resultset_completed); // the resultset should always be completed if PgSQL_Result_to_PgSQL_wire is called + if (transfer_started == false) { // we have all the resultset when PgSQL_Result_to_PgSQL_wire was called + if (qpo && qpo->cache_ttl > 0 && is_tuple == true) { // the resultset should be cached + /*if (mysql_errno(pgsql) == 0 && + (mysql_warning_count(pgsql) == 0 || + mysql_thread___query_cache_handle_warnings == 1)) { // no errors + if ( + (qpo->cache_empty_result == 1) + || ( + (qpo->cache_empty_result == -1) + && + (thread->variables.query_cache_stores_empty_result || query_result->num_rows) + ) + ) { + client_myds->resultset->copy_add(client_myds->PSarrayOUT, 0, client_myds->PSarrayOUT->len); + client_myds->resultset_length = query_result->resultset_size; + unsigned char* aa = client_myds->resultset2buffer(false); + while (client_myds->resultset->len) client_myds->resultset->remove_index(client_myds->resultset->len - 1, NULL); + bool deprecate_eof_active = client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + GloQC->set( + client_myds->myconn->userinfo->hash, + (const unsigned char*)CurrentQuery.QueryPointer, + CurrentQuery.QueryLength, + aa, + client_myds->resultset_length, + thread->curtime / 1000, + thread->curtime / 1000, + thread->curtime / 1000 + qpo->cache_ttl, + deprecate_eof_active + ); + l_free(client_myds->resultset_length, aa); + client_myds->resultset_length = 0; + } + }*/ + } + } + } else { // if query result is empty, means there was an error before query result was generated + + if (!_conn->is_error_present()) + assert(0); // if query result is empty, there should be an error present in connection. + + if (_myds && _myds->killed_at) { + if (_myds->kill_type == 0) { + client_myds->myprot.generate_error_packet(true, true, (char*)"Query execution was interrupted, query_timeout exceeded", + PGSQL_ERROR_CODES::ERRCODE_QUERY_CANCELED, false); + //PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, _conn->parent->myhgc->hid, _conn->parent->address, _conn->parent->port, 1907); + } + else { + client_myds->myprot.generate_error_packet(true, true, (char*)"Query execution was interrupted", + PGSQL_ERROR_CODES::ERRCODE_QUERY_CANCELED, false); + //PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, _conn->parent->myhgc->hid, _conn->parent->address, _conn->parent->port, 1317); + } + } + else { + client_myds->myprot.generate_error_packet(true, true, _conn->get_error_message().c_str(), _conn->get_error_code(), false); + //PgHGM->p_update_pgsql_error_counter(p_pgsql_error_type::proxysql, _conn->parent->myhgc->hid, _conn->parent->address, _conn->parent->port, 1907); + } + + /*int myerrno = mysql_errno(pgsql); + if (myerrno == 0) { + unsigned int num_rows = mysql_affected_rows(pgsql); + uint16_t setStatus = (active_transactions ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + if (pgsql->server_status & SERVER_MORE_RESULTS_EXIST) + setStatus |= SERVER_MORE_RESULTS_EXIST; + setStatus |= (pgsql->server_status & ~SERVER_STATUS_AUTOCOMMIT); // get flags from server_status but ignore autocommit + setStatus = setStatus & ~SERVER_STATUS_CURSOR_EXISTS; // Do not send cursor #1128 + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, client_myds->pkt_sid + 1, num_rows, pgsql->insert_id, setStatus, warning_count, pgsql->info); + //client_myds->pkt_sid++; + } + else { + // error + char sqlstate[10]; + sprintf(sqlstate, "%s", mysql_sqlstate(pgsql)); + if (_myds && _myds->killed_at) { // see case #750 + if (_myds->kill_type == 0) { + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, 1907, sqlstate, (char*)"Query execution was interrupted, query_timeout exceeded"); + } + else { + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, 1317, sqlstate, (char*)"Query execution was interrupted"); + } + } + else { + client_myds->myprot.generate_pkt_ERR(true, NULL, NULL, client_myds->pkt_sid + 1, mysql_errno(pgsql), sqlstate, mysql_error(pgsql)); + } + //client_myds->pkt_sid++; + } + */ + } +} + +void PgSQL_Session::SQLite3_to_MySQL(SQLite3_result* result, char* error, int affected_rows, MySQL_Protocol* myprot, bool in_transaction, bool deprecate_eof_active) { + assert(myprot); + MySQL_Data_Stream* myds = myprot->get_myds(); + myds->DSS = STATE_QUERY_SENT_DS; + int sid = 1; + if (result) { + myprot->generate_pkt_column_count(true, NULL, NULL, sid, result->columns); sid++; + for (int i = 0; i < result->columns; i++) { + myprot->generate_pkt_field(true, NULL, NULL, sid, (char*)"", (char*)"", (char*)"", result->column_definition[i]->name, (char*)"", 33, 15, MYSQL_TYPE_VAR_STRING, 1, 0x1f, false, 0, NULL); + sid++; + } + myds->DSS = STATE_COLUMN_DEFINITION; + unsigned int nTrx = 0; + uint16_t setStatus = 0; + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + if (in_transaction == false) { + nTrx = NumActiveTransactions(); + setStatus |= (nTrx ? SERVER_STATUS_IN_TRANS : 0); + } + else { + // this is for SQLite3 Server + if (session_type == PROXYSQL_SESSION_SQLITE) { + //if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + } + else { + // for sessions that are not SQLITE . Admin and Clickhouse . + // default + setStatus |= SERVER_STATUS_AUTOCOMMIT; + } + setStatus |= SERVER_STATUS_IN_TRANS; + } + if (!deprecate_eof_active) { + myprot->generate_pkt_EOF(true, NULL, NULL, sid, 0, setStatus); sid++; + } + + char** p = (char**)malloc(sizeof(char*) * result->columns); + unsigned long* l = (unsigned long*)malloc(sizeof(unsigned long*) * result->columns); + + MySQL_ResultSet query_result{}; + query_result.buffer_init(myprot); + + for (int r = 0; r < result->rows_count; r++) { + for (int i = 0; i < result->columns; i++) { + l[i] = result->rows[r]->sizes[i]; + p[i] = result->rows[r]->fields[i]; + } + sid = myprot->generate_pkt_row3(&query_result, NULL, sid, result->columns, l, p, 0); sid++; + } + + query_result.buffer_to_PSarrayOut(); + query_result.get_resultset(myds->PSarrayOUT); + + myds->DSS = STATE_ROW; + + if (deprecate_eof_active) { + myprot->generate_pkt_OK(true, NULL, NULL, sid, 0, 0, setStatus, 0, NULL, true); sid++; + } + else { + myprot->generate_pkt_EOF(true, NULL, NULL, sid, 0, setStatus); sid++; + } + + myds->DSS = STATE_SLEEP; + free(l); + free(p); + + } + else { // no result set + if (error) { + // there was an error + if (strcmp(error, (char*)"database is locked") == 0) { + client_myds->myprot.generate_error_packet(true, true, error, + PGSQL_ERROR_CODES::ERRCODE_T_R_DEADLOCK_DETECTED, false); + } + else { + client_myds->myprot.generate_error_packet(true, true, error, + PGSQL_ERROR_CODES::ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION, false); + } + } + else { + // no error, DML succeeded + unsigned int nTrx = 0; + uint16_t setStatus = 0; + if (in_transaction == false) { + nTrx = NumActiveTransactions(); + setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + } + else { + // this is for SQLite3 Server + if (session_type == PROXYSQL_SESSION_SQLITE) { + //if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + } + else { + // for sessions that are not SQLITE . Admin and Clickhouse . + // default + setStatus |= SERVER_STATUS_AUTOCOMMIT; + } + setStatus |= SERVER_STATUS_IN_TRANS; + } + myprot->generate_pkt_OK(true, NULL, NULL, sid, affected_rows, 0, setStatus, 0, NULL); + } + myds->DSS = STATE_SLEEP; + } +} + +unsigned long long PgSQL_Session::IdleTime() { + unsigned long long ret = 0; + if (client_myds == 0) return 0; + if (status != WAITING_CLIENT_DATA && status != CONNECTING_CLIENT) return 0; + int idx = client_myds->poll_fds_idx; + unsigned long long last_sent = thread->mypolls.last_sent[idx]; + unsigned long long last_recv = thread->mypolls.last_recv[idx]; + unsigned long long last_time = (last_sent > last_recv ? last_sent : last_recv); + if (thread->curtime > last_time) { + ret = thread->curtime - last_time; + } + return ret; +} + + + +// this is called either from RequestEnd(), or at the end of executing +// prepared statements +void PgSQL_Session::LogQuery(PgSQL_Data_Stream* myds) { + // we need to access statistics before calling CurrentQuery.end() + // so we track the time here + CurrentQuery.end_time = thread->curtime; + + if (qpo) { + if (qpo->log == 1) { + GloPgSQL_Logger->log_request(this, myds); // we send for logging only if logging is enabled for this query + } + else { + if (qpo->log == -1) { + if (pgsql_thread___eventslog_default_log == 1) { + GloPgSQL_Logger->log_request(this, myds); // we send for logging only if enabled by default + } + } + } + } +} +void PgSQL_Session::RequestEnd(PgSQL_Data_Stream* myds) { + // check if multiplexing needs to be disabled + char* qdt = NULL; + + if (status != PROCESSING_STMT_EXECUTE) { + qdt = CurrentQuery.get_digest_text(); + } + else { + qdt = CurrentQuery.stmt_info->digest_text; + } + + if (qdt && myds && myds->myconn) { + myds->myconn->ProcessQueryAndSetStatusFlags(qdt); + } + + switch (status) { + case PROCESSING_STMT_EXECUTE: + case PROCESSING_STMT_PREPARE: + // if a prepared statement is executed, LogQuery was already called + break; + default: + if (session_fast_forward == false) { + LogQuery(myds); + } + break; + } + + GloPgQPro->delete_QP_out(qpo); + // if there is an associated myds, clean its status + if (myds) { + // if there is a pgsql connection, clean its status + if (myds->myconn) { + myds->myconn->async_free_result(); + myds->myconn->compute_unknown_transaction_status(); + } + myds->free_mysql_real_query(); + } + if (session_fast_forward == false) { + // reset status of the session + status = WAITING_CLIENT_DATA; + if (client_myds) { + // reset status of client data stream + client_myds->DSS = STATE_SLEEP; + // finalize the query + CurrentQuery.end(); + } + } + started_sending_data_to_client = false; + previous_hostgroup = current_hostgroup; +} + + +// this function tries to report all the memory statistics related to the sessions +void PgSQL_Session::Memory_Stats() { + if (thread == NULL) + return; + unsigned int i; + unsigned long long backend = 0; + unsigned long long frontend = 0; + unsigned long long internal = 0; + internal += sizeof(PgSQL_Session); + if (qpo) + internal += sizeof(Query_Processor_Output); + if (client_myds) { + internal += sizeof(PgSQL_Data_Stream); + if (client_myds->queueIN.buffer) + frontend += QUEUE_T_DEFAULT_SIZE; + if (client_myds->queueOUT.buffer) + frontend += QUEUE_T_DEFAULT_SIZE; + if (client_myds->myconn) { + internal += sizeof(PgSQL_Connection); + } + if (client_myds->PSarrayIN) { + internal += client_myds->PSarrayIN->total_size(); + } + if (client_myds->PSarrayIN) { + if (session_fast_forward == true) { + internal += client_myds->PSarrayOUT->total_size(); + } else { + internal += client_myds->PSarrayOUT->total_size(PGSQL_RESULTSET_BUFLEN); + internal += client_myds->resultset->total_size(PGSQL_RESULTSET_BUFLEN); + } + } + } + for (i = 0; i < mybes->len; i++) { + PgSQL_Backend* _mybe = (PgSQL_Backend*)mybes->index(i); + internal += sizeof(PgSQL_Backend); + if (_mybe->server_myds) { + internal += sizeof(PgSQL_Data_Stream); + if (_mybe->server_myds->queueIN.buffer) + backend += QUEUE_T_DEFAULT_SIZE; + if (_mybe->server_myds->queueOUT.buffer) + backend += QUEUE_T_DEFAULT_SIZE; + if (_mybe->server_myds->myconn) { + PgSQL_Connection* myconn = _mybe->server_myds->myconn; + internal += sizeof(PgSQL_Connection); + if (myconn->is_connected()) { + //backend += sizeof(MYSQL); + //backend += myconn->pgsql->net.max_packet; + backend += myconn->get_memory_usage(); + //backend += (4096 * 15); // ASYNC_CONTEXT_DEFAULT_STACK_SIZE + } + if (myconn->query_result) { + backend += myconn->query_result->current_size(); + } + } + } + } + thread->status_variables.stvar[st_var_mysql_backend_buffers_bytes] += backend; + thread->status_variables.stvar[st_var_mysql_frontend_buffers_bytes] += frontend; + thread->status_variables.stvar[st_var_mysql_session_internal_bytes] += internal; +} + + +void PgSQL_Session::create_new_session_and_reset_connection(PgSQL_Data_Stream* _myds) { + PgSQL_Data_Stream* new_myds = NULL; + PgSQL_Connection* mc = _myds->myconn; + // we remove the connection from the original data stream + _myds->detach_connection(); + _myds->unplug_backend(); + + // we create a brand new session, a new data stream, and attach the connection to it + PgSQL_Session* new_sess = new PgSQL_Session(); + new_sess->mybe = new_sess->find_or_create_backend(mc->parent->myhgc->hid); + + new_myds = new_sess->mybe->server_myds; + new_myds->attach_connection(mc); + new_myds->assign_fd_from_mysql_conn(); + new_myds->myds_type = MYDS_BACKEND; + new_sess->to_process = 1; + new_myds->wait_until = thread->curtime + pgsql_thread___connect_timeout_server * 1000; // max_timeout + mc->last_time_used = thread->curtime; + new_myds->myprot.init(&new_myds, new_myds->myconn->userinfo, NULL); + new_sess->status = RESETTING_CONNECTION; + mc->async_state_machine = ASYNC_IDLE; // may not be true, but is used to correctly perform error handling + mc->auto_increment_delay_token = 0; + new_myds->DSS = STATE_MARIADB_QUERY; + thread->register_session_connection_handler(new_sess, true); + if (new_myds->mypolls == NULL) { + thread->mypolls.add(POLLIN | POLLOUT, new_myds->fd, new_myds, thread->curtime); + } + int rc = new_sess->handler(); + if (rc == -1) { + unsigned int sess_idx = thread->mysql_sessions->len - 1; + thread->unregister_session(sess_idx); + delete new_sess; + } +} + +bool PgSQL_Session::handle_command_query_kill(PtrSize_t* pkt) { + /*unsigned char command_type = *((unsigned char*)pkt->ptr + sizeof(mysql_hdr)); + if (CurrentQuery.QueryParserArgs.digest_text) { + if (command_type == _MYSQL_COM_QUERY) { + if (client_myds && client_myds->myconn) { + PgSQL_Connection* mc = client_myds->myconn; + if (mc->userinfo && mc->userinfo->username) { + if (CurrentQuery.PgQueryCmd == PGSQL_QUERY_KILL) { + char* qu = query_strip_comments((char*)pkt->ptr + 1 + sizeof(mysql_hdr), pkt->size - 1 - sizeof(mysql_hdr), + pgsql_thread___query_digests_lowercase); + string nq = string(qu, strlen(qu)); + re2::RE2::Options* opt2 = new re2::RE2::Options(RE2::Quiet); + opt2->set_case_sensitive(false); + char* pattern = (char*)"^KILL\\s+(CONNECTION |QUERY |)\\s*(\\d+)\\s*$"; + re2::RE2* re = new RE2(pattern, *opt2); + int id = 0; + string tk; + RE2::FullMatch(nq, *re, &tk, &id); + delete re; + delete opt2; + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 2, "filtered query= \"%s\"\n", qu); + free(qu); + if (id) { + int tki = -1; + if (tk.c_str()) { + if ((strlen(tk.c_str()) == 0) || (strcasecmp(tk.c_str(), "CONNECTION ") == 0)) { + tki = 0; + } + else { + if (strcasecmp(tk.c_str(), "QUERY ") == 0) { + tki = 1; + } + } + } + if (tki >= 0) { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 2, "Killing %s %d\n", (tki == 0 ? "CONNECTION" : "QUERY"), id); + GloPTH->kill_connection_or_query(id, (tki == 0 ? false : true), mc->userinfo->username); + client_myds->DSS = STATE_QUERY_SENT_NET; + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus = SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + RequestEnd(NULL); + l_free(pkt->size, pkt->ptr); + return true; + } + } + } + } + } + } + }*/ + return false; +} + +#if 0 +void PgSQL_Session::add_ldap_comment_to_pkt(PtrSize_t* _pkt) { + if (GloMyLdapAuth == NULL) + return; + if (use_ldap_auth == false) + return; + if (client_myds == NULL || client_myds->myconn == NULL || client_myds->myconn->userinfo == NULL) + return; + if (client_myds->myconn->userinfo->fe_username == NULL) + return; + char* fe = client_myds->myconn->userinfo->fe_username; + char* a = (char*)" /* %s=%s */"; + char* b = (char*)malloc(strlen(a) + strlen(fe) + strlen(mysql_thread___add_ldap_user_comment)); + sprintf(b, a, mysql_thread___add_ldap_user_comment, fe); + PtrSize_t _new_pkt; + _new_pkt.ptr = malloc(strlen(b) + _pkt->size); + memcpy(_new_pkt.ptr, _pkt->ptr, 5); + unsigned char* _c = (unsigned char*)_new_pkt.ptr; + _c += 5; + void* idx = memchr((char*)_pkt->ptr + 5, ' ', _pkt->size - 5); + if (idx) { + size_t first_word_len = (char*)idx - (char*)_pkt->ptr - 5; + if (((char*)_pkt->ptr + 5)[0] == '/' && ((char*)_pkt->ptr + 5)[1] == '*') { + void* comment_endpos = memmem(static_cast(_pkt->ptr) + 7, _pkt->size - 7, "*/", strlen("*/")); + + if (comment_endpos == NULL || idx < comment_endpos) { + b[1] = ' '; + b[2] = ' '; + b[strlen(b) - 1] = ' '; + b[strlen(b) - 2] = ' '; + } + } + memcpy(_c, (char*)_pkt->ptr + 5, first_word_len); + _c += first_word_len; + memcpy(_c, b, strlen(b)); + _c += strlen(b); + memcpy(_c, (char*)idx, _pkt->size - 5 - first_word_len); + } + else { + memcpy(_c, (char*)_pkt->ptr + 5, _pkt->size - 5); + _c += _pkt->size - 5; + memcpy(_c, b, strlen(b)); + } + l_free(_pkt->size, _pkt->ptr); + _pkt->size = _pkt->size + strlen(b); + _pkt->ptr = _new_pkt.ptr; + free(b); + CurrentQuery.QueryLength = _pkt->size - 5; + CurrentQuery.QueryPointer = (unsigned char*)_pkt->ptr + 5; +} +#endif // 0 + +void PgSQL_Session::finishQuery(PgSQL_Data_Stream* myds, PgSQL_Connection* myconn, bool prepared_stmt_with_no_params) { + myds->myconn->reduce_auto_increment_delay_token(); + if (locked_on_hostgroup >= 0) { + if (qpo->multiplex == -1) { + myds->myconn->set_status(true, STATUS_MYSQL_CONNECTION_NO_MULTIPLEX); + } + } + + const bool is_active_transaction = myds->myconn->IsActiveTransaction(); + const bool multiplex_disabled_by_status = myds->myconn->MultiplexDisabled(false); + + const bool multiplex_delayed = myds->myconn->auto_increment_delay_token > 0; + const bool multiplex_delayed_with_timeout = + !multiplex_disabled_by_status && multiplex_delayed && pgsql_thread___auto_increment_delay_multiplex_timeout_ms > 0; + + const bool multiplex_disabled = !multiplex_disabled_by_status && (!multiplex_delayed || multiplex_delayed_with_timeout); + const bool conn_is_reusable = myds->myconn->reusable == true && !is_active_transaction && multiplex_disabled; + + if (pgsql_thread___multiplexing && conn_is_reusable) { + if ((pgsql_thread___connection_delay_multiplex_ms || multiplex_delayed_with_timeout) && mirror == false) { + if (multiplex_delayed_with_timeout) { + uint64_t delay_multiplex_us = pgsql_thread___connection_delay_multiplex_ms * 1000; + uint64_t auto_increment_delay_us = pgsql_thread___auto_increment_delay_multiplex_timeout_ms * 1000; + uint64_t delay_us = delay_multiplex_us > auto_increment_delay_us ? delay_multiplex_us : auto_increment_delay_us; + + myds->wait_until = thread->curtime + delay_us; + } else { + myds->wait_until = thread->curtime + pgsql_thread___connection_delay_multiplex_ms * 1000; + } + + myconn->async_state_machine = ASYNC_IDLE; + myconn->multiplex_delayed = true; + myds->DSS = STATE_MARIADB_GENERIC; + } else if (prepared_stmt_with_no_params == true) { // see issue #1432 + myconn->async_state_machine = ASYNC_IDLE; + myds->DSS = STATE_MARIADB_GENERIC; + myds->wait_until = 0; + myconn->multiplex_delayed = false; + } else { + myconn->multiplex_delayed = false; + myds->wait_until = 0; + myds->DSS = STATE_NOT_INITIALIZED; + if (mysql_thread___autocommit_false_not_reusable && myds->myconn->IsAutoCommit() == false) { + create_new_session_and_reset_connection(myds); + } + else { + myds->return_MySQL_Connection_To_Pool(); + } + } + if (transaction_persistent == true) { + transaction_persistent_hostgroup = -1; + } + } + else { + myconn->multiplex_delayed = false; + myconn->compute_unknown_transaction_status(); + myconn->async_state_machine = ASYNC_IDLE; + myds->DSS = STATE_MARIADB_GENERIC; + if (transaction_persistent == true) { + if (transaction_persistent_hostgroup == -1) { // change only if not set already, do not allow to change it again + if (myds->myconn->IsActiveTransaction() == true) { // only active transaction is important here. Ignore other criterias + transaction_persistent_hostgroup = current_hostgroup; + } + } + else { + if (myds->myconn->IsActiveTransaction() == false) { // a transaction just completed + transaction_persistent_hostgroup = -1; + } + } + } + } +} + + +bool PgSQL_Session::known_query_for_locked_on_hostgroup(uint64_t digest) { + bool ret = false; + switch (digest) { + case 1732998280766099668ULL: // "SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT" + case 3748394912237323598ULL: // "SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS" + case 14407184196285870219ULL: // "SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION" + case 16906282918371515167ULL: // "SET @OLD_TIME_ZONE=@@TIME_ZONE" + case 15781568104089880179ULL: // "SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0" + case 5915334213354374281ULL: // "SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0" + case 7837089204483965579ULL: // "SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO'" + case 4312882378746554890ULL: // "SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0" + case 4379922288366515816ULL: // "SET @rocksdb_get_is_supported = IF (@rocksdb_has_p_s_session_variables, 'SELECT COUNT(*) INTO @rocksdb_is_supported FROM performance_schema.session_variables WHERE VARIABLE_NAME... + case 12687634401278615449ULL: // "SET @rocksdb_enable_bulk_load = IF (@rocksdb_is_supported, 'SET SESSION rocksdb_bulk_load = 1', 'SET @rocksdb_dummy_bulk_load = 0')" + case 15991633859978935883ULL: // "SET @MYSQLDUMP_TEMP_LOG_BIN = @@SESSION.SQL_LOG_BIN" + case 10636751085721966716ULL: // "SET @@GLOBAL.GTID_PURGED=?" + case 15976043181199829579ULL: // "SET SQL_QUOTE_SHOW_CREATE=?" + case 12094956190640701942ULL: // "SET SESSION information_schema_stats_expiry=0" + /* + case ULL: // + case ULL: // + case ULL: // + case ULL: // + case ULL: // + */ + ret = true; + break; + default: + break; + } + return ret; +} + + + +void PgSQL_Session::unable_to_parse_set_statement(bool* lock_hostgroup) { + // we couldn't parse the query + string query_str = string((char*)CurrentQuery.QueryPointer, CurrentQuery.QueryLength); + string digest_str = string(CurrentQuery.get_digest_text()); + string& nqn = (pgsql_thread___parse_failure_logs_digest == true ? digest_str : query_str); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Locking hostgroup for query %s\n", query_str.c_str()); + if (qpo->multiplex == -1) { + // we have no rule about this SET statement. We set hostgroup locking + if (locked_on_hostgroup < 0) { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "SET query to cause setting lock_hostgroup: %s\n", nqn.c_str()); + if (known_query_for_locked_on_hostgroup(CurrentQuery.QueryParserArgs.digest)) { + proxy_info("Setting lock_hostgroup for SET query: %s\n", nqn.c_str()); + } + else { + if (client_myds && client_myds->addr.addr) { + proxy_warning("Unable to parse unknown SET query from client %s:%d. Setting lock_hostgroup. Please report a bug for future enhancements:%s\n", client_myds->addr.addr, client_myds->addr.port, nqn.c_str()); + } + else { + proxy_warning("Unable to parse unknown SET query. Setting lock_hostgroup. Please report a bug for future enhancements:%s\n", nqn.c_str()); + } + } + *lock_hostgroup = true; + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "SET query to cause setting lock_hostgroup, but already set: %s\n", nqn.c_str()); + if (known_query_for_locked_on_hostgroup(CurrentQuery.QueryParserArgs.digest)) { + //proxy_info("Setting lock_hostgroup for SET query: %s\n", nqn.c_str()); + } + else { + if (client_myds && client_myds->addr.addr) { + proxy_warning("Unable to parse unknown SET query from client %s:%d. Setting lock_hostgroup. Please report a bug for future enhancements:%s\n", client_myds->addr.addr, client_myds->addr.port, nqn.c_str()); + } + else { + proxy_warning("Unable to parse unknown SET query. Setting lock_hostgroup. Please report a bug for future enhancements:%s\n", nqn.c_str()); + } + } + } + } + else { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, + "Unable to parse SET query but NOT setting lock_hostgroup %s\n", query_str.c_str()); + } +} + +#if 0 +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_RESET(PtrSize_t& pkt) { + uint32_t stmt_global_id = 0; + memcpy(&stmt_global_id, (char*)pkt.ptr + 5, sizeof(uint32_t)); + SLDH->reset(stmt_global_id); + l_free(pkt.size, pkt.ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + unsigned int nTrx = NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, 0, 0, setStatus, 0, NULL); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; +} + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_CLOSE(PtrSize_t& pkt) { + uint32_t client_global_id = 0; + memcpy(&client_global_id, (char*)pkt.ptr + 5, sizeof(uint32_t)); + // FIXME: no input validation + uint64_t stmt_global_id = 0; + stmt_global_id = client_myds->myconn->local_stmts->find_global_stmt_id_from_client(client_global_id); + SLDH->reset(client_global_id); + if (stmt_global_id) { + sess_STMTs_meta->erase(stmt_global_id); + } + client_myds->myconn->local_stmts->client_close(client_global_id); + l_free(pkt.size, pkt.ptr); + // FIXME: this is not complete. Counters should be decreased + thread->status_variables.stvar[st_var_frontend_stmt_close]++; + thread->status_variables.stvar[st_var_queries]++; + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; +} + + +void PgSQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_STMT_SEND_LONG_DATA(PtrSize_t& pkt) { + // FIXME: no input validation + uint32_t stmt_global_id = 0; + memcpy(&stmt_global_id, (char*)pkt.ptr + 5, sizeof(uint32_t)); + uint32_t stmt_param_id = 0; + memcpy(&stmt_param_id, (char*)pkt.ptr + 9, sizeof(uint16_t)); + SLDH->add(stmt_global_id, stmt_param_id, (char*)pkt.ptr + 11, pkt.size - 11); + client_myds->DSS = STATE_SLEEP; + status = WAITING_CLIENT_DATA; + l_free(pkt.size, pkt.ptr); +} +#endif // 0 + +void PgSQL_Session::detected_broken_connection(const char* file, unsigned int line, const char* func, const char* action, PgSQL_Connection* myconn, bool verbose) { + + const char* code = PgSQL_Error_Helper::get_error_code(PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION);; + const char* msg = "Detected offline server prior to statement execution"; + + if (myconn->is_error_present() == true) { + code = myconn->get_error_code_str(); + msg = myconn->get_error_message().c_str(); + } + + unsigned long long last_used = thread->curtime - myconn->last_time_used; + last_used /= 1000; + if (verbose) { + proxy_error_inline(file, line, func, "Detected a broken connection while %s on (%d,%s,%d,%lu) , FD (Conn:%d , MyDS:%d) , user %s , last_used %llums ago : %s, %s\n", action, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, myconn->get_mysql_thread_id(), myconn->myds->fd, myconn->fd, myconn->userinfo->username, last_used, code, msg); + } else { + proxy_error_inline(file, line, func, "Detected a broken connection while %s on (%d,%s,%d,%lu) , user %s , last_used %llums ago : %s, %s\n", action, myconn->parent->myhgc->hid, myconn->parent->address, myconn->parent->port, myconn->get_mysql_thread_id(), myconn->userinfo->username, last_used, code, msg); + } +} + +void PgSQL_Session::generate_status_one_hostgroup(int hid, std::string& s) { + SQLite3_result* resultset = PgHGM->SQL3_Connection_Pool(false, &hid); + json j_res; + if (resultset->rows_count) { + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + json j; // one json for each row + for (int i = 0; i < resultset->columns; i++) { + // using the format j["name"] == "value" + j[resultset->column_definition[i]->name] = (r->fields[i] ? std::string(r->fields[i]) : std::string("(null)")); + } + j_res.push_back(j); // the row json is added to the final json + } + } + else { + j_res = json::array(); + } + s = j_res.dump(); + delete resultset; +} + +void PgSQL_Session::reset_warning_hostgroup_flag_and_release_connection() +{ + if (warning_in_hg > -1) { + // if we've reached this point, it means that warning was found in the previous query, but the + // current executed query is not 'SHOW WARNINGS' or 'SHOW COUNT(*) FROM WARNINGS', so we can safely reset warning_in_hg and + // return connection back to the connection pool. + PgSQL_Backend* _mybe = find_backend(warning_in_hg); + if (_mybe) { + PgSQL_Data_Stream* myds = _mybe->server_myds; + if (myds && myds->myconn) { + myds->myconn->warning_count = 0; + myds->myconn->set_status(false, STATUS_MYSQL_CONNECTION_HAS_WARNINGS); + if ((myds->myconn->reusable == true) && myds->myconn->IsActiveTransaction() == false && myds->myconn->MultiplexDisabled() == false) { + myds->return_MySQL_Connection_To_Pool(); + } + } + } + warning_in_hg = -1; + } +} + +/** + * @brief Sets the previous status of the PgSQL session according to the current status, with an option to allow EXECUTE statements. + * + * This method updates the previous status of the PgSQL session based on its current status. It employs a switch statement + * to determine the current status and then pushes the corresponding status value onto the `previous_status` stack. If the + * `allow_execute` parameter is set to true and the current status is `PROCESSING_STMT_EXECUTE`, the method pushes this status + * onto the stack; otherwise, it skips pushing the status for EXECUTE statements. If the current status does not match any known + * status value (which should not occur under normal circumstances), the method asserts to indicate a programming error. + * It currently works with only 3 possible status: + * - PROCESSING_QUERY + * - PROCESSING_STMT_PREPARE + * - PROCESSING_STMT_EXECUTE + * + * @param allow_execute A boolean value indicating whether to allow the status of EXECUTE statements to be pushed onto the + * `previous_status` stack. If set to true, the method will include EXECUTE statements in the session's status history. + * + * @return void. + * @note This method assumes that the `status` member variable has been properly initialized with one of the predefined + * status values. + * @note This method is primarily used to maintain a history of the session's previous states for later reference or + * recovery purposes. + * @note The LCOV_EXCL_START and LCOV_EXCL_STOP directives are used to exclude the assert statement from code coverage + * analysis because the condition should not occur during normal execution and is included as a safeguard against + * programming errors. + */ +void PgSQL_Session::set_previous_status_mode3(bool allow_execute) { + switch (status) { + case PROCESSING_QUERY: + previous_status.push(PROCESSING_QUERY); + break; + case PROCESSING_STMT_PREPARE: + previous_status.push(PROCESSING_STMT_PREPARE); + break; + case PROCESSING_STMT_EXECUTE: + if (allow_execute == true) { + previous_status.push(PROCESSING_STMT_EXECUTE); + break; + } + default: + // LCOV_EXCL_START + assert(0); // Assert to indicate an unexpected status value + break; + // LCOV_EXCL_STOP + } +} diff --git a/lib/PgSQL_Thread.cpp b/lib/PgSQL_Thread.cpp new file mode 100644 index 0000000000..53f1d5a837 --- /dev/null +++ b/lib/PgSQL_Thread.cpp @@ -0,0 +1,5402 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + +//#define __CLASS_STANDARD_MYSQL_THREAD_H + +#include +#include + +#include "PgSQL_HostGroups_Manager.h" +#include "prometheus_helpers.h" +#define PGSQL_THREAD_IMPLEMENTATION +#include "proxysql.h" +#include "cpp.h" +#include "PgSQL_Thread.h" +#include +#include +#include "re2/re2.h" +#include "re2/regexp.h" + +#include "PgSQL_Data_Stream.h" +#include "PgSQL_Query_Processor.h" +#include "StatCounters.h" +#include "MySQL_PreparedStatement.h" +#include "PgSQL_Logger.hpp" + +#include + +using std::vector; +using std::function; + +#ifdef DEBUG +static PgSQL_Session* sess_stopat; +#endif + +#ifdef epoll_create1 +#define EPOLL_CREATE epoll_create1(0) +#else +#define EPOLL_CREATE epoll_create(1) +#endif + +#define PROXYSQL_LISTEN_LEN 1024 +#define MIN_THREADS_FOR_MAINTENANCE 8 + +extern PgSQL_Query_Processor* GloPgQPro; +extern PgSQL_Threads_Handler* GloPTH; +extern MySQL_Monitor* GloMyMon; +extern PgSQL_Logger* GloPgSQL_Logger; + +typedef struct mythr_st_vars { + enum PgSQL_Thread_status_variable v_idx; + p_th_counter::metric m_idx; + char* name; + uint32_t conv; +} mythr_st_vars_t; + +typedef struct mythr_g_st_vars { + enum PgSQL_Thread_status_variable v_idx; + p_th_gauge::metric m_idx; + char* name; + uint32_t conv; +} mythr_g_st_vars_t; + +// Note: the order here is not important. +mythr_st_vars_t PgSQL_Thread_status_variables_counter_array[]{ + /*{st_var_backend_stmt_prepare, p_th_counter::com_backend_stmt_prepare, (char*)"Com_backend_stmt_prepare"}, + { st_var_backend_stmt_execute, p_th_counter::com_backend_stmt_execute, (char*)"Com_backend_stmt_execute" }, + { st_var_backend_stmt_close, p_th_counter::com_backend_stmt_close, (char*)"Com_backend_stmt_close" }, + { st_var_frontend_stmt_prepare, p_th_counter::com_frontend_stmt_prepare, (char*)"Com_frontend_stmt_prepare" }, + { st_var_frontend_stmt_execute, p_th_counter::com_frontend_stmt_execute, (char*)"Com_frontend_stmt_execute" }, + { st_var_frontend_stmt_close, p_th_counter::com_frontend_stmt_close, (char*)"Com_frontend_stmt_close" }, + { st_var_queries, p_th_counter::questions, (char*)"Questions" }, + { st_var_queries_slow, p_th_counter::slow_queries, (char*)"Slow_queries" }, + { st_var_queries_gtid, p_th_counter::gtid_consistent_queries, (char*)"GTID_consistent_queries" }, + { st_var_gtid_session_collected,p_th_counter::gtid_session_collected, (char*)"GTID_session_collected" }, + { st_var_queries_backends_bytes_recv, p_th_counter::queries_backends_bytes_recv, (char*)"Queries_backends_bytes_recv" }, + { st_var_queries_backends_bytes_sent, p_th_counter::queries_backends_bytes_sent, (char*)"Queries_backends_bytes_sent" }, + { st_var_queries_frontends_bytes_recv, p_th_counter::queries_frontends_bytes_recv, (char*)"Queries_frontends_bytes_recv" }, + { st_var_queries_frontends_bytes_sent, p_th_counter::queries_frontends_bytes_sent, (char*)"Queries_frontends_bytes_sent" }, + { st_var_query_processor_time , p_th_counter::query_processor_time_nsec, (char*)"Query_Processor_time_nsec", 1000 * 1000 * 1000 }, + { st_var_backend_query_time , p_th_counter::backend_query_time_nsec, (char*)"Backend_query_time_nsec", 1000 * 1000 * 1000 }, + { st_var_ConnPool_get_conn_latency_awareness , p_th_counter::connpool_get_conn_latency_awareness, (char*)"ConnPool_get_conn_latency_awareness" }, + { st_var_ConnPool_get_conn_immediate, p_th_counter::connpool_get_conn_immediate, (char*)"ConnPool_get_conn_immediate" }, + { st_var_ConnPool_get_conn_success, p_th_counter::connpool_get_conn_success, (char*)"ConnPool_get_conn_success" }, + { st_var_ConnPool_get_conn_failure, p_th_counter::connpool_get_conn_failure, (char*)"ConnPool_get_conn_failure" }, + { st_var_killed_connections, p_th_counter::mysql_killed_backend_connections, (char*)"mysql_killed_backend_connections" }, + { st_var_killed_queries, p_th_counter::mysql_killed_backend_queries, (char*)"mysql_killed_backend_queries" }, + { st_var_hostgroup_locked_set_cmds, p_th_counter::hostgroup_locked_set_cmds, (char*)"hostgroup_locked_set_cmds" }, + { st_var_hostgroup_locked_queries, p_th_counter::hostgroup_locked_queries, (char*)"hostgroup_locked_queries" }, + { st_var_unexpected_com_quit, p_th_counter::mysql_unexpected_frontend_com_quit,(char*)"mysql_unexpected_frontend_com_quit" }, + { st_var_unexpected_packet, p_th_counter::mysql_unexpected_frontend_packets,(char*)"mysql_unexpected_frontend_packets" }, + { st_var_queries_with_max_lag_ms__total_wait_time_us , p_th_counter::queries_with_max_lag_ms__total_wait_time_us, (char*)"queries_with_max_lag_ms__total_wait_time_us" }, + { st_var_queries_with_max_lag_ms__delayed , p_th_counter::queries_with_max_lag_ms__delayed, (char*)"queries_with_max_lag_ms__delayed" }, + { st_var_queries_with_max_lag_ms, p_th_counter::queries_with_max_lag_ms, (char*)"queries_with_max_lag_ms" }, + { st_var_backend_lagging_during_query,p_th_counter::backend_lagging_during_query, (char*)"backend_lagging_during_query" }, + { st_var_backend_offline_during_query,p_th_counter::backend_offline_during_query, (char*)"backend_offline_during_query" }, + { st_var_aws_aurora_replicas_skipped_during_query , p_th_counter::aws_aurora_replicas_skipped_during_query, (char*)"get_aws_aurora_replicas_skipped_during_query" }, + { st_var_automatic_detected_sqli, p_th_counter::automatic_detected_sql_injection, (char*)"automatic_detected_sql_injection" }, + { st_var_mysql_whitelisted_sqli_fingerprint,p_th_counter::mysql_whitelisted_sqli_fingerprint, (char*)"mysql_whitelisted_sqli_fingerprint" }, + { st_var_max_connect_timeout_err, p_th_counter::max_connect_timeouts, (char*)"max_connect_timeouts" }, + { st_var_generated_pkt_err, p_th_counter::generated_error_packets, (char*)"generated_error_packets" }, + { st_var_client_host_error_killed_connections, p_th_counter::client_host_error_killed_connections, (char*)"client_host_error_killed_connections" },*/ +}; + +mythr_g_st_vars_t PgSQL_Thread_status_variables_gauge_array[]{ + /*{st_var_hostgroup_locked, p_th_gauge::client_connections_hostgroup_locked, (char*)"Client_Connections_hostgroup_locked"}*/ +}; + +extern mysql_variable_st mysql_tracked_variables[]; + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + + +#ifdef DEBUG +#define DEB "_DEBUG" +#else +#define DEB "" +#endif /* DEBUG */ +#define MYSQL_THREAD_VERSION "0.2.0902" DEB + + +#define DEFAULT_NUM_THREADS 4 +#define DEFAULT_STACK_SIZE 1024*1024 + +#define SESSIONS_FOR_CONNECTIONS_HANDLER 64 + +__thread unsigned int __thread_PgSQL_Thread_Variables_version; + +volatile static unsigned int __global_PgSQL_Thread_Variables_version; + + +PgSQL_Listeners_Manager::PgSQL_Listeners_Manager() { + ifaces = new PtrArray(); +} +PgSQL_Listeners_Manager::~PgSQL_Listeners_Manager() { + while (ifaces->len) { + iface_info* ifi = (iface_info*)ifaces->remove_index_fast(0); + shutdown(ifi->fd, SHUT_RDWR); + close(ifi->fd); + if (ifi->port == 0) { + unlink(ifi->address); + } + delete ifi; + } + delete ifaces; + ifaces = NULL; +} + +int PgSQL_Listeners_Manager::add(const char* iface, unsigned int num_threads, int** perthrsocks) { + for (unsigned int i = 0; i < ifaces->len; i++) { + iface_info* ifi = (iface_info*)ifaces->index(i); + if (strcmp(ifi->iface, iface) == 0) { + return -1; + } + } + + char* address = NULL; char* port = NULL; + int s = -1; + char* h = NULL; + bool is_ipv6 = false; + + if (*(char*)iface == '[') { + is_ipv6 = true; + char* p = strchr((char*)iface, ']'); + if (p == NULL) { + proxy_error("Invalid IPv6 address: %s\n", iface); + return -1; + } + h = (char*)++iface; // remove first '[' + *p = '\0'; + iface = p++; // remove last ']' + address = h; + port = ++p; // remove ':' + } + else { + c_split_2(iface, ":", &address, &port); + } + +#ifdef SO_REUSEPORT + if (GloVars.global.reuseport == false) { + s = (atoi(port) ? listen_on_port(address, atoi(port), PROXYSQL_LISTEN_LEN) : listen_on_unix(address, PROXYSQL_LISTEN_LEN)); + } + else { + if (atoi(port) == 0) { + s = listen_on_unix(address, PROXYSQL_LISTEN_LEN); + } + else { + // for TCP we will use SO_REUSEPORT + int* l_perthrsocks = (int*)malloc(sizeof(int) * num_threads); + unsigned int i; + for (i = 0; i < num_threads; i++) { + s = listen_on_port(address, atoi(port), PROXYSQL_LISTEN_LEN, true); + ioctl_FIONBIO(s, 1); + iface_info* ifi = new iface_info((char*)iface, address, atoi(port), s); + ifaces->add(ifi); + l_perthrsocks[i] = s; + } + *perthrsocks = l_perthrsocks; + s = 0; + } + } +#else + s = (atoi(port) ? listen_on_port(address, atoi(port), PROXYSQL_LISTEN_LEN) : listen_on_unix(address, PROXYSQL_LISTEN_LEN)); +#endif /* SO_REUSEPORT */ + if (s == -1) { + if (is_ipv6 == false) { + free(address); + free(port); + } + + return s; + } + if (s > 0) { + ioctl_FIONBIO(s, 1); + iface_info* ifi = new iface_info((char*)iface, address, atoi(port), s); + ifaces->add(ifi); + } + if (is_ipv6 == false) { + free(address); + free(port); + } + + return s; +} + +int PgSQL_Listeners_Manager::find_idx(const char* iface) { + for (unsigned int i = 0; i < ifaces->len; i++) { + iface_info* ifi = (iface_info*)ifaces->index(i); + if (strcmp(ifi->iface, iface) == 0) { + return i; + } + } + return -1; +} + +iface_info* PgSQL_Listeners_Manager::find_iface_from_fd(int fd) { + for (unsigned int i = 0; i < ifaces->len; i++) { + iface_info* ifi = (iface_info*)ifaces->index(i); + if (ifi->fd == fd) { + return ifi; + } + } + return NULL; +} + +int PgSQL_Listeners_Manager::find_idx(const char* address, int port) { + for (unsigned int i = 0; i < ifaces->len; i++) { + iface_info* ifi = (iface_info*)ifaces->index(i); + if (strcmp(ifi->address, address) == 0 && ifi->port == port) { + return i; + } + } + return -1; +} + +int PgSQL_Listeners_Manager::get_fd(unsigned int idx) { + iface_info* ifi = (iface_info*)ifaces->index(idx); + return ifi->fd; +} + +void PgSQL_Listeners_Manager::del(unsigned int idx) { + iface_info* ifi = (iface_info*)ifaces->remove_index_fast(idx); + if (ifi->port == 0) { + unlink(ifi->address); + } + delete ifi; +} + +static char* pgsql_thread_variables_names[] = { + (char*)"authentication_method", + (char*)"shun_on_failures", + (char*)"shun_recovery_time_sec", + (char*)"unshun_algorithm", + (char*)"query_retries_on_failure", + (char*)"client_host_cache_size", + (char*)"client_host_error_counts", + (char*)"connect_retries_on_failure", + (char*)"connect_retries_delay", + (char*)"connection_delay_multiplex_ms", + (char*)"connection_max_age_ms", + (char*)"connect_timeout_client", + (char*)"connect_timeout_server", + (char*)"connect_timeout_server_max", + (char*)"enable_client_deprecate_eof", + (char*)"enable_server_deprecate_eof", + (char*)"enable_load_data_local_infile", + (char*)"eventslog_filename", + (char*)"eventslog_filesize", + (char*)"eventslog_default_log", + (char*)"eventslog_format", + (char*)"auditlog_filename", + (char*)"auditlog_filesize", + //(char *)"default_charset", // removed in 2.0.13 . Obsoleted previously using MySQL_Variables instead + (char*)"handle_unknown_charset", + (char*)"free_connections_pct", + (char*)"connection_warming", +#ifdef IDLE_THREADS + (char*)"session_idle_ms", +#endif // IDLE_THREADS + (char*)"have_ssl", + (char*)"have_compress", + (char*)"interfaces", + (char*)"log_mysql_warnings_enabled", + (char*)"monitor_enabled", + (char*)"monitor_history", + (char*)"monitor_connect_interval", + (char*)"monitor_connect_timeout", + (char*)"monitor_ping_interval", + (char*)"monitor_ping_max_failures", + (char*)"monitor_ping_timeout", + (char*)"monitor_aws_rds_topology_discovery_interval", + (char*)"monitor_read_only_interval", + (char*)"monitor_read_only_timeout", + (char*)"monitor_read_only_max_timeout_count", + (char*)"monitor_replication_lag_group_by_host", + (char*)"monitor_replication_lag_interval", + (char*)"monitor_replication_lag_timeout", + (char*)"monitor_replication_lag_count", + (char*)"monitor_groupreplication_healthcheck_interval", + (char*)"monitor_groupreplication_healthcheck_timeout", + (char*)"monitor_groupreplication_healthcheck_max_timeout_count", + (char*)"monitor_groupreplication_max_transactions_behind_count", + (char*)"monitor_groupreplication_max_transactions_behind_for_read_only", + (char*)"monitor_galera_healthcheck_interval", + (char*)"monitor_galera_healthcheck_timeout", + (char*)"monitor_galera_healthcheck_max_timeout_count", + (char*)"monitor_username", + (char*)"monitor_password", + (char*)"monitor_replication_lag_use_percona_heartbeat", + (char*)"monitor_query_interval", + (char*)"monitor_query_timeout", + (char*)"monitor_slave_lag_when_null", + (char*)"monitor_threads_min", + (char*)"monitor_threads_max", + (char*)"monitor_threads_queue_maxsize", + (char*)"monitor_local_dns_cache_ttl", + (char*)"monitor_local_dns_cache_refresh_interval", + (char*)"monitor_local_dns_resolver_queue_maxsize", + (char*)"monitor_wait_timeout", + (char*)"monitor_writer_is_also_reader", + (char*)"max_allowed_packet", + (char*)"tcp_keepalive_time", + (char*)"use_tcp_keepalive", + (char*)"automatic_detect_sqli", + (char*)"firewall_whitelist_enabled", + (char*)"firewall_whitelist_errormsg", + (char*)"throttle_connections_per_sec_to_hostgroup", + (char*)"max_transaction_idle_time", + (char*)"max_transaction_time", + (char*)"multiplexing", + (char*)"log_unhealthy_connections", + (char*)"enforce_autocommit_on_reads", + (char*)"autocommit_false_not_reusable", + (char*)"autocommit_false_is_transaction", + (char*)"verbose_query_error", + (char*)"hostgroup_manager_verbose", + (char*)"binlog_reader_connect_retry_msec", + (char*)"threshold_query_length", + (char*)"threshold_resultset_size", + (char*)"query_digests_max_digest_length", + (char*)"query_digests_max_query_length", + (char*)"query_digests_grouping_limit", + (char*)"query_digests_groups_grouping_limit", + (char*)"query_rules_fast_routing_algorithm", + (char*)"wait_timeout", + (char*)"throttle_max_bytes_per_second_to_client", + (char*)"throttle_ratio_server_to_client", + (char*)"max_connections", + (char*)"max_stmts_per_connection", + (char*)"max_stmts_cache", + (char*)"mirror_max_concurrency", + (char*)"mirror_max_queue_length", + (char*)"default_max_latency_ms", + (char*)"default_query_delay", + (char*)"default_query_timeout", + (char*)"query_processor_iterations", + (char*)"query_processor_regex", + (char*)"set_query_lock_on_hostgroup", + (char*)"set_parser_algorithm", + (char*)"auto_increment_delay_multiplex", + (char*)"auto_increment_delay_multiplex_timeout_ms", + (char*)"long_query_time", + (char*)"query_cache_size_MB", + (char*)"query_cache_soft_ttl_pct", + (char*)"query_cache_handle_warnings", + (char*)"ping_interval_server_msec", + (char*)"ping_timeout_server", + (char*)"default_schema", + (char*)"poll_timeout", + (char*)"poll_timeout_on_failure", + (char*)"server_capabilities", + (char*)"server_version", + (char*)"default_client_encoding", + (char*)"keep_multiplexing_variables", + (char*)"kill_backend_connection_when_disconnect", + (char*)"client_session_track_gtid", + (char*)"sessions_sort", +#ifdef IDLE_THREADS + (char*)"session_idle_show_processlist", +#endif // IDLE_THREADS + (char*)"show_processlist_extended", + (char*)"commands_stats", + (char*)"query_digests", + (char*)"query_digests_lowercase", + (char*)"query_digests_replace_null", + (char*)"query_digests_no_digits", + (char*)"query_digests_normalize_digest_text", + (char*)"query_digests_track_hostname", + (char*)"query_digests_keep_comment", + (char*)"parse_failure_logs_digest", + (char*)"servers_stats", + (char*)"default_reconnect", +#ifdef DEBUG + (char*)"session_debug", +#endif /* DEBUG */ + (char*)"ssl_p2s_ca", + (char*)"ssl_p2s_capath", + (char*)"ssl_p2s_cert", + (char*)"ssl_p2s_key", + (char*)"ssl_p2s_cipher", + (char*)"ssl_p2s_crl", + (char*)"ssl_p2s_crlpath", + (char*)"stacksize", + (char*)"threads", + (char*)"init_connect", + (char*)"ldap_user_variable", + (char*)"add_ldap_user_comment", + (char*)"default_session_track_gtids", + (char*)"min_num_servers_lantency_awareness", + (char*)"aurora_max_lag_ms_only_read_from_replicas", + (char*)"stats_time_backend_query", + (char*)"stats_time_query_processor", + (char*)"query_cache_stores_empty_result", + (char*)"data_packets_history_size", + (char*)"handle_warnings", + NULL +}; + +using metric_name = std::string; +using metric_help = std::string; +using metric_tags = std::map; + +using th_counter_tuple = +std::tuple< + p_th_counter::metric, + metric_name, + metric_help, + metric_tags +>; + +using th_gauge_tuple = +std::tuple< + p_th_gauge::metric, + metric_name, + metric_help, + metric_tags +>; + +using th_counter_vector = std::vector; +using th_gauge_vector = std::vector; + +/** + * @brief Metrics map holding the metrics for the PgSQL_Thread module. + * + * @note Many metrics in this map, share a common "id name", because + * they differ only by label, because of this, HELP is shared between + * them. For better visual identification of this groups they are + * separated using a line separator comment. + */ +const std::tuple +th_metrics_map = std::make_tuple( + th_counter_vector{ + // ==================================================================== + std::make_tuple( + p_th_counter::queries_backends_bytes_sent, + "proxysql_queries_backends_bytes_total", + "Total number of bytes (sent|received) in backend connections.", + metric_tags { + { "traffic_flow", "sent" } + } + ), + std::make_tuple( + p_th_counter::queries_backends_bytes_recv, + "proxysql_queries_backends_bytes_total", + "Total number of bytes (sent|received) in backend connections.", + metric_tags { + { "traffic_flow", "received" } + } + ), + // ==================================================================== + + // ==================================================================== + std::make_tuple( + p_th_counter::queries_frontends_bytes_sent, + "proxysql_queries_frontends_bytes_total", + "Total number of bytes (sent|received) in frontend connections.", + metric_tags { + { "traffic_flow", "sent" } + } + ), + std::make_tuple( + p_th_counter::queries_frontends_bytes_recv, + "proxysql_queries_frontends_bytes_total", + "Total number of bytes (sent|received) in frontend connections.", + metric_tags { + { "traffic_flow", "received" } + } + ), + // ==================================================================== + + std::make_tuple( + p_th_counter::query_processor_time_nsec, + "proxysql_query_processor_time_seconds_total", + "The time spent inside the \"Query Processor\" to determine what action needs to be taken with the query (internal module).", + metric_tags {} + ), + std::make_tuple( + p_th_counter::backend_query_time_nsec, + "proxysql_backend_query_time_seconds_total", + "Time spent making network calls to communicate with the backends.", + metric_tags {} + ), + + // ==================================================================== + std::make_tuple( + p_th_counter::com_backend_stmt_prepare, + "proxysql_com_backend_stmt_total", + "Represents the number of statements (PREPARE|EXECUTE|CLOSE) executed by ProxySQL against the backends.", + metric_tags { + { "op", "prepare" } + } + ), + std::make_tuple( + p_th_counter::com_backend_stmt_execute, + "proxysql_com_backend_stmt_total", + "Represents the number of statements (PREPARE|EXECUTE|CLOSE) executed by ProxySQL against the backends.", + metric_tags { + { "op", "execute" } + } + ), + std::make_tuple( + p_th_counter::com_backend_stmt_close, + "proxysql_com_backend_stmt_total", + "Represents the number of statements (PREPARE|EXECUTE|CLOSE) executed by ProxySQL against the backends.", + metric_tags { + { "op", "close" } + } + ), + // ==================================================================== + + // ==================================================================== + std::make_tuple( + p_th_counter::com_frontend_stmt_prepare, + "proxysql_com_frontend_stmt_total", + "Represents the number of statements (PREPARE|EXECUTE|CLOSE) executed by clients.", + metric_tags { + { "op", "prepare" } + } + ), + std::make_tuple( + p_th_counter::com_frontend_stmt_execute, + "proxysql_com_frontend_stmt_total", + "Represents the number of statements (PREPARE|EXECUTE|CLOSE) executed by clients.", + metric_tags { + { "op", "execute" } + } + ), + std::make_tuple( + p_th_counter::com_frontend_stmt_close, + "proxysql_com_frontend_stmt_total", + "Represents the number of statements (PREPARE|EXECUTE|CLOSE) executed by clients.", + metric_tags { + { "op", "close" } + } + ), + // ==================================================================== + + std::make_tuple( + p_th_counter::questions, + "proxysql_questions_total", + "The total number of client requests / statements executed.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::slow_queries, + "proxysql_slow_queries_total", + "The total number of queries with an execution time greater than \"mysql-long_query_time\" milliseconds.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::gtid_consistent_queries, + "proxysql_gtid_consistent_queries_total", + "Total queries with GTID consistent read.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::gtid_session_collected, + "proxysql_gtid_session_collected_total", + "Total queries with GTID session state.", + metric_tags {} + ), + + // ==================================================================== + std::make_tuple( + p_th_counter::connpool_get_conn_latency_awareness, + "proxysql_connpool_get_conn_success_latency_awareness_total", + "The connection was picked using the latency awareness algorithm.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::connpool_get_conn_immediate, + "proxysql_connpool_get_conn_success_immediate_total", + "The connection is provided from per-thread cache.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::connpool_get_conn_success, + "proxysql_connpool_get_conn_success_total", + "The session is able to get a connection, either from per-thread cache or connection pool.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::connpool_get_conn_failure, + "proxysql_connpool_get_conn_failure_total", + "The connection pool cannot provide any connection.", + metric_tags {} + ), + // ==================================================================== + + std::make_tuple( + p_th_counter::generated_error_packets, + "proxysql_generated_error_packets_total", + "Total generated error packets.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::max_connect_timeouts, + "proxysql_max_connect_timeouts_total", + "Maximum connection timeout reached when trying to connect to backend sever.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::backend_lagging_during_query, + "proxysql_backend_lagging_during_query_total", + "Query failed because server was shunned due to lag.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::backend_offline_during_query, + "proxysql_backend_offline_during_query_total", + "Query failed because server was offline.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::queries_with_max_lag_ms, + "proxysql_queries_with_max_lag_total", + "Received queries that have a 'max_lag' attribute.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::queries_with_max_lag_ms__delayed, + "proxysql_queries_with_max_lag__delayed_total", + "Query delayed because no connection was selected due to 'max_lag' annotation.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::queries_with_max_lag_ms__total_wait_time_us, + "proxysql_queries_with_max_lag__total_wait_time_total", + "Total waited time due to connection selection because of 'max_lag' annotation.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::mysql_unexpected_frontend_com_quit, + "proxysql_mysql_unexpected_frontend_com_quit_total", + "Unexpected 'COM_QUIT' received from the client.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::hostgroup_locked_set_cmds, + "proxysql_hostgroup_locked_set_cmds_total", + "Total number of connections that have been locked in a hostgroup.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::hostgroup_locked_queries, + "proxysql_hostgroup_locked_queries_total", + "Query blocked because connection is locked into some hostgroup but is trying to reach other.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::mysql_unexpected_frontend_packets, + "proxysql_mysql_unexpected_frontend_packets_total", + "Unexpected packet received from client.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::aws_aurora_replicas_skipped_during_query, + "proxysql_aws_aurora_replicas_skipped_during_query_total", + "Replicas skipped due to current lag being higher than 'max_lag' annotation.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::automatic_detected_sql_injection, + "proxysql_automatic_detected_sql_injection_total", + "Blocked a detected 'sql injection' attempt.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::mysql_whitelisted_sqli_fingerprint, + "proxysql_mysql_whitelisted_sqli_fingerprint_total", + "Detected a whitelisted 'sql injection' fingerprint.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::mysql_killed_backend_connections, + "proxysql_mysql_killed_backend_connections_total", + "Number of backend connection killed.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::mysql_killed_backend_queries, + "proxysql_mysql_killed_backend_queries_total", + "Killed backend queries.", + metric_tags {} + ), + std::make_tuple( + p_th_counter::client_host_error_killed_connections, + "proxysql_client_host_error_killed_connections", + "Killed client connections because address exceeded 'client_host_error_counts'.", + metric_tags {} + ) + }, + th_gauge_vector{ + std::make_tuple( + p_th_gauge::active_transactions, + "proxysql_active_transactions", + "Provides a count of how many client connection are currently processing a transaction.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::client_connections_non_idle, + "proxysql_client_connections_non_idle", + "Number of client connections that are currently handled by the main worker threads.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::client_connections_hostgroup_locked, + "proxysql_client_connections_hostgroup_locked", + "Number of client connection locked to a specific hostgroup.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_backend_buffers_bytes, + "proxysql_mysql_backend_buffers_bytes", + "Buffers related to backend connections if \"fast_forward\" is used (0 means fast_forward is not used).", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_frontend_buffers_bytes, + "proxysql_mysql_frontend_buffers_bytes", + "Buffers related to frontend connections (read/write buffers and other queues).", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_session_internal_bytes, + "proxysql_mysql_session_internal_bytes", + "Other memory used by ProxySQL to handle MySQL Sessions.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mirror_concurrency, + "proxysql_mirror_concurrency", + "Mirror current concurrency", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mirror_queue_lengths, + "proxysql_mirror_queue_lengths", + "Mirror queue length", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_thread_workers, + "proxysql_mysql_thread_workers", + "Number of MySQL Thread workers i.e. 'mysql-threads'", + metric_tags {} + ), + // global_variables + std::make_tuple( + p_th_gauge::mysql_wait_timeout, + "proxysql_mysql_wait_timeout", + "If a proxy session has been idle for more than this threshold, the proxy will kill the session.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_max_connections, + "proxysql_mysql_max_connections", + "The maximum number of client connections that the proxy can handle.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_enabled, + "proxysql_mysql_monitor_enabled", + "Enables or disables MySQL Monitor.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_ping_interval, + "proxysql_mysql_monitor_ping_interval", + "How frequently a ping check is performed, in seconds.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_ping_timeout, + "proxysql_mysql_monitor_ping_timeout_seconds", + "Ping timeout in seconds.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_ping_max_failures, + "proxysql_mysql_monitor_ping_max_failures", + "Reached maximum ping attempts from monitor.", + metric_tags {} + ), + std::make_tuple ( + p_th_gauge::mysql_monitor_aws_rds_topology_discovery_interval, + "proxysql_mysql_monitor_aws_rds_topology_discovery_interval", + "How frequently a topology discovery is performed, e.g. a value of 500 means one topology discovery every 500 read-only checks ", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_read_only_interval, + "proxysql_mysql_monitor_read_only_interval_seconds", + "How frequently a read only check is performed, in seconds.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_read_only_timeout, + "proxysql_mysql_monitor_read_only_timeout_seconds", + "Read only check timeout in seconds.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_writer_is_also_reader, + "proxysql_mysql_monitor_writer_is_also_reader", + "Encodes different behaviors for nodes depending on their 'READ_ONLY' flag value.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_replication_lag_group_by_host, + "proxysql_monitor_replication_lag_group_by_host", + "Encodes different replication lag check if the same server is in multiple hostgroups.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_replication_lag_interval, + "proxysql_mysql_monitor_replication_lag_interval_seconds", + "How frequently a replication lag check is performed, in seconds.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_replication_lag_timeout, + "proxysql_mysql_monitor_replication_lag_timeout_seconds", + "Replication lag check timeout in seconds.", + metric_tags {} + ), + std::make_tuple( + p_th_gauge::mysql_monitor_history, + "proxysql_mysql_monitor_history_timeout_seconds", + "The duration for which the events for the checks made by the Monitor module are kept, in seconds.", + metric_tags {} + ) + } +); + +PgSQL_Threads_Handler::PgSQL_Threads_Handler() { +#ifdef DEBUG + if (glovars.has_debug == false) { +#else + if (glovars.has_debug == true) { +#endif /* DEBUG */ + // LCOV_EXCL_START + perror("Incompatible debugging version"); + exit(EXIT_FAILURE); + // LCOV_EXCL_STOP + } + num_threads = 0; + pgsql_threads = NULL; +#ifdef IDLE_THREADS + pgsql_threads_idles = NULL; +#endif // IDLE_THREADS + stacksize = 0; + shutdown_ = 0; + bootstrapping_listeners = true; + pthread_rwlock_init(&rwlock, NULL); + pthread_attr_init(&attr); + // Zero initialize all variables + memset(&variables, 0, sizeof(variables)); + + variables.authentication_method = (int)AUTHENTICATION_METHOD::SASL_SCRAM_SHA_256; //SCRAM Authentication method + variables.server_version = strdup((char*)"16.1"); + variables.default_client_encoding = strdup((char*)"UTF8"); + variables.shun_on_failures = 5; + variables.shun_recovery_time_sec = 10; + variables.unshun_algorithm = 0; + variables.query_retries_on_failure = 1; + variables.client_host_cache_size = 0; + variables.client_host_error_counts = 0; + variables.handle_warnings = 1; + variables.connect_retries_on_failure = 10; + variables.connection_delay_multiplex_ms = 0; + variables.connection_max_age_ms = 0; + variables.connect_timeout_client = 10000; + variables.connect_timeout_server = 1000; + variables.connect_timeout_server_max = 10000; + variables.free_connections_pct = 10; + variables.connect_retries_delay = 1; + variables.monitor_enabled = true; + variables.monitor_history = 7200000; // changed in 2.6.0 : was 600000 + variables.monitor_connect_interval = 120000; + variables.monitor_connect_timeout = 600; + variables.monitor_ping_interval = 8000; + variables.monitor_ping_max_failures = 3; + variables.monitor_ping_timeout = 1000; + variables.monitor_aws_rds_topology_discovery_interval=1000; + variables.monitor_read_only_interval = 1000; + variables.monitor_read_only_timeout = 800; + variables.monitor_read_only_max_timeout_count = 3; + variables.monitor_replication_lag_group_by_host = false; + variables.monitor_replication_lag_interval = 10000; + variables.monitor_replication_lag_timeout = 1000; + variables.monitor_replication_lag_count = 1; + variables.monitor_groupreplication_healthcheck_interval = 5000; + variables.monitor_groupreplication_healthcheck_timeout = 800; + variables.monitor_groupreplication_healthcheck_max_timeout_count = 3; + variables.monitor_groupreplication_max_transactions_behind_count = 3; + variables.monitor_groupreplication_max_transactions_behind_for_read_only = 1; + variables.monitor_galera_healthcheck_interval = 5000; + variables.monitor_galera_healthcheck_timeout = 800; + variables.monitor_galera_healthcheck_max_timeout_count = 3; + variables.monitor_query_interval = 60000; + variables.monitor_query_timeout = 100; + variables.monitor_slave_lag_when_null = 60; + variables.monitor_threads_min = 8; + variables.monitor_threads_max = 128; + variables.monitor_threads_queue_maxsize = 128; + variables.monitor_local_dns_cache_ttl = 300000; + variables.monitor_local_dns_cache_refresh_interval = 60000; + variables.monitor_local_dns_resolver_queue_maxsize = 128; + variables.monitor_username = strdup((char*)"monitor"); + variables.monitor_password = strdup((char*)"monitor"); + variables.monitor_replication_lag_use_percona_heartbeat = strdup((char*)""); + variables.monitor_wait_timeout = true; + variables.monitor_writer_is_also_reader = true; + variables.max_allowed_packet = 64 * 1024 * 1024; + variables.automatic_detect_sqli = false; + variables.firewall_whitelist_enabled = false; + variables.firewall_whitelist_errormsg = strdup((char*)"Firewall blocked this query"); + variables.use_tcp_keepalive = true; // changed in 2.6.0 , was false + variables.tcp_keepalive_time = 120; // changed in 2.6.0 , was 0 + variables.throttle_connections_per_sec_to_hostgroup = 1000000; + variables.max_transaction_idle_time = 4 * 3600 * 1000; + variables.max_transaction_time = 4 * 3600 * 1000; + variables.hostgroup_manager_verbose = 1; + variables.binlog_reader_connect_retry_msec = 3000; + variables.threshold_query_length = 512 * 1024; + variables.threshold_resultset_size = 4 * 1024 * 1024; + variables.query_digests_max_digest_length = 2 * 1024; + variables.query_digests_max_query_length = 65000; // legacy default + variables.query_rules_fast_routing_algorithm = 1; + variables.wait_timeout = 8 * 3600 * 1000; + variables.throttle_max_bytes_per_second_to_client = 0; + variables.throttle_ratio_server_to_client = 0; + variables.connection_warming = false; + variables.max_connections = 10 * 1000; + variables.max_stmts_per_connection = 20; + variables.max_stmts_cache = 10000; + variables.mirror_max_concurrency = 16; + variables.mirror_max_queue_length = 32000; + variables.default_max_latency_ms = 1 * 1000; // by default, the maximum allowed latency for a host is 1000ms + variables.default_query_delay = 0; + variables.default_query_timeout = 24 * 3600 * 1000; + variables.query_processor_iterations = 0; + variables.query_processor_regex = 1; + variables.set_query_lock_on_hostgroup = 1; + variables.set_parser_algorithm = 2; // before 2.6.0 this was 1 + variables.auto_increment_delay_multiplex = 5; + variables.auto_increment_delay_multiplex_timeout_ms = 10000; + variables.long_query_time = 1000; + variables.query_cache_size_MB = 256; + variables.query_cache_soft_ttl_pct = 0; + variables.query_cache_handle_warnings = 0; + variables.init_connect = NULL; + variables.ldap_user_variable = NULL; + variables.add_ldap_user_comment = NULL; + for (int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + variables.default_variables[i] = strdup(mysql_tracked_variables[i].default_value); + } + variables.default_session_track_gtids = strdup((char*)MYSQL_DEFAULT_SESSION_TRACK_GTIDS); + variables.ping_interval_server_msec = 10000; + variables.ping_timeout_server = 200; + variables.default_schema = strdup((char*)"information_schema"); + variables.handle_unknown_charset = 1; + variables.interfaces = strdup((char*)""); + variables.eventslog_filename = strdup((char*)""); // proxysql-mysql-eventslog is recommended + variables.eventslog_filesize = 100 * 1024 * 1024; + variables.eventslog_default_log = 0; + variables.eventslog_format = 1; + variables.auditlog_filename = strdup((char*)""); + variables.auditlog_filesize = 100 * 1024 * 1024; + //variables.server_capabilities=CLIENT_FOUND_ROWS | CLIENT_PROTOCOL_41 | CLIENT_IGNORE_SIGPIPE | CLIENT_TRANSACTIONS | CLIENT_SECURE_CONNECTION | CLIENT_CONNECT_WITH_DB; + // major upgrade in 2.0.0 + variables.server_capabilities = CLIENT_MYSQL | CLIENT_FOUND_ROWS | CLIENT_PROTOCOL_41 | CLIENT_IGNORE_SIGPIPE | CLIENT_TRANSACTIONS | CLIENT_SECURE_CONNECTION | CLIENT_CONNECT_WITH_DB | CLIENT_PLUGIN_AUTH;; + variables.poll_timeout = 2000; + variables.poll_timeout_on_failure = 100; + variables.have_compress = true; + variables.have_ssl = true; // changed in 2.6.0 , was false by default for performance reason + variables.commands_stats = true; + variables.multiplexing = true; + variables.log_unhealthy_connections = true; + variables.enforce_autocommit_on_reads = false; + variables.autocommit_false_not_reusable = false; + variables.autocommit_false_is_transaction = false; + variables.verbose_query_error = false; + variables.query_digests = true; + variables.query_digests_lowercase = false; + variables.query_digests_replace_null = false; + variables.query_digests_no_digits = false; + variables.query_digests_normalize_digest_text = false; + variables.query_digests_track_hostname = false; + variables.query_digests_keep_comment = false; + variables.parse_failure_logs_digest = false; + variables.min_num_servers_lantency_awareness = 1000; + variables.aurora_max_lag_ms_only_read_from_replicas = 2; + variables.stats_time_backend_query = false; + variables.stats_time_query_processor = false; + variables.query_cache_stores_empty_result = true; + variables.kill_backend_connection_when_disconnect = true; + variables.client_session_track_gtid = true; + variables.sessions_sort = true; +#ifdef IDLE_THREADS + variables.session_idle_ms = 1; + variables.session_idle_show_processlist = true; +#endif // IDLE_THREADS + variables.show_processlist_extended = 0; + variables.servers_stats = true; + variables.default_reconnect = true; + variables.ssl_p2s_ca = NULL; + variables.ssl_p2s_capath = NULL; + variables.ssl_p2s_cert = NULL; + variables.ssl_p2s_key = NULL; + variables.ssl_p2s_cipher = NULL; + variables.ssl_p2s_crl = NULL; + variables.ssl_p2s_crlpath = NULL; + variables.keep_multiplexing_variables = strdup((char*)"tx_isolation,transaction_isolation,version"); +#ifdef DEBUG + variables.session_debug = true; +#endif /*debug */ + variables.query_digests_grouping_limit = 3; + variables.query_digests_groups_grouping_limit = 10; // changed in 2.6.0 , was 0 + variables.enable_client_deprecate_eof = true; + variables.enable_server_deprecate_eof = true; + variables.enable_load_data_local_infile = false; + variables.log_mysql_warnings_enabled = false; + variables.data_packets_history_size = 0; + // status variables + status_variables.mirror_sessions_current = 0; + __global_PgSQL_Thread_Variables_version = 1; + MLM = new PgSQL_Listeners_Manager(); + + // Initialize prometheus metrics + //init_prometheus_counter_array(th_metrics_map, this->status_variables.p_counter_array); + //init_prometheus_gauge_array(th_metrics_map, this->status_variables.p_gauge_array); + + // Init client_host_cache mutex + pthread_mutex_init(&mutex_client_host_cache, NULL); + } + +unsigned int PgSQL_Threads_Handler::get_global_version() { + return __sync_fetch_and_add(&__global_PgSQL_Thread_Variables_version, 0); +} + +int PgSQL_Threads_Handler::listener_add(const char* address, int port) { + char* s = (char*)malloc(strlen(address) + 32); + sprintf(s, "%s:%d", address, port); + int ret = listener_add((const char*)s); + free(s); + return ret; +} + +int PgSQL_Threads_Handler::listener_add(const char* iface) { + int rc; + int* perthrsocks = NULL;; + rc = MLM->add(iface, num_threads, &perthrsocks); + if (rc > -1) { + unsigned int i; + if (perthrsocks == NULL) { + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + while (!__sync_bool_compare_and_swap(&thr->mypolls.pending_listener_add, 0, rc)) { + usleep(10); // pause a bit + } + } + } + else { + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + while (!__sync_bool_compare_and_swap(&thr->mypolls.pending_listener_add, 0, perthrsocks[i])) { + usleep(10); // pause a bit + } + } + free(perthrsocks); + } + } + return rc; +} + +int PgSQL_Threads_Handler::listener_del(const char* iface) { + int idx; + while ((idx = MLM->find_idx(iface)) >= 0) { + unsigned int i; + int fd = MLM->get_fd(idx); + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; +#ifdef SO_REUSEPORT + if (GloVars.global.reuseport) + while (!__sync_bool_compare_and_swap(&thr->mypolls.pending_listener_del, 0, -1)); + else +#endif + while (!__sync_bool_compare_and_swap(&thr->mypolls.pending_listener_del, 0, fd)); + } + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + while (__sync_fetch_and_add(&thr->mypolls.pending_listener_del, 0)); + } + MLM->del(idx); +#ifdef SO_REUSEPORT + if (GloVars.global.reuseport) { + continue; + } +#endif + shutdown(fd, SHUT_RDWR); + close(fd); + } + return 0; +} + +void PgSQL_Threads_Handler::wrlock() { + pthread_rwlock_wrlock(&rwlock); +} + +void PgSQL_Threads_Handler::wrunlock() { + pthread_rwlock_unlock(&rwlock); +} + +void PgSQL_Threads_Handler::commit() { + __sync_add_and_fetch(&__global_PgSQL_Thread_Variables_version, 1); + proxy_debug(PROXY_DEBUG_MYSQL_SERVER, 1, "Increasing version number to %d - all threads will notice this and refresh their variables\n", __global_PgSQL_Thread_Variables_version); +} + +char* PgSQL_Threads_Handler::get_variable_string(char* name) { + if (!strncmp(name, "monitor_", 8)) { + if (!strcmp(name, "monitor_username")) return strdup(variables.monitor_username); + if (!strcmp(name, "monitor_password")) return strdup(variables.monitor_password); + if (!strcmp(name, "monitor_replication_lag_use_percona_heartbeat")) return strdup(variables.monitor_replication_lag_use_percona_heartbeat); + } + if (!strncmp(name, "ssl_", 4)) { + if (!strcmp(name, "ssl_p2s_ca")) { + if (variables.ssl_p2s_ca == NULL || strlen(variables.ssl_p2s_ca) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_ca); + } + } + if (!strcmp(name, "ssl_p2s_cert")) { + if (variables.ssl_p2s_cert == NULL || strlen(variables.ssl_p2s_cert) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_cert); + } + } + if (!strcmp(name, "ssl_p2s_capath")) { + if (variables.ssl_p2s_capath == NULL || strlen(variables.ssl_p2s_capath) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_capath); + } + } + if (!strcmp(name, "ssl_p2s_key")) { + if (variables.ssl_p2s_key == NULL || strlen(variables.ssl_p2s_key) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_key); + } + } + if (!strcmp(name, "ssl_p2s_cipher")) { + if (variables.ssl_p2s_cipher == NULL || strlen(variables.ssl_p2s_cipher) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_cipher); + } + } + if (!strcmp(name, "ssl_p2s_crl")) { + if (variables.ssl_p2s_crl == NULL || strlen(variables.ssl_p2s_crl) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_crl); + } + } + if (!strcmp(name, "ssl_p2s_crlpath")) { + if (variables.ssl_p2s_crlpath == NULL || strlen(variables.ssl_p2s_crlpath) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_crlpath); + } + } + } + if (!strcmp(name, "firewall_whitelist_errormsg")) { + if (variables.firewall_whitelist_errormsg == NULL || strlen(variables.firewall_whitelist_errormsg) == 0) { + return NULL; + } + else { + return strdup(variables.firewall_whitelist_errormsg); + } + } + if (!strcmp(name, "init_connect")) { + if (variables.init_connect == NULL || strlen(variables.init_connect) == 0) { + return NULL; + } + else { + return strdup(variables.init_connect); + } + } + if (!strcmp(name, "ldap_user_variable")) { + if (variables.ldap_user_variable == NULL || strlen(variables.ldap_user_variable) == 0) { + return NULL; + } + else { + return strdup(variables.ldap_user_variable); + } + } + if (!strcmp(name, "add_ldap_user_comment")) { + if (variables.add_ldap_user_comment == NULL || strlen(variables.add_ldap_user_comment) == 0) { + return NULL; + } + else { + return strdup(variables.add_ldap_user_comment); + } + } + if (!strncmp(name, "default_", 8)) { + for (int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (mysql_tracked_variables[i].is_global_variable == false) + continue; + char buf[128]; + sprintf(buf, "default_%s", mysql_tracked_variables[i].internal_variable_name); + if (!strcmp(name, buf)) { + if (variables.default_variables[i] == NULL) { + variables.default_variables[i] = strdup(mysql_tracked_variables[i].default_value); + } + return strdup(variables.default_variables[i]); + } + } + if (!strcmp(name, "default_session_track_gtids")) { + if (variables.default_session_track_gtids == NULL) { + variables.default_session_track_gtids = strdup((char*)MYSQL_DEFAULT_SESSION_TRACK_GTIDS); + } + return strdup(variables.default_session_track_gtids); + } + if (!strcmp(name, "default_schema")) return strdup(variables.default_schema); + } + if (!strcmp(name, "server_version")) return strdup(variables.server_version); + if (!strcmp(name, "default_client_encoding")) return strdup(variables.default_client_encoding); + if (!strcmp(name, "eventslog_filename")) return strdup(variables.eventslog_filename); + if (!strcmp(name, "auditlog_filename")) return strdup(variables.auditlog_filename); + if (!strcmp(name, "interfaces")) return strdup(variables.interfaces); + if (!strcmp(name, "keep_multiplexing_variables")) return strdup(variables.keep_multiplexing_variables); + // LCOV_EXCL_START + proxy_error("Not existing variable: %s\n", name); assert(0); + return NULL; + // LCOV_EXCL_STOP +} + +uint16_t PgSQL_Threads_Handler::get_variable_uint16(char* name) { + if (!strcasecmp(name, "server_capabilities")) return variables.server_capabilities; + // LCOV_EXCL_START + proxy_error("Not existing variable: %s\n", name); assert(0); + return 0; + // LCOV_EXCL_STOP +} + +int PgSQL_Threads_Handler::get_variable_int(const char* name) { + // convert name to string, and lowercase + std::string nameS = string(name); + std::transform(nameS.begin(), nameS.end(), nameS.begin(), [](unsigned char c) { return std::tolower(c); }); + { + // integer variable + std::unordered_map>::const_iterator it = VariablesPointers_int.find(nameS); + if (it != VariablesPointers_int.end()) { + int* v = std::get<0>(it->second); + return *v; + } + } + { + // bool variable + std::unordered_map>::const_iterator it = VariablesPointers_bool.find(nameS); + if (it != VariablesPointers_bool.end()) { + bool* v = std::get<0>(it->second); + int a = (int)*v; + return a; + } + } + + + //VALGRIND_DISABLE_ERROR_REPORTING; + if (!strcmp(name, "stacksize")) return (stacksize ? stacksize : DEFAULT_STACK_SIZE); + // LCOV_EXCL_START + proxy_error("Not existing variable: %s\n", name); assert(0); + return 0; + // LCOV_EXCL_STOP +//VALGRIND_ENABLE_ERROR_REPORTING; +} + +char* PgSQL_Threads_Handler::get_variable(char* name) { // this is the public function, accessible from admin + //VALGRIND_DISABLE_ERROR_REPORTING; +#define INTBUFSIZE 4096 + char intbuf[INTBUFSIZE]; + + // convert name to string, and lowercase + std::string nameS = string(name); + std::transform(nameS.begin(), nameS.end(), nameS.begin(), [](unsigned char c) { return std::tolower(c); }); + + { + // integer variable + std::unordered_map>::const_iterator it = VariablesPointers_int.find(nameS); + if (it != VariablesPointers_int.end()) { + int* v = std::get<0>(it->second); + sprintf(intbuf, "%d", *v); + return strdup(intbuf); + } + } + { + // bool variable + std::unordered_map>::const_iterator it = VariablesPointers_bool.find(nameS); + if (it != VariablesPointers_bool.end()) { + bool* v = std::get<0>(it->second); + return strdup((*v ? "true" : "false")); + } + } + + + if (!strcasecmp(name, "firewall_whitelist_errormsg")) { + if (variables.firewall_whitelist_errormsg == NULL || strlen(variables.firewall_whitelist_errormsg) == 0) { + return NULL; + } + else { + return strdup(variables.firewall_whitelist_errormsg); + } + } + if (!strcasecmp(name, "init_connect")) { + if (variables.init_connect == NULL || strlen(variables.init_connect) == 0) { + return NULL; + } + else { + return strdup(variables.init_connect); + } + } + if (!strcasecmp(name, "ldap_user_variable")) { + if (variables.ldap_user_variable == NULL || strlen(variables.ldap_user_variable) == 0) { + return NULL; + } + else { + return strdup(variables.ldap_user_variable); + } + } + if (!strcasecmp(name, "add_ldap_user_comment")) { + if (variables.add_ldap_user_comment == NULL || strlen(variables.add_ldap_user_comment) == 0) { + return NULL; + } + else { + return strdup(variables.add_ldap_user_comment); + } + } + if (!strcasecmp(name, "default_session_track_gtids")) { + if (variables.default_session_track_gtids == NULL) { + variables.default_session_track_gtids = strdup((char*)MYSQL_DEFAULT_SESSION_TRACK_GTIDS); + } + return strdup(variables.default_session_track_gtids); + } + if (strlen(name) > 8) { + if (strncmp(name, "default_", 8) == 0) { + for (unsigned int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (mysql_tracked_variables[i].is_global_variable) { + size_t var_len = strlen(mysql_tracked_variables[i].internal_variable_name); + if (strlen(name) == (var_len + 8)) { + if (!strncmp(name + 8, mysql_tracked_variables[i].internal_variable_name, var_len)) { + return strdup(variables.default_variables[i]); + } + } + } + } + } + } + if (!strcasecmp(name, "firewall_whitelist_errormsg")) return strdup(variables.firewall_whitelist_errormsg); + if (!strcasecmp(name, "server_version")) return strdup(variables.server_version); + if (!strcasecmp(name, "default_client_encoding")) return strdup(variables.default_client_encoding); + if (!strcasecmp(name, "auditlog_filename")) return strdup(variables.auditlog_filename); + if (!strcasecmp(name, "eventslog_filename")) return strdup(variables.eventslog_filename); + if (!strcasecmp(name, "default_schema")) return strdup(variables.default_schema); + if (!strcasecmp(name, "keep_multiplexing_variables")) return strdup(variables.keep_multiplexing_variables); + if (!strcasecmp(name, "interfaces")) return strdup(variables.interfaces); + if (!strcasecmp(name, "server_capabilities")) { + // FIXME : make it human readable + sprintf(intbuf, "%d", variables.server_capabilities); + return strdup(intbuf); + } + // SSL variables + if (!strncasecmp(name, "ssl_", 4)) { + if (!strcasecmp(name, "ssl_p2s_ca")) { + if (variables.ssl_p2s_ca == NULL || strlen(variables.ssl_p2s_ca) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_ca); + } + } + if (!strcasecmp(name, "ssl_p2s_capath")) { + if (variables.ssl_p2s_capath == NULL || strlen(variables.ssl_p2s_capath) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_capath); + } + } + if (!strcasecmp(name, "ssl_p2s_cert")) { + if (variables.ssl_p2s_cert == NULL || strlen(variables.ssl_p2s_cert) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_cert); + } + } + if (!strcasecmp(name, "ssl_p2s_key")) { + if (variables.ssl_p2s_key == NULL || strlen(variables.ssl_p2s_key) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_key); + } + } + if (!strcasecmp(name, "ssl_p2s_cipher")) { + if (variables.ssl_p2s_cipher == NULL || strlen(variables.ssl_p2s_cipher) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_cipher); + } + } + if (!strcasecmp(name, "ssl_p2s_crl")) { + if (variables.ssl_p2s_crl == NULL || strlen(variables.ssl_p2s_crl) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_crl); + } + } + if (!strcasecmp(name, "ssl_p2s_crlpath")) { + if (variables.ssl_p2s_crlpath == NULL || strlen(variables.ssl_p2s_crlpath) == 0) { + return NULL; + } + else { + return strdup(variables.ssl_p2s_crlpath); + } + } + } + // monitor variables + if (!strncasecmp(name, "monitor_", 8)) { + if (!strcasecmp(name, "monitor_username")) return strdup(variables.monitor_username); + if (!strcasecmp(name, "monitor_password")) return strdup(variables.monitor_password); + if (!strcasecmp(name, "monitor_replication_lag_use_percona_heartbeat")) return strdup(variables.monitor_replication_lag_use_percona_heartbeat); + } + if (!strcasecmp(name, "threads")) { + sprintf(intbuf, "%d", (num_threads ? num_threads : DEFAULT_NUM_THREADS)); + return strdup(intbuf); + } + if (!strcasecmp(name, "stacksize")) { + sprintf(intbuf, "%d", (int)(stacksize ? stacksize : DEFAULT_STACK_SIZE)); + return strdup(intbuf); + } + + return NULL; + //VALGRIND_ENABLE_ERROR_REPORTING; +} + + + +bool PgSQL_Threads_Handler::set_variable(char* name, const char* value) { // this is the public function, accessible from admin + // IN: + // name: variable name + // value: variable value + // + // OUT: + // false: unable to change the variable value, either because doesn't exist, or because out of range, or read only + // true: variable value changed + // + if (!value) return false; + size_t vallen = strlen(value); + + + // convert name to string, and lowercase + std::string nameS = string(name); + std::transform(nameS.begin(), nameS.end(), nameS.begin(), [](unsigned char c) { return std::tolower(c); }); + { + // integer variable ? + std::unordered_map>::const_iterator it = VariablesPointers_int.find(nameS); + if (it != VariablesPointers_int.end()) { + // Log warnings for variables with possibly wrong values + if (nameS == "auto_increment_delay_multiplex_timeout_ms") { + int intv = atoi(value); + if (intv <= 60) { + proxy_warning("'mysql-auto_increment_delay_multiplex_timeout_ms' is set to a low value: %ums. Remember value is in 'ms'\n", intv); + } + } + if (nameS == "query_rules_fast_routing_algorithm") { + if (GloPgQPro) { + int intv = atoi(value); + if (intv >= std::get<1>(it->second) && intv <= std::get<2>(it->second)) { + GloPgQPro->wrlock(); + GloPgQPro->query_rules_fast_routing_algorithm = intv; + GloPgQPro->wrunlock(); + } + } + } + bool special_variable = std::get<3>(it->second); // if special_variable is true, min and max values are ignored, and more input validation is needed + if (special_variable == false) { + int intv = atoi(value); + if (intv >= std::get<1>(it->second) && intv <= std::get<2>(it->second)) { + int* v = std::get<0>(it->second); + *v = intv; + return true; + } + return false; + } + else { + // we need to perform input validation + } + } + } + { + // boolean variable ? + std::unordered_map>::const_iterator it = VariablesPointers_bool.find(nameS); + if (it != VariablesPointers_bool.end()) { + bool special_variable = std::get<1>(it->second); // if special_variable is true, more input validation is needed + if (special_variable == false) { + bool* v = std::get<0>(it->second); + if (strcasecmp(value, "true") == 0 || strcasecmp(value, "1") == 0) { + *v = true; + return true; + } + if (strcasecmp(value, "false") == 0 || strcasecmp(value, "0") == 0) { + *v = false; + return true; + } + return false; + } + else { + // we need to perform input validation + } + } + } + + // monitor variables + if (!strncasecmp(name, "monitor_", 8)) { + if (!strcasecmp(name, "monitor_username")) { + if (vallen) { + free(variables.monitor_username); + variables.monitor_username = strdup(value); + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "monitor_password")) { + free(variables.monitor_password); + variables.monitor_password = strdup(value); + return true; + } + if (!strcasecmp(name, "monitor_replication_lag_use_percona_heartbeat")) { + if (vallen == 0) { // empty string + free(variables.monitor_replication_lag_use_percona_heartbeat); + variables.monitor_replication_lag_use_percona_heartbeat = strdup((value)); + return true; + } + else { + re2::RE2::Options* opt2 = new re2::RE2::Options(RE2::Quiet); + opt2->set_case_sensitive(false); + char* patt = (char*)"`?([a-z\\d_]+)`?\\.`?([a-z\\d_]+)`?"; + RE2* re = new RE2(patt, *opt2); + bool rc = false; + rc = RE2::FullMatch(value, *re); + delete re; + delete opt2; + if (rc) { + free(variables.monitor_replication_lag_use_percona_heartbeat); + variables.monitor_replication_lag_use_percona_heartbeat = strdup(value); + return true; + } + else { + proxy_error("%s is an invalid value for %s, not matching regex \"%s\"\n", value, name, patt); + } + } + return false; + } + } + if (!strcasecmp(name, "binlog_reader_connect_retry_msec")) { + int intv = atoi(value); + if (intv >= 200 && intv <= 120000) { + __sync_lock_test_and_set(&variables.binlog_reader_connect_retry_msec, intv); + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "wait_timeout")) { + int intv = atoi(value); + if (intv >= 0 && intv <= 20 * 24 * 3600 * 1000) { + variables.wait_timeout = intv; + if (variables.wait_timeout < 5000) { + proxy_warning("mysql-wait_timeout is set to a low value: %ums\n", variables.wait_timeout); + } + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "eventslog_format")) { + int intv = atoi(value); + if (intv >= 1 && intv <= 2) { + if (variables.eventslog_format != intv) { + // if we are switching format, we need to switch file too + if (GloPgSQL_Logger) { + proxy_info("Switching query logging format from %d to %d\n", variables.eventslog_format, intv); + GloPgSQL_Logger->flush_log(); + } + variables.eventslog_format = intv; + } + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "default_schema")) { + if (vallen) { + free(variables.default_schema); + variables.default_schema = strdup(value); + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "interfaces")) { + if (vallen && strlen(variables.interfaces) == 0) { + free(variables.interfaces); + variables.interfaces = strdup(value); + return true; + } + else { + if (vallen && strcmp(value, variables.interfaces) == 0) { + return true; + } + else { + return false; + } + } + } + if (!strcasecmp(name, "server_version")) { + if (vallen) { + free(variables.server_version); + variables.server_version = strdup(value); + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "default_client_encoding")) { + if (vallen) { + free(variables.default_client_encoding); + variables.default_client_encoding = strdup(value); + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "init_connect")) { + if (variables.init_connect) free(variables.init_connect); + variables.init_connect = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.init_connect = strdup(value); + } + return true; + } + if (!strcasecmp(name, "firewall_whitelist_errormsg")) { + if (variables.firewall_whitelist_errormsg) free(variables.firewall_whitelist_errormsg); + variables.firewall_whitelist_errormsg = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.firewall_whitelist_errormsg = strdup(value); + } + return true; + } + if (!strcasecmp(name, "ldap_user_variable")) { + if (variables.ldap_user_variable) free(variables.ldap_user_variable); + variables.ldap_user_variable = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.ldap_user_variable = strdup(value); + } + return true; + } + if (!strcasecmp(name, "add_ldap_user_comment")) { + if (variables.add_ldap_user_comment) free(variables.add_ldap_user_comment); + variables.add_ldap_user_comment = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.add_ldap_user_comment = strdup(value); + } + return true; + } + + if (!strcasecmp(name, "default_session_track_gtids")) { + if (variables.default_session_track_gtids) free(variables.default_session_track_gtids); + variables.default_session_track_gtids = NULL; + if (vallen) { + // we only accept 2 value for session_track_gtids = OFF or OWN_GTID + if (strcasecmp(value, (char*)"OFF") == 0) { + // for convention, we stored the value as uppercase + variables.default_session_track_gtids = strdup((char*)"OFF"); + return true; + } + else if (strcasecmp(value, (char*)"OWN_GTID") == 0) { + // for convention, we stored the value as uppercase + variables.default_session_track_gtids = strdup((char*)"OWN_GTID"); + return true; + } + } + return false; // we couldn't set it to a valid value. It will be reset to default + } + + if (!strncmp(name, "default_", 8)) { + for (int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (mysql_tracked_variables[i].is_global_variable == false) + continue; + char buf[128]; + sprintf(buf, "default_%s", mysql_tracked_variables[i].internal_variable_name); + if (!strcmp(name, buf)) { + if (variables.default_variables[i]) free(variables.default_variables[i]); + variables.default_variables[i] = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.default_variables[i] = strdup(value); + } + if (variables.default_variables[i] == NULL) + variables.default_variables[i] = strdup(mysql_tracked_variables[i].default_value); + return true; + } + } + } + + + if (!strcasecmp(name, "keep_multiplexing_variables")) { + if (vallen) { + free(variables.keep_multiplexing_variables); + variables.keep_multiplexing_variables = strdup(value); + return true; + } + else { + return false; + } + } + // SSL proxy to server variables + if (!strcasecmp(name, "ssl_p2s_ca")) { + if (variables.ssl_p2s_ca) free(variables.ssl_p2s_ca); + variables.ssl_p2s_ca = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.ssl_p2s_ca = strdup(value); + } + return true; + } + if (!strcasecmp(name, "ssl_p2s_capath")) { + if (variables.ssl_p2s_capath) free(variables.ssl_p2s_capath); + variables.ssl_p2s_capath = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.ssl_p2s_capath = strdup(value); + } + return true; + } + if (!strcasecmp(name, "ssl_p2s_cert")) { + if (variables.ssl_p2s_cert) free(variables.ssl_p2s_cert); + variables.ssl_p2s_cert = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.ssl_p2s_cert = strdup(value); + } + return true; + } + if (!strcasecmp(name, "ssl_p2s_key")) { + if (variables.ssl_p2s_key) free(variables.ssl_p2s_key); + variables.ssl_p2s_key = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.ssl_p2s_key = strdup(value); + } + return true; + } + if (!strcasecmp(name, "ssl_p2s_cipher")) { + if (variables.ssl_p2s_cipher) free(variables.ssl_p2s_cipher); + variables.ssl_p2s_cipher = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.ssl_p2s_cipher = strdup(value); + } + return true; + } + if (!strcasecmp(name, "ssl_p2s_crl")) { + if (variables.ssl_p2s_crl) free(variables.ssl_p2s_crl); + variables.ssl_p2s_crl = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.ssl_p2s_crl = strdup(value); + } + return true; + } + if (!strcasecmp(name, "ssl_p2s_crlpath")) { + if (variables.ssl_p2s_crlpath) free(variables.ssl_p2s_crlpath); + variables.ssl_p2s_crlpath = NULL; + if (vallen) { + if (strcmp(value, "(null)")) + variables.ssl_p2s_crlpath = strdup(value); + } + return true; + } + + if (!strcasecmp(name, "auditlog_filename")) { + if (value[strlen(value) - 1] == '/') { + proxy_error("%s is an invalid value for auditlog_filename, please specify a filename not just the path\n", value); + return false; + } + else if (value[0] == '/') { + char* full_path = strdup(value); + char* eval_dirname = dirname(full_path); + DIR* eventlog_dir = opendir(eval_dirname); + if (eventlog_dir) { + closedir(eventlog_dir); + free(variables.auditlog_filename); + variables.auditlog_filename = strdup(value); + free(full_path); + return true; + } + else { + proxy_error("%s is an invalid value for auditlog_filename path, the directory cannot be accessed\n", eval_dirname); + return false; + } + } + else { + free(variables.auditlog_filename); + variables.auditlog_filename = strdup(value); + return true; + } + } + if (!strcasecmp(name, "eventslog_filename")) { + if (value[strlen(value) - 1] == '/') { + proxy_error("%s is an invalid value for eventslog_filename, please specify a filename not just the path\n", value); + return false; + } + else if (value[0] == '/') { + char* full_path = strdup(value); + char* eval_dirname = dirname(full_path); + DIR* eventlog_dir = opendir(eval_dirname); + if (eventlog_dir) { + closedir(eventlog_dir); + free(variables.eventslog_filename); + variables.eventslog_filename = strdup(value); + free(full_path); + return true; + } + else { + proxy_error("%s is an invalid value for eventslog_filename path, the directory cannot be accessed\n", eval_dirname); + return false; + } + } + else { + free(variables.eventslog_filename); + variables.eventslog_filename = strdup(value); + return true; + } + } + if (!strcasecmp(name, "server_capabilities")) { + // replaced atoi() with strtoul() to have a 32 bit result + uint32_t intv = strtoul(value, NULL, 10); + if (intv > 10) { + // Note that: + // - some capabilities are changed at runtime while performing the handshake with the client + // - even if we support 32 bits capabilities, many of them do not have any real meaning for proxysql (not supported) + variables.server_capabilities = intv; + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "stacksize")) { + int intv = atoi(value); + if (intv >= 256 * 1024 && intv <= 4 * 1024 * 1024) { + stacksize = intv; + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "threads")) { + unsigned int intv = atoi(value); + if ((num_threads == 0 || num_threads == intv || pgsql_threads == NULL) && intv > 0 && intv < 256) { + num_threads = intv; + //this->status_variables.p_gauge_array[p_th_gauge::mysql_thread_workers]->Set(intv); + return true; + } + else { + return false; + } + } + if (!strcasecmp(name, "have_compress")) { + if (strcasecmp(value, "true") == 0 || strcasecmp(value, "1") == 0) { + variables.have_compress = true; + variables.server_capabilities |= CLIENT_COMPRESS; + return true; + } + if (strcasecmp(value, "false") == 0 || strcasecmp(value, "0") == 0) { + variables.have_compress = false; + variables.server_capabilities &= ~CLIENT_COMPRESS; + return true; + } + return false; + } + if (!strcasecmp(name, "have_ssl")) { + if (strcasecmp(value, "true") == 0 || strcasecmp(value, "1") == 0) { + variables.have_ssl = true; + variables.server_capabilities |= CLIENT_SSL; + return true; + } + if (strcasecmp(value, "false") == 0 || strcasecmp(value, "0") == 0) { + variables.have_ssl = false; + variables.server_capabilities &= ~CLIENT_SSL; + return true; + } + return false; + } + if (!strcasecmp(name, "forward_autocommit")) { + if (strcasecmp(value, "true") == 0 || strcasecmp(value, "1") == 0) { + proxy_error("Variable mysql-forward_autocommit is deprecated. See issue #3253\n"); + return false; + } + return false; + } + if (!strcasecmp(name, "data_packets_history_size")) { + int intv = atoi(value); + if (intv >= 0 && intv < INT_MAX) { + variables.data_packets_history_size = intv; + GloVars.global.data_packets_history_size = intv; + return true; + } + else { + return false; + } + } + return false; +} + + +// return variables from both pgsql_thread_variables_names AND mysql_tracked_variables +char** PgSQL_Threads_Handler::get_variables_list() { + + + // initialize VariablesPointers_bool + // it is safe to do it here because get_variables_list() is the first function called during start time + if (VariablesPointers_bool.size() == 0) { + VariablesPointers_bool["autocommit_false_is_transaction"] = make_tuple(&variables.autocommit_false_is_transaction, false); + VariablesPointers_bool["autocommit_false_not_reusable"] = make_tuple(&variables.autocommit_false_not_reusable, false); + VariablesPointers_bool["automatic_detect_sqli"] = make_tuple(&variables.automatic_detect_sqli, false); + VariablesPointers_bool["client_session_track_gtid"] = make_tuple(&variables.client_session_track_gtid, false); + VariablesPointers_bool["commands_stats"] = make_tuple(&variables.commands_stats, false); + VariablesPointers_bool["connection_warming"] = make_tuple(&variables.connection_warming, false); + VariablesPointers_bool["default_reconnect"] = make_tuple(&variables.default_reconnect, false); + VariablesPointers_bool["enable_client_deprecate_eof"] = make_tuple(&variables.enable_client_deprecate_eof, false); + VariablesPointers_bool["enable_server_deprecate_eof"] = make_tuple(&variables.enable_server_deprecate_eof, false); + VariablesPointers_bool["enable_load_data_local_infile"] = make_tuple(&variables.enable_load_data_local_infile, false); + VariablesPointers_bool["enforce_autocommit_on_reads"] = make_tuple(&variables.enforce_autocommit_on_reads, false); + VariablesPointers_bool["firewall_whitelist_enabled"] = make_tuple(&variables.firewall_whitelist_enabled, false); + VariablesPointers_bool["kill_backend_connection_when_disconnect"] = make_tuple(&variables.kill_backend_connection_when_disconnect, false); + VariablesPointers_bool["log_mysql_warnings_enabled"] = make_tuple(&variables.log_mysql_warnings_enabled, false); + VariablesPointers_bool["log_unhealthy_connections"] = make_tuple(&variables.log_unhealthy_connections, false); + VariablesPointers_bool["monitor_enabled"] = make_tuple(&variables.monitor_enabled, false); + VariablesPointers_bool["monitor_replication_lag_group_by_host"] = make_tuple(&variables.monitor_replication_lag_group_by_host, false); + VariablesPointers_bool["monitor_wait_timeout"] = make_tuple(&variables.monitor_wait_timeout, false); + VariablesPointers_bool["monitor_writer_is_also_reader"] = make_tuple(&variables.monitor_writer_is_also_reader, false); + VariablesPointers_bool["multiplexing"] = make_tuple(&variables.multiplexing, false); + VariablesPointers_bool["query_cache_stores_empty_result"] = make_tuple(&variables.query_cache_stores_empty_result, false); + VariablesPointers_bool["query_digests"] = make_tuple(&variables.query_digests, false); + VariablesPointers_bool["query_digests_lowercase"] = make_tuple(&variables.query_digests_lowercase, false); + VariablesPointers_bool["query_digests_replace_null"] = make_tuple(&variables.query_digests_replace_null, false); + VariablesPointers_bool["query_digests_no_digits"] = make_tuple(&variables.query_digests_no_digits, false); + VariablesPointers_bool["query_digests_normalize_digest_text"] = make_tuple(&variables.query_digests_normalize_digest_text, false); + VariablesPointers_bool["query_digests_track_hostname"] = make_tuple(&variables.query_digests_track_hostname, false); + VariablesPointers_bool["query_digests_keep_comment"] = make_tuple(&variables.query_digests_keep_comment, false); + VariablesPointers_bool["parse_failure_logs_digest"] = make_tuple(&variables.parse_failure_logs_digest, false); + VariablesPointers_bool["servers_stats"] = make_tuple(&variables.servers_stats, false); + VariablesPointers_bool["sessions_sort"] = make_tuple(&variables.sessions_sort, false); + VariablesPointers_bool["stats_time_backend_query"] = make_tuple(&variables.stats_time_backend_query, false); + VariablesPointers_bool["stats_time_query_processor"] = make_tuple(&variables.stats_time_query_processor, false); + VariablesPointers_bool["use_tcp_keepalive"] = make_tuple(&variables.use_tcp_keepalive, false); + VariablesPointers_bool["verbose_query_error"] = make_tuple(&variables.verbose_query_error, false); +#ifdef IDLE_THREADS + VariablesPointers_bool["session_idle_show_processlist"] = make_tuple(&variables.session_idle_show_processlist, false); +#endif // IDLE_THREADS +#ifdef DEBUG + VariablesPointers_bool["session_debug"] = make_tuple(&variables.session_debug, false); +#endif /* DEBUG */ + // variables with special variable == true + // the input validation for these variables MUST be EXPLICIT + VariablesPointers_bool["have_compress"] = make_tuple(&variables.have_compress, true); + VariablesPointers_bool["have_ssl"] = make_tuple(&variables.have_ssl, true); + } + + + // initialize VariablesPointers_int + // it is safe to do it here because get_variables_list() is the first function called during start time + if (VariablesPointers_int.size() == 0) { + VariablesPointers_int["authentication_method"] = make_tuple(&variables.authentication_method, 0, 4, false); + // Monitor variables + VariablesPointers_int["monitor_history"] = make_tuple(&variables.monitor_history, 1000, 7 * 24 * 3600 * 1000, false); + + VariablesPointers_int["monitor_connect_interval"] = make_tuple(&variables.monitor_connect_interval, 100, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_connect_timeout"] = make_tuple(&variables.monitor_connect_timeout, 100, 600 * 1000, false); + + VariablesPointers_int["monitor_ping_interval"] = make_tuple(&variables.monitor_ping_interval, 100, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_ping_timeout"] = make_tuple(&variables.monitor_ping_timeout, 100, 600 * 1000, false); + VariablesPointers_int["monitor_ping_max_failures"] = make_tuple(&variables.monitor_ping_max_failures, 1, 1000 * 1000, false); + + VariablesPointers_int["monitor_aws_rds_topology_discovery_interval"] = make_tuple(&variables.monitor_aws_rds_topology_discovery_interval, 1, 100000, false); + VariablesPointers_int["monitor_read_only_interval"] = make_tuple(&variables.monitor_read_only_interval, 100, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_read_only_timeout"] = make_tuple(&variables.monitor_read_only_timeout, 100, 600 * 1000, false); + VariablesPointers_int["monitor_read_only_max_timeout_count"] = make_tuple(&variables.monitor_read_only_max_timeout_count, 1, 1000 * 1000, false); + + VariablesPointers_int["monitor_replication_lag_interval"] = make_tuple(&variables.monitor_replication_lag_interval, 100, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_replication_lag_timeout"] = make_tuple(&variables.monitor_replication_lag_timeout, 100, 600 * 1000, false); + VariablesPointers_int["monitor_replication_lag_count"] = make_tuple(&variables.monitor_replication_lag_count, 1, 10, false); + + VariablesPointers_int["monitor_groupreplication_healthcheck_interval"] = make_tuple(&variables.monitor_groupreplication_healthcheck_interval, 100, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_groupreplication_healthcheck_timeout"] = make_tuple(&variables.monitor_groupreplication_healthcheck_timeout, 100, 600 * 1000, false); + VariablesPointers_int["monitor_groupreplication_healthcheck_max_timeout_count"] = make_tuple(&variables.monitor_groupreplication_healthcheck_max_timeout_count, 1, 10, false); + VariablesPointers_int["monitor_groupreplication_max_transactions_behind_count"] = make_tuple(&variables.monitor_groupreplication_max_transactions_behind_count, 1, 10, false); + VariablesPointers_int["monitor_groupreplication_max_transactions_behind_for_read_only"] = make_tuple(&variables.monitor_groupreplication_max_transactions_behind_for_read_only, 0, 2, false); + + VariablesPointers_int["monitor_galera_healthcheck_interval"] = make_tuple(&variables.monitor_galera_healthcheck_interval, 50, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_galera_healthcheck_timeout"] = make_tuple(&variables.monitor_galera_healthcheck_timeout, 50, 600 * 1000, false); + VariablesPointers_int["monitor_galera_healthcheck_max_timeout_count"] = make_tuple(&variables.monitor_galera_healthcheck_max_timeout_count, 1, 10, false); + + VariablesPointers_int["monitor_query_interval"] = make_tuple(&variables.monitor_query_interval, 100, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_query_timeout"] = make_tuple(&variables.monitor_query_timeout, 100, 600 * 1000, false); + + VariablesPointers_int["monitor_threads_min"] = make_tuple(&variables.monitor_threads_min, 2, 256, false); + VariablesPointers_int["monitor_threads_max"] = make_tuple(&variables.monitor_threads_max, 4, 1024, false); + + VariablesPointers_int["monitor_slave_lag_when_null"] = make_tuple(&variables.monitor_slave_lag_when_null, 0, 604800, false); + VariablesPointers_int["monitor_threads_queue_maxsize"] = make_tuple(&variables.monitor_threads_queue_maxsize, 16, 1024, false); + + VariablesPointers_int["monitor_local_dns_cache_ttl"] = make_tuple(&variables.monitor_local_dns_cache_ttl, 0, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_local_dns_cache_refresh_interval"] = make_tuple(&variables.monitor_local_dns_cache_refresh_interval, 0, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["monitor_local_dns_resolver_queue_maxsize"] = make_tuple(&variables.monitor_local_dns_resolver_queue_maxsize, 16, 1024, false); + // mirroring + VariablesPointers_int["mirror_max_concurrency"] = make_tuple(&variables.mirror_max_concurrency, 1, 8 * 1024, false); + VariablesPointers_int["mirror_max_queue_length"] = make_tuple(&variables.mirror_max_queue_length, 0, 1024 * 1024, false); + // query processor and query digest + VariablesPointers_int["auto_increment_delay_multiplex"] = make_tuple(&variables.auto_increment_delay_multiplex, 0, 1000000, false); + VariablesPointers_int["auto_increment_delay_multiplex_timeout_ms"] = make_tuple(&variables.auto_increment_delay_multiplex_timeout_ms, 0, 3600 * 1000, false); + VariablesPointers_int["default_query_delay"] = make_tuple(&variables.default_query_delay, 0, 3600 * 1000, false); + VariablesPointers_int["default_query_timeout"] = make_tuple(&variables.default_query_timeout, 1000, 20 * 24 * 3600 * 1000, false); + VariablesPointers_int["query_digests_grouping_limit"] = make_tuple(&variables.query_digests_grouping_limit, 1, 2089, false); + VariablesPointers_int["query_digests_groups_grouping_limit"] = make_tuple(&variables.query_digests_groups_grouping_limit, 0, 2089, false); + VariablesPointers_int["query_digests_max_digest_length"] = make_tuple(&variables.query_digests_max_digest_length, 16, 1 * 1024 * 1024, false); + VariablesPointers_int["query_digests_max_query_length"] = make_tuple(&variables.query_digests_max_query_length, 16, 1 * 1024 * 1024, false); + VariablesPointers_int["query_rules_fast_routing_algorithm"] = make_tuple(&variables.query_rules_fast_routing_algorithm, 1, 2, false); + VariablesPointers_int["query_processor_iterations"] = make_tuple(&variables.query_processor_iterations, 0, 1000 * 1000, false); + VariablesPointers_int["query_processor_regex"] = make_tuple(&variables.query_processor_regex, 1, 2, false); + VariablesPointers_int["query_retries_on_failure"] = make_tuple(&variables.query_retries_on_failure, 0, 1000, false); + VariablesPointers_int["set_query_lock_on_hostgroup"] = make_tuple(&variables.set_query_lock_on_hostgroup, 0, 1, false); + VariablesPointers_int["set_parser_algorithm"] = make_tuple(&variables.set_parser_algorithm, 1, 2, false); + + // throttle + VariablesPointers_int["throttle_connections_per_sec_to_hostgroup"] = make_tuple(&variables.throttle_connections_per_sec_to_hostgroup, 1, 100 * 1000 * 1000, false); + VariablesPointers_int["throttle_max_bytes_per_second_to_client"] = make_tuple(&variables.throttle_max_bytes_per_second_to_client, 0, 2147483647, false); + VariablesPointers_int["throttle_ratio_server_to_client"] = make_tuple(&variables.throttle_ratio_server_to_client, 0, 100, false); + // backend management + VariablesPointers_int["default_max_latency_ms"] = make_tuple(&variables.default_max_latency_ms, 0, 20 * 24 * 3600 * 1000, false); + VariablesPointers_int["free_connections_pct"] = make_tuple(&variables.free_connections_pct, 0, 100, false); + VariablesPointers_int["poll_timeout"] = make_tuple(&variables.poll_timeout, 10, 20000, false); + VariablesPointers_int["poll_timeout_on_failure"] = make_tuple(&variables.poll_timeout_on_failure, 10, 20000, false); + VariablesPointers_int["shun_on_failures"] = make_tuple(&variables.shun_on_failures, 0, 10000000, false); + VariablesPointers_int["shun_recovery_time_sec"] = make_tuple(&variables.shun_recovery_time_sec, 0, 3600 * 24 * 365, false); + VariablesPointers_int["unshun_algorithm"] = make_tuple(&variables.unshun_algorithm, 0, 1, false); + VariablesPointers_int["hostgroup_manager_verbose"] = make_tuple(&variables.hostgroup_manager_verbose, 0, 3, false); + VariablesPointers_int["tcp_keepalive_time"] = make_tuple(&variables.tcp_keepalive_time, 0, 7200, false); + VariablesPointers_int["min_num_servers_lantency_awareness"] = make_tuple(&variables.min_num_servers_lantency_awareness, 0, 10000, false); + VariablesPointers_int["aurora_max_lag_ms_only_read_from_replicas"] = make_tuple(&variables.aurora_max_lag_ms_only_read_from_replicas, 0, 100, false); + // connection management + VariablesPointers_int["connect_retries_on_failure"] = make_tuple(&variables.connect_retries_on_failure, 0, 1000, false); + VariablesPointers_int["connect_retries_delay"] = make_tuple(&variables.connect_retries_delay, 0, 10000, false); + VariablesPointers_int["connect_timeout_client"] = make_tuple(&variables.connect_timeout_client, 500, 3600 * 1000, false); + VariablesPointers_int["connect_timeout_server"] = make_tuple(&variables.connect_timeout_server, 10, 120 * 1000, false); + VariablesPointers_int["connect_timeout_server_max"] = make_tuple(&variables.connect_timeout_server_max, 10, 3600 * 1000, false); + VariablesPointers_int["connection_delay_multiplex_ms"] = make_tuple(&variables.connection_delay_multiplex_ms, 0, 300 * 1000, false); + VariablesPointers_int["connection_max_age_ms"] = make_tuple(&variables.connection_max_age_ms, 0, 3600 * 24 * 1000, false); + VariablesPointers_int["handle_unknown_charset"] = make_tuple(&variables.handle_unknown_charset, 0, HANDLE_UNKNOWN_CHARSET__MAX_HANDLE_VALUE, false); + VariablesPointers_int["ping_interval_server_msec"] = make_tuple(&variables.ping_interval_server_msec, 1000, 7 * 24 * 3600 * 1000, false); + VariablesPointers_int["ping_timeout_server"] = make_tuple(&variables.ping_timeout_server, 10, 600 * 1000, false); + VariablesPointers_int["client_host_cache_size"] = make_tuple(&variables.client_host_cache_size, 0, 1024 * 1024, false); + VariablesPointers_int["client_host_error_counts"] = make_tuple(&variables.client_host_error_counts, 0, 1024 * 1024, false); + VariablesPointers_int["handle_warnings"] = make_tuple(&variables.handle_warnings, 0, 1, false); + + // logs + VariablesPointers_int["auditlog_filesize"] = make_tuple(&variables.auditlog_filesize, 1024 * 1024, 1 * 1024 * 1024 * 1024, false); + VariablesPointers_int["eventslog_filesize"] = make_tuple(&variables.eventslog_filesize, 1024 * 1024, 1 * 1024 * 1024 * 1024, false); + VariablesPointers_int["eventslog_default_log"] = make_tuple(&variables.eventslog_default_log, 0, 1, false); + // various + VariablesPointers_int["long_query_time"] = make_tuple(&variables.long_query_time, 0, 20 * 24 * 3600 * 1000, false); + VariablesPointers_int["max_allowed_packet"] = make_tuple(&variables.max_allowed_packet, 8192, 1024 * 1024 * 1024, false); + VariablesPointers_int["max_connections"] = make_tuple(&variables.max_connections, 1, 1000 * 1000, false); + VariablesPointers_int["max_stmts_per_connection"] = make_tuple(&variables.max_stmts_per_connection, 1, 1024, false); + VariablesPointers_int["max_stmts_cache"] = make_tuple(&variables.max_stmts_cache, 128, 1024 * 1024, false); + VariablesPointers_int["max_transaction_idle_time"] = make_tuple(&variables.max_transaction_idle_time, 1000, 20 * 24 * 3600 * 1000, false); + VariablesPointers_int["max_transaction_time"] = make_tuple(&variables.max_transaction_time, 1000, 20 * 24 * 3600 * 1000, false); + VariablesPointers_int["query_cache_size_mb"] = make_tuple(&variables.query_cache_size_MB, 0, 1024 * 10240, false); + VariablesPointers_int["query_cache_soft_ttl_pct"] = make_tuple(&variables.query_cache_soft_ttl_pct, 0, 100, false); + VariablesPointers_int["query_cache_handle_warnings"] = make_tuple(&variables.query_cache_handle_warnings, 0, 1, false); + +#ifdef IDLE_THREADS + VariablesPointers_int["session_idle_ms"] = make_tuple(&variables.session_idle_ms, 1, 3600 * 1000, false); +#endif // IDLE_THREADS + VariablesPointers_int["show_processlist_extended"] = make_tuple(&variables.show_processlist_extended, 0, 2, false); + VariablesPointers_int["threshold_query_length"] = make_tuple(&variables.threshold_query_length, 1024, 1 * 1024 * 1024 * 1024, false); + VariablesPointers_int["threshold_resultset_size"] = make_tuple(&variables.threshold_resultset_size, 1024, 1 * 1024 * 1024 * 1024, false); + + // variables with special variable == true + // the input validation for these variables MUST be EXPLICIT + VariablesPointers_int["binlog_reader_connect_retry_msec"] = make_tuple(&variables.binlog_reader_connect_retry_msec, 0, 0, true); + VariablesPointers_int["eventslog_format"] = make_tuple(&variables.eventslog_format, 0, 0, true); + VariablesPointers_int["wait_timeout"] = make_tuple(&variables.wait_timeout, 0, 0, true); + VariablesPointers_int["data_packets_history_size"] = make_tuple(&variables.data_packets_history_size, 0, 0, true); + + } + + + const size_t l = sizeof(pgsql_thread_variables_names) / sizeof(char*); + unsigned int i; + size_t ltv = 0; + for (i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (mysql_tracked_variables[i].is_global_variable) + ltv++; + } + char** ret = (char**)malloc(sizeof(char*) * (l + ltv)); // not adding + 1 because pgsql_thread_variables_names is already NULL terminated + size_t fv = 0; + for (i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (mysql_tracked_variables[i].is_global_variable) { + char* m = (char*)malloc(strlen(mysql_tracked_variables[i].internal_variable_name) + 1 + strlen((char*)"default_")); + sprintf(m, "default_%s", mysql_tracked_variables[i].internal_variable_name); + ret[fv] = m; + fv++; + } + } + // this is an extra check. + assert(fv == ltv); + for (i = ltv; i < l + ltv - 1; i++) { + ret[i] = (strdup(pgsql_thread_variables_names[i - ltv])); + } + ret[l + ltv - 1] = NULL; // last value + return ret; +} + +// Returns true if the given name is the name of an existing mysql variable +// scan both pgsql_thread_variables_names AND mysql_tracked_variables +bool PgSQL_Threads_Handler::has_variable(const char* name) { + if (strlen(name) > 8) { + if (strncmp(name, "default_", 8) == 0) { + for (unsigned int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (mysql_tracked_variables[i].is_global_variable) { + size_t var_len = strlen(mysql_tracked_variables[i].internal_variable_name); + if (strlen(name) == (var_len + 8)) { + if (!strncmp(name + 8, mysql_tracked_variables[i].internal_variable_name, var_len)) { + return true; + } + } + } + } + } + } + size_t no_vars = sizeof(pgsql_thread_variables_names) / sizeof(char*); + for (unsigned int i = 0; i < no_vars - 1; ++i) { + size_t var_len = strlen(pgsql_thread_variables_names[i]); + if (strlen(name) == var_len && !strncmp(name, pgsql_thread_variables_names[i], var_len)) { + return true; + } + } + return false; +} + +void PgSQL_Threads_Handler::print_version() { + fprintf(stderr, "Standard MySQL Threads Handler rev. %s -- %s -- %s\n", MYSQL_THREAD_VERSION, __FILE__, __TIMESTAMP__); +} + +void PgSQL_Threads_Handler::init(unsigned int num, size_t stack) { + if (stack) { + stacksize = stack; + } + else { + if (stacksize == 0) stacksize = DEFAULT_STACK_SIZE; + } + if (num) { + num_threads = num; + //this->status_variables.p_gauge_array[p_th_gauge::mysql_thread_workers]->Set(num); + } + else { + if (num_threads == 0) { + num_threads = DEFAULT_NUM_THREADS; //default + //this->status_variables.p_gauge_array[p_th_gauge::mysql_thread_workers]->Set(DEFAULT_NUM_THREADS); + } + } + int rc = pthread_attr_setstacksize(&attr, stacksize); + assert(rc == 0); + pgsql_threads = (proxysql_pgsql_thread_t*)calloc(num_threads, sizeof(proxysql_pgsql_thread_t)); +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) + pgsql_threads_idles = (proxysql_pgsql_thread_t*)calloc(num_threads, sizeof(proxysql_pgsql_thread_t)); +#endif // IDLE_THREADS +} + +proxysql_pgsql_thread_t* PgSQL_Threads_Handler::create_thread(unsigned int tn, void* (*start_routine) (void*), bool idles) { + if (idles == false) { + if (pthread_create(&pgsql_threads[tn].thread_id, &attr, start_routine, &pgsql_threads[tn]) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } +#ifdef IDLE_THREADS + } + else { + if (GloVars.global.idle_threads) { + if (pthread_create(&pgsql_threads_idles[tn].thread_id, &attr, start_routine, &pgsql_threads_idles[tn]) != 0) { + // LCOV_EXCL_START + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } +#endif // IDLE_THREADS + } + return NULL; +} + +void PgSQL_Threads_Handler::shutdown_threads() { + unsigned int i; + shutdown_ = 1; + if (pgsql_threads) { + for (i = 0; i < num_threads; i++) { + if (pgsql_threads[i].worker) { + pthread_mutex_lock(&pgsql_threads[i].worker->thread_mutex); + pgsql_threads[i].worker->shutdown = 1; + pthread_mutex_unlock(&pgsql_threads[i].worker->thread_mutex); + } + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + for (i = 0; i < num_threads; i++) { + if (pgsql_threads_idles[i].worker) { + pthread_mutex_lock(&pgsql_threads[i].worker->thread_mutex); + pgsql_threads_idles[i].worker->shutdown = 1; + pthread_mutex_unlock(&pgsql_threads[i].worker->thread_mutex); + } + } + } +#endif /* IDLE_THREADS */ + signal_all_threads(1); + for (i = 0; i < num_threads; i++) { + if (pgsql_threads[i].worker) + pthread_join(pgsql_threads[i].thread_id, NULL); +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + if (pgsql_threads_idles[i].worker) + pthread_join(pgsql_threads_idles[i].thread_id, NULL); + } +#endif /* IDLE_THREADS */ + } + } +} + +void PgSQL_Threads_Handler::start_listeners() { + // we set bootstrapping_listeners to true + // In this way PgSQL_Thread will knows there are more listeners to add + // and it will continue looping until all listeners are added + bootstrapping_listeners = true; + char* _tmp = NULL; + _tmp = GloPTH->get_variable((char*)"interfaces"); + if (strlen(_tmp) == 0) { + GloPTH->set_variable((char*)"interfaces", (char*)"0.0.0.0:6133"); + } + free(_tmp); + tokenizer_t tok; + tokenizer(&tok, variables.interfaces, ";", TOKENIZER_NO_EMPTIES); + const char* token; + for (token = tokenize(&tok); token; token = tokenize(&tok)) { + listener_add((char*)token); + } + free_tokenizer(&tok); + // no more listeners to add + bootstrapping_listeners = false; +} + +void PgSQL_Threads_Handler::stop_listeners() { + if (variables.interfaces == NULL || strlen(variables.interfaces) == 0) + return; + tokenizer_t tok; + tokenizer(&tok, variables.interfaces, ";", TOKENIZER_NO_EMPTIES); + const char* token; + for (token = tokenize(&tok); token; token = tokenize(&tok)) { + listener_del((char*)token); + } + free_tokenizer(&tok); +} + +/** + * @brief Gets the client address stored in 'client_addr' member as + * an string if available. If member 'client_addr' is NULL, returns an + * empty string. + * + * @return Either an string holding the string representation of internal + * member 'client_addr', or empty string if this member is NULL. + */ +static std::string get_client_addr(struct sockaddr* client_addr) { + char buf[INET6_ADDRSTRLEN]; + std::string str_client_addr{}; + + if (client_addr == NULL) { + return str_client_addr; + } + + switch (client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)client_addr; + inet_ntop(client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + str_client_addr = std::string{ buf }; + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)client_addr; + inet_ntop(client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + str_client_addr = std::string{ buf }; + break; + } + default: + str_client_addr = std::string{ "localhost" }; + break; + } + + return str_client_addr; +} + +PgSQL_Client_Host_Cache_Entry PgSQL_Threads_Handler::find_client_host_cache(struct sockaddr* client_sockaddr) { + PgSQL_Client_Host_Cache_Entry entry{ 0, 0 }; + // Client_sockaddr **shouldn't** ever by 'NULL', no matter the + // 'session_type' in from which this function is called. Because + // `PgSQL_Session::client_myds::client_addr` should **always** be + // initialized before `handler` is called. + assert(client_sockaddr != NULL); + if (client_sockaddr->sa_family != AF_INET && client_sockaddr->sa_family != AF_INET6) { + return entry; + } + std::string client_addr = get_client_addr(client_sockaddr); + if (client_addr == "127.0.0.1") { + return entry; + } + + pthread_mutex_lock(&mutex_client_host_cache); + auto found_entry = client_host_cache.find(client_addr); + if (found_entry != client_host_cache.end()) { + entry = found_entry->second; + } + pthread_mutex_unlock(&mutex_client_host_cache); + + return entry; +} + +/** + * @brief Number of columns for representing a 'PgSQL_Client_Host_Cache_Entry' + * in a 'SQLite3_result'. + */ +const int CLIENT_HOST_CACHE_COLUMNS = 3; + +/** + * @brief Helper function that converts a given client address and a + * 'PgSQL_Client_Host_Cache_Entry', into a row for a 'SQLite3_result' for + * table 'STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE'. + * + * @param address The client address to be added to the resulset row. + * @param entry The 'PgSQL_Client_Host_Cache_Entry' to be added to the resulset + * row. + * + * @return A pointer array holding the values for each of the columns of the + * row. It should be freed through helper function 'free_client_host_cache_row'. + */ +char** client_host_cache_entry_row( + const std::string address, const PgSQL_Client_Host_Cache_Entry & entry +) { + // INET6_ADDRSTRLEN length should be enough for holding any member: + // { address: MAX INET6_ADDRSTRLEN, updated_at: uint64_t, error_count: uint32_t } + char buff[INET6_ADDRSTRLEN]; + char** row = + static_cast(malloc(sizeof(char*) * CLIENT_HOST_CACHE_COLUMNS)); + + time_t __now = time(NULL); + unsigned long long curtime = monotonic_time(); + time_t last_updated = __now - curtime / 1000000 + entry.updated_at / 1000000; + + row[0] = strdup(address.c_str()); + sprintf(buff, "%u", entry.error_count); + row[1] = strdup(buff); + sprintf(buff, "%lu", last_updated); + row[2] = strdup(buff); + + return row; +} + +/** + * @brief Helper function to free the row returned by + * 'client_host_cache_entry_row'. + * + * @param row The pointer array holding the row values to be freed. + */ +static void free_client_host_cache_row(char** row) { + for (int i = 0; i < CLIENT_HOST_CACHE_COLUMNS; i++) { + free(row[i]); + } + free(row); +} + +SQLite3_result* PgSQL_Threads_Handler::get_client_host_cache(bool reset) { + SQLite3_result* result = new SQLite3_result(CLIENT_HOST_CACHE_COLUMNS); + + pthread_mutex_lock(&mutex_client_host_cache); + result->add_column_definition(SQLITE_TEXT, "client_address"); + result->add_column_definition(SQLITE_TEXT, "error_count"); + result->add_column_definition(SQLITE_TEXT, "last_updated"); + + for (const auto& cache_entry : client_host_cache) { + char** row = client_host_cache_entry_row(cache_entry.first, cache_entry.second); + result->add_row(row); + free_client_host_cache_row(row); + } + + if (reset) { + client_host_cache.clear(); + } + + pthread_mutex_unlock(&mutex_client_host_cache); + return result; +} + +void PgSQL_Threads_Handler::update_client_host_cache(struct sockaddr* client_sockaddr, bool error) { + // Client_sockaddr **shouldn't** ever by 'NULL', no matter the + // 'session_type' in from which this function is called. Because + // `PgSQL_Session::client_myds::client_addr` should **always** be + // initialized before `handler` is called. + assert(client_sockaddr != NULL); + if (client_sockaddr->sa_family != AF_INET && client_sockaddr->sa_family != AF_INET6) { + return; + } + std::string client_addr = get_client_addr(client_sockaddr); + if (client_addr == "127.0.0.1") { + return; + } + + if (error) { + pthread_mutex_lock(&mutex_client_host_cache); + // If the cache is full, find the oldest entry on it, and update/remove it. + if ( + pgsql_thread___client_host_cache_size && + client_host_cache.size() >= static_cast(pgsql_thread___client_host_cache_size) + ) { + auto older_elem = std::min_element( + client_host_cache.begin(), + client_host_cache.end(), + [](const std::pair& f_entry, + const std::pair& s_entry) + { + return f_entry.second.updated_at < s_entry.second.updated_at; + } + ); + if (older_elem != client_host_cache.end()) { + if (older_elem->first != client_addr) { + client_host_cache.erase(older_elem); + } + } + } + + // Find the entry for the client, and update/insert it. + auto cache_entry = client_host_cache.find(client_addr); + if (cache_entry != client_host_cache.end()) { + cache_entry->second.error_count += 1; + cache_entry->second.updated_at = monotonic_time(); + } + else { + // Notice than the value of 'pgsql_thread___client_host_cache_size' can + // change at runtime. Due to this, we should only insert when the size of the + // cache is smaller than this value, otherwise we could end in situations in + // which cache doesn't shrink after it's size is reduced at runtime. + if (client_host_cache.size() < static_cast(pgsql_thread___client_host_cache_size)) { + PgSQL_Client_Host_Cache_Entry new_entry{ monotonic_time(), 1 }; + client_host_cache.insert({ client_addr, new_entry }); + } + } + pthread_mutex_unlock(&mutex_client_host_cache); + } + else { + pthread_mutex_lock(&mutex_client_host_cache); + client_host_cache.erase(client_addr); + pthread_mutex_unlock(&mutex_client_host_cache); + } +} + +void PgSQL_Threads_Handler::flush_client_host_cache() { + pthread_mutex_lock(&mutex_client_host_cache); + client_host_cache.clear(); + pthread_mutex_unlock(&mutex_client_host_cache); +} + +PgSQL_Threads_Handler::~PgSQL_Threads_Handler() { + if (variables.monitor_username) { free(variables.monitor_username); variables.monitor_username = NULL; } + if (variables.monitor_password) { free(variables.monitor_password); variables.monitor_password = NULL; } + if (variables.monitor_replication_lag_use_percona_heartbeat) { + free(variables.monitor_replication_lag_use_percona_heartbeat); + variables.monitor_replication_lag_use_percona_heartbeat = NULL; + } + if (variables.default_schema) free(variables.default_schema); + if (variables.interfaces) free(variables.interfaces); + if (variables.server_version) free(variables.server_version); + if (variables.default_client_encoding) free(variables.default_client_encoding); + if (variables.keep_multiplexing_variables) free(variables.keep_multiplexing_variables); + if (variables.firewall_whitelist_errormsg) free(variables.firewall_whitelist_errormsg); + if (variables.init_connect) free(variables.init_connect); + if (variables.ldap_user_variable) free(variables.ldap_user_variable); + if (variables.add_ldap_user_comment) free(variables.add_ldap_user_comment); + if (variables.default_session_track_gtids) free(variables.default_session_track_gtids); + if (variables.eventslog_filename) free(variables.eventslog_filename); + if (variables.auditlog_filename) free(variables.auditlog_filename); + if (variables.ssl_p2s_ca) free(variables.ssl_p2s_ca); + if (variables.ssl_p2s_capath) free(variables.ssl_p2s_capath); + if (variables.ssl_p2s_cert) free(variables.ssl_p2s_cert); + if (variables.ssl_p2s_key) free(variables.ssl_p2s_key); + if (variables.ssl_p2s_cipher) free(variables.ssl_p2s_cipher); + if (variables.ssl_p2s_crl) free(variables.ssl_p2s_crl); + if (variables.ssl_p2s_crlpath) free(variables.ssl_p2s_crlpath); + + for (int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (variables.default_variables[i]) { + free(variables.default_variables[i]); + variables.default_variables[i] = NULL; + } + } + free(pgsql_threads); + pgsql_threads = NULL; +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + free(pgsql_threads_idles); + pgsql_threads_idles = NULL; + } +#endif // IDLE_THREADS + delete MLM; + MLM = NULL; +} + +PgSQL_Thread::~PgSQL_Thread() { + + if (mysql_sessions) { + while (mysql_sessions->len) { + PgSQL_Session* sess = (PgSQL_Session*)mysql_sessions->remove_index_fast(0); + if (sess->session_type == PROXYSQL_SESSION_ADMIN || sess->session_type == PROXYSQL_SESSION_STATS) { + char _buf[1024]; + sprintf(_buf, "%s:%d:%s()", __FILE__, __LINE__, __func__); + if (GloPgSQL_Logger) { GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_CLOSE, sess, NULL, _buf); } + } + delete sess; + } + delete mysql_sessions; + mysql_sessions = NULL; + GloPgQPro->end_thread(); // only for real threads + } + + if (mirror_queue_mysql_sessions) { + while (mirror_queue_mysql_sessions->len) { + PgSQL_Session* sess = (PgSQL_Session*)mirror_queue_mysql_sessions->remove_index_fast(0); + delete sess; + } + delete mirror_queue_mysql_sessions; + mirror_queue_mysql_sessions = NULL; + } + + if (mirror_queue_mysql_sessions_cache) { + while (mirror_queue_mysql_sessions_cache->len) { + PgSQL_Session* sess = (PgSQL_Session*)mirror_queue_mysql_sessions_cache->remove_index_fast(0); + delete sess; + } + delete mirror_queue_mysql_sessions_cache; + mirror_queue_mysql_sessions_cache = NULL; + } + +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + if (idle_mysql_sessions) { + while (idle_mysql_sessions->len) { + PgSQL_Session* sess = (PgSQL_Session*)idle_mysql_sessions->remove_index_fast(0); + delete sess; + } + delete idle_mysql_sessions; + } + + if (resume_mysql_sessions) { + while (resume_mysql_sessions->len) { + PgSQL_Session* sess = (PgSQL_Session*)resume_mysql_sessions->remove_index_fast(0); + delete sess; + } + delete resume_mysql_sessions; + } + + if (myexchange.idle_mysql_sessions) { + while (myexchange.idle_mysql_sessions->len) { + PgSQL_Session* sess = (PgSQL_Session*)myexchange.idle_mysql_sessions->remove_index_fast(0); + delete sess; + } + delete myexchange.idle_mysql_sessions; + } + + if (myexchange.resume_mysql_sessions) { + while (myexchange.resume_mysql_sessions->len) { + PgSQL_Session* sess = (PgSQL_Session*)myexchange.resume_mysql_sessions->remove_index_fast(0); + delete sess; + } + delete myexchange.resume_mysql_sessions; + } + } +#endif // IDLE_THREADS + + if (cached_connections) { + return_local_connections(); + delete cached_connections; + } + + unsigned int i; + for (i = 0; i < mypolls.len; i++) { + if ( + mypolls.myds[i] && // fix bug #278 . This should be caused by not initialized datastreams used to ping the backend + mypolls.myds[i]->myds_type == MYDS_LISTENER) { + delete mypolls.myds[i]; + } + } + + if (my_idle_conns) + free(my_idle_conns); + + if (mysql_thread___monitor_username) { free(mysql_thread___monitor_username); mysql_thread___monitor_username = NULL; } + if (mysql_thread___monitor_password) { free(mysql_thread___monitor_password); mysql_thread___monitor_password = NULL; } + if (mysql_thread___monitor_replication_lag_use_percona_heartbeat) { + free(mysql_thread___monitor_replication_lag_use_percona_heartbeat); + mysql_thread___monitor_replication_lag_use_percona_heartbeat = NULL; + } + //if (pgsql_thread___default_schema) { free(pgsql_thread___default_schema); pgsql_thread___default_schema = NULL; } + if (pgsql_thread___keep_multiplexing_variables) { free(pgsql_thread___keep_multiplexing_variables); pgsql_thread___keep_multiplexing_variables = NULL; } + if (pgsql_thread___firewall_whitelist_errormsg) { free(pgsql_thread___firewall_whitelist_errormsg); pgsql_thread___firewall_whitelist_errormsg = NULL; } + if (pgsql_thread___init_connect) { free(pgsql_thread___init_connect); pgsql_thread___init_connect = NULL; } + //if (mysql_thread___ldap_user_variable) { free(mysql_thread___ldap_user_variable); mysql_thread___ldap_user_variable = NULL; } + //if (mysql_thread___add_ldap_user_comment) { free(mysql_thread___add_ldap_user_comment); mysql_thread___add_ldap_user_comment = NULL; } + //if (mysql_thread___default_session_track_gtids) { free(mysql_thread___default_session_track_gtids); mysql_thread___default_session_track_gtids = NULL; } + + if (pgsql_thread___server_version) { free(pgsql_thread___server_version); pgsql_thread___server_version = NULL; } + if (pgsql_thread___default_client_encoding) { free(pgsql_thread___default_client_encoding); pgsql_thread___default_client_encoding = NULL; } + + for (int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (mysql_thread___default_variables[i]) { + free(mysql_thread___default_variables[i]); + mysql_thread___default_variables[i] = NULL; + } + } + + if (pgsql_thread___eventslog_filename) { free(pgsql_thread___eventslog_filename); pgsql_thread___eventslog_filename = NULL; } + if (pgsql_thread___auditlog_filename) { free(pgsql_thread___auditlog_filename); pgsql_thread___auditlog_filename = NULL; } + if (pgsql_thread___ssl_p2s_ca) { free(pgsql_thread___ssl_p2s_ca); pgsql_thread___ssl_p2s_ca = NULL; } + if (pgsql_thread___ssl_p2s_capath) { free(pgsql_thread___ssl_p2s_capath); pgsql_thread___ssl_p2s_capath = NULL; } + if (pgsql_thread___ssl_p2s_cert) { free(pgsql_thread___ssl_p2s_cert); pgsql_thread___ssl_p2s_cert = NULL; } + if (pgsql_thread___ssl_p2s_key) { free(pgsql_thread___ssl_p2s_key); pgsql_thread___ssl_p2s_key = NULL; } + if (pgsql_thread___ssl_p2s_cipher) { free(pgsql_thread___ssl_p2s_cipher); pgsql_thread___ssl_p2s_cipher = NULL; } + if (pgsql_thread___ssl_p2s_crl) { free(pgsql_thread___ssl_p2s_crl); pgsql_thread___ssl_p2s_crl = NULL; } + if (pgsql_thread___ssl_p2s_crlpath) { free(pgsql_thread___ssl_p2s_crlpath); pgsql_thread___ssl_p2s_crlpath = NULL; } + + if (match_regexes) { + Session_Regex* sr = NULL; + sr = match_regexes[0]; + delete sr; + sr = match_regexes[1]; + delete sr; + sr = match_regexes[2]; + delete sr; + sr = match_regexes[3]; + delete sr; + free(match_regexes); + match_regexes = NULL; + } + if (thr_SetParser != NULL) { + delete thr_SetParser; + thr_SetParser = NULL; + } + +} + +bool PgSQL_Thread::init() { + int i; + mysql_sessions = new PtrArray(); + mirror_queue_mysql_sessions = new PtrArray(); + mirror_queue_mysql_sessions_cache = new PtrArray(); + cached_connections = new PtrArray(); + assert(mysql_sessions); + +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + idle_mysql_sessions = new PtrArray(); + resume_mysql_sessions = new PtrArray(); + + myexchange.idle_mysql_sessions = new PtrArray(); + myexchange.resume_mysql_sessions = new PtrArray(); + pthread_mutex_init(&myexchange.mutex_idles, NULL); + pthread_mutex_init(&myexchange.mutex_resumes, NULL); + assert(idle_mysql_sessions); + assert(resume_mysql_sessions); + } +#endif // IDLE_THREADS + + pthread_mutex_init(&kq.m, NULL); + + shutdown = 0; + my_idle_conns = (PgSQL_Connection**)malloc(sizeof(PgSQL_Connection*) * SESSIONS_FOR_CONNECTIONS_HANDLER); + memset(my_idle_conns, 0, sizeof(PgSQL_Connection*) * SESSIONS_FOR_CONNECTIONS_HANDLER); + GloPgQPro->init_thread(); + refresh_variables(); + i = pipe(pipefd); + ioctl_FIONBIO(pipefd[0], 1); + ioctl_FIONBIO(pipefd[1], 1); + mypolls.add(POLLIN, pipefd[0], NULL, 0); + assert(i == 0); + + thr_SetParser = new SetParser(""); + match_regexes = (Session_Regex**)malloc(sizeof(Session_Regex*) * 4); + // match_regexes[0]=new Session_Regex((char *)"^SET (|SESSION |@@|@@session.)SQL_LOG_BIN( *)(:|)=( *)"); + match_regexes[0] = NULL; // NOTE: historically we used match_regexes[0] for SET SQL_LOG_BIN . Not anymore + + std::stringstream ss; + ss << "^SET (|SESSION |@@|@@session.|@@local.)`?(" << pgsql_variables.variables_regexp << "SESSION_TRACK_GTIDS|TX_ISOLATION|TX_READ_ONLY|TRANSACTION_ISOLATION|TRANSACTION_READ_ONLY)`?( *)(:|)=( *)"; + match_regexes[1] = new Session_Regex((char*)ss.str().c_str()); + + match_regexes[2] = new Session_Regex((char*)"^SET(?: +)(|SESSION +)TRANSACTION(?: +)(?:(?:(ISOLATION(?: +)LEVEL)(?: +)(REPEATABLE(?: +)READ|READ(?: +)COMMITTED|READ(?: +)UNCOMMITTED|SERIALIZABLE))|(?:(READ)(?: +)(WRITE|ONLY)))"); + match_regexes[3] = new Session_Regex((char*)"^(set)(?: +)((charset)|(character +set))(?: )"); + + return true; +} + +struct pollfd* PgSQL_Thread::get_pollfd(unsigned int i) { + return &mypolls.fds[i]; +} + +void PgSQL_Thread::poll_listener_add(int sock) { + PgSQL_Data_Stream* listener_DS = new PgSQL_Data_Stream(); + listener_DS->myds_type = MYDS_LISTENER; + listener_DS->fd = sock; + + proxy_debug(PROXY_DEBUG_NET, 1, "Created listener %p for socket %d\n", listener_DS, sock); + mypolls.add(POLLIN, sock, listener_DS, monotonic_time()); +} + +void PgSQL_Thread::poll_listener_del(int sock) { + int i = mypolls.find_index(sock); + if (i >= 0) { + PgSQL_Data_Stream* myds = mypolls.myds[i]; + mypolls.remove_index_fast(i); +#ifdef SO_REUSEPORT + if (GloVars.global.reuseport) +#else + myds->fd = -1; // this to prevent that delete myds will shutdown the fd; +#endif + delete myds; + } +} + +void PgSQL_Thread::unregister_session(int idx) { + if (mysql_sessions == NULL) return; + proxy_debug(PROXY_DEBUG_NET, 1, "Thread=%p, Session=%p -- Unregistered session\n", this, mysql_sessions->index(idx)); + mysql_sessions->remove_index_fast(idx); +} + + +// this function was inline in PgSQL_Thread::run() +void PgSQL_Thread::run___get_multiple_idle_connections(int& num_idles) { + int i; + num_idles = PgHGM->get_multiple_idle_connections(-1, curtime - pgsql_thread___ping_interval_server_msec * 1000, my_idle_conns, SESSIONS_FOR_CONNECTIONS_HANDLER); + for (i = 0; i < num_idles; i++) { + PgSQL_Data_Stream* myds; + PgSQL_Connection* mc = my_idle_conns[i]; + PgSQL_Session* sess = new PgSQL_Session(); + sess->mybe = sess->find_or_create_backend(mc->parent->myhgc->hid); + + myds = sess->mybe->server_myds; + myds->attach_connection(mc); + myds->assign_fd_from_mysql_conn(); + myds->myds_type = MYDS_BACKEND; + + sess->to_process = 1; + myds->wait_until = curtime + pgsql_thread___ping_timeout_server * 1000; // max_timeout + mc->last_time_used = curtime; + myds->myprot.init(&myds, myds->myconn->userinfo, NULL); + sess->status = PINGING_SERVER; + myds->DSS = STATE_MARIADB_PING; + register_session_connection_handler(sess, true); + int rc = sess->handler(); + if (rc == -1) { + unsigned int sess_idx = mysql_sessions->len - 1; + unregister_session(sess_idx); + delete sess; + } + } + processing_idles = true; + last_processing_idles = curtime; +} + +// this function was inline in PgSQL_Thread::run() +void PgSQL_Thread::run___cleanup_mirror_queue() { + unsigned int l = (unsigned int)pgsql_thread___mirror_max_concurrency; + if (mirror_queue_mysql_sessions_cache->len > l) { + while (mirror_queue_mysql_sessions_cache->len > mirror_queue_mysql_sessions->len && mirror_queue_mysql_sessions_cache->len > l) { + PgSQL_Session* newsess = (PgSQL_Session*)mirror_queue_mysql_sessions_cache->remove_index_fast(0); + __sync_add_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1); + //GloPTH->status_variables.p_gauge_array[p_th_gauge::mirror_concurrency]->Increment(); + delete newsess; + } + } +} + +// main loop +void PgSQL_Thread::run() { + unsigned int n; + int rc; + +#ifdef IDLE_THREADS + bool idle_maintenance_thread = epoll_thread; + if (idle_maintenance_thread) { + // we check if it is the first time we are called + if (efd == -1) { + efd = EPOLL_CREATE; + int fd = pipefd[0]; + struct epoll_event event; + memset(&event, 0, sizeof(event)); // let's make valgrind happy + event.events = EPOLLIN; + event.data.u32 = 0; // special value to point to the pipe + epoll_ctl(efd, EPOLL_CTL_ADD, fd, &event); + } + } +#endif // IDLE_THREADS + + curtime = monotonic_time(); + atomic_curtime = curtime; + + pthread_mutex_lock(&thread_mutex); + while (shutdown == 0) { + +#ifdef IDLE_THREADS + if (idle_maintenance_thread) { + goto __run_skip_1; + } +#endif // IDLE_THREADS + + int num_idles; + if (processing_idles == true && (last_processing_idles < curtime - pgsql_thread___ping_timeout_server * 1000)) { + processing_idles = false; + } + if (processing_idles == false && (last_processing_idles < curtime - pgsql_thread___ping_interval_server_msec * 1000)) { + run___get_multiple_idle_connections(num_idles); + } + +#ifdef IDLE_THREADS + __run_skip_1 : + + if (idle_maintenance_thread) { + idle_thread_gets_sessions_from_worker_thread(); + goto __run_skip_1a; + } +#endif // IDLE_THREADS + + handle_mirror_queue_mysql_sessions(); + + ProcessAllMyDS_BeforePoll(); + +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + if (idle_maintenance_thread == false) { + int r = rand() % (GloPTH->num_threads); + PgSQL_Thread* thr = GloPTH->pgsql_threads_idles[r].worker; + worker_thread_assigns_sessions_to_idle_thread(thr); + worker_thread_gets_sessions_from_idle_thread(); + } + } + + + __run_skip_1a: +#endif // IDLE_THREADS + + pthread_mutex_unlock(&thread_mutex); + if (unlikely(mypolls.bootstrapping_listeners == true)) { + while ( // spin here if ... + (n = __sync_add_and_fetch(&mypolls.pending_listener_add, 0)) // there is a new listener to add + || + (GloPTH->bootstrapping_listeners == true) // PgSQL_Thread_Handlers has more listeners to configure + ) { + if (n) { + poll_listener_add(n); + assert(__sync_bool_compare_and_swap(&mypolls.pending_listener_add, n, 0)); + } + else { + if (GloPTH->bootstrapping_listeners == false) { + // we stop looping + mypolls.bootstrapping_listeners = false; + } + } +#ifdef DEBUG + usleep(5 + rand() % 10); +#endif + } + } + + proxy_debug(PROXY_DEBUG_NET, 7, "poll_timeout=%u\n", mypolls.poll_timeout); + if (pgsql_thread___wait_timeout == 0) { + // we should be going into PAUSE mode + if (mypolls.poll_timeout == 0 || mypolls.poll_timeout > 100000) { + mypolls.poll_timeout = 100000; + } + } + proxy_debug(PROXY_DEBUG_NET, 7, "poll_timeout=%u\n", mypolls.poll_timeout); + + + // flush mysql log file + GloPgSQL_Logger->flush(); + + pre_poll_time = curtime; + int ttw = (mypolls.poll_timeout ? (mypolls.poll_timeout / 1000 < (unsigned int)pgsql_thread___poll_timeout ? mypolls.poll_timeout / 1000 : pgsql_thread___poll_timeout) : pgsql_thread___poll_timeout); +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads && idle_maintenance_thread) { + memset(events, 0, sizeof(struct epoll_event) * MY_EPOLL_THREAD_MAXEVENTS); // let's make valgrind happy. It also seems that needs to be zeroed anyway + // we call epoll() + rc = epoll_wait(efd, events, MY_EPOLL_THREAD_MAXEVENTS, pgsql_thread___poll_timeout); + } + else { +#endif // IDLE_THREADS + //this is the only portion of code not protected by a global mutex + proxy_debug(PROXY_DEBUG_NET, 5, "Calling poll with timeout %d\n", ttw); + // poll is called with a timeout of mypolls.poll_timeout if set , or pgsql_thread___poll_timeout + rc = poll(mypolls.fds, mypolls.len, ttw); + proxy_debug(PROXY_DEBUG_NET, 5, "%s\n", "Returning poll"); +#ifdef IDLE_THREADS + } +#endif // IDLE_THREADS + + if (unlikely(maintenance_loop == true)) { + while ((n = __sync_add_and_fetch(&mypolls.pending_listener_del, 0))) { // spin here + if (static_cast(n) == -1) { + for (unsigned int i = 0; i < mypolls.len; i++) { + if (mypolls.myds[i] && mypolls.myds[i]->myds_type == MYDS_LISTENER) { + poll_listener_del(mypolls.myds[i]->fd); + } + } + } + else { + poll_listener_del(n); + } + assert(__sync_bool_compare_and_swap(&mypolls.pending_listener_del, n, 0)); + } + } + + pthread_mutex_lock(&thread_mutex); + if (shutdown == 1) { return; } + mypolls.poll_timeout = 0; // always reset this to 0 . If a session needs a specific timeout, it will set this one + + curtime = monotonic_time(); + atomic_curtime = curtime; + + poll_timeout_bool = false; + if ( +#ifdef IDLE_THREADS + idle_maintenance_thread == false && +#endif // IDLE_THREADS + (curtime >= (pre_poll_time + ttw))) { + poll_timeout_bool = true; + } + unsigned long long maintenance_interval = 1000000; // hardcoded value for now +#ifdef IDLE_THREADS + if (idle_maintenance_thread) { + maintenance_interval = maintenance_interval * 2; + } +#endif // IDLE_THREADS + if (curtime > last_maintenance_time + maintenance_interval) { + last_maintenance_time = curtime; + maintenance_loop = true; + servers_table_version_previous = servers_table_version_current; + servers_table_version_current = PgHGM->get_servers_table_version(); + } + else { + maintenance_loop = false; + } + + handle_kill_queues(); + + // update polls statistics + mypolls.loops++; + mypolls.loop_counters->incr(curtime / 1000000); + + if (maintenance_loop == true +#ifdef IDLE_THREADS + // in case of idle thread + // do not run any mirror cleanup and do not + // update query processor stats + && idle_maintenance_thread == false +#endif // IDLE_THREADS + ) { + // house keeping + run___cleanup_mirror_queue(); + GloPgQPro->update_query_processor_stats(); + } + + if (rc == -1 && errno == EINTR) + // poll() timeout, try again + continue; + if (rc == -1) { + // LCOV_EXCL_START + // error , exit + perror("poll()"); + exit(EXIT_FAILURE); + // LCOV_EXCL_STOP + } + + if (__sync_add_and_fetch(&__global_PgSQL_Thread_Variables_version, 0) > __thread_PgSQL_Thread_Variables_version) { + refresh_variables(); + } + + run_SetAllSession_ToProcess0(); + +#ifdef IDLE_THREADS + // here we handle epoll_wait() + if (GloVars.global.idle_threads && idle_maintenance_thread) { + if (rc) { + int i; + for (i = 0; i < rc; i++) { + if (events[i].data.u32) { + idle_thread_prepares_session_to_send_to_worker_thread(i); + } + } + // FIXME: this loop seems suboptimal, it can be combined with the previous one + for (i = 0; i < rc; i++) { + if (events[i].events == EPOLLIN && events[i].data.u32 == 0) { + unsigned char c; + int fd = pipefd[0]; + if (read(fd, &c, 1) <= 0) { + } + else { + //i=rc; + maintenance_loop = true; + } + } + } + } + if (mysql_sessions->len && maintenance_loop) { + if (curtime == last_maintenance_time) { + idle_thread_to_kill_idle_sessions(); + } + } + goto __run_skip_2; + } +#endif // IDLE_THREADS + + ProcessAllMyDS_AfterPoll(); + +#ifdef IDLE_THREADS + __run_skip_2 : + if (GloVars.global.idle_threads && idle_maintenance_thread) { + // this is an idle thread + unsigned int w = rand() % (GloPTH->num_threads); + PgSQL_Thread* thr = GloPTH->pgsql_threads[w].worker; + if (resume_mysql_sessions->len) { + idle_thread_assigns_sessions_to_worker_thread(thr); + } + else { + idle_thread_check_if_worker_thread_has_unprocess_resumed_sessions_and_signal_it(thr); + } + } + else { +#endif // IDLE_THREADS + // iterate through all sessions and process the session logic + process_all_sessions(); + + return_local_connections(); +#ifdef IDLE_THREADS + } +#endif // IDLE_THREADS + } +} +// end of ::run() + +#ifdef IDLE_THREADS +void PgSQL_Thread::idle_thread_to_kill_idle_sessions() { +#define SESS_TO_SCAN 128 + if (mysess_idx + SESS_TO_SCAN > mysql_sessions->len) { + mysess_idx = 0; + } + unsigned int i; + unsigned long long min_idle = 0; + if (curtime > (unsigned long long)pgsql_thread___wait_timeout * 1000) { + min_idle = curtime - (unsigned long long)pgsql_thread___wait_timeout * 1000; + } + for (i = 0; i < SESS_TO_SCAN && mysess_idx < mysql_sessions->len; i++) { + uint32_t sess_pos = mysess_idx; + PgSQL_Session* mysess = (PgSQL_Session*)mysql_sessions->index(sess_pos); + if (mysess->idle_since < min_idle || mysess->killed == true) { + mysess->killed = true; + PgSQL_Data_Stream* tmp_myds = mysess->client_myds; + int dsidx = tmp_myds->poll_fds_idx; + //fprintf(stderr,"Removing session %p, DS %p idx %d\n",mysess,tmp_myds,dsidx); + mypolls.remove_index_fast(dsidx); + tmp_myds->mypolls = NULL; + mysess->thread = NULL; + // we first delete the association in sessmap + sessmap.erase(mysess->thread_session_id); + if (mysql_sessions->len > 1) { + // take the last element and adjust the map + PgSQL_Session* mysess_last = (PgSQL_Session*)mysql_sessions->index(mysql_sessions->len - 1); + if (mysess->thread_session_id != mysess_last->thread_session_id) + sessmap[mysess_last->thread_session_id] = sess_pos; + } + unregister_session(sess_pos); + resume_mysql_sessions->add(mysess); + epoll_ctl(efd, EPOLL_CTL_DEL, tmp_myds->fd, NULL); + } + mysess_idx++; + } +} + +void PgSQL_Thread::idle_thread_prepares_session_to_send_to_worker_thread(int i) { + // NOTE: not sure why, sometime events returns odd values. If set, we take it out as normal worker threads know how to handle it + if (events[i].events) { + uint32_t sess_thr_id = events[i].data.u32; + uint32_t sess_pos = sessmap[sess_thr_id]; + PgSQL_Session* mysess = (PgSQL_Session*)mysql_sessions->index(sess_pos); + PgSQL_Data_Stream* tmp_myds = mysess->client_myds; + int dsidx = tmp_myds->poll_fds_idx; + //fprintf(stderr,"Removing session %p, DS %p idx %d\n",mysess,tmp_myds,dsidx); + mypolls.remove_index_fast(dsidx); + tmp_myds->mypolls = NULL; + mysess->thread = NULL; + // we first delete the association in sessmap + sessmap.erase(mysess->thread_session_id); + if (mysql_sessions->len > 1) { + // take the last element and adjust the map + PgSQL_Session* mysess_last = (PgSQL_Session*)mysql_sessions->index(mysql_sessions->len - 1); + if (mysess->thread_session_id != mysess_last->thread_session_id) + sessmap[mysess_last->thread_session_id] = sess_pos; + } + unregister_session(sess_pos); + resume_mysql_sessions->add(mysess); + epoll_ctl(efd, EPOLL_CTL_DEL, tmp_myds->fd, NULL); + } +} + +void PgSQL_Thread::idle_thread_check_if_worker_thread_has_unprocess_resumed_sessions_and_signal_it(PgSQL_Thread * thr) { + pthread_mutex_lock(&thr->myexchange.mutex_resumes); + if (shutdown == 0 && thr->shutdown == 0 && thr->myexchange.resume_mysql_sessions->len) { + unsigned char c = 0; + int fd = thr->pipefd[1]; + if (write(fd, &c, 1) == -1) { + //proxy_error("Error while signaling maintenance thread\n"); + } + } + pthread_mutex_unlock(&thr->myexchange.mutex_resumes); +} + +void PgSQL_Thread::idle_thread_assigns_sessions_to_worker_thread(PgSQL_Thread * thr) { + bool send_signal = false; + // send_signal variable will control if we need to signal or not + // the worker thread + pthread_mutex_lock(&thr->myexchange.mutex_resumes); + if (shutdown == 0 && thr->shutdown == 0) + if (resume_mysql_sessions->len) { + while (resume_mysql_sessions->len) { + PgSQL_Session* mysess = (PgSQL_Session*)resume_mysql_sessions->remove_index_fast(0); + thr->myexchange.resume_mysql_sessions->add(mysess); + } + send_signal = true; // signal only if there are sessions to resume + } + pthread_mutex_unlock(&thr->myexchange.mutex_resumes); + if (send_signal) { // signal only if there are sessions to resume + unsigned char c = 0; + //PgSQL_Thread *thr=GloPTH->pgsql_threads[w].worker; + // we signal the thread to inform there are sessions + int fd = thr->pipefd[1]; + if (write(fd, &c, 1) == -1) { + //proxy_error("Error while signaling maintenance thread\n"); + } + } +} + +void PgSQL_Thread::worker_thread_assigns_sessions_to_idle_thread(PgSQL_Thread * thr) { + if (shutdown == 0 && thr->shutdown == 0 && idle_mysql_sessions->len) { + pthread_mutex_lock(&thr->myexchange.mutex_idles); + bool empty_queue = true; + if (thr->myexchange.idle_mysql_sessions->len) { + // there are already sessions in the queues. We assume someone already notified worker 0 + empty_queue = false; + } + while (idle_mysql_sessions->len) { + PgSQL_Session* mysess = (PgSQL_Session*)idle_mysql_sessions->remove_index_fast(0); + thr->myexchange.idle_mysql_sessions->add(mysess); + } + pthread_mutex_unlock(&thr->myexchange.mutex_idles); + if (empty_queue == true) { + unsigned char c = 1; + int fd = thr->pipefd[1]; + if (write(fd, &c, 1) == -1) { + //proxy_error("Error while signaling maintenance thread\n"); + } + } + } +} + +void PgSQL_Thread::worker_thread_gets_sessions_from_idle_thread() { + pthread_mutex_lock(&myexchange.mutex_resumes); + if (myexchange.resume_mysql_sessions->len) { + //unsigned int maxsess=GloPTH->resume_mysql_sessions->len; + while (myexchange.resume_mysql_sessions->len) { + PgSQL_Session* mysess = (PgSQL_Session*)myexchange.resume_mysql_sessions->remove_index_fast(0); + register_session(this, mysess, false); + PgSQL_Data_Stream* myds = mysess->client_myds; + mypolls.add(POLLIN, myds->fd, myds, monotonic_time()); + } + } + pthread_mutex_unlock(&myexchange.mutex_resumes); +} +#endif // IDLE_THREADS + + +bool PgSQL_Thread::process_data_on_data_stream(PgSQL_Data_Stream * myds, unsigned int n) { + if (mypolls.fds[n].revents) { +#ifdef IDLE_THREADS + if (myds->myds_type == MYDS_FRONTEND) { + if (epoll_thread) { + mypolls.remove_index_fast(n); + myds->mypolls = NULL; + unsigned int i; + for (i = 0; i < mysql_sessions->len; i++) { + PgSQL_Session* mysess = (PgSQL_Session*)mysql_sessions->index(i); + if (mysess == myds->sess) { + mysess->thread = NULL; + unregister_session(i); + //exit_cond=true; + resume_mysql_sessions->add(myds->sess); + return false; + } + } + } + } +#endif // IDLE_THREADS + mypolls.last_recv[n] = curtime; + myds->revents = mypolls.fds[n].revents; + myds->sess->to_process = 1; + assert(myds->sess->status != session_status___NONE); + } + else { + // no events + if (myds->wait_until && curtime > myds->wait_until) { + // timeout + myds->sess->to_process = 1; + assert(myds->sess->status != session_status___NONE); + } + else { + if (myds->sess->pause_until && curtime > myds->sess->pause_until) { + // timeout + myds->sess->to_process = 1; + } + } + } + if (myds->myds_type == MYDS_BACKEND && myds->sess->status != FAST_FORWARD) { + if (mypolls.fds[n].revents) { + // this part of the code fixes an important bug + // if a connection in use but idle (ex: running a transaction) + // get data, immediately destroy the session + // + // this can happen, for example, with a low wait_timeout and running transaction + if (myds->sess->status == WAITING_CLIENT_DATA) { + if (myds->myconn->async_state_machine == ASYNC_IDLE) { + proxy_warning("Detected broken idle connection on %s:%d\n", myds->myconn->parent->address, myds->myconn->parent->port); + myds->destroy_MySQL_Connection_From_Pool(false); + myds->sess->set_unhealthy(); + return false; + } + } + } + return true; + } + if (mypolls.fds[n].revents) { + if (mypolls.myds[n]->DSS < STATE_MARIADB_BEGIN || mypolls.myds[n]->DSS > STATE_MARIADB_END) { + // only if we aren't using MariaDB Client Library + int rb = 0; + do { + rb = myds->read_from_net(); + if (rb > 0 && myds->myds_type == MYDS_FRONTEND) { + status_variables.stvar[st_var_queries_frontends_bytes_recv] += rb; + } + myds->read_pkts(); + + if (rb > 0 && myds->myds_type == MYDS_BACKEND) { + if (myds->sess->session_fast_forward) { + if (myds->encrypted == true) { // we are in fast_forward mode and encrypted == true + // PMC-10004 + // we probably should use SSL_pending() and/or SSL_has_pending() to determine + // if there is more data to be read, but it doesn't seem to be working. + // Therefore we try to call read_from_net() again as long as there is data. + // Previously we hardcoded 16KB but it seems that it can return in smaller + // chunks of 4KB. + // We finally removed the chunk size as it seems that any size is possible. +/* + int sslp = SSL_pending(myds->ssl); + int sslhp = SSL_has_pending(myds->ssl); + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p: in fast_forward mode and SSL read %d bytes , SSL_pending: %d bytes , SSL_has_pending: %d\n", myds->sess, rb, sslp, sslhp); +*/ + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p, DataStream=%p , thread_session_id=%u -- in fast_forward mode and SSL read %d bytes\n", myds->sess, myds, myds->sess->thread_session_id, rb); + while (rb > 0) { + rb = myds->read_from_net(); + if (rb > 0 && myds->myds_type == MYDS_FRONTEND) { + status_variables.stvar[st_var_queries_frontends_bytes_recv] += rb; + } + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p, DataStream=%p -- in fast_forward mode and SSL read %d bytes\n", myds->sess, myds, rb); + myds->read_pkts(); + } + rb = 0; // exit loop + } + else { // we are in fast_forward mode and encrypted == false + struct pollfd _fds; + nfds_t _nfds = 1; + _fds.fd = mypolls.fds[n].fd; + _fds.events = POLLIN; + _fds.revents = 0; + int _rc = poll(&_fds, _nfds, 0); + if ((_rc > 0) && _fds.revents == POLLIN) { + // there is more data + myds->revents = _fds.revents; + } + else { + rb = 0; // exit loop + } + } + } + else { + rb = 0; // exit loop + } + } + else { + bool set_rb_zero = true; + if (rb > 0 && myds->myds_type == MYDS_FRONTEND) { + if (myds->encrypted == true) { + if (SSL_is_init_finished(myds->ssl)) { + if (myds->data_in_rbio()) { + set_rb_zero = false; + } + } + } + } + if (set_rb_zero) + rb = 0; // exit loop + } + } while (rb > 0); + + } + else { + if (mypolls.fds[n].revents) { + myds->myconn->handler(mypolls.fds[n].revents); + } + } + if ((mypolls.fds[n].events & POLLOUT) + && + ((mypolls.fds[n].revents & POLLERR) || (mypolls.fds[n].revents & POLLHUP)) + ) { + myds->set_net_failure(); + } + myds->check_data_flow(); + } + + + if (myds->active == 0) { + if (myds->sess->client_myds == myds) { + proxy_debug(PROXY_DEBUG_NET, 1, "Session=%p, DataStream=%p -- Deleting FD %d\n", myds->sess, myds, myds->fd); + myds->sess->set_unhealthy(); + } + else { + // if this is a backend with fast_forward, set unhealthy + // if this is a backend without fast_forward, do not set unhealthy: it will be handled by client library + if (myds->sess->session_fast_forward) { // if fast forward + if (myds->myds_type == MYDS_BACKEND) { // and backend + myds->sess->set_unhealthy(); // set unhealthy + } + } + } + } + return true; +} + + +// this function was inline in PgSQL_Thread::process_all_sessions() +void PgSQL_Thread::ProcessAllSessions_CompletedMirrorSession(unsigned int& n, PgSQL_Session * sess) { + unregister_session(n); + n--; + unsigned int l = (unsigned int)pgsql_thread___mirror_max_concurrency; + if (mirror_queue_mysql_sessions->len * 0.3 > l) l = mirror_queue_mysql_sessions->len * 0.3; + if (mirror_queue_mysql_sessions_cache->len <= l) { + bool to_cache = true; + if (sess->mybe) { + if (sess->mybe->server_myds) { + to_cache = false; + } + } + if (to_cache) { + __sync_sub_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1); + //GloPTH->status_variables.p_gauge_array[p_th_gauge::mirror_concurrency]->Decrement(); + mirror_queue_mysql_sessions_cache->add(sess); + } + else { + delete sess; + } + } + else { + delete sess; + } +} + + +// this function was inline in PgSQL_Thread::process_all_sessions() +void PgSQL_Thread::ProcessAllSessions_MaintenanceLoop(PgSQL_Session * sess, unsigned long long sess_time, unsigned int& total_active_transactions_) { + unsigned int numTrx = 0; + total_active_transactions_ += sess->active_transactions; + sess->to_process = 1; + if ((sess_time / 1000 > (unsigned long long)pgsql_thread___max_transaction_idle_time) || (sess_time / 1000 > (unsigned long long)pgsql_thread___wait_timeout)) { + //numTrx = sess->NumActiveTransactions(); + numTrx = sess->active_transactions; + if (numTrx) { + // the session has idle transactions, kill it + if (sess_time / 1000 > (unsigned long long)pgsql_thread___max_transaction_idle_time) { + sess->killed = true; + if (sess->client_myds) { + proxy_warning("Killing client connection %s:%d because of (possible) transaction idle for %llums\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, sess_time / 1000); + } + } + } + else { + // the session is idle, kill it + if (sess_time / 1000 > (unsigned long long)pgsql_thread___wait_timeout) { + sess->killed = true; + if (sess->client_myds) { + proxy_warning("Killing client connection %s:%d because inactive for %llums\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, sess_time / 1000); + } + } + } + } + else { + if (sess->active_transactions > 0) { + // here is all the logic related to max_transaction_time + unsigned long long trx_started = sess->transaction_started_at; + if (trx_started > 0 && curtime > trx_started) { + unsigned long long trx_time = curtime - trx_started; + unsigned long long trx_time_ms = trx_time / 1000; + if (trx_time_ms > (unsigned long long)pgsql_thread___max_transaction_time) { + sess->killed = true; + if (sess->client_myds) { + proxy_warning("Killing client connection %s:%d because of (possible) transaction running for %llums\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, trx_time_ms); + } + } + } + } + } + if (servers_table_version_current != servers_table_version_previous) { // bug fix for #1085 + // Immediatelly kill all client connections using an OFFLINE node when session_fast_forward == true + if (sess->session_fast_forward) { + if (sess->HasOfflineBackends()) { + sess->killed = true; + proxy_warning("Killing client connection %s:%d due to 'session_fast_forward' and offline backends\n", sess->client_myds->addr.addr, sess->client_myds->addr.port); + } + } + else { + // Search for connections that should be terminated, and simulate data in them + // the following 2 lines of code replace the previous 2 lines + // instead of killing the sessions, fails the backend connections + if (sess->SetEventInOfflineBackends()) { + sess->to_process = 1; + } + } + } + + // Perform the maintenance for expired connections on the session + if (pgsql_thread___multiplexing) { + const auto auto_incr_delay_multiplex_check = [curtime = this->curtime](PgSQL_Connection* myconn) -> bool { + const uint64_t multiplex_timeout_ms = pgsql_thread___auto_increment_delay_multiplex_timeout_ms; + const bool multiplex_delayed_enabled = multiplex_timeout_ms != 0 && myconn->auto_increment_delay_token > 0; + const bool timeout_expired = multiplex_delayed_enabled && myconn->myds->wait_until != 0 && myconn->myds->wait_until < curtime; + return timeout_expired; + }; + + const auto conn_delay_multiplex = [curtime = this->curtime](PgSQL_Connection* myconn) -> bool { + const bool multiplex_delayed = pgsql_thread___connection_delay_multiplex_ms != 0 && myconn->multiplex_delayed == true; + const bool timeout_expired = multiplex_delayed && myconn->myds->wait_until != 0 && myconn->myds->wait_until < curtime; + return timeout_expired; + }; + + const vector> expire_conn_checks{ + auto_incr_delay_multiplex_check, + conn_delay_multiplex + }; + + sess->update_expired_conns(expire_conn_checks); + } +} + +void PgSQL_Thread::process_all_sessions() { + unsigned int n; + unsigned int total_active_transactions_ = 0; +#ifdef IDLE_THREADS + bool idle_maintenance_thread = epoll_thread; +#endif // IDLE_THREADS + int rc; + bool sess_sort = pgsql_thread___sessions_sort; +#ifdef IDLE_THREADS + if (idle_maintenance_thread) { + sess_sort = false; + } +#endif // IDLE_THREADS + if (sess_sort && mysql_sessions->len > 3) { + ProcessAllSessions_SortingSessions(); + } + for (n = 0; n < mysql_sessions->len; n++) { + PgSQL_Session* sess = (PgSQL_Session*)mysql_sessions->index(n); +#ifdef DEBUG + if (sess == sess_stopat) { + sess_stopat = sess; + } +#endif + if (sess->mirror == true) { // this is a mirror session + if (sess->status == WAITING_CLIENT_DATA) { // the mirror session has completed + ProcessAllSessions_CompletedMirrorSession(n, sess); + continue; + } + } + if (sess->status == CONNECTING_CLIENT) { + unsigned long long sess_time = sess->IdleTime(); + if (sess_time / 1000 > (unsigned long long)pgsql_thread___connect_timeout_client) { + proxy_warning("Closing not established client connection %s:%d after %llums\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, sess_time / 1000); + sess->healthy = 0; + if (pgsql_thread___client_host_cache_size) { + GloPTH->update_client_host_cache(sess->client_myds->client_addr, true); + } + } + } + if (maintenance_loop) { + unsigned long long sess_time = sess->IdleTime(); +#ifdef IDLE_THREADS + if (idle_maintenance_thread == false) +#endif // IDLE_THREADS + { + ProcessAllSessions_MaintenanceLoop(sess, sess_time, total_active_transactions_); + } +#ifdef IDLE_THREADS + else + { + if ((sess_time / 1000 > (unsigned long long)pgsql_thread___wait_timeout)) { + sess->killed = true; + sess->to_process = 1; + proxy_warning("Killing client connection %s:%d because inactive for %llums\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, sess_time / 1000); + } + } +#endif // IDLE_THREADS + } + else { + // NOTE: we used the special value -1 to inform PgSQL_Session::handler() to recompute it + // removing this logic in 2.0.15 + //sess->active_transactions = -1; + } + if (sess->healthy == 0) { + char _buf[1024]; + if (sess->client_myds) { + if (pgsql_thread___log_unhealthy_connections) { + if (sess->session_fast_forward == false) { + proxy_warning( + "Closing unhealthy client connection %s:%d\n", sess->client_myds->addr.addr, + sess->client_myds->addr.port + ); + } + else { + proxy_warning( + "Closing 'fast_forward' client connection %s:%d\n", sess->client_myds->addr.addr, + sess->client_myds->addr.port + ); + } + } + } + sprintf(_buf, "%s:%d:%s()", __FILE__, __LINE__, __func__); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_CLOSE, sess, NULL, _buf); + unregister_session(n); + n--; + delete sess; + } + else { + if (sess->to_process == 1) { + if (sess->pause_until <= curtime) { + rc = sess->handler(); + //total_active_transactions_+=sess->active_transactions; + if (rc == -1 || sess->killed == true) { + char _buf[1024]; + if (sess->client_myds && sess->killed) + proxy_warning("Closing killed client connection %s:%d\n", sess->client_myds->addr.addr, sess->client_myds->addr.port); + sprintf(_buf, "%s:%d:%s()", __FILE__, __LINE__, __func__); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_CLOSE, sess, NULL, _buf); + unregister_session(n); + n--; + delete sess; + } + } + } + else { + if (sess->killed == true) { + // this is a special cause, if killed the session needs to be executed no matter if paused + sess->handler(); + char _buf[1024]; + if (sess->client_myds) + proxy_warning("Closing killed client connection %s:%d\n", sess->client_myds->addr.addr, sess->client_myds->addr.port); + sprintf(_buf, "%s:%d:%s()", __FILE__, __LINE__, __func__); + GloPgSQL_Logger->log_audit_entry(PROXYSQL_MYSQL_AUTH_CLOSE, sess, NULL, _buf); + unregister_session(n); + n--; + delete sess; + } + } + } + } + if (maintenance_loop) { + unsigned int total_active_transactions_tmp; + total_active_transactions_tmp = __sync_add_and_fetch(&status_variables.active_transactions, 0); + __sync_bool_compare_and_swap(&status_variables.active_transactions, total_active_transactions_tmp, total_active_transactions_); + } +} + +void PgSQL_Thread::refresh_variables() { + pthread_mutex_lock(&GloVars.global.ext_glomth_mutex); + if (GloPTH == NULL) { + return; + } + GloPTH->wrlock(); + __thread_PgSQL_Thread_Variables_version = __global_PgSQL_Thread_Variables_version; + pgsql_thread___authentication_method = GloPTH->get_variable_int((char*)"authentication_method"); + pgsql_thread___show_processlist_extended = GloPTH->get_variable_int((char*)"show_processlist_extended"); + pgsql_thread___max_connections = GloPTH->get_variable_int((char*)"max_connections"); + pgsql_thread___use_tcp_keepalive = (bool)GloPTH->get_variable_int((char*)"use_tcp_keepalive"); + pgsql_thread___tcp_keepalive_time = GloPTH->get_variable_int((char*)"tcp_keepalive_time"); + pgsql_thread___throttle_connections_per_sec_to_hostgroup = GloPTH->get_variable_int((char*)"throttle_connections_per_sec_to_hostgroup"); + pgsql_thread___max_transaction_idle_time = GloPTH->get_variable_int((char*)"max_transaction_idle_time"); + pgsql_thread___max_transaction_time = GloPTH->get_variable_int((char*)"max_transaction_time"); + pgsql_thread___threshold_query_length = GloPTH->get_variable_int((char*)"threshold_query_length"); + pgsql_thread___threshold_resultset_size = GloPTH->get_variable_int((char*)"threshold_resultset_size"); + pgsql_thread___poll_timeout = GloPTH->get_variable_int((char*)"poll_timeout"); + pgsql_thread___poll_timeout_on_failure = GloPTH->get_variable_int((char*)"poll_timeout_on_failure"); + pgsql_thread___wait_timeout = GloPTH->get_variable_int((char*)"wait_timeout"); + pgsql_thread___client_host_cache_size = GloPTH->get_variable_int((char*)"client_host_cache_size"); + pgsql_thread___client_host_error_counts = GloPTH->get_variable_int((char*)"client_host_error_counts"); + pgsql_thread___connect_retries_on_failure = GloPTH->get_variable_int((char*)"connect_retries_on_failure"); + pgsql_thread___connect_retries_delay = GloPTH->get_variable_int((char*)"connect_retries_delay"); + pgsql_thread___multiplexing = (bool)GloPTH->get_variable_int((char*)"multiplexing"); + pgsql_thread___connection_delay_multiplex_ms = GloPTH->get_variable_int((char*)"connection_delay_multiplex_ms"); + pgsql_thread___connection_max_age_ms = GloPTH->get_variable_int((char*)"connection_max_age_ms"); + pgsql_thread___connect_timeout_client = GloPTH->get_variable_int((char*)"connect_timeout_client"); + pgsql_thread___connect_timeout_server = GloPTH->get_variable_int((char*)"connect_timeout_server"); + pgsql_thread___connect_timeout_server_max = GloPTH->get_variable_int((char*)"connect_timeout_server_max"); + pgsql_thread___connection_warming = (bool)GloPTH->get_variable_int((char*)"connection_warming"); + pgsql_thread___log_unhealthy_connections = (bool)GloPTH->get_variable_int((char*)"log_unhealthy_connections"); + pgsql_thread___throttle_max_bytes_per_second_to_client = GloPTH->get_variable_int((char*)"throttle_max_bytes_per_second_to_client"); + pgsql_thread___throttle_ratio_server_to_client = GloPTH->get_variable_int((char*)"throttle_ratio_server_to_client"); + pgsql_thread___shun_on_failures = GloPTH->get_variable_int((char*)"shun_on_failures"); + pgsql_thread___shun_recovery_time_sec = GloPTH->get_variable_int((char*)"shun_recovery_time_sec"); + pgsql_thread___hostgroup_manager_verbose = GloPTH->get_variable_int((char*)"hostgroup_manager_verbose"); + pgsql_thread___default_max_latency_ms = GloPTH->get_variable_int((char*)"default_max_latency_ms"); + pgsql_thread___unshun_algorithm = GloPTH->get_variable_int((char*)"unshun_algorithm"); + pgsql_thread___free_connections_pct = GloPTH->get_variable_int((char*)"free_connections_pct"); + pgsql_thread___kill_backend_connection_when_disconnect = (bool)GloPTH->get_variable_int((char*)"kill_backend_connection_when_disconnect"); + pgsql_thread___max_allowed_packet = GloPTH->get_variable_int((char*)"max_allowed_packet"); + pgsql_thread___set_query_lock_on_hostgroup = GloPTH->get_variable_int((char*)"set_query_lock_on_hostgroup"); + pgsql_thread___verbose_query_error = (bool)GloPTH->get_variable_int((char*)"verbose_query_error"); +#ifdef IDLE_THREADS + pgsql_thread___session_idle_ms = GloPTH->get_variable_int((char*)"session_idle_ms"); +#endif // IDLE_THREADS + pgsql_thread___long_query_time = GloPTH->get_variable_int((char*)"long_query_time"); + pgsql_thread___set_parser_algorithm = GloPTH->get_variable_int((char*)"set_parser_algorithm"); + pgsql_thread___parse_failure_logs_digest = (bool)GloPTH->get_variable_int((char*)"parse_failure_logs_digest"); + pgsql_thread___auto_increment_delay_multiplex = GloPTH->get_variable_int((char*)"auto_increment_delay_multiplex"); + pgsql_thread___auto_increment_delay_multiplex_timeout_ms = GloPTH->get_variable_int((char*)"auto_increment_delay_multiplex_timeout_ms"); + pgsql_thread___default_query_delay = GloPTH->get_variable_int((char*)"default_query_delay"); + pgsql_thread___default_query_timeout = GloPTH->get_variable_int((char*)"default_query_timeout"); + pgsql_thread___query_retries_on_failure = GloPTH->get_variable_int((char*)"query_retries_on_failure"); + pgsql_thread___ping_interval_server_msec = GloPTH->get_variable_int((char*)"ping_interval_server_msec"); + pgsql_thread___ping_timeout_server = GloPTH->get_variable_int((char*)"ping_timeout_server"); + pgsql_thread___mirror_max_concurrency = GloPTH->get_variable_int((char*)"mirror_max_concurrency"); + pgsql_thread___mirror_max_queue_length = GloPTH->get_variable_int((char*)"mirror_max_queue_length"); + pgsql_thread___sessions_sort = (bool)GloPTH->get_variable_int((char*)"sessions_sort"); + pgsql_thread___show_processlist_extended = GloPTH->get_variable_int((char*)"show_processlist_extended"); + pgsql_thread___servers_stats = (bool)GloPTH->get_variable_int((char*)"servers_stats"); + pgsql_thread___default_reconnect = (bool)GloPTH->get_variable_int((char*)"default_reconnect"); + + pgsql_thread___automatic_detect_sqli = (bool)GloPTH->get_variable_int((char*)"automatic_detect_sqli"); + + pgsql_thread___firewall_whitelist_enabled = (bool)GloPTH->get_variable_int((char*)"firewall_whitelist_enabled"); + pgsql_thread___query_digests_max_digest_length = GloPTH->get_variable_int((char*)"query_digests_max_digest_length"); + pgsql_thread___query_digests_max_query_length = GloPTH->get_variable_int((char*)"query_digests_max_query_length"); + pgsql_thread___query_processor_iterations = GloPTH->get_variable_int((char*)"query_processor_iterations"); + pgsql_thread___query_processor_regex = GloPTH->get_variable_int((char*)"query_processor_regex"); + + mysql_thread___query_cache_size_MB = GloPTH->get_variable_int((char*)"query_cache_size_MB"); + mysql_thread___query_cache_soft_ttl_pct = GloPTH->get_variable_int((char*)"query_cache_soft_ttl_pct"); + mysql_thread___query_cache_handle_warnings = GloPTH->get_variable_int((char*)"query_cache_handle_warnings"); + /* + mysql_thread___max_stmts_per_connection = GloPTH->get_variable_int((char*)"max_stmts_per_connection"); + mysql_thread___max_stmts_cache = GloPTH->get_variable_int((char*)"max_stmts_cache"); + + */ + if (mysql_thread___monitor_username) free(mysql_thread___monitor_username); + mysql_thread___monitor_username = GloPTH->get_variable_string((char*)"monitor_username"); + if (mysql_thread___monitor_password) free(mysql_thread___monitor_password); + mysql_thread___monitor_password = GloPTH->get_variable_string((char*)"monitor_password"); + /*if (mysql_thread___monitor_replication_lag_use_percona_heartbeat) free(mysql_thread___monitor_replication_lag_use_percona_heartbeat); + mysql_thread___monitor_replication_lag_use_percona_heartbeat = GloPTH->get_variable_string((char*)"monitor_replication_lag_use_percona_heartbeat"); + + mysql_thread___monitor_wait_timeout = (bool)GloPTH->get_variable_int((char*)"monitor_wait_timeout"); + mysql_thread___monitor_writer_is_also_reader = (bool)GloPTH->get_variable_int((char*)"monitor_writer_is_also_reader"); + mysql_thread___monitor_enabled = (bool)GloPTH->get_variable_int((char*)"monitor_enabled"); + mysql_thread___monitor_history = GloPTH->get_variable_int((char*)"monitor_history"); + mysql_thread___monitor_connect_interval = GloPTH->get_variable_int((char*)"monitor_connect_interval"); + mysql_thread___monitor_connect_timeout = GloPTH->get_variable_int((char*)"monitor_connect_timeout"); + mysql_thread___monitor_ping_interval = GloPTH->get_variable_int((char*)"monitor_ping_interval"); + mysql_thread___monitor_ping_max_failures = GloPTH->get_variable_int((char*)"monitor_ping_max_failures"); + mysql_thread___monitor_ping_timeout = GloPTH->get_variable_int((char*)"monitor_ping_timeout"); + mysql_thread___monitor_aws_rds_topology_discovery_interval = GloPTH->get_variable_int((char *)"monitor_aws_rds_topology_discovery_interval"); + mysql_thread___monitor_read_only_interval = GloPTH->get_variable_int((char*)"monitor_read_only_interval"); + mysql_thread___monitor_read_only_timeout = GloPTH->get_variable_int((char*)"monitor_read_only_timeout"); + mysql_thread___monitor_read_only_max_timeout_count = GloPTH->get_variable_int((char*)"monitor_read_only_max_timeout_count"); + mysql_thread___monitor_replication_lag_group_by_host = (bool)GloPTH->get_variable_int((char*)"monitor_replication_lag_group_by_host"); + mysql_thread___monitor_replication_lag_interval = GloPTH->get_variable_int((char*)"monitor_replication_lag_interval"); + mysql_thread___monitor_replication_lag_timeout = GloPTH->get_variable_int((char*)"monitor_replication_lag_timeout"); + mysql_thread___monitor_replication_lag_count = GloPTH->get_variable_int((char*)"monitor_replication_lag_count"); + mysql_thread___monitor_groupreplication_healthcheck_interval = GloPTH->get_variable_int((char*)"monitor_groupreplication_healthcheck_interval"); + mysql_thread___monitor_groupreplication_healthcheck_timeout = GloPTH->get_variable_int((char*)"monitor_groupreplication_healthcheck_timeout"); + mysql_thread___monitor_groupreplication_healthcheck_max_timeout_count = GloPTH->get_variable_int((char*)"monitor_groupreplication_healthcheck_max_timeout_count"); + mysql_thread___monitor_groupreplication_max_transactions_behind_count = GloPTH->get_variable_int((char*)"monitor_groupreplication_max_transactions_behind_count"); + mysql_thread___monitor_groupreplication_max_transaction_behind_for_read_only = GloPTH->get_variable_int((char*)"monitor_groupreplication_max_transactions_behind_for_read_only"); + mysql_thread___monitor_galera_healthcheck_interval = GloPTH->get_variable_int((char*)"monitor_galera_healthcheck_interval"); + mysql_thread___monitor_galera_healthcheck_timeout = GloPTH->get_variable_int((char*)"monitor_galera_healthcheck_timeout"); + mysql_thread___monitor_galera_healthcheck_max_timeout_count = GloPTH->get_variable_int((char*)"monitor_galera_healthcheck_max_timeout_count"); + mysql_thread___monitor_query_interval = GloPTH->get_variable_int((char*)"monitor_query_interval"); + mysql_thread___monitor_query_timeout = GloPTH->get_variable_int((char*)"monitor_query_timeout"); + mysql_thread___monitor_slave_lag_when_null = GloPTH->get_variable_int((char*)"monitor_slave_lag_when_null"); + mysql_thread___monitor_threads_min = GloPTH->get_variable_int((char*)"monitor_threads_min"); + mysql_thread___monitor_threads_max = GloPTH->get_variable_int((char*)"monitor_threads_max"); + mysql_thread___monitor_threads_queue_maxsize = GloPTH->get_variable_int((char*)"monitor_threads_queue_maxsize"); + mysql_thread___monitor_local_dns_cache_ttl = GloPTH->get_variable_int((char*)"monitor_local_dns_cache_ttl"); + mysql_thread___monitor_local_dns_cache_refresh_interval = GloPTH->get_variable_int((char*)"monitor_local_dns_cache_refresh_interval"); + mysql_thread___monitor_local_dns_resolver_queue_maxsize = GloPTH->get_variable_int((char*)"monitor_local_dns_resolver_queue_maxsize"); + */ + if (pgsql_thread___firewall_whitelist_errormsg) free(pgsql_thread___firewall_whitelist_errormsg); + pgsql_thread___firewall_whitelist_errormsg = GloPTH->get_variable_string((char*)"firewall_whitelist_errormsg"); + /* + if (mysql_thread___ldap_user_variable) free(mysql_thread___ldap_user_variable); + mysql_thread___ldap_user_variable = GloPTH->get_variable_string((char*)"ldap_user_variable"); + if (mysql_thread___add_ldap_user_comment) free(mysql_thread___add_ldap_user_comment); + mysql_thread___add_ldap_user_comment = GloPTH->get_variable_string((char*)"add_ldap_user_comment"); + if (mysql_thread___default_session_track_gtids) free(mysql_thread___default_session_track_gtids); + mysql_thread___default_session_track_gtids = GloPTH->get_variable_string((char*)"default_session_track_gtids"); + + for (int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + if (mysql_thread___default_variables[i]) { + free(mysql_thread___default_variables[i]); + mysql_thread___default_variables[i] = NULL; + } + char buf[128]; + if (mysql_tracked_variables[i].is_global_variable) { + sprintf(buf, "default_%s", mysql_tracked_variables[i].internal_variable_name); + mysql_thread___default_variables[i] = GloPTH->get_variable_string(buf); + } + } +*/ + if (pgsql_thread___init_connect) free(pgsql_thread___init_connect); + pgsql_thread___init_connect = GloPTH->get_variable_string((char*)"init_connect"); + + // SSL proxy to server + if (pgsql_thread___ssl_p2s_ca) free(pgsql_thread___ssl_p2s_ca); + pgsql_thread___ssl_p2s_ca = GloPTH->get_variable_string((char*)"ssl_p2s_ca"); + if (pgsql_thread___ssl_p2s_capath) free(pgsql_thread___ssl_p2s_capath); + pgsql_thread___ssl_p2s_capath = GloPTH->get_variable_string((char*)"ssl_p2s_capath"); + if (pgsql_thread___ssl_p2s_cert) free(pgsql_thread___ssl_p2s_cert); + pgsql_thread___ssl_p2s_cert = GloPTH->get_variable_string((char*)"ssl_p2s_cert"); + if (pgsql_thread___ssl_p2s_key) free(pgsql_thread___ssl_p2s_key); + pgsql_thread___ssl_p2s_key = GloPTH->get_variable_string((char*)"ssl_p2s_key"); + if (pgsql_thread___ssl_p2s_cipher) free(pgsql_thread___ssl_p2s_cipher); + pgsql_thread___ssl_p2s_cipher = GloPTH->get_variable_string((char*)"ssl_p2s_cipher"); + if (pgsql_thread___ssl_p2s_crl) free(pgsql_thread___ssl_p2s_crl); + pgsql_thread___ssl_p2s_crl = GloPTH->get_variable_string((char*)"ssl_p2s_crl"); + if (pgsql_thread___ssl_p2s_crlpath) free(pgsql_thread___ssl_p2s_crlpath); + pgsql_thread___ssl_p2s_crlpath = GloPTH->get_variable_string((char*)"ssl_p2s_crlpath"); + + if (pgsql_thread___server_version) free(pgsql_thread___server_version); + pgsql_thread___server_version = GloPTH->get_variable_string((char*)"server_version"); + if (pgsql_thread___default_client_encoding) free(pgsql_thread___default_client_encoding); + pgsql_thread___default_client_encoding = GloPTH->get_variable_string((char*)"default_client_encoding"); + + pgsql_thread___have_ssl = (bool)GloPTH->get_variable_int((char*)"have_ssl"); + + if (pgsql_thread___eventslog_filename) free(pgsql_thread___eventslog_filename); + pgsql_thread___eventslog_filesize = GloPTH->get_variable_int((char*)"eventslog_filesize"); + pgsql_thread___eventslog_default_log = GloPTH->get_variable_int((char*)"eventslog_default_log"); + pgsql_thread___eventslog_format = GloPTH->get_variable_int((char*)"eventslog_format"); + pgsql_thread___eventslog_filename = GloPTH->get_variable_string((char*)"eventslog_filename"); + if (pgsql_thread___auditlog_filename) free(pgsql_thread___auditlog_filename); + pgsql_thread___auditlog_filesize = GloPTH->get_variable_int((char*)"auditlog_filesize"); + pgsql_thread___auditlog_filename = GloPTH->get_variable_string((char*)"auditlog_filename"); + + GloPgSQL_Logger->events_set_base_filename(); // both filename and filesize are set here + GloPgSQL_Logger->audit_set_base_filename(); // both filename and filesize are set here + + //if (pgsql_thread___default_schema) free(pgsql_thread___default_schema); + //pgsql_thread___default_schema = GloPTH->get_variable_string((char*)"default_schema"); + if (pgsql_thread___keep_multiplexing_variables) free(pgsql_thread___keep_multiplexing_variables); + pgsql_thread___keep_multiplexing_variables = GloPTH->get_variable_string((char*)"keep_multiplexing_variables"); + + /* + mysql_thread___server_capabilities = GloPTH->get_variable_uint16((char*)"server_capabilities"); + mysql_thread___handle_unknown_charset = GloPTH->get_variable_int((char*)"handle_unknown_charset"); + mysql_thread___have_compress = (bool)GloPTH->get_variable_int((char*)"have_compress"); + + mysql_thread___enforce_autocommit_on_reads = (bool)GloPTH->get_variable_int((char*)"enforce_autocommit_on_reads"); + mysql_thread___autocommit_false_not_reusable = (bool)GloPTH->get_variable_int((char*)"autocommit_false_not_reusable"); + mysql_thread___autocommit_false_is_transaction = (bool)GloPTH->get_variable_int((char*)"autocommit_false_is_transaction"); + */ + pgsql_thread___commands_stats = (bool)GloPTH->get_variable_int((char*)"commands_stats"); + pgsql_thread___query_digests = (bool)GloPTH->get_variable_int((char*)"query_digests"); + pgsql_thread___query_digests_lowercase = (bool)GloPTH->get_variable_int((char*)"query_digests_lowercase"); + pgsql_thread___query_digests_replace_null = (bool)GloPTH->get_variable_int((char*)"query_digests_replace_null"); + pgsql_thread___query_digests_no_digits = (bool)GloPTH->get_variable_int((char*)"query_digests_no_digits"); + pgsql_thread___query_digests_normalize_digest_text = (bool)GloPTH->get_variable_int((char*)"query_digests_normalize_digest_text"); + pgsql_thread___query_digests_track_hostname = (bool)GloPTH->get_variable_int((char*)"query_digests_track_hostname"); + pgsql_thread___query_digests_grouping_limit = (int)GloPTH->get_variable_int((char*)"query_digests_grouping_limit"); + pgsql_thread___query_digests_groups_grouping_limit = (int)GloPTH->get_variable_int((char*)"query_digests_groups_grouping_limit"); + pgsql_thread___query_digests_keep_comment = (bool)GloPTH->get_variable_int((char*)"query_digests_keep_comment"); + /* + variables.min_num_servers_lantency_awareness = GloPTH->get_variable_int((char*)"min_num_servers_lantency_awareness"); + variables.aurora_max_lag_ms_only_read_from_replicas = GloPTH->get_variable_int((char*)"aurora_max_lag_ms_only_read_from_replicas"); + variables.stats_time_backend_query = (bool)GloPTH->get_variable_int((char*)"stats_time_backend_query"); + variables.stats_time_query_processor = (bool)GloPTH->get_variable_int((char*)"stats_time_query_processor"); + variables.query_cache_stores_empty_result = (bool)GloPTH->get_variable_int((char*)"query_cache_stores_empty_result"); + + mysql_thread___client_session_track_gtid = (bool)GloPTH->get_variable_int((char*)"client_session_track_gtid"); + +#ifdef IDLE_THREADS + mysql_thread___session_idle_show_processlist = (bool)GloPTH->get_variable_int((char*)"session_idle_show_processlist"); +#endif // IDLE_THREADS + + mysql_thread___enable_client_deprecate_eof = (bool)GloPTH->get_variable_int((char*)"enable_client_deprecate_eof"); + mysql_thread___enable_server_deprecate_eof = (bool)GloPTH->get_variable_int((char*)"enable_server_deprecate_eof"); + */ + pgsql_thread___enable_load_data_local_infile = (bool)GloPTH->get_variable_int((char*)"enable_load_data_local_infile"); + /*mysql_thread___log_mysql_warnings_enabled = (bool)GloPTH->get_variable_int((char*)"log_mysql_warnings_enabled"); + mysql_thread___client_host_cache_size = GloPTH->get_variable_int((char*)"client_host_cache_size"); + mysql_thread___client_host_error_counts = GloPTH->get_variable_int((char*)"client_host_error_counts"); + mysql_thread___handle_warnings = GloPTH->get_variable_int((char*)"handle_warnings"); +#ifdef DEBUG + mysql_thread___session_debug = (bool)GloPTH->get_variable_int((char*)"session_debug"); +#endif // DEBUG +*/ + GloPTH->wrunlock(); + pthread_mutex_unlock(&GloVars.global.ext_glomth_mutex); +} + +PgSQL_Thread::PgSQL_Thread() { + pthread_mutex_init(&thread_mutex, NULL); + my_idle_conns = NULL; + cached_connections = NULL; + mysql_sessions = NULL; + mirror_queue_mysql_sessions = NULL; + mirror_queue_mysql_sessions_cache = NULL; +#ifdef IDLE_THREADS + efd = -1; + epoll_thread = false; + mysess_idx = 0; + idle_mysql_sessions = NULL; + resume_mysql_sessions = NULL; + myexchange.idle_mysql_sessions = NULL; + myexchange.resume_mysql_sessions = NULL; +#endif // IDLE_THREADS + processing_idles = false; + last_processing_idles = 0; + __thread_PgSQL_Thread_Variables_version = 0; + pgsql_thread___server_version = NULL; + pgsql_thread___default_client_encoding = NULL; + pgsql_thread___have_ssl = true; + //pgsql_thread___default_schema = NULL; + pgsql_thread___init_connect = NULL; + //pgsql_thread___ldap_user_variable = NULL; + //pgsql_thread___add_ldap_user_comment = NULL; + pgsql_thread___eventslog_filename = NULL; + pgsql_thread___auditlog_filename = NULL; + + // SSL proxy to server + pgsql_thread___ssl_p2s_ca = NULL; + pgsql_thread___ssl_p2s_capath = NULL; + pgsql_thread___ssl_p2s_cert = NULL; + pgsql_thread___ssl_p2s_key = NULL; + pgsql_thread___ssl_p2s_cipher = NULL; + pgsql_thread___ssl_p2s_crl = NULL; + pgsql_thread___ssl_p2s_crlpath = NULL; + + last_maintenance_time = 0; + last_move_to_idle_thread_time = 0; + maintenance_loop = true; + + servers_table_version_previous = 0; + servers_table_version_current = 0; + + status_variables.active_transactions = 0; + + for (unsigned int i = 0; i < PG_st_var_END; i++) { + status_variables.stvar[i] = 0; + } + match_regexes = NULL; + + variables.min_num_servers_lantency_awareness = 1000; + variables.aurora_max_lag_ms_only_read_from_replicas = 2; + variables.stats_time_backend_query = false; + variables.stats_time_query_processor = false; + variables.query_cache_stores_empty_result = true; + + for (int i = 0; i < SQL_NAME_LAST_LOW_WM; i++) { + mysql_thread___default_variables[i] = NULL; + } + shutdown = 0; + thr_SetParser = NULL; +} + +void PgSQL_Thread::register_session_connection_handler(PgSQL_Session * _sess, bool _new) { + _sess->thread = this; + _sess->connections_handler = true; + assert(_new); + mysql_sessions->add(_sess); +} + +void PgSQL_Thread::unregister_session_connection_handler(int idx, bool _new) { + assert(_new); + mysql_sessions->remove_index_fast(idx); +} + +void PgSQL_Thread::listener_handle_new_connection(PgSQL_Data_Stream * myds, unsigned int n) { + int c; + union { + struct sockaddr_in in; + struct sockaddr_in6 in6; + } custom_sockaddr; + struct sockaddr* addr = (struct sockaddr*)malloc(sizeof(custom_sockaddr)); + socklen_t addrlen = sizeof(custom_sockaddr); + memset(addr, 0, sizeof(custom_sockaddr)); + if (GloPTH->num_threads > 1) { + // there are more than 1 thread . We pause for a little bit to avoid all connections to be handled by the same thread +#ifdef SO_REUSEPORT + if (GloVars.global.reuseport == false) { // only if reuseport is not enabled + //usleep(10+rand()%50); + } +#else + //usleep(10+rand()%50); +#endif /* SO_REUSEPORT */ + } + c = accept(myds->fd, addr, &addrlen); + if (c > -1) { // accept() succeeded + if (pgsql_thread___client_host_cache_size) { + PgSQL_Client_Host_Cache_Entry client_host_entry = + GloPTH->find_client_host_cache(addr); + if ( + client_host_entry.updated_at != 0 && + client_host_entry.error_count >= static_cast(pgsql_thread___client_host_error_counts) + ) { + std::string client_addr = get_client_addr(addr); + proxy_error( + "Closing connection because client '%s' reached 'pgsql-client_host_error_counts': %d\n", + client_addr.c_str(), pgsql_thread___client_host_error_counts + ); + close(c); + free(addr); + status_variables.stvar[st_var_client_host_error_killed_connections] += 1; + return; + } + } + + // create a new client connection + mypolls.fds[n].revents = 0; + PgSQL_Session* sess = create_new_session_and_client_data_stream(c); + __sync_add_and_fetch(&PgHGM->status.client_connections_created, 1); + if (__sync_add_and_fetch(&PgHGM->status.client_connections, 1) > pgsql_thread___max_connections) { + sess->max_connections_reached = true; + } + sess->client_myds->client_addrlen = addrlen; + sess->client_myds->client_addr = addr; + + switch (sess->client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)sess->client_myds->client_addr; + char buf[INET_ADDRSTRLEN]; + inet_ntop(sess->client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + sess->client_myds->addr.addr = strdup(buf); + sess->client_myds->addr.port = htons(ipv4->sin_port); + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)sess->client_myds->client_addr; + char buf[INET6_ADDRSTRLEN]; + inet_ntop(sess->client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + sess->client_myds->addr.addr = strdup(buf); + sess->client_myds->addr.port = htons(ipv6->sin6_port); + break; + } + default: + sess->client_myds->addr.addr = strdup("localhost"); + break; + } + + iface_info* ifi = NULL; + ifi = GloPTH->MLM_find_iface_from_fd(myds->fd); // here we try to get the info about the proxy bind address + if (ifi) { + sess->client_myds->proxy_addr.addr = strdup(ifi->address); + sess->client_myds->proxy_addr.port = ifi->port; + } + //sess->client_myds->myprot.generate_pkt_initial_handshake(true, NULL, NULL, &sess->thread_session_id, true); + + sess->client_myds->DSS = STATE_SERVER_HANDSHAKE; + sess->status = CONNECTING_CLIENT; + + ioctl_FIONBIO(sess->client_myds->fd, 1); + mypolls.add(POLLIN | POLLOUT, sess->client_myds->fd, sess->client_myds, curtime); + proxy_debug(PROXY_DEBUG_NET, 1, "Session=%p -- Adding client FD %d\n", sess, sess->client_myds->fd); + + // we now enforce sending the 'initial handshake packet' as soon as it's generated. This + // is done to prevent situations in which a client sends a packet *before* receiving + // this 'initial handshake', leading to invalid state in dataflow, since it will be + // data in both ends of the datastream. For more details see #3342. + //sess->writeout(); + } + else { + free(addr); + // if we arrive here, accept() failed + // because multiple threads try to handle the same incoming connection, this is OK + } +} + +SQLite3_result* PgSQL_Threads_Handler::SQL3_GlobalStatus(bool _memory) { + const int colnum = 2; + char buf[256]; + char** pta = (char**)malloc(sizeof(char*) * colnum); + if (_memory == true) { + Get_Memory_Stats(); + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping PgSQL Global Status\n"); + SQLite3_result* result = new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT, "Variable_Name"); + result->add_column_definition(SQLITE_TEXT, "Variable_Value"); + // NOTE: as there is no string copy, we do NOT free pta[0] and pta[1] + { // uptime + unsigned long long t1 = monotonic_time(); + pta[0] = (char*)"ProxySQL_Uptime"; + sprintf(buf, "%llu", (t1 - GloVars.global.start_time) / 1000 / 1000); + pta[1] = buf; + result->add_row(pta); + } + { // Active Transactions + pta[0] = (char*)"Active_Transactions"; + sprintf(buf, "%u", get_active_transations()); + pta[1] = buf; + result->add_row(pta); + } + { // Connections created + pta[0] = (char*)"Client_Connections_aborted"; + sprintf(buf, "%lu", PgHGM->status.client_connections_aborted); + pta[1] = buf; + result->add_row(pta); + } + { // Connections + pta[0] = (char*)"Client_Connections_connected"; + sprintf(buf, "%d", PgHGM->status.client_connections); + pta[1] = buf; + result->add_row(pta); + } + { // Connections created + pta[0] = (char*)"Client_Connections_created"; + sprintf(buf, "%lu", PgHGM->status.client_connections_created); + pta[1] = buf; + result->add_row(pta); + } + { + // Connections + pta[0] = (char*)"Server_Connections_aborted"; + sprintf(buf, "%lu", PgHGM->status.server_connections_aborted); + pta[1] = buf; + result->add_row(pta); + } + { + // Connections + pta[0] = (char*)"Server_Connections_connected"; + sprintf(buf, "%lu", PgHGM->status.server_connections_connected); + pta[1] = buf; + result->add_row(pta); + } + { + // Connections + pta[0] = (char*)"Server_Connections_created"; + sprintf(buf, "%lu", PgHGM->status.server_connections_created); + pta[1] = buf; + result->add_row(pta); + } + { + // Connections delayed + pta[0] = (char*)"Server_Connections_delayed"; + sprintf(buf, "%lu", PgHGM->status.server_connections_delayed); + pta[1] = buf; + result->add_row(pta); + } +#ifdef IDLE_THREADS + { // Connections non idle + pta[0] = (char*)"Client_Connections_non_idle"; + sprintf(buf, "%u", get_non_idle_client_connections()); + pta[1] = buf; + result->add_row(pta); + } +#endif // IDLE_THREADS + { // PgSQL Backend buffers bytes + pta[0] = (char*)"pgsql_backend_buffers_bytes"; + sprintf(buf, "%llu", get_pgsql_backend_buffers_bytes()); + pta[1] = buf; + result->add_row(pta); + } + { // PgSQL Frontend buffers bytes + pta[0] = (char*)"pgsql_frontend_buffers_bytes"; + sprintf(buf, "%llu", get_pgsql_frontend_buffers_bytes()); + pta[1] = buf; + result->add_row(pta); + } + { // PgSQL Frontend buffers bytes + pta[0] = (char*)"pgsql_session_internal_bytes"; + sprintf(buf, "%llu", get_pgsql_session_internal_bytes()); + pta[1] = buf; + result->add_row(pta); + } + /*{ // Queries autocommit + pta[0] = (char*)"Com_autocommit"; + sprintf(buf, "%llu", PgHGM->status.autocommit_cnt); + pta[1] = buf; + result->add_row(pta); + } + { // Queries filtered autocommit + pta[0] = (char*)"Com_autocommit_filtered"; + sprintf(buf, "%llu", PgHGM->status.autocommit_cnt_filtered); + pta[1] = buf; + result->add_row(pta); + }*/ + { // Queries commit + pta[0] = (char*)"Commit"; + sprintf(buf, "%llu", PgHGM->status.commit_cnt); + pta[1] = buf; + result->add_row(pta); + } + { // Queries filtered commit + pta[0] = (char*)"Commit_filtered"; + sprintf(buf, "%llu", PgHGM->status.commit_cnt_filtered); + pta[1] = buf; + result->add_row(pta); + } + { // Queries rollback + pta[0] = (char*)"Rollback"; + sprintf(buf, "%llu", PgHGM->status.rollback_cnt); + pta[1] = buf; + result->add_row(pta); + } + { // Queries filtered rollback + pta[0] = (char*)"Rollback_filtered"; + sprintf(buf, "%llu", PgHGM->status.rollback_cnt_filtered); + pta[1] = buf; + result->add_row(pta); + } + { // Queries backend RESET_CONNECTION + pta[0] = (char*)"Backend_reset_connection"; + sprintf(buf, "%llu", PgHGM->status.backend_reset_connection); + pta[1] = buf; + result->add_row(pta); + } + /* { // Queries backend INIT DB + pta[0] = (char*)"Com_backend_init_db"; + sprintf(buf, "%llu", PgHGM->status.backend_init_db); + pta[1] = buf; + result->add_row(pta); + }*/ + { // Queries backend SET client_encoding + pta[0] = (char*)"Backend_set_client_encoding"; + sprintf(buf, "%llu", PgHGM->status.backend_set_client_encoding); + pta[1] = buf; + result->add_row(pta); + } + /* { // Queries frontend INIT DB + pta[0] = (char*)"Com_frontend_init_db"; + sprintf(buf, "%llu", PgHGM->status.frontend_init_db); + pta[1] = buf; + result->add_row(pta); + }*/ + { // Queries frontend SET client_encoding + pta[0] = (char*)"Frontend_set_client_encoding"; + sprintf(buf, "%llu", PgHGM->status.frontend_set_client_encoding); + pta[1] = buf; + result->add_row(pta); + } + /* { // Queries frontend USE DB + pta[0] = (char*)"Com_frontend_use_db"; + sprintf(buf, "%llu", PgHGM->status.frontend_use_db); + pta[1] = buf; + result->add_row(pta); + }*/ +/* + for (unsigned int i = 0; i < sizeof(PgSQL_Thread_status_variables_counter_array) / sizeof(mythr_st_vars_t); i++) { + if (PgSQL_Thread_status_variables_counter_array[i].name) { + if (strlen(PgSQL_Thread_status_variables_counter_array[i].name)) { + pta[0] = PgSQL_Thread_status_variables_counter_array[i].name; + unsigned long long stvar = + get_status_variable( + PgSQL_Thread_status_variables_counter_array[i].v_idx, + PgSQL_Thread_status_variables_counter_array[i].m_idx, + PgSQL_Thread_status_variables_counter_array[i].conv + ); + sprintf(buf, "%llu", stvar); + pta[1] = buf; + result->add_row(pta); + } + } + } + // Gauge variables + for (unsigned int i = 0; i < sizeof(PgSQL_Thread_status_variables_gauge_array) / sizeof(mythr_g_st_vars_t); i++) { + if (PgSQL_Thread_status_variables_gauge_array[i].name) { + if (strlen(PgSQL_Thread_status_variables_gauge_array[i].name)) { + pta[0] = PgSQL_Thread_status_variables_gauge_array[i].name; + unsigned long long stvar = + get_status_variable( + PgSQL_Thread_status_variables_gauge_array[i].v_idx, + PgSQL_Thread_status_variables_gauge_array[i].m_idx, + PgSQL_Thread_status_variables_gauge_array[i].conv + ); + sprintf(buf, "%llu", stvar); + pta[1] = buf; + result->add_row(pta); + } + } + } +*/ + { // Mirror current concurrency + pta[0] = (char*)"Mirror_concurrency"; + sprintf(buf, "%u", status_variables.mirror_sessions_current); + pta[1] = buf; + result->add_row(pta); + } + { // Mirror queue length + pta[0] = (char*)"Mirror_queue_length"; + sprintf(buf, "%llu", get_total_mirror_queue()); + pta[1] = buf; + result->add_row(pta); + } + { // Queries that are SELECT for update or equivalent + pta[0] = (char*)"Selects_for_update__autocommit0"; + sprintf(buf, "%llu", PgHGM->status.select_for_update_or_equivalent); + pta[1] = buf; + result->add_row(pta); + } + { // Servers_table_version + pta[0] = (char*)"Servers_table_version"; + sprintf(buf, "%u", PgHGM->get_servers_table_version()); + pta[1] = buf; + result->add_row(pta); + } + { // MySQL Threads workers + pta[0] = (char*)"PgSQL_Thread_Workers"; + sprintf(buf, "%d", num_threads); + pta[1] = buf; + result->add_row(pta); + } + { // Access_Denied_Wrong_Password + pta[0] = (char*)"Access_Denied_Wrong_Password"; + sprintf(buf, "%llu", PgHGM->status.access_denied_wrong_password); + pta[1] = buf; + result->add_row(pta); + } + { // Access_Denied_Max_Connections + pta[0] = (char*)"Access_Denied_Max_Connections"; + sprintf(buf, "%llu", PgHGM->status.access_denied_max_connections); + pta[1] = buf; + result->add_row(pta); + } + { // Access_Denied_Max_User_Connections + pta[0] = (char*)"Access_Denied_Max_User_Connections"; + sprintf(buf, "%llu", PgHGM->status.access_denied_max_user_connections); + pta[1] = buf; + result->add_row(pta); + } + /*if (GloMyMon) { + { // MySQL Monitor workers + pta[0] = (char*)"MySQL_Monitor_Workers"; + sprintf(buf, "%d", (variables.monitor_enabled ? GloMyMon->num_threads : 0)); + pta[1] = buf; + result->add_row(pta); + } + { // MySQL Monitor workers + pta[0] = (char*)"MySQL_Monitor_Workers_Aux"; + sprintf(buf, "%d", (variables.monitor_enabled ? GloMyMon->aux_threads : 0)); + pta[1] = buf; + result->add_row(pta); + } + { // MySQL Monitor workers + pta[0] = (char*)"MySQL_Monitor_Workers_Started"; + sprintf(buf, "%d", (variables.monitor_enabled ? GloMyMon->started_threads : 0)); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_connect_check_OK"; + sprintf(buf, "%llu", GloMyMon->connect_check_OK); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_connect_check_ERR"; + sprintf(buf, "%llu", GloMyMon->connect_check_ERR); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_ping_check_OK"; + sprintf(buf, "%llu", GloMyMon->ping_check_OK); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_ping_check_ERR"; + sprintf(buf, "%llu", GloMyMon->ping_check_ERR); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_read_only_check_OK"; + sprintf(buf, "%llu", GloMyMon->read_only_check_OK); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_read_only_check_ERR"; + sprintf(buf, "%llu", GloMyMon->read_only_check_ERR); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_replication_lag_check_OK"; + sprintf(buf, "%llu", GloMyMon->replication_lag_check_OK); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_replication_lag_check_ERR"; + sprintf(buf, "%llu", GloMyMon->replication_lag_check_ERR); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_dns_cache_queried"; + sprintf(buf, "%llu", GloMyMon->dns_cache_queried); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_dns_cache_lookup_success"; + sprintf(buf, "%llu", GloMyMon->dns_cache_lookup_success); + pta[1] = buf; + result->add_row(pta); + } + { + pta[0] = (char*)"MySQL_Monitor_dns_cache_record_updated"; + sprintf(buf, "%llu", GloMyMon->dns_cache_record_updated); + pta[1] = buf; + result->add_row(pta); + } + }*/ + free(pta); + return result; +} + + +void PgSQL_Threads_Handler::Get_Memory_Stats() { + unsigned int i; + unsigned int j; + j = num_threads; +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + j += num_threads; + } +#endif // IDLE_THREADS + for (i = 0; i < j; i++) { + PgSQL_Thread* thr = NULL; + if (i < num_threads && pgsql_threads) { + thr = (PgSQL_Thread*)pgsql_threads[i].worker; +#ifdef IDLE_THREADS + } + else { + if (GloVars.global.idle_threads && pgsql_threads_idles) { + thr = (PgSQL_Thread*)pgsql_threads_idles[i - num_threads].worker; + } +#endif // IDLE_THREADS + } + if (thr == NULL) return; // quick exit, at least one thread is not ready + pthread_mutex_lock(&thr->thread_mutex); + thr->Get_Memory_Stats(); + pthread_mutex_unlock(&thr->thread_mutex); + } +} + +SQLite3_result* PgSQL_Threads_Handler::SQL3_Processlist() { + const int colnum = 16; + char port[NI_MAXSERV]; + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping PgSQL Processlist\n"); + SQLite3_result* result = new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT, "ThreadID"); + result->add_column_definition(SQLITE_TEXT, "SessionID"); + result->add_column_definition(SQLITE_TEXT, "user"); + result->add_column_definition(SQLITE_TEXT, "database"); + result->add_column_definition(SQLITE_TEXT, "cli_host"); + result->add_column_definition(SQLITE_TEXT, "cli_port"); + result->add_column_definition(SQLITE_TEXT, "hostgroup"); + result->add_column_definition(SQLITE_TEXT, "l_srv_host"); + result->add_column_definition(SQLITE_TEXT, "l_srv_port"); + result->add_column_definition(SQLITE_TEXT, "srv_host"); + result->add_column_definition(SQLITE_TEXT, "srv_port"); + result->add_column_definition(SQLITE_TEXT, "command"); + result->add_column_definition(SQLITE_TEXT, "time_ms"); + result->add_column_definition(SQLITE_TEXT, "info"); + result->add_column_definition(SQLITE_TEXT, "status_flags"); + result->add_column_definition(SQLITE_TEXT, "extended_info"); + unsigned int i; + unsigned int i2; + // signal_all_threads(1); + i2 = num_threads; +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + i2 += num_threads; + } +#endif // IDLE_THREADS + + for (i = 0; i < i2; i++) { + PgSQL_Thread* thr = NULL; + if (i < num_threads && pgsql_threads) { + thr = (PgSQL_Thread*)pgsql_threads[i].worker; +#ifdef IDLE_THREADS + } + else { + if (GloVars.global.idle_threads && mysql_thread___session_idle_show_processlist && pgsql_threads_idles) { + thr = (PgSQL_Thread*)pgsql_threads_idles[i - num_threads].worker; + } +#endif // IDLE_THREADS + } + if (thr == NULL) break; // quick exit, at least one thread is not ready + pthread_mutex_lock(&thr->thread_mutex); + unsigned int j; + for (j = 0; j < thr->mysql_sessions->len; j++) { + PgSQL_Session* sess = (PgSQL_Session*)thr->mysql_sessions->pdata[j]; + if (sess->client_myds) { + char buf[1024]; + char** pta = (char**)malloc(sizeof(char*) * colnum); + sprintf(buf, "%d", i); + pta[0] = strdup(buf); + sprintf(buf, "%u", sess->thread_session_id); + pta[1] = strdup(buf); + PgSQL_Connection_userinfo* ui = sess->client_myds->myconn->userinfo; + pta[2] = NULL; + pta[3] = NULL; + if (ui) { + if (ui->username) { + pta[2] = strdup(ui->username); + } + else { + pta[2] = strdup("unauthenticated user"); + } + if (ui->dbname) { + pta[3] = strdup(ui->dbname); + } + } + + if (sess->mirror == false) { + switch (sess->client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)sess->client_myds->client_addr; + inet_ntop(sess->client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + pta[4] = strdup(buf); + sprintf(port, "%d", ntohs(ipv4->sin_port)); + pta[5] = strdup(port); + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)sess->client_myds->client_addr; + inet_ntop(sess->client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + pta[4] = strdup(buf); + sprintf(port, "%d", ntohs(ipv6->sin6_port)); + pta[5] = strdup(port); + break; + } + default: + pta[4] = strdup("localhost"); + pta[5] = NULL; + break; + } + } + else { + pta[4] = strdup("mirror_internal"); + pta[5] = NULL; + } + sprintf(buf, "%d", sess->current_hostgroup); + pta[6] = strdup(buf); + if (sess->mybe && sess->mybe->server_myds && sess->mybe->server_myds->myconn) { + PgSQL_Connection* mc = sess->mybe->server_myds->myconn; + + + struct sockaddr addr; + socklen_t addr_len = sizeof(struct sockaddr); + memset(&addr, 0, addr_len); + int rc; + rc = getsockname(mc->fd, &addr, &addr_len); + if (rc == 0) { + switch (addr.sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)&addr; + inet_ntop(addr.sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + pta[7] = strdup(buf); + sprintf(port, "%d", ntohs(ipv4->sin_port)); + pta[8] = strdup(port); + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)&addr; + inet_ntop(addr.sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + pta[7] = strdup(buf); + sprintf(port, "%d", ntohs(ipv6->sin6_port)); + pta[8] = strdup(port); + break; + } + default: + pta[7] = strdup("localhost"); + pta[8] = NULL; + break; + } + } + else { + pta[7] = NULL; + pta[8] = NULL; + } + + sprintf(buf, "%s", mc->parent->address); + pta[9] = strdup(buf); + sprintf(buf, "%d", mc->parent->port); + pta[10] = strdup(buf); + if (sess->CurrentQuery.stmt_info == NULL) { // text protocol + if (mc->query.length) { + pta[13] = (char*)malloc(mc->query.length + 1); + strncpy(pta[13], mc->query.ptr, mc->query.length); + pta[13][mc->query.length] = '\0'; + } + else { + pta[13] = NULL; + } + } + else { // prepared statement + MySQL_STMT_Global_info* si = sess->CurrentQuery.stmt_info; + if (si->query_length) { + pta[13] = (char*)malloc(si->query_length + 1); + strncpy(pta[13], si->query, si->query_length); + pta[13][si->query_length] = '\0'; + } + else { + pta[13] = NULL; + } + } + sprintf(buf, "%d", mc->status_flags); + pta[14] = strdup(buf); + } + else { + pta[7] = NULL; + pta[8] = NULL; + pta[9] = NULL; + pta[10] = NULL; + pta[13] = NULL; + pta[14] = NULL; + } + switch (sess->status) { + case CONNECTING_SERVER: + pta[11] = strdup("Connect"); + break; + case PROCESSING_QUERY: + if (sess->pause_until > sess->thread->curtime) { + pta[11] = strdup("Delay"); + } + else { + pta[11] = strdup("Query"); + } + break; + case WAITING_CLIENT_DATA: + pta[11] = strdup("Sleep"); + break; + case CHANGING_USER_SERVER: + pta[11] = strdup("Changing user server"); + break; + case CHANGING_USER_CLIENT: + pta[11] = strdup("Change user client"); + break; + case RESETTING_CONNECTION: + pta[11] = strdup("Resetting connection"); + break; + case RESETTING_CONNECTION_V2: + pta[11] = strdup("Resetting connection V2"); + break; + //case CHANGING_SCHEMA: + // pta[11] = strdup("InitDB"); + // break; + case PROCESSING_STMT_EXECUTE: + pta[11] = strdup("Execute"); + break; + case PROCESSING_STMT_PREPARE: + pta[11] = strdup("Prepare"); + break; + case CONNECTING_CLIENT: + pta[11] = strdup("Connecting client"); + break; + case PINGING_SERVER: + pta[11] = strdup("Pinging server"); + break; + case WAITING_SERVER_DATA: + pta[11] = strdup("Waiting server data"); + break; + //case CHANGING_CHARSET: + // pta[11] = strdup("Changing charset"); + // break; + //case CHANGING_AUTOCOMMIT: + // pta[11] = strdup("Changing autocommit"); + // break; + case SETTING_INIT_CONNECT: + pta[11] = strdup("Setting init connect"); + break; + /* + case SETTING_SQL_LOG_BIN: + pta[11]=strdup("Set log bin"); + break; + case SETTING_SQL_MODE: + pta[11]=strdup("Set SQL mode"); + break; + case SETTING_TIME_ZONE: + pta[11]=strdup("Set TZ"); + break; + */ + case SETTING_VARIABLE: + { + int idx = sess->changing_variable_idx; + if (idx < SQL_NAME_LAST_HIGH_WM) { + char buf[128]; + sprintf(buf, "Setting variable %s", mysql_tracked_variables[idx].set_variable_name); + pta[11] = strdup(buf); + } + else { + pta[11] = strdup("Setting variable"); + } + } + break; + case FAST_FORWARD: + pta[11] = strdup("Fast forward"); + break; + case session_status___NONE: + pta[11] = strdup("None"); + break; + default: + sprintf(buf, "%d", sess->status); + pta[11] = strdup(buf); + break; + } + if (sess->mirror == false) { + int idx = sess->client_myds->poll_fds_idx; + unsigned long long last_sent = sess->thread->mypolls.last_sent[idx]; + unsigned long long last_recv = sess->thread->mypolls.last_recv[idx]; + unsigned long long last_time = (last_sent > last_recv ? last_sent : last_recv); + if (last_time > sess->thread->curtime) { + last_time = sess->thread->curtime; + } + sprintf(buf, "%llu", (sess->thread->curtime - last_time) / 1000); + } + else { + // for mirror session we only consider the start time + sprintf(buf, "%llu", (sess->thread->curtime - sess->start_time) / 1000); + } + pta[12] = strdup(buf); + + pta[15] = NULL; + if (pgsql_thread___show_processlist_extended) { + json j; + sess->generate_proxysql_internal_session_json(j); + if (pgsql_thread___show_processlist_extended == 2) { + std::string s = j.dump(4, ' ', false, json::error_handler_t::replace); + pta[15] = strdup(s.c_str()); + } + else { + std::string s = j.dump(-1, ' ', false, json::error_handler_t::replace); + pta[15] = strdup(s.c_str()); + } + } + result->add_row(pta); + unsigned int k; + for (k = 0; k < colnum; k++) { + if (pta[k]) + free(pta[k]); + } + free(pta); + } + } + pthread_mutex_unlock(&thr->thread_mutex); + } + return result; +} + +void PgSQL_Threads_Handler::signal_all_threads(unsigned char _c) { + unsigned int i; + unsigned char c = _c; + if (pgsql_threads == 0) return; + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr == NULL) return; // quick exit, at least one thread is not ready + int fd = thr->pipefd[1]; + if (write(fd, &c, 1) == -1) { + proxy_error("Error during write in signal_all_threads()\n"); + } + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads_idles[i].worker; + if (thr == NULL) return; // quick exit, at least one thread is not ready + int fd = thr->pipefd[1]; + if (write(fd, &c, 1) == -1) { + proxy_error("Error during write in signal_all_threads()\n"); + } + } +#endif // IDLE_THREADS +} + +void PgSQL_Threads_Handler::kill_connection_or_query(uint32_t _thread_session_id, bool query, char* username) { + unsigned int i; + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + thr_id_usr* tu = (thr_id_usr*)malloc(sizeof(thr_id_usr)); + tu->id = _thread_session_id; + tu->username = strdup(username); + pthread_mutex_lock(&thr->kq.m); + if (query) { + thr->kq.query_ids.push_back(tu); + } + else { + thr->kq.conn_ids.push_back(tu); + } + pthread_mutex_unlock(&thr->kq.m); + + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads_idles[i].worker; + thr_id_usr* tu = (thr_id_usr*)malloc(sizeof(thr_id_usr)); + tu->id = _thread_session_id; + tu->username = strdup(username); + pthread_mutex_lock(&thr->kq.m); + if (query) { + thr->kq.query_ids.push_back(tu); + } + else { + thr->kq.conn_ids.push_back(tu); + } + pthread_mutex_unlock(&thr->kq.m); + } + } +#endif + signal_all_threads(0); +} + +bool PgSQL_Threads_Handler::kill_session(uint32_t _thread_session_id) { + bool ret = false; + unsigned int i; + signal_all_threads(1); + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + pthread_mutex_lock(&thr->thread_mutex); + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads_idles[i].worker; + pthread_mutex_lock(&thr->thread_mutex); + } +#endif // IDLE_THREADS + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + unsigned int j; + for (j = 0; j < thr->mysql_sessions->len; j++) { + PgSQL_Session* sess = (PgSQL_Session*)thr->mysql_sessions->pdata[j]; + if (sess->thread_session_id == _thread_session_id) { + sess->killed = true; + ret = true; + goto __exit_kill_session; + } + } + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads_idles[i].worker; + unsigned int j; + for (j = 0; j < thr->mysql_sessions->len; j++) { + PgSQL_Session* sess = (PgSQL_Session*)thr->mysql_sessions->pdata[j]; + if (sess->thread_session_id == _thread_session_id) { + sess->killed = true; + ret = true; + goto __exit_kill_session; + } + } + } +#endif // IDLE_THREADS +__exit_kill_session: + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + pthread_mutex_unlock(&thr->thread_mutex); + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) + for (i = 0; i < num_threads; i++) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads_idles[i].worker; + pthread_mutex_unlock(&thr->thread_mutex); + } +#endif // IDLE_THREADS + return ret; +} + +unsigned long long PgSQL_Threads_Handler::get_total_mirror_queue() { + if ((__sync_fetch_and_add(&status_variables.threads_initialized, 0) == 0) || this->shutdown_) return 0; + unsigned long long q = 0; + unsigned int i; + for (i = 0; i < num_threads; i++) { + if (pgsql_threads) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr) + q += thr->mirror_queue_mysql_sessions->len; // this is a dirty read + } + } + //this->status_variables.p_gauge_array[p_th_gauge::mirror_queue_lengths]->Set(q); + + return q; +} + + +/* +unsigned long long PgSQL_Threads_Handler::get_status_variable( + enum PgSQL_Thread_status_variable v_idx, + p_th_counter::metric m_idx, + unsigned long long conv +) { + if ((__sync_fetch_and_add(&status_variables.threads_initialized, 0) == 0) || this->shutdown_) return 0; + unsigned long long q = 0; + unsigned int i; + for (i = 0; i < num_threads; i++) { + if (pgsql_threads) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->status_variables.stvar[v_idx], 0); + } + } + if (m_idx != p_th_counter::__size) { + const auto& cur_val = status_variables.p_counter_array[m_idx]->Value(); + double final_val = 0; + + if (conv != 0) { + final_val = (q - (cur_val * conv)) / conv; + } + else { + final_val = q - cur_val; + } + + status_variables.p_counter_array[m_idx]->Increment(final_val); + } + return q; +} +*/ + +/* +unsigned long long PgSQL_Threads_Handler::get_status_variable( + enum PgSQL_Thread_status_variable v_idx, + p_th_gauge::metric m_idx, + unsigned long long conv +) { + if ((__sync_fetch_and_add(&status_variables.threads_initialized, 0) == 0) || this->shutdown_) return 0; + unsigned long long q = 0; + unsigned int i; + for (i = 0; i < num_threads; i++) { + if (pgsql_threads) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->status_variables.stvar[v_idx], 0); + } + } + if (m_idx != p_th_gauge::__size) { + double final_val = 0; + + if (conv != 0) { + final_val = q / static_cast(conv); + } + else { + final_val = q; + } + + status_variables.p_gauge_array[m_idx]->Set(final_val); + } + return q; + +} +*/ + +unsigned int PgSQL_Threads_Handler::get_active_transations() { + if ((__sync_fetch_and_add(&status_variables.threads_initialized, 0) == 0) || this->shutdown_) return 0; + unsigned long long q = 0; + unsigned int i; + for (i = 0; i < num_threads; i++) { + if (pgsql_threads) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->status_variables.active_transactions, 0); + } + } + //this->status_variables.p_gauge_array[p_th_gauge::active_transactions]->Set(q); + + return q; +} + +#ifdef IDLE_THREADS +unsigned int PgSQL_Threads_Handler::get_non_idle_client_connections() { + if ((__sync_fetch_and_add(&status_variables.threads_initialized, 0) == 0) || this->shutdown_) return 0; + unsigned long long q = 0; + unsigned int i; + for (i = 0; i < num_threads; i++) { + if (pgsql_threads) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->mysql_sessions->len, 0); + } + } + //this->status_variables.p_gauge_array[p_th_gauge::client_connections_non_idle]->Set(q); + + return q; +} +#endif // IDLE_THREADS + +unsigned long long PgSQL_Threads_Handler::get_pgsql_backend_buffers_bytes() { + if ((__sync_fetch_and_add(&status_variables.threads_initialized, 0) == 0) || this->shutdown_) return 0; + unsigned long long q = 0; + unsigned int i; + for (i = 0; i < num_threads; i++) { + if (pgsql_threads) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->status_variables.stvar[st_var_mysql_backend_buffers_bytes], 0); + } + } + //const auto& cur_val = this->status_variables.p_counter_array[p_th_gauge::mysql_backend_buffers_bytes]->Value(); + //this->status_variables.p_counter_array[p_th_gauge::mysql_backend_buffers_bytes]->Increment(q - cur_val); + + return q; +} + +unsigned long long PgSQL_Threads_Handler::get_pgsql_frontend_buffers_bytes() { + if ((__sync_fetch_and_add(&status_variables.threads_initialized, 0) == 0) || this->shutdown_) return 0; + unsigned long long q = 0; + unsigned int i; + for (i = 0; i < num_threads; i++) { + if (pgsql_threads) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->status_variables.stvar[st_var_mysql_frontend_buffers_bytes], 0); + } + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) + for (i = 0; i < num_threads; i++) { + if (pgsql_threads_idles) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads_idles[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->status_variables.stvar[st_var_mysql_frontend_buffers_bytes], 0); + } + } +#endif // IDLE_THREADS + //this->status_variables.p_counter_array[p_th_gauge::mysql_frontend_buffers_bytes]->Increment(q); + + return q; +} + +unsigned long long PgSQL_Threads_Handler::get_pgsql_session_internal_bytes() { + if ((__sync_fetch_and_add(&status_variables.threads_initialized, 0) == 0) || this->shutdown_) return 0; + unsigned long long q = 0; + unsigned int i; + for (i = 0; i < num_threads; i++) { + if (pgsql_threads) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->status_variables.stvar[st_var_mysql_session_internal_bytes], 0); + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) + if (pgsql_threads_idles) { + PgSQL_Thread* thr = (PgSQL_Thread*)pgsql_threads_idles[i].worker; + if (thr) + q += __sync_fetch_and_add(&thr->status_variables.stvar[st_var_mysql_session_internal_bytes], 0); + } +#endif // IDLE_THREADS + } + //this->status_variables.p_gauge_array[p_th_gauge::mysql_session_internal_bytes]->Set(q); + + return q; +} + +void PgSQL_Threads_Handler::p_update_metrics() { + get_total_mirror_queue(); + get_active_transations(); +#ifdef IDLE_THREADS + get_non_idle_client_connections(); +#endif // IDLE_THREADS + get_pgsql_backend_buffers_bytes(); + get_pgsql_frontend_buffers_bytes(); + get_pgsql_session_internal_bytes(); +/* + for (unsigned int i = 0; i < sizeof(PgSQL_Thread_status_variables_counter_array) / sizeof(mythr_st_vars_t); i++) { + if (PgSQL_Thread_status_variables_counter_array[i].name) { + get_status_variable( + PgSQL_Thread_status_variables_counter_array[i].v_idx, + PgSQL_Thread_status_variables_counter_array[i].m_idx, + PgSQL_Thread_status_variables_counter_array[i].conv + ); + } + } + // Gauge variables + for (unsigned int i = 0; i < sizeof(PgSQL_Thread_status_variables_gauge_array) / sizeof(mythr_g_st_vars_t); i++) { + if (PgSQL_Thread_status_variables_gauge_array[i].name) { + get_status_variable( + PgSQL_Thread_status_variables_gauge_array[i].v_idx, + PgSQL_Thread_status_variables_gauge_array[i].m_idx, + PgSQL_Thread_status_variables_gauge_array[i].conv + ); + } + } +*/ +/* + this->status_variables.p_gauge_array[p_th_gauge::mysql_wait_timeout]->Set(this->variables.wait_timeout); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_ping_interval]->Set(this->variables.monitor_ping_interval / 1000.0); + this->status_variables.p_gauge_array[p_th_gauge::mysql_max_connections]->Set(this->variables.max_connections); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_enabled]->Set(this->variables.monitor_enabled); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_ping_timeout]->Set(this->variables.monitor_ping_timeout / 1000.0); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_ping_max_failures]->Set(this->variables.monitor_ping_max_failures); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_aws_rds_topology_discovery_interval]->Set(this->variables.monitor_aws_rds_topology_discovery_interval); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_read_only_interval]->Set(this->variables.monitor_read_only_interval/1000.0); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_read_only_timeout]->Set(this->variables.monitor_read_only_timeout/1000.0); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_writer_is_also_reader]->Set(this->variables.monitor_writer_is_also_reader); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_replication_lag_group_by_host]->Set(this->variables.monitor_replication_lag_group_by_host); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_replication_lag_interval]->Set(this->variables.monitor_replication_lag_interval / 1000.0); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_replication_lag_timeout]->Set(this->variables.monitor_replication_lag_timeout / 1000.0); + this->status_variables.p_gauge_array[p_th_gauge::mysql_monitor_history]->Set(this->variables.monitor_history / 1000.0); +*/ +} + +void PgSQL_Thread::Get_Memory_Stats() { + unsigned int i; + status_variables.stvar[st_var_mysql_backend_buffers_bytes] = 0; + status_variables.stvar[st_var_mysql_frontend_buffers_bytes] = 0; + status_variables.stvar[st_var_mysql_session_internal_bytes] = sizeof(PgSQL_Thread); + if (mysql_sessions) { + status_variables.stvar[st_var_mysql_session_internal_bytes] += (mysql_sessions->size) * sizeof(PgSQL_Session*); + if (epoll_thread == false) { + for (i = 0; i < mysql_sessions->len; i++) { + PgSQL_Session* sess = (PgSQL_Session*)mysql_sessions->index(i); + sess->Memory_Stats(); + } + } + else { + status_variables.stvar[st_var_mysql_frontend_buffers_bytes] += (mysql_sessions->len * QUEUE_T_DEFAULT_SIZE * 2); + status_variables.stvar[st_var_mysql_session_internal_bytes] += (mysql_sessions->len * sizeof(PgSQL_Connection)); +#if !defined(__FreeBSD__) && !defined(__APPLE__) + status_variables.stvar[st_var_mysql_session_internal_bytes] += ((sizeof(int) + sizeof(int) + sizeof(std::_Rb_tree_node_base)) * mysql_sessions->len); +#else + status_variables.stvar[st_var_mysql_session_internal_bytes] += ((sizeof(int) + sizeof(int) + 32) * mysql_sessions->len); +#endif + } + } +} + + +PgSQL_Connection* PgSQL_Thread::get_MyConn_local(unsigned int _hid, PgSQL_Session * sess, char* gtid_uuid, uint64_t gtid_trxid, int max_lag_ms) { + // some sanity check + if (sess == NULL) return NULL; + if (sess->client_myds == NULL) return NULL; + if (sess->client_myds->myconn == NULL) return NULL; + if (sess->client_myds->myconn->userinfo == NULL) return NULL; + unsigned int i; + std::vector parents; // this is a vector of srvers that needs to be excluded in case gtid_uuid is used + PgSQL_Connection* c = NULL; + for (i = 0; i < cached_connections->len; i++) { + c = (PgSQL_Connection*)cached_connections->index(i); + if (c->parent->myhgc->hid == _hid && sess->client_myds->myconn->has_same_connection_options(c)) { // options are all identical + if ( + (gtid_uuid == NULL) || // gtid_uuid is not used + (gtid_uuid && find(parents.begin(), parents.end(), c->parent) == parents.end()) // the server is currently not excluded + ) { + PgSQL_Connection* client_conn = sess->client_myds->myconn; + if (c->requires_RESETTING_CONNECTION(client_conn) == false) { // RESETTING CONNECTION is not required + unsigned int not_match = 0; // number of not matching session variables + c->number_of_matching_session_variables(client_conn, not_match); + if (not_match == 0) { // all session variables match + if (gtid_uuid) { // gtid_uuid is used + // we first check if we already excluded this parent (MySQL Server) + PgSQL_SrvC* mysrvc = c->parent; + std::vector::iterator it; + it = find(parents.begin(), parents.end(), mysrvc); + if (it != parents.end()) { + // we didn't exclude this server (yet?) + bool gtid_found = false; +#if 0 + gtid_found = PgHGM->gtid_exists(mysrvc, gtid_uuid, gtid_trxid); +#endif // 0 + if (gtid_found) { // this server has the correct GTID + c = (PgSQL_Connection*)cached_connections->remove_index_fast(i); + return c; + } else { + parents.push_back(mysrvc); // stop evaluating this server + } + } + } else { // gtid_is not used + if (max_lag_ms >= 0) { + if ((unsigned int)max_lag_ms < (c->parent->aws_aurora_current_lag_us / 1000)) { + status_variables.stvar[st_var_aws_aurora_replicas_skipped_during_query]++; + continue; + } + } + // return the connection + c = (PgSQL_Connection*)cached_connections->remove_index_fast(i); + return c; + } + } + + } + } + } + } + return NULL; +} + +void PgSQL_Thread::push_MyConn_local(PgSQL_Connection * c) { + PgSQL_SrvC* mysrvc = NULL; + mysrvc = (PgSQL_SrvC*)c->parent; + // reset insert_id #1093 + //c->pgsql->insert_id = 0; + if (mysrvc->status == MYSQL_SERVER_STATUS_ONLINE) { + if (c->async_state_machine == ASYNC_IDLE) { + cached_connections->add(c); + return; // all went well + } + } + PgHGM->push_MyConn_to_pool(c); +} + +void PgSQL_Thread::return_local_connections() { + if (cached_connections->len == 0) { + return; + } + /* + PgSQL_Connection **ca=(PgSQL_Connection **)malloc(sizeof(PgSQL_Connection *)*(cached_connections->len+1)); + unsigned int i=0; + */ + // ca[i]=NULL; + PgHGM->push_MyConn_to_pool_array((PgSQL_Connection**)cached_connections->pdata, cached_connections->len); + // free(ca); + while (cached_connections->len) { + cached_connections->remove_index_fast(0); + } +} + +void PgSQL_Thread::Scan_Sessions_to_Kill_All() { + if (kq.conn_ids.size() + kq.query_ids.size()) { + Scan_Sessions_to_Kill(mysql_sessions); + } +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + if (kq.conn_ids.size() + kq.query_ids.size()) { + Scan_Sessions_to_Kill(idle_mysql_sessions); + } + if (kq.conn_ids.size() + kq.query_ids.size()) { + Scan_Sessions_to_Kill(resume_mysql_sessions); + } + if (kq.conn_ids.size() + kq.query_ids.size()) { + pthread_mutex_lock(&myexchange.mutex_idles); + Scan_Sessions_to_Kill(myexchange.idle_mysql_sessions); + pthread_mutex_unlock(&myexchange.mutex_idles); + } + if (kq.conn_ids.size() + kq.query_ids.size()) { + pthread_mutex_lock(&myexchange.mutex_resumes); + Scan_Sessions_to_Kill(myexchange.resume_mysql_sessions); + pthread_mutex_unlock(&myexchange.mutex_resumes); + } + } +#endif + for (std::vector::iterator it = kq.conn_ids.begin(); it != kq.conn_ids.end(); ++it) { + thr_id_usr* t = *it; + free(t->username); + free(t); + } + for (std::vector::iterator it = kq.query_ids.begin(); it != kq.query_ids.end(); ++it) { + thr_id_usr* t = *it; + free(t->username); + free(t); + } + kq.conn_ids.clear(); + kq.query_ids.clear(); +} + +void PgSQL_Thread::Scan_Sessions_to_Kill(PtrArray * mysess) { + for (unsigned int n = 0; n < mysess->len && (kq.conn_ids.size() + kq.query_ids.size()); n++) { + PgSQL_Session* _sess = (PgSQL_Session*)mysess->index(n); + bool cont = true; + for (std::vector::iterator it = kq.conn_ids.begin(); cont && it != kq.conn_ids.end(); ++it) { + thr_id_usr* t = *it; + if (t->id == _sess->thread_session_id) { + if (_sess->client_myds) { + if (strcmp(t->username, _sess->client_myds->myconn->userinfo->username) == 0) { + _sess->killed = true; + } + } + cont = false; + free(t->username); + free(t); + kq.conn_ids.erase(it); + } + } + for (std::vector::iterator it = kq.query_ids.begin(); cont && it != kq.query_ids.end(); ++it) { + thr_id_usr* t = *it; + if (t->id == _sess->thread_session_id) { + proxy_info("Killing query %d\n", t->id); + if (_sess->client_myds) { + if (strcmp(t->username, _sess->client_myds->myconn->userinfo->username) == 0) { + if (_sess->mybe) { + if (_sess->mybe->server_myds) { + _sess->mybe->server_myds->wait_until = curtime; + _sess->mybe->server_myds->kill_type = 1; + } + } + } + } + cont = false; + free(t->username); + free(t); + kq.query_ids.erase(it); + } + } + } +} + +#ifdef IDLE_THREADS +void PgSQL_Thread::idle_thread_gets_sessions_from_worker_thread() { + pthread_mutex_lock(&myexchange.mutex_idles); + while (myexchange.idle_mysql_sessions->len) { + PgSQL_Session* mysess = (PgSQL_Session*)myexchange.idle_mysql_sessions->remove_index_fast(0); + register_session(this, mysess, false); + PgSQL_Data_Stream* myds = mysess->client_myds; + mypolls.add(POLLIN, myds->fd, myds, monotonic_time()); + // add in epoll() + struct epoll_event event; + memset(&event, 0, sizeof(event)); // let's make valgrind happy + event.data.u32 = mysess->thread_session_id; + event.events = EPOLLIN; + epoll_ctl(efd, EPOLL_CTL_ADD, myds->fd, &event); + // we map thread_id -> position in mysql_session (end of the list) + sessmap[mysess->thread_session_id] = mysql_sessions->len - 1; + //fprintf(stderr,"Adding session %p idx, DS %p idx %d\n",mysess,myds,myds->poll_fds_idx); + } + pthread_mutex_unlock(&myexchange.mutex_idles); +} +#endif // IDLE_THREADS + +void PgSQL_Thread::handle_mirror_queue_mysql_sessions() { + while (mirror_queue_mysql_sessions->len) { + if (__sync_add_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1) > (unsigned int)pgsql_thread___mirror_max_concurrency) { + __sync_sub_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1); + //goto __mysql_thread_exit_add_mirror; // we can't add more mirror sessions at runtime + return; + } + else { + int idx; + idx = fastrand() % (mirror_queue_mysql_sessions->len); + PgSQL_Session* newsess = (PgSQL_Session*)mirror_queue_mysql_sessions->remove_index_fast(idx); + register_session(this, newsess); + newsess->handler(); // execute immediately + if (newsess->status == WAITING_CLIENT_DATA) { // the mirror session has completed + unregister_session(mysql_sessions->len - 1); + unsigned int l = (unsigned int)pgsql_thread___mirror_max_concurrency; + if (mirror_queue_mysql_sessions->len * 0.3 > l) l = mirror_queue_mysql_sessions->len * 0.3; + if (mirror_queue_mysql_sessions_cache->len <= l) { + bool to_cache = true; + if (newsess->mybe) { + if (newsess->mybe->server_myds) { + to_cache = false; + } + } + if (to_cache) { + __sync_sub_and_fetch(&GloPTH->status_variables.mirror_sessions_current, 1); + mirror_queue_mysql_sessions_cache->add(newsess); + } + else { + delete newsess; + } + } + else { + delete newsess; + } + } + //newsess->to_process=0; + } + } +} + +void PgSQL_Thread::handle_kill_queues() { + pthread_mutex_lock(&kq.m); + if (kq.conn_ids.size() + kq.query_ids.size()) { + Scan_Sessions_to_Kill_All(); + maintenance_loop = true; + } + pthread_mutex_unlock(&kq.m); +} diff --git a/lib/PgSQL_Variables.cpp b/lib/PgSQL_Variables.cpp new file mode 100644 index 0000000000..4a9a89dabb --- /dev/null +++ b/lib/PgSQL_Variables.cpp @@ -0,0 +1,657 @@ +#include "PgSQL_Variables.h" +#include "proxysql.h" + +#include "PgSQL_Session.h" +#include "PgSQL_Data_Stream.h" +#ifndef SPOOKYV2 +#include "SpookyV2.h" +#define SPOOKYV2 +#endif + +#include + + +static inline char is_digit(char c) { + if(c >= '0' && c <= '9') + return 1; + return 0; +} + +#include "proxysql_find_charset.h" + +pgsql_verify_var PgSQL_Variables::verifiers[SQL_NAME_LAST_HIGH_WM]; +pgsql_update_var PgSQL_Variables::updaters[SQL_NAME_LAST_HIGH_WM]; + + +PgSQL_Variables::PgSQL_Variables() { + // add here all the variables we want proxysql to recognize, but ignore + ignore_vars.push_back("interactive_timeout"); + ignore_vars.push_back("wait_timeout"); + ignore_vars.push_back("net_read_timeout"); + ignore_vars.push_back("net_write_timeout"); + ignore_vars.push_back("net_buffer_length"); + ignore_vars.push_back("read_buffer_size"); + ignore_vars.push_back("read_rnd_buffer_size"); + // NOTE: This variable has been temporarily ignored. Check issues #3442 and #3441. + ignore_vars.push_back("session_track_schema"); + variables_regexp = ""; + for (auto i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + // we initialized all the internal_variable_name if set to NULL + if (mysql_tracked_variables[i].internal_variable_name == NULL) { + mysql_tracked_variables[i].internal_variable_name = mysql_tracked_variables[i].set_variable_name; + } + } +/* + NOTE: + make special ATTENTION that the order in mysql_variable_name + and mysql_tracked_variables[] is THE SAME + NOTE: + PgSQL_Variables::PgSQL_Variables() has a built-in check to make sure that the order is correct, + and that variables are in alphabetical order +*/ + for (int i = SQL_NAME_LAST_LOW_WM; i < SQL_NAME_LAST_HIGH_WM; i++) { + assert(i == mysql_tracked_variables[i].idx); + if (i > SQL_NAME_LAST_LOW_WM+1) { + assert(strcmp(mysql_tracked_variables[i].set_variable_name, mysql_tracked_variables[i-1].set_variable_name) > 0); + } + } + for (auto i = 0; i < SQL_NAME_LAST_HIGH_WM; i++) { + if (i == SQL_CHARACTER_SET || i == SQL_CHARACTER_ACTION || i == SQL_SET_NAMES) { + PgSQL_Variables::updaters[i] = NULL; + PgSQL_Variables::verifiers[i] = NULL; + } + else if (i == SQL_SQL_LOG_BIN) { +// PgSQL_Variables::verifiers[i] = verify_server_variable; +// PgSQL_Variables::updaters[i] = logbin_update_server_variable; + } else { + PgSQL_Variables::verifiers[i] = verify_server_variable; + PgSQL_Variables::updaters[i] = update_server_variable; + } + if (mysql_tracked_variables[i].status == SETTING_VARIABLE) { + variables_regexp += mysql_tracked_variables[i].set_variable_name; + variables_regexp += "|"; + } + } + for (std::vector::iterator it=ignore_vars.begin(); it != ignore_vars.end(); it++) { + variables_regexp += *it; + variables_regexp += "|"; + } +} + +PgSQL_Variables::~PgSQL_Variables() {} + +bool PgSQL_Variables::client_set_hash_and_value(PgSQL_Session* session, int idx, const std::string& value, uint32_t hash) { + if (!session || !session->client_myds || !session->client_myds->myconn) { + proxy_warning("Session validation failed\n"); + return false; + } + + session->client_myds->myconn->var_hash[idx] = hash; + if (session->client_myds->myconn->variables[idx].value) { + free(session->client_myds->myconn->variables[idx].value); + } + session->client_myds->myconn->variables[idx].value = strdup(value.c_str()); + + return true; +} + +void PgSQL_Variables::client_reset_value(PgSQL_Session* session, int idx) { + if (!session || !session->client_myds || !session->client_myds->myconn) { + proxy_warning("Session validation failed\n"); + return; + } + + PgSQL_Connection *client_conn = session->client_myds->myconn; + + if (client_conn->var_hash[idx] != 0) { + client_conn->var_hash[idx] = 0; + if (client_conn->variables[idx].value) { + free(client_conn->variables[idx].value); + client_conn->variables[idx].value = NULL; + } + // we now regererate dynamic_variables_idx + client_conn->reorder_dynamic_variables_idx(); + } +} +void PgSQL_Variables::server_set_hash_and_value(PgSQL_Session* session, int idx, const char* value, uint32_t hash) { + if (!session || !session->mybe || !session->mybe->server_myds || !session->mybe->server_myds->myconn || !value) { + proxy_warning("Session validation failed\n"); + return; + } + + session->mybe->server_myds->myconn->var_hash[idx] = hash; + if (session->mybe->server_myds->myconn->variables[idx].value) { + free(session->mybe->server_myds->myconn->variables[idx].value); + } + session->mybe->server_myds->myconn->variables[idx].value = strdup(value); +} + + +/** + * @brief Set the supplied value for the session variable specified by the supplied + * index into the supplied client session. + * + * @details There are two session variables which require special handling: + * - 'SET NAMES' + * - 'SET CHARACTER SET' + * + * For the second case 'SET CHARACTER SET' we forget about the values for: + * - 'SQL_CHARACTER_SET_CONNECTION' + * - 'SQL_COLLATION_CONNECTION' + * + * This is done because 'character_set_database' is not known when the set happens and + * because we work under the assumption that if a client request 'SET CHARACTER SET' + * doesn't require a specific 'collation_connection' and 'character_set_connection". + * Furthermore, 'character_set_database' is deprecated since MySQL 5.7 and will only + * be usable as an immutable session variable in the future. For reference see: + * + * - 'https://dev.pgsql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_character_set_database + * + * Due to this, *it's expected behavior* to see that a connection that sets 'SET CHARACTER SET' + * has a variant 'collation_connection' and 'character_set_connection' depending on the + * backend connection that is retrieved from the connection pool. If the 'collation_connection' + * and 'character_set_connection' variables are relevant and should never change, + * 'SET NAMES' should be used. + * + * @param session The client session which variable value is going to be modified. + * @param idx The index of the session variable to modify. + * @param value The session variable value to be set. + * + * @return 'true' in case of success, 'false' otherwise. + */ +bool PgSQL_Variables::client_set_value(PgSQL_Session* session, int idx, const std::string& value) { + if (!session || !session->client_myds || !session->client_myds->myconn) { + proxy_warning("Session validation failed\n"); + return false; + } + + /* Process SET NAMES and SET CHARSET commands + * The character_set_client, character_set_results, character_set_connection variables are set here + * During multiplexing/query execution these values will be used to set corresponding backend variables + * The charset used in SET NAMES and SET CHARSET is not used in setting backend chrsets + */ + switch (idx) { + case SQL_CHARACTER_ACTION: + // SET NAMES command from client + if (value == "1") { + if (pgsql_variables.client_get_value(session, SQL_CHARACTER_SET)) { + const char* value = pgsql_variables.client_get_value(session, SQL_CHARACTER_SET); + uint32_t hash = pgsql_variables.client_get_hash(session, SQL_CHARACTER_SET); + + pgsql_variables.client_set_hash_and_value(session, SQL_CHARACTER_SET_RESULTS, value, hash); + pgsql_variables.client_set_hash_and_value(session, SQL_CHARACTER_SET_CLIENT, value, hash); + pgsql_variables.client_set_hash_and_value(session, SQL_CHARACTER_SET_CONNECTION, value, hash); + pgsql_variables.client_set_hash_and_value(session, SQL_COLLATION_CONNECTION, value, hash); + } + } + // SET CHARSET command from client + else if (value == "2") { + if (pgsql_variables.client_get_value(session, SQL_CHARACTER_SET)) { + const char* value = pgsql_variables.client_get_value(session, SQL_CHARACTER_SET); + uint32_t hash = pgsql_variables.client_get_hash(session, SQL_CHARACTER_SET); + + pgsql_variables.client_set_hash_and_value(session, SQL_CHARACTER_SET_RESULTS, value, hash); + pgsql_variables.client_set_hash_and_value(session, SQL_CHARACTER_SET_CLIENT, value, hash); + + // Setting connection and collation connection to NULL + // because we do not know database character set + pgsql_variables.client_set_hash_and_value(session, SQL_CHARACTER_SET_CONNECTION, "", 0); + pgsql_variables.client_set_hash_and_value(session, SQL_COLLATION_CONNECTION, "", 0); + } + } + } + + session->client_myds->myconn->var_hash[idx] = SpookyHash::Hash32(value.c_str(),strlen(value.c_str()),10); + if (session->client_myds->myconn->variables[idx].value) { + free(session->client_myds->myconn->variables[idx].value); + } + session->client_myds->myconn->variables[idx].value = strdup(value.c_str()); + // we now regererate dynamic_variables_idx + session->client_myds->myconn->reorder_dynamic_variables_idx(); + return true; +} + +const char* PgSQL_Variables::client_get_value(PgSQL_Session* session, int idx) const { + assert(session); + assert(session->client_myds); + assert(session->client_myds->myconn); + return session->client_myds->myconn->variables[idx].value; +} + +uint32_t PgSQL_Variables::client_get_hash(PgSQL_Session* session, int idx) const { + assert(session); + assert(session->client_myds); + assert(session->client_myds->myconn); + return session->client_myds->myconn->var_hash[idx]; +} + +void PgSQL_Variables::server_set_value(PgSQL_Session* session, int idx, const char* value) { + assert(session); + assert(session->mybe); + assert(session->mybe->server_myds); + assert(session->mybe->server_myds->myconn); + if (!value) return; // FIXME: I am not sure about this implementation . If value == NULL , show the variable be reset? + session->mybe->server_myds->myconn->var_hash[idx] = SpookyHash::Hash32(value,strlen(value),10); + + if (session->mybe->server_myds->myconn->variables[idx].value) { + free(session->mybe->server_myds->myconn->variables[idx].value); + } + session->mybe->server_myds->myconn->variables[idx].value = strdup(value); + // we now regererate dynamic_variables_idx + session->mybe->server_myds->myconn->reorder_dynamic_variables_idx(); +} + +void PgSQL_Variables::server_reset_value(PgSQL_Session* session, int idx) { + assert(session); + assert(session->mybe); + assert(session->mybe->server_myds); + assert(session->mybe->server_myds->myconn); + + PgSQL_Connection *backend_conn = session->mybe->server_myds->myconn; + + if (backend_conn->var_hash[idx] != 0) { + backend_conn->var_hash[idx] = 0; + if (backend_conn->variables[idx].value) { + free(backend_conn->variables[idx].value); + backend_conn->variables[idx].value = NULL; + } + // we now regererate dynamic_variables_idx + backend_conn->reorder_dynamic_variables_idx(); + } +} + +const char* PgSQL_Variables::server_get_value(PgSQL_Session* session, int idx) const { + assert(session); + assert(session->mybe); + assert(session->mybe->server_myds); + assert(session->mybe->server_myds->myconn); + return session->mybe->server_myds->myconn->variables[idx].value; +} + +uint32_t PgSQL_Variables::server_get_hash(PgSQL_Session* session, int idx) const { + assert(session); + assert(session->mybe); + assert(session->mybe->server_myds); + assert(session->mybe->server_myds->myconn); + return session->mybe->server_myds->myconn->var_hash[idx]; +} + +bool PgSQL_Variables::update_variable(PgSQL_Session* session, session_status status, int &_rc) { + int idx = SQL_NAME_LAST_HIGH_WM; + if (session->status == SETTING_VARIABLE) { + // if status is SETTING_VARIABLE , what variable needs to be changed is defined in changing_variable_idx + idx = session->changing_variable_idx; + } else { + for (int i=0; iclient_myds->myconn->var_hash[idx]; + auto server_hash = session->mybe->server_myds->myconn->var_hash[idx]; + if (client_hash && client_hash != server_hash) { + ret = verifiers[idx](session, idx, client_hash, server_hash); + } + } + return ret; +} + +bool validate_charset(PgSQL_Session* session, int idx, int &_rc) { + if (idx == SQL_CHARACTER_SET || idx == SQL_CHARACTER_SET_CLIENT || idx == SQL_CHARACTER_SET_RESULTS || + idx == SQL_CHARACTER_SET_CONNECTION || idx == SQL_CHARACTER_SET_DATABASE || idx == SQL_COLLATION_CONNECTION) { + PgSQL_Data_Stream *myds = session->mybe->server_myds; + PgSQL_Connection *myconn = myds->myconn; + char msg[128]; + const MARIADB_CHARSET_INFO *ci = NULL; + const char* replace_collation = NULL; + const char* not_supported_collation = NULL; + unsigned int replace_collation_nr = 0; + std::stringstream ss; + int charset = atoi(pgsql_variables.client_get_value(session, idx)); + if (charset >= 255 && myconn->pgsql->server_version[0] != '8') { + switch(mysql_thread___handle_unknown_charset) { + case HANDLE_UNKNOWN_CHARSET__DISCONNECT_CLIENT: + snprintf(msg,sizeof(msg),"Can't initialize character set %s", pgsql_variables.client_get_value(session, idx)); + proxy_error("Can't initialize character set on %s, %d: Error %d (%s). Closing client connection %s:%d.\n", + myconn->parent->address, myconn->parent->port, 2019, msg, session->client_myds->addr.addr, session->client_myds->addr.port); + myds->destroy_MySQL_Connection_From_Pool(false); + myds->fd=0; + _rc=-1; + return false; + case HANDLE_UNKNOWN_CHARSET__REPLACE_WITH_DEFAULT_VERBOSE: + ci = proxysql_find_charset_nr(charset); + if (!ci) { + // LCOV_EXCL_START + proxy_error("Cannot find character set [%s]\n", pgsql_variables.client_get_value(session, idx)); + assert(0); + // LCOV_EXCL_STOP + } + not_supported_collation = ci->name; + + if (idx == SQL_COLLATION_CONNECTION) { + ci = proxysql_find_charset_collate(mysql_thread___default_variables[idx]); + } else { + if (mysql_thread___default_variables[idx]) { + ci = proxysql_find_charset_name(mysql_thread___default_variables[idx]); + } else { + ci = proxysql_find_charset_name(mysql_thread___default_variables[SQL_CHARACTER_SET]); + } + } + + if (!ci) { + // LCOV_EXCL_START + proxy_error("Cannot find character set [%s]\n", mysql_thread___default_variables[idx]); + assert(0); + // LCOV_EXCL_STOP + } + replace_collation = ci->name; + replace_collation_nr = ci->nr; + + proxy_warning("Server doesn't support collation (%s) %s. Replacing it with the configured default (%d) %s. Client %s:%d\n", + pgsql_variables.client_get_value(session, idx), not_supported_collation, + replace_collation_nr, replace_collation, session->client_myds->addr.addr, session->client_myds->addr.port); + + ss << replace_collation_nr; + pgsql_variables.client_set_value(session, idx, ss.str()); + _rc=0; + return true; + case HANDLE_UNKNOWN_CHARSET__REPLACE_WITH_DEFAULT: + if (idx == SQL_COLLATION_CONNECTION) { + ci = proxysql_find_charset_collate(mysql_thread___default_variables[idx]); + } else { + if (mysql_thread___default_variables[idx]) { + ci = proxysql_find_charset_name(mysql_thread___default_variables[idx]); + } else { + ci = proxysql_find_charset_name(mysql_thread___default_variables[SQL_CHARACTER_SET]); + } + } + + if (!ci) { + // LCOV_EXCL_START + proxy_error("Cannot filnd charset [%s]\n", mysql_thread___default_variables[idx]); + assert(0); + // LCOV_EXCL_STOP + } + replace_collation_nr = ci->nr; + + ss << replace_collation_nr; + pgsql_variables.client_set_value(session, idx, ss.str()); + _rc=0; + return true; + default: + proxy_error("Wrong configuration of the handle_unknown_charset\n"); + _rc=-1; + return false; + } + } + } + _rc=0; + return true; +} + +bool update_server_variable(PgSQL_Session* session, int idx, int &_rc) { + bool no_quote = true; + if (mysql_tracked_variables[idx].quote) no_quote = false; + bool st = mysql_tracked_variables[idx].set_transaction; + const char * set_var_name = mysql_tracked_variables[idx].set_variable_name; + bool ret = false; + + /* Validating that charset is less than 255 for mysqld version <8.0 + */ + if (!validate_charset(session, idx, _rc)) { + return false; + } + + /* character set variables store collation id in the char* string, but we set character_set_% command + * uses character set name or collation name. This branch convert collation id to character set name + * or collation name for further execution on backend + */ + if (idx==SQL_CHARACTER_SET_RESULTS) { + const MARIADB_CHARSET_INFO *ci = NULL; + ci = proxysql_find_charset_nr(atoi(pgsql_variables.client_get_value(session, SQL_CHARACTER_SET_RESULTS))); + + if (!ci) { + if (!strcmp(pgsql_variables.client_get_value(session, SQL_CHARACTER_SET_RESULTS), "NULL")) { + pgsql_variables.server_set_value(session, idx, pgsql_variables.client_get_value(session, idx)); + ret = session->handler_again___status_SETTING_GENERIC_VARIABLE(&_rc, set_var_name, "NULL", no_quote, st); + } else if (!strcmp(pgsql_variables.client_get_value(session, SQL_CHARACTER_SET_RESULTS), "binary")) { + pgsql_variables.server_set_value(session, idx, pgsql_variables.client_get_value(session, idx)); + ret = session->handler_again___status_SETTING_GENERIC_VARIABLE(&_rc, set_var_name, "binary", no_quote, st); + } + } else { + pgsql_variables.server_set_value(session, idx, pgsql_variables.client_get_value(session, idx)); + ret = session->handler_again___status_SETTING_GENERIC_VARIABLE(&_rc, set_var_name, ci->csname, no_quote, st); + } + } else if (idx==SQL_COLLATION_CONNECTION) { + const MARIADB_CHARSET_INFO *ci = NULL; + ci = proxysql_find_charset_nr(atoi(pgsql_variables.client_get_value(session, SQL_COLLATION_CONNECTION))); + + if (ci) { + std::stringstream ss; + ss << ci->nr; + + pgsql_variables.server_set_value(session, idx, pgsql_variables.client_get_value(session, idx)); + ret = session->handler_again___status_SETTING_GENERIC_VARIABLE(&_rc, set_var_name, ci->name, no_quote, st); + } + } else if (idx==SQL_CHARACTER_SET_CONNECTION) { + const MARIADB_CHARSET_INFO *ci = NULL; + ci = proxysql_find_charset_nr(atoi(pgsql_variables.client_get_value(session, SQL_CHARACTER_SET_CONNECTION))); + + if (ci) { + unsigned int nr = ci->nr; + std::stringstream ss; + ss << nr; + + pgsql_variables.server_set_value(session, idx, pgsql_variables.client_get_value(session, idx)); + ret = session->handler_again___status_SETTING_GENERIC_VARIABLE(&_rc, set_var_name, ci->csname, no_quote, st); + } + } else if (idx==SQL_CHARACTER_SET_CLIENT || idx==SQL_CHARACTER_SET_DATABASE) { + const MARIADB_CHARSET_INFO *ci = NULL; + ci = proxysql_find_charset_nr(atoi(pgsql_variables.client_get_value(session, idx))); + + std::stringstream ss; + ss << ci->nr; + pgsql_variables.server_set_value(session, idx, pgsql_variables.client_get_value(session, idx)); + ret = session->handler_again___status_SETTING_GENERIC_VARIABLE(&_rc, set_var_name, ci->csname, no_quote, st); + } else { + pgsql_variables.server_set_value(session, idx, pgsql_variables.client_get_value(session, idx)); + ret = session->handler_again___status_SETTING_GENERIC_VARIABLE(&_rc, set_var_name, pgsql_variables.server_get_value(session, idx), no_quote, st); + } + return ret; +} + +bool verify_set_names(PgSQL_Session* session) { + uint32_t client_charset_hash = pgsql_variables.client_get_hash(session, SQL_CHARACTER_SET_CLIENT); + if (client_charset_hash == 0) + return false; + + uint32_t results_charset_hash = pgsql_variables.client_get_hash(session, SQL_CHARACTER_SET_RESULTS); + if (client_charset_hash != results_charset_hash) + return false; + + uint32_t connection_charset_hash = pgsql_variables.client_get_hash(session, SQL_CHARACTER_SET_CONNECTION); + if (client_charset_hash != connection_charset_hash) + return false; + + uint32_t collation_hash = pgsql_variables.client_get_hash(session, SQL_COLLATION_CONNECTION); + if (client_charset_hash != collation_hash) + return false; + + if (client_charset_hash != pgsql_variables.server_get_hash(session, SQL_CHARACTER_SET_CLIENT) || + results_charset_hash != pgsql_variables.server_get_hash(session, SQL_CHARACTER_SET_RESULTS) || + connection_charset_hash != pgsql_variables.server_get_hash(session, SQL_CHARACTER_SET_CONNECTION) || + collation_hash != pgsql_variables.server_get_hash(session, SQL_COLLATION_CONNECTION)) { + + switch(session->status) { // this switch can be replaced with a simple previous_status.push(status), but it is here for readibility + case PROCESSING_QUERY: + session->previous_status.push(PROCESSING_QUERY); + break; + case PROCESSING_STMT_PREPARE: + session->previous_status.push(PROCESSING_STMT_PREPARE); + break; + case PROCESSING_STMT_EXECUTE: + session->previous_status.push(PROCESSING_STMT_EXECUTE); + break; + default: + // LCOV_EXCL_START + proxy_error("Wrong status %d\n", session->status); + assert(0); + break; + // LCOV_EXCL_STOP + } + session->set_status(SETTING_SET_NAMES); + uint32_t hash = pgsql_variables.client_get_hash(session, SQL_CHARACTER_SET_CLIENT); + const char* value = pgsql_variables.client_get_value(session, SQL_CHARACTER_SET_CLIENT); + pgsql_variables.server_set_hash_and_value(session, SQL_CHARACTER_SET_CLIENT, value, hash); + pgsql_variables.server_set_hash_and_value(session, SQL_CHARACTER_SET_RESULTS, value, hash); + pgsql_variables.server_set_hash_and_value(session, SQL_CHARACTER_SET_CONNECTION, value, hash); + pgsql_variables.server_set_hash_and_value(session, SQL_COLLATION_CONNECTION, value, hash); + pgsql_variables.client_set_hash_and_value(session, SQL_CHARACTER_SET, value, hash); + pgsql_variables.server_set_hash_and_value(session, SQL_CHARACTER_SET, value, hash); + return true; + } + + return false; +} + +inline bool verify_server_variable(PgSQL_Session* session, int idx, uint32_t client_hash, uint32_t server_hash) { + if (client_hash && client_hash != server_hash) { + // Edge case for set charset command, because we do not know database character set + // for now we are setting connection and collation to empty + if (idx == SQL_CHARACTER_SET_CONNECTION || idx == SQL_COLLATION_CONNECTION ) { + if (pgsql_variables.client_get_hash(session, idx) == 0) { + pgsql_variables.server_set_hash_and_value(session, idx, "", 0); + return false; + } + } + // this variable is relevant only if status == SETTING_VARIABLE + session->changing_variable_idx = (enum mysql_variable_name)idx; + switch(session->status) { // this switch can be replaced with a simple previous_status.push(status), but it is here for readibility + case PROCESSING_QUERY: + session->previous_status.push(PROCESSING_QUERY); + break; + case PROCESSING_STMT_PREPARE: + session->previous_status.push(PROCESSING_STMT_PREPARE); + break; + case PROCESSING_STMT_EXECUTE: + session->previous_status.push(PROCESSING_STMT_EXECUTE); + break; + default: + // LCOV_EXCL_START + proxy_error("Wrong status %d\n", session->status); + assert(0); + break; + // LCOV_EXCL_STOP + } + session->set_status(mysql_tracked_variables[idx].status); + pgsql_variables.server_set_value(session, idx, pgsql_variables.client_get_value(session, idx)); + return true; + } + return false; +} + +#if 0 +bool logbin_update_server_variable(PgSQL_Session* session, int idx, int &_rc) { + return session->handler_again___status_SETTING_SQL_LOG_BIN(&_rc); +} +#endif // 0 + +bool PgSQL_Variables::parse_variable_boolean(PgSQL_Session *sess, int idx, string& value1, bool * lock_hostgroup) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Processing SET %s value %s\n", mysql_tracked_variables[idx].set_variable_name, value1.c_str()); + int __tmp_value = -1; + if ( + (strcasecmp(value1.c_str(),(char *)"0")==0) || + (strcasecmp(value1.c_str(),(char *)"false")==0) || + (strcasecmp(value1.c_str(),(char *)"off")==0) + ) { + __tmp_value = 0; + } else { + if ( + (strcasecmp(value1.c_str(),(char *)"1")==0) || + (strcasecmp(value1.c_str(),(char *)"true")==0) || + (strcasecmp(value1.c_str(),(char *)"on")==0) + ) { + __tmp_value = 1; + } + } + + if (__tmp_value >= 0) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 7, "Processing SET %s value %s\n", mysql_tracked_variables[idx].set_variable_name, value1.c_str()); + uint32_t var_value_int=SpookyHash::Hash32(value1.c_str(),value1.length(),10); + if (pgsql_variables.client_get_hash(sess, idx) != var_value_int) { + if (__tmp_value == 0) { + if (!pgsql_variables.client_set_value(sess, idx, "OFF")) + return false; + } else { + if (!pgsql_variables.client_set_value(sess, idx, "ON")) + return false; + } + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Changing connection %s to %s\n", mysql_tracked_variables[idx].set_variable_name, value1.c_str()); + } + } else { + sess->unable_to_parse_set_statement(lock_hostgroup); + return false; + } + return true; +} + + + +bool PgSQL_Variables::parse_variable_number(PgSQL_Session *sess, int idx, string& value1, bool * lock_hostgroup) { + int vl = strlen(value1.c_str()); + const char *v = value1.c_str(); + bool only_digit_chars = true; + for (int i=0; iunable_to_parse_set_statement(lock_hostgroup); + return false; + } + return true; +} + diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index f80826327d..23d3091b28 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include // std::cout #include // std::stringstream #include @@ -9,7 +13,11 @@ #include "prometheus/counter.h" #include "openssl/ssl.h" #include "openssl/err.h" + +#include "Base_Thread.h" + #include "MySQL_HostGroups_Manager.h" +#include "PgSQL_HostGroups_Manager.h" #include "mysql.h" #include "proxysql_admin.h" #include "re2/re2.h" @@ -22,14 +30,18 @@ #include "cpp.h" #include "MySQL_Data_Stream.h" -#include "query_processor.h" +#include "PgSQL_Data_Stream.h" +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" #include "ProxySQL_HTTP_Server.hpp" // HTTP server #include "MySQL_Authentication.hpp" +#include "PgSQL_Authentication.h" #include "MySQL_LDAP_Authentication.hpp" #include "MySQL_PreparedStatement.h" #include "ProxySQL_Cluster.hpp" #include "ProxySQL_Statistics.hpp" #include "MySQL_Logger.hpp" +#include "PgSQL_Logger.hpp" #include "SQLite3_Server.h" #include "Web_Interface.hpp" @@ -64,6 +76,9 @@ #include +#include "PgSQL_Protocol.h" +//#include "usual/time.h" + using std::string; using std::unique_ptr; @@ -97,6 +112,8 @@ struct MHD_Daemon *Admin_HTTP_Server; extern ProxySQL_Statistics *GloProxyStats; +int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg); + extern char *ssl_key_fp; extern char *ssl_cert_fp; extern char *ssl_ca_fp; @@ -107,6 +124,10 @@ char * proxysql_version = NULL; #include "proxysql_find_charset.h" +template ::value, bool>::type = true> +T j_get_srv_default_int_val( +const json& j, uint32_t hid, const string& key, const function& val_check); + static const vector mysql_servers_tablenames = { "mysql_servers", @@ -118,13 +139,29 @@ static const vector mysql_servers_tablenames = { "mysql_servers_ssl_params", }; +static const vector pgsql_servers_tablenames = { + "pgsql_servers", + "pgsql_replication_hostgroups", +// "pgsql_group_replication_hostgroups", +// "pgsql_galera_hostgroups", +// "pgsql_aws_aurora_hostgroups", + "pgsql_hostgroup_attributes", +}; + static const vector mysql_firewall_tablenames = { "mysql_firewall_whitelist_users", "mysql_firewall_whitelist_rules", "mysql_firewall_whitelist_sqli_fingerprints", }; +static const vector pgsql_firewall_tablenames = { + "pgsql_firewall_whitelist_users", + "pgsql_firewall_whitelist_rules", + "pgsql_firewall_whitelist_sqli_fingerprints", +}; + static const vector mysql_query_rules_tablenames = { "mysql_query_rules", "mysql_query_rules_fast_routing" }; +static const vector pgsql_query_rules_tablenames = { "pgsql_query_rules", "pgsql_query_rules_fast_routing" }; static const vector scheduler_tablenames = { "scheduler" }; static const vector proxysql_servers_tablenames = { "proxysql_servers" }; static const vector restapi_tablenames = { "restapi_routes" }; @@ -136,6 +173,9 @@ static unordered_map&> module_tablenames = { { "scheduler", scheduler_tablenames }, { "proxysql_servers", proxysql_servers_tablenames }, { "restapi", restapi_tablenames }, + { "pgsql_servers", pgsql_servers_tablenames }, + { "pgsql_firewall", pgsql_firewall_tablenames }, + { "pgsql_query_rules", pgsql_query_rules_tablenames }, }; static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, const string& p2, const string& p3) { @@ -205,11 +245,11 @@ static int round_intv_to_time_interval(const char* name, int _intv) { } while (0) -typedef struct _arg_mysql_adm_t { +typedef struct _arg_proxysql_adm_t { struct sockaddr * addr; socklen_t addr_size; int client_t; -} arg_mysql_adm; +} arg_proxysql_adm; void StringToHex(unsigned char *string, unsigned char *hexstring, size_t l) { unsigned char ch; @@ -233,14 +273,6 @@ void StringToHex(unsigned char *string, unsigned char *hexstring, size_t l) { } } -static int int_cmp(const void *a, const void *b) { - const unsigned long long *ia = (const unsigned long long *)a; - const unsigned long long *ib = (const unsigned long long *)b; - if (*ia < *ib) return -1; - if (*ia > *ib) return 1; - return 0; -} - struct cpu_timer { cpu_timer() { @@ -265,45 +297,28 @@ char *s_strdup(char *s) { return ret; } +int admin_load_main_=0; +bool admin_nostart_=false; -static char *sha1_pass_hex(char *sha1_pass) { // copied from MySQL_Protocol.cpp - if (sha1_pass==NULL) return NULL; - // previous code is commented. Uncomment all to perform validation -// char *buff=(char *)malloc(SHA_DIGEST_LENGTH*2+2); -// buff[0]='*'; -// buff[SHA_DIGEST_LENGTH*2+1]='\0'; -// int i; -// uint8_t a; -// for (i=0;i map_test_mysql_firewall_whitelist_rules; -char rand_del[6]; +//pthread_mutex_t test_mysql_firewall_whitelist_mutex = PTHREAD_MUTEX_INITIALIZER; +//std::unordered_map map_test_mysql_firewall_whitelist_rules; +//char rand_del[6]; //static int http_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, size_t *upload_data_size, void **ptr) { MHD_Result http_handler(void *cls, struct MHD_Connection *connection, const char *url, const char *method, const char *version, const char *upload_data, long unsigned int *upload_data_size, void **ptr) { @@ -338,269 +353,7 @@ MHD_Result http_handler(void *cls, struct MHD_Connection *connection, const char #define LINESIZE 2048 -// mysql_servers in v1.1.0 -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_1_0 "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" - -// mysql_servers in v1.2.0e -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_0e "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" - -// mysql_servers in v1.2.2 -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_2 "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" - -// mysql_servers in v1.4.4 -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_4_4 "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" - -// mysql_servers in v2.0.0 -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0a "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT CHECK (gtid_port <> port) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" - -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0b "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_port INT CHECK (gtid_port <> port) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" - -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0c "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK (gtid_port <> port AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" - -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_11 "CREATE TABLE mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK ((gtid_port <> port OR gtid_port=0) AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" - -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_11 - -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS_V2_6_0 "CREATE TABLE mysql_servers_ssl_params (hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , username VARCHAR NOT NULL DEFAULT '' , ssl_ca VARCHAR NOT NULL DEFAULT '' , ssl_cert VARCHAR NOT NULL DEFAULT '' , ssl_key VARCHAR NOT NULL DEFAULT '' , ssl_capath VARCHAR NOT NULL DEFAULT '' , ssl_crl VARCHAR NOT NULL DEFAULT '' , ssl_crlpath VARCHAR NOT NULL DEFAULT '' , ssl_cipher VARCHAR NOT NULL DEFAULT '' , tls_version VARCHAR NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port, username) )" - -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS_V2_6_0 - -#define ADMIN_SQLITE_TABLE_MYSQL_USERS_V1_3_0 "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 0 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" -#define ADMIN_SQLITE_TABLE_MYSQL_USERS_V1_4_0 "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" -#define ADMIN_SQLITE_TABLE_MYSQL_USERS_V2_0_0 "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" -#define ADMIN_SQLITE_TABLE_MYSQL_USERS_V2_1_0 "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" -#define ADMIN_SQLITE_TABLE_MYSQL_USERS ADMIN_SQLITE_TABLE_MYSQL_USERS_V2_1_0 - -#define ADMIN_SQLITE_RUNTIME_MYSQL_USERS "CREATE TABLE runtime_mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 1 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '', comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" - -#define ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING_V2_0_0 "CREATE TABLE mysql_ldap_mapping (priority INTEGER CHECK (priority >= 1 AND priority <= 1000000) PRIMARY KEY , frontend_entity VARCHAR NOT NULL , backend_entity VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (frontend_entity))" -#define ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING_V2_0_0 - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_LDAP_MAPPING "CREATE TABLE runtime_mysql_ldap_mapping (priority INTEGER PRIMARY KEY NOT NULL , frontend_entity VARCHAR NOT NULL , backend_entity VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (frontend_entity))" - -#define ADMIN_SQLITE_RUNTIME_CHECKSUMS_VALUES "CREATE TABLE runtime_checksums_values (name VARCHAR NOT NULL , version INT NOT NULL , epoch INT NOT NULL , checksum VARCHAR NOT NULL , PRIMARY KEY (name))" - -// mysql_query_rules in v1.1.0 -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_1_0 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , delay INT UNSIGNED , error_msg VARCHAR , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" - -// mysql_query_rules in v1.2.0a -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0a "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" - -// mysql_query_rules in v1.2.0g -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0g "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" - -// mysql_query_rules in v1.2.2 -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_2 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -// mysql_query_rules in v1.3.1 -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_3_1 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -//mysql_query_rules in v1.4.0 + next_query_flagIN -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0a "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0b "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_1 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0a "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0b "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0c "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT , replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0d "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0e "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_1_0 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535) , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0) , replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_1_0 -//#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0b - - -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING "CREATE TABLE mysql_query_rules_fast_routing (username VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , destination_hostgroup INT CHECK (destination_hostgroup >= 0) NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, schemaname, flagIN) )" - -#define ADMIN_SQLITE_TABLE_GLOBAL_SETTINGS "CREATE TABLE global_settings (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" - -#define ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES "CREATE TABLE global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" - -#define ADMIN_SQLITE_RUNTIME_GLOBAL_VARIABLES "CREATE TABLE runtime_global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" - -//#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR , UNIQUE (reader_hostgroup))" - -// mysql_replication_hostgroups in v1.0 -#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_0 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , UNIQUE (reader_hostgroup))" - -// mysql_replication_hostgroups in v1.2.2 -#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_2_2 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR , UNIQUE (reader_hostgroup))" - -// mysql_replication_hostgroups in v1.4.5 -#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_4_5 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" - -// mysql_replication_hostgroups in v2.0.0 -#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V2_0_0 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" - -// mysql_replication_hostgroups in v2.0.8 -#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V2_0_8 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only','read_only|innodb_read_only','read_only&innodb_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" - -// mysql_replication_hostgroups current -#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V2_0_8 - -#define ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS "CREATE TABLE mysql_collations (Id INTEGER NOT NULL PRIMARY KEY , Collation VARCHAR NOT NULL , Charset VARCHAR NOT NULL , `Default` VARCHAR NOT NULL)" - -#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES_V2_0_15 "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" - -#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES_v2_1_0 "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , timeout_ms INTEGER CHECK (timeout_ms>=100 AND timeout_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" - -#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES ADMIN_SQLITE_TABLE_RESTAPI_ROUTES_v2_1_0 - -#define ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES "CREATE TABLE runtime_restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , timeout_ms INTEGER CHECK (timeout_ms>=100 AND timeout_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" - -#define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" - -#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_0 "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" - -#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2a "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY(id))" - -#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2b "CREATE TABLE scheduler (id INTEGER NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY(id))" - -#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2c "CREATE TABLE scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" - - -#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS_v209 "CREATE TABLE mysql_firewall_whitelist_users (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , mode VARCHAR CHECK (mode IN ('OFF','DETECTING','PROTECTING')) NOT NULL DEFAULT ('OFF') , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address) )" - -#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS_v209 - -#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES_v209 "CREATE TABLE mysql_firewall_whitelist_rules (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , digest VARCHAR NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address, schemaname, flagIN, digest) )" - -#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES_v209 - -#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS_v209 "CREATE TABLE mysql_firewall_whitelist_sqli_fingerprints (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , fingerprint VARCHAR NOT NULL , PRIMARY KEY (fingerprint) )" - -#define ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS_v209 - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_USERS "CREATE TABLE runtime_mysql_firewall_whitelist_users (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , mode VARCHAR CHECK (mode IN ('OFF','DETECTING','PROTECTING')) NOT NULL DEFAULT ('OFF') , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address) )" - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_RULES "CREATE TABLE runtime_mysql_firewall_whitelist_rules (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , digest VARCHAR NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, client_address, schemaname, flagIN, digest) )" - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS "CREATE TABLE runtime_mysql_firewall_whitelist_sqli_fingerprints (active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , fingerprint VARCHAR NOT NULL , PRIMARY KEY (fingerprint) )" - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT CHECK (hostgroup_id>=0) NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , gtid_port INT CHECK ((gtid_port <> port OR gtid_port=0) AND gtid_port >= 0 AND gtid_port <= 65535) NOT NULL DEFAULT 0 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0 AND weight <=10000000) NOT NULL DEFAULT 1 , compression INT CHECK (compression IN(0,1)) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS "CREATE TABLE runtime_mysql_servers_ssl_params (hostname VARCHAR NOT NULL , port INT CHECK (port >= 0 AND port <= 65535) NOT NULL DEFAULT 3306 , username VARCHAR NOT NULL DEFAULT '' , ssl_ca VARCHAR NOT NULL DEFAULT '' , ssl_cert VARCHAR NOT NULL DEFAULT '' , ssl_key VARCHAR NOT NULL DEFAULT '' , ssl_capath VARCHAR NOT NULL DEFAULT '' , ssl_crl VARCHAR NOT NULL DEFAULT '' , ssl_crlpath VARCHAR NOT NULL DEFAULT '' , ssl_cipher VARCHAR NOT NULL DEFAULT '' , tls_version VARCHAR NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port, username) )" - - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only','read_only|innodb_read_only','read_only&innodb_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '', UNIQUE (reader_hostgroup))" - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES "CREATE TABLE runtime_mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , attributes VARCHAR CHECK (JSON_VALID(attributes) OR attributes = '') NOT NULL DEFAULT '' , comment VARCHAR)" - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES_FAST_ROUTING "CREATE TABLE runtime_mysql_query_rules_fast_routing (username VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , destination_hostgroup INT CHECK (destination_hostgroup >= 0) NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, schemaname, flagIN) )" - -#define ADMIN_SQLITE_TABLE_RUNTIME_SCHEDULER "CREATE TABLE runtime_scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" - -#define STATS_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE stats_mysql_query_rules (rule_id INTEGER PRIMARY KEY , hits INT NOT NULL)" -#define STATS_SQLITE_TABLE_MYSQL_USERS "CREATE TABLE stats_mysql_users (username VARCHAR PRIMARY KEY , frontend_connections INT NOT NULL , frontend_max_connections INT NOT NULL)" -#define STATS_SQLITE_TABLE_MYSQL_COMMANDS_COUNTERS "CREATE TABLE stats_mysql_commands_counters (Command VARCHAR NOT NULL PRIMARY KEY , Total_Time_us INT NOT NULL , Total_cnt INT NOT NULL , cnt_100us INT NOT NULL , cnt_500us INT NOT NULL , cnt_1ms INT NOT NULL , cnt_5ms INT NOT NULL , cnt_10ms INT NOT NULL , cnt_50ms INT NOT NULL , cnt_100ms INT NOT NULL , cnt_500ms INT NOT NULL , cnt_1s INT NOT NULL , cnt_5s INT NOT NULL , cnt_10s INT NOT NULL , cnt_INFs)" -#define STATS_SQLITE_TABLE_MYSQL_PROCESSLIST "CREATE TABLE stats_mysql_processlist (ThreadID INT NOT NULL , SessionID INTEGER PRIMARY KEY , user VARCHAR , db VARCHAR , cli_host VARCHAR , cli_port INT , hostgroup INT , l_srv_host VARCHAR , l_srv_port INT , srv_host VARCHAR , srv_port INT , command VARCHAR , time_ms INT NOT NULL , info VARCHAR , status_flags INT , extended_info VARCHAR)" -#define STATS_SQLITE_TABLE_MYSQL_CONNECTION_POOL "CREATE TABLE stats_mysql_connection_pool (hostgroup INT , srv_host VARCHAR , srv_port INT , status VARCHAR , ConnUsed INT , ConnFree INT , ConnOK INT , ConnERR INT , MaxConnUsed INT , Queries INT , Queries_GTID_sync INT , Bytes_data_sent INT , Bytes_data_recv INT , Latency_us INT)" - -#define STATS_SQLITE_TABLE_MYSQL_CONNECTION_POOL_RESET "CREATE TABLE stats_mysql_connection_pool_reset (hostgroup INT , srv_host VARCHAR , srv_port INT , status VARCHAR , ConnUsed INT , ConnFree INT , ConnOK INT , ConnERR INT , MaxConnUsed INT , Queries INT , Queries_GTID_sync INT , Bytes_data_sent INT , Bytes_data_recv INT , Latency_us INT)" - -#define STATS_SQLITE_TABLE_MYSQL_FREE_CONNECTIONS "CREATE TABLE stats_mysql_free_connections (fd INT NOT NULL , hostgroup INT NOT NULL , srv_host VARCHAR NOT NULL , srv_port INT NOT NULL , user VARCHAR NOT NULL , schema VARCHAR , init_connect VARCHAR , time_zone VARCHAR , sql_mode VARCHAR , autocommit VARCHAR , idle_ms INT , statistics VARCHAR , mysql_info VARCHAR)" - -#define STATS_SQLITE_TABLE_MYSQL_QUERY_DIGEST "CREATE TABLE stats_mysql_query_digest (hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, schemaname, username, client_address, digest))" - -#define STATS_SQLITE_TABLE_MYSQL_QUERY_DIGEST_RESET "CREATE TABLE stats_mysql_query_digest_reset (hostgroup INT , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, schemaname, username, client_address, digest))" - -#define STATS_SQLITE_TABLE_MYSQL_GLOBAL "CREATE TABLE stats_mysql_global (Variable_Name VARCHAR NOT NULL PRIMARY KEY , Variable_Value VARCHAR NOT NULL)" - -#define STATS_SQLITE_TABLE_MEMORY_METRICS "CREATE TABLE stats_memory_metrics (Variable_Name VARCHAR NOT NULL PRIMARY KEY , Variable_Value VARCHAR NOT NULL)" - -#define STATS_SQLITE_TABLE_MYSQL_GTID_EXECUTED "CREATE TABLE stats_mysql_gtid_executed (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_executed VARCHAR , events INT NOT NULL)" - -#define STATS_SQLITE_TABLE_MYSQL_ERRORS "CREATE TABLE stats_mysql_errors (hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , errno INT NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , last_error VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup, hostname, port, username, schemaname, errno) )" -#define STATS_SQLITE_TABLE_MYSQL_ERRORS_RESET "CREATE TABLE stats_mysql_errors_reset (hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , errno INT NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , last_error VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup, hostname, port, username, schemaname, errno) )" - -#define STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE "CREATE TABLE stats_mysql_client_host_cache (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)" -#define STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE_RESET "CREATE TABLE stats_mysql_client_host_cache_reset (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)" - -#ifdef DEBUG -#define ADMIN_SQLITE_TABLE_DEBUG_LEVELS "CREATE TABLE debug_levels (module VARCHAR NOT NULL PRIMARY KEY , verbosity INT NOT NULL DEFAULT 0)" -#define ADMIN_SQLITE_TABLE_DEBUG_FILTERS "CREATE TABLE debug_filters (filename VARCHAR NOT NULL , line INT NOT NULL , funct VARCHAR NOT NULL , PRIMARY KEY (filename, line, funct) )" -#endif /* DEBUG */ - -#define ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS_V1_4 "CREATE TABLE mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" - -#define ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS_V2_0_0 "CREATE TABLE mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" - -#define ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS_V2_0_0 - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" - -#define ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS_V2_0_0a "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" - -#define ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS_V2_0_0b "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" - -#define ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS_V2_0_0b - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE runtime_mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1,2)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" - -#define ADMIN_SQLITE_TABLE_COREDUMP_FILTERS "CREATE TABLE coredump_filters (filename VARCHAR NOT NULL , line INT NOT NULL , PRIMARY KEY (filename, line) )" - -#define ADMIN_SQLITE_RUNTIME_COREDUMP_FILTERS "CREATE TABLE runtime_coredump_filters (filename VARCHAR NOT NULL , line INT NOT NULL , PRIMARY KEY (filename, line) )" - -// AWS Aurora - -#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_8 "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , comment VARCHAR , UNIQUE (reader_hostgroup))" - -#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_9 "CREATE TABLE mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , comment VARCHAR , UNIQUE (reader_hostgroup))" - -#define ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_9 - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS "CREATE TABLE runtime_mysql_aws_aurora_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , aurora_port INT NOT NUlL DEFAULT 3306 , domain_name VARCHAR NOT NULL CHECK (SUBSTR(domain_name,1,1) = '.') , max_lag_ms INT NOT NULL CHECK (max_lag_ms>= 10 AND max_lag_ms <= 600000) DEFAULT 600000 , check_interval_ms INT NOT NULL CHECK (check_interval_ms >= 100 AND check_interval_ms <= 600000) DEFAULT 1000 , check_timeout_ms INT NOT NULL CHECK (check_timeout_ms >= 80 AND check_timeout_ms <= 3000) DEFAULT 800 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , new_reader_weight INT CHECK (new_reader_weight >= 0 AND new_reader_weight <=10000000) NOT NULL DEFAULT 1 , add_lag_ms INT NOT NULL CHECK (add_lag_ms >= 0 AND add_lag_ms <= 600000) DEFAULT 30 , min_lag_ms INT NOT NULL CHECK (min_lag_ms >= 0 AND min_lag_ms <= 600000) DEFAULT 30 , lag_num_checks INT NOT NULL CHECK (lag_num_checks >= 1 AND lag_num_checks <= 16) DEFAULT 1 , comment VARCHAR , UNIQUE (reader_hostgroup))" - -#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" - -#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" - -#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_6_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" - -//#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_6_0 - -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE runtime_mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , hostgroup_settings VARCHAR CHECK (JSON_VALID(hostgroup_settings) OR hostgroup_settings = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')" - -#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_6_0 - -// Cluster solution - -#define ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS "CREATE TABLE proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" - -#define ADMIN_SQLITE_TABLE_RUNTIME_PROXYSQL_SERVERS "CREATE TABLE runtime_proxysql_servers (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostname, port) )" - -#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CLIENTS_STATUS "CREATE TABLE stats_proxysql_servers_clients_status (uuid VARCHAR NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , admin_mysql_ifaces VARCHAR NOT NULL , last_seen_at INT NOT NULL , PRIMARY KEY (uuid, hostname, port) )" - -#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS "CREATE TABLE stats_proxysql_servers_status (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , master VARCHAR NOT NULL , global_version INT NOT NULL , check_age_us INT NOT NULL , ping_time_us INT NOT NULL, checks_OK INT NOT NULL , checks_ERR INT NOT NULL , PRIMARY KEY (hostname, port) )" - -#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_METRICS "CREATE TABLE stats_proxysql_servers_metrics (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , response_time_ms INT NOT NULL , Uptime_s INT NOT NULL , last_check_ms INT NOT NULL , Queries INT NOT NULL , Client_Connections_connected INT NOT NULL , Client_Connections_created INT NOT NULL , PRIMARY KEY (hostname, port) )" - -#define STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CHECKSUMS "CREATE TABLE stats_proxysql_servers_checksums (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 6032 , name VARCHAR NOT NULL , version INT NOT NULL , epoch INT NOT NULL , checksum VARCHAR NOT NULL , changed_at INT NOT NULL , updated_at INT NOT NULL , diff_check INT NOT NULL , PRIMARY KEY (hostname, port, name) )" - -#define STATS_SQLITE_TABLE_PROXYSQL_MESSAGE_METRICS "CREATE TABLE stats_proxysql_message_metrics (message_id VARCHAR NOT NULL , filename VARCHAR NOT NULL , line INT CHECK (line >= 0) NOT NULL DEFAULT 0 , func VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , PRIMARY KEY (filename, line, func) )" - -#define STATS_SQLITE_TABLE_PROXYSQL_MESSAGE_METRICS_RESET "CREATE TABLE stats_proxysql_message_metrics_reset (message_id VARCHAR NOT NULL , filename VARCHAR NOT NULL , line INT CHECK (line >= 0) NOT NULL DEFAULT 0 , func VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , PRIMARY KEY (filename, line, func) )" - -#ifdef PROXYSQLCLICKHOUSE -// ClickHouse Tables - -#define ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS_141 "CREATE TABLE clickhouse_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username))" - -#define ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS_141 - -#define ADMIN_SQLITE_TABLE_RUNTIME_CLICKHOUSE_USERS "CREATE TABLE runtime_clickhouse_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username))" -#endif /* PROXYSQLCLICKHOUSE */ - - -#define ADMIN_SQLITE_TABLE_STATS_MYSQL_PREPARED_STATEMENTS_INFO "CREATE TABLE stats_mysql_prepared_statements_info (global_stmt_id INT NOT NULL , schemaname VARCHAR NOT NULL , username VARCHAR NOT NULL , digest VARCHAR NOT NULL , ref_count_client INT NOT NULL , ref_count_server INT NOT NULL , num_columns INT NOT NULL, num_params INT NOT NULL, query VARCHAR NOT NULL)" +#include "ProxySQL_Admin_Tables_Definitions.h" static char * admin_variables_names[]= { (char *)"admin_credentials", @@ -612,6 +365,7 @@ static char * admin_variables_names[]= { (char *)"stats_system_cpu", (char *)"stats_system_memory", (char *)"mysql_ifaces", + (char *)"pgsql_ifaces", (char *)"telnet_admin_ifaces", (char *)"telnet_stats_ifaces", (char *)"refresh_interval", @@ -732,6 +486,12 @@ admin_metrics_map = std::make_tuple( "MySQL listener paused because of PROXYSQL PAUSE.", metric_tags {} ), + std::make_tuple( + p_admin_gauge::pgsql_listener_paused, + "proxysql_pgsql_listener_paused", + "PgSQL listener paused because of PROXYSQL PAUSE.", + metric_tags {} + ), // memory metrics std::make_tuple ( p_admin_gauge::connpool_memory_bytes, @@ -815,7 +575,7 @@ admin_metrics_map = std::make_tuple( metric_tags {} ), std::make_tuple ( - // TODO: Check why 'global_mysql_firewall_whitelist_users_result___size' never updated + // TODO: Check why 'global_firewall_whitelist_users_result___size' never updated p_admin_gauge::mysql_firewall_users_config, "proxysql_mysql_firewall_users_config_bytes", "Full 'mysql_firewall_users' config 'resultset' size.", @@ -927,9 +687,9 @@ admin_metrics_map = std::make_tuple( } ); -static ProxySQL_Admin *SPA=NULL; +ProxySQL_Admin *SPA=NULL; -static void * (*child_func[3]) (void *arg); +void * (*child_func[3]) (void *arg); const std::vector LOAD_ADMIN_VARIABLES_TO_MEMORY = { @@ -990,13 +750,50 @@ const std::vector SAVE_MYSQL_VARIABLES_TO_MEMORY = { "SAVE MYSQL VARIABLES FROM RUNTIME" , "SAVE MYSQL VARIABLES FROM RUN" }; +// PgSQL +const std::vector LOAD_PGSQL_SERVERS_FROM_MEMORY = { + "LOAD PGSQL SERVERS FROM MEMORY" , + "LOAD PGSQL SERVERS FROM MEM" , + "LOAD PGSQL SERVERS TO RUNTIME" , + "LOAD PGSQL SERVERS TO RUN" }; + +const std::vector SAVE_PGSQL_SERVERS_TO_MEMORY = { + "SAVE PGSQL SERVERS TO MEMORY" , + "SAVE PGSQL SERVERS TO MEM" , + "SAVE PGSQL SERVERS FROM RUNTIME" , + "SAVE PGSQL SERVERS FROM RUN" }; + +const std::vector LOAD_PGSQL_USERS_FROM_MEMORY = { + "LOAD PGSQL USERS FROM MEMORY" , + "LOAD PGSQL USERS FROM MEM" , + "LOAD PGSQL USERS TO RUNTIME" , + "LOAD PGSQL USERS TO RUN" }; + +const std::vector SAVE_PGSQL_USERS_TO_MEMORY = { + "SAVE PGSQL USERS TO MEMORY" , + "SAVE PGSQL USERS TO MEM" , + "SAVE PGSQL USERS FROM RUNTIME" , + "SAVE PGSQL USERS FROM RUN" }; + +const std::vector LOAD_PGSQL_VARIABLES_FROM_MEMORY = { + "LOAD PGSQL VARIABLES FROM MEMORY" , + "LOAD PGSQL VARIABLES FROM MEM" , + "LOAD PGSQL VARIABLES TO RUNTIME" , + "LOAD PGSQL VARIABLES TO RUN" }; + +const std::vector SAVE_PGSQL_VARIABLES_TO_MEMORY = { + "SAVE PGSQL VARIABLES TO MEMORY" , + "SAVE PGSQL VARIABLES TO MEM" , + "SAVE PGSQL VARIABLES FROM RUNTIME" , + "SAVE PGSQL VARIABLES FROM RUN" }; +// const std::vector LOAD_COREDUMP_FROM_MEMORY = { "LOAD COREDUMP FROM MEMORY" , "LOAD COREDUMP FROM MEM" , "LOAD COREDUMP TO RUNTIME" , "LOAD COREDUMP TO RUN" }; -static unordered_map, vector>> load_save_disk_commands; +unordered_map, vector>> load_save_disk_commands; static void generate_load_save_disk_commands(std::vector& vec1, std::vector& vec2, const string& name) { string s; @@ -1021,46 +818,7 @@ static void generate_load_save_disk_commands(const string& name, const string& c } -bool is_admin_command_or_alias(const std::vector& cmds, char *query_no_space, int query_no_space_length) { - for (std::vector::const_iterator it=cmds.begin(); it!=cmds.end(); ++it) { - if ((unsigned int)query_no_space_length==it->length() && !strncasecmp(it->c_str(), query_no_space, query_no_space_length)) { - proxy_info("Received %s command\n", query_no_space); - return true; - } - } - return false; -} - -bool FlushCommandWrapper(MySQL_Session *sess, const std::vector& cmds, char *query_no_space, int query_no_space_length, const string& name, const string& direction) { - if ( is_admin_command_or_alias(cmds, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA = GloAdmin; - SPA->flush_GENERIC__from_to(name, direction); -#ifdef DEBUG - string msg = "Loaded " + name + " "; - if (direction == "memory_to_disk") - msg += "from MEMORY to DISK"; - else if (direction == "disk_to_memory") - msg += "from DISK to MEMORY"; - else - assert(0); - msg += "\n"; - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s", msg.c_str()); -#endif // DEBUG - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return true; - } - return false; -} - -bool FlushCommandWrapper(MySQL_Session *sess, const string& modname, char *query_no_space, int query_no_space_length) { - assert(load_save_disk_commands.find(modname) != load_save_disk_commands.end()); - tuple, vector>& t = load_save_disk_commands[modname]; - if (FlushCommandWrapper(sess, get<1>(t), query_no_space, query_no_space_length, modname, "disk_to_memory") == true) - return true; - if (FlushCommandWrapper(sess, get<2>(t), query_no_space, query_no_space_length, modname, "memory_to_disk") == true) - return true; - return false; -} +bool is_admin_command_or_alias(const std::vector& cmds, char *query_no_space, int query_no_space_length); incoming_servers_t::incoming_servers_t() {} @@ -1094,14 +852,31 @@ mysql_servers_v2_checksum_t::mysql_servers_v2_checksum_t() : epoch(0) {} mysql_servers_v2_checksum_t::mysql_servers_v2_checksum_t(const std::string& checksum, time_t epoch) : value(checksum), epoch(epoch) {} -bootstrap_info_t::~bootstrap_info_t() { - if (servers != nullptr) { - mysql_free_result(servers); - } - if (users != nullptr) { - mysql_free_result(users); - } -} + +runtime_pgsql_servers_checksum_t::runtime_pgsql_servers_checksum_t() : epoch(0) {} + +runtime_pgsql_servers_checksum_t::runtime_pgsql_servers_checksum_t(const std::string& checksum, time_t epoch) : + value(checksum), epoch(epoch) {} + +pgsql_servers_v2_checksum_t::pgsql_servers_v2_checksum_t() : epoch(0) {} + +pgsql_servers_v2_checksum_t::pgsql_servers_v2_checksum_t(const std::string& checksum, time_t epoch) : + value(checksum), epoch(epoch) {} + +incoming_pgsql_servers_t::incoming_pgsql_servers_t() {} + +incoming_pgsql_servers_t::incoming_pgsql_servers_t( + SQLite3_result* incoming_pgsql_servers_v2, + SQLite3_result* incoming_replication_hostgroups, + SQLite3_result* incoming_hostgroup_attributes, + SQLite3_result* runtime_pgsql_servers +) : + incoming_pgsql_servers_v2(incoming_pgsql_servers_v2), + incoming_replication_hostgroups(incoming_replication_hostgroups), + incoming_hostgroup_attributes(incoming_hostgroup_attributes), + runtime_pgsql_servers(runtime_pgsql_servers) +{} + peer_runtime_mysql_servers_t::peer_runtime_mysql_servers_t() : resultset(nullptr), checksum() {} @@ -1117,36 +892,20 @@ peer_mysql_servers_v2_t::peer_mysql_servers_v2_t( ) : resultset(resultset), checksum(checksum) {} -int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { - int r = 0; - if (!GloQPro) return 0; - if (use_swap == false) { - SQLite3_result * resultset=NULL; - if (reset==true) { - resultset=GloQPro->get_query_digests_reset(); - } else { - resultset=GloQPro->get_query_digests(); - } - if (resultset==NULL) return 0; - r = resultset->rows_count; - delete resultset; - } else { - umap_query_digest uqd; - umap_query_digest_text uqdt; - GloQPro->get_query_digests_reset(&uqd, &uqdt); - r = uqd.size(); - for (std::unordered_map::iterator it=uqd.begin(); it!=uqd.end(); ++it) { - QP_query_digest_stats * qds = (QP_query_digest_stats *)it->second; - delete qds; - } - uqd.erase(uqd.begin(),uqd.end()); - for (std::unordered_map::iterator it=uqdt.begin(); it!=uqdt.end(); ++it) { - free(it->second); - } - uqdt.erase(uqdt.begin(),uqdt.end()); - } - return r; -} +peer_runtime_pgsql_servers_t::peer_runtime_pgsql_servers_t() : resultset(nullptr), checksum() {} + +peer_runtime_pgsql_servers_t::peer_runtime_pgsql_servers_t( + SQLite3_result* resultset, const runtime_pgsql_servers_checksum_t& checksum +) : resultset(resultset), checksum(checksum) +{} + +peer_pgsql_servers_v2_t::peer_pgsql_servers_v2_t() : resultset(nullptr), checksum() {} + +peer_pgsql_servers_v2_t::peer_pgsql_servers_v2_t( + SQLite3_result* resultset, const pgsql_servers_v2_checksum_t& checksum +) : resultset(resultset), checksum(checksum) +{} + ProxySQL_Config& ProxySQL_Admin::proxysql_config() { static ProxySQL_Config instance = ProxySQL_Config(admindb); @@ -1164,13 +923,19 @@ ProxySQL_Restapi& ProxySQL_Admin::proxysql_restapi() { return instance; } +template int ProxySQL_Admin::FlushDigestTableToDisk(SQLite3DB *_db) { - int r = 0; - if (!GloQPro) return 0; + umap_query_digest uqd; umap_query_digest_text uqdt; - GloQPro->get_query_digests_reset(&uqd, &uqdt); - r = uqd.size(); + if constexpr (pt == SERVER_TYPE_MYSQL) { + if (!GloMyQPro) return 0; + GloMyQPro->get_query_digests_reset(&uqd, &uqdt); + } else if constexpr (pt == SERVER_TYPE_PGSQL) { + if (!GloPgQPro) return 0; + GloPgQPro->get_query_digests_reset(&uqd, &uqdt); + } + int r = uqd.size(); SQLite3DB * sdb = _db; sdb->execute("BEGIN"); int rc; @@ -1179,8 +944,15 @@ int ProxySQL_Admin::FlushDigestTableToDisk(SQLite3DB *_db) { char *query1=NULL; char *query32=NULL; std::string query32s = ""; - query1=(char *)"INSERT INTO history_mysql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; - query32s = "INSERT INTO history_mysql_query_digest VALUES " + generate_multi_rows_query(32,15); + + if constexpr (pt == SERVER_TYPE_MYSQL) { + query1 = (char*)"INSERT INTO history_mysql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; + query32s = "INSERT INTO history_mysql_query_digest VALUES " + generate_multi_rows_query(32, 15); + } else if constexpr (pt == SERVER_TYPE_PGSQL) { + query1 = (char*)"INSERT INTO history_pgsql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)"; + query32s = "INSERT INTO history_pgsql_query_digest VALUES " + generate_multi_rows_query(32, 15); + } + query32 = (char *)query32s.c_str(); rc = sdb->prepare_v2(query1, &statement1); ASSERT_SQLITE_OK(rc, sdb); @@ -1299,2215 +1071,555 @@ int ProxySQL_Admin::FlushDigestTableToDisk(SQLite3DB *_db) { return r; } -bool ProxySQL_Test___Refresh_MySQL_Variables(unsigned int cnt) { - MySQL_Thread *mysql_thr=new MySQL_Thread(); - mysql_thr->curtime=monotonic_time(); - for (unsigned int i = 0; i < cnt ; i++) { - mysql_thr->refresh_variables(); - } - delete mysql_thr; - return true; -} +#include "Admin_ifaces.h" -int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg) { - int r = 0; - r = GloQPro->purge_query_digests(async_purge, parallel, msg); - return r; -} +admin_main_loop_listeners S_amll; -int ProxySQL_Test___GenerateRandomQueryInDigestTable(int n) { - //unsigned long long queries=n; - //queries *= 1000; - MySQL_Session *sess = new MySQL_Session(); - // When the session is destroyed, client_connections is automatically decreased. - // Because this is not a real connection, we artificially increase - // client_connections - __sync_fetch_and_add(&MyHGM->status.client_connections,1); - sess->client_myds = new MySQL_Data_Stream(); - sess->client_myds->fd=0; - sess->client_myds->init(MYDS_FRONTEND, sess, sess->client_myds->fd); - MySQL_Connection *myconn=new MySQL_Connection(); - sess->client_myds->attach_connection(myconn); - myconn->set_is_client(); // this is used for prepared statements - //unsigned long long cur = monotonic_time(); - SQP_par_t qp; - qp.first_comment=NULL; - qp.query_prefix=NULL; - qp.digest_text = (char *)malloc(1024); - MySQL_Connection_userinfo ui; - char * username_buf = (char *)malloc(32); - char * schemaname_buf = (char *)malloc(64); - //ui.username = username_buf; - //ui.schemaname = schemaname_buf; - strcpy(username_buf,"user_name_"); - strcpy(schemaname_buf,"shard_name_"); - bool orig_norm = mysql_thread___query_digests_normalize_digest_text; - for (int i=0; i ? AND a.c IN (?,?,?) ORDER BY k,l DESC LIMIT ?",i, j); - int digest_text_length = strlen(qp.digest_text); - qp.digest=SpookyHash::Hash64(qp.digest_text, digest_text_length, 0); - for (int k=0; k<10; k++) { - //sprintf(username_buf,"user_%d",k%10); - int _a = fastrand(); - int _k = _a%20; - int _j = _a%7; - for (int _i=0 ; _i<_k ; _i++) { - username_buf[10+_i]='0' + (_j+_i)%10; - } - username_buf[10+_k]='\0'; - for (int l=0; l<10; l++) { - //if (fastrand()%100==0) { - // sprintf(schemaname_buf,"long_shard_name_shard_whatever_%d",l%10); - //} else { - // sprintf(schemaname_buf,"shard_%d",l%10); - //} - int _a = fastrand(); - int _k = _a%30; - int _j = _a%11; - for (int _i=0 ; _i<_k ; _i++) { - schemaname_buf[11+_i]='0' + (_j+_i)%10; - } - schemaname_buf[11+_k]='\0'; - ui.set(username_buf, NULL, schemaname_buf, NULL); - int hg = 0; - uint64_t hash2; - SpookyHash myhash; - myhash.Init(19,3); - myhash.Update(ui.username,strlen(ui.username)); - myhash.Update(&qp.digest,sizeof(qp.digest)); - myhash.Update(ui.schemaname,strlen(ui.schemaname)); - myhash.Update(&hg,sizeof(hg)); - myhash.Final(&qp.digest_total,&hash2); - //update_query_digest(qp, sess->current_hostgroup, ui, t, sess->thread->curtime, NULL, sess); - GloQPro->update_query_digest(&qp,hg,&ui,fastrand(),0,NULL,sess); - } - } - } + +template +bool admin_handler_command_kill_connection(char *query_no_space, unsigned int query_no_space_length, S* sess, ProxySQL_Admin *pa) { + uint32_t id=atoi(query_no_space+16); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Trying to kill session %u\n", id); + bool rc=GloMTH->kill_session(id); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (rc) { + SPA->send_ok_msg_to_client(sess, NULL, 0, query_no_space); + } else { + char buf[1024]; + sprintf(buf,"Unknown thread id: %u", id); + SPA->send_error_msg_to_client(sess, buf); } - delete sess; - mysql_thread___query_digests_normalize_digest_text = orig_norm; - return n*1000; + return false; } +void flush_logs_handler() { + GloAdmin->flush_logs(); +} -typedef struct _main_args { - int nfds; - struct pollfd *fds; - int *callback_func; - volatile int *shutdown; -} main_args; - -typedef struct _ifaces_desc_t { - char **mysql_ifaces; - char **telnet_admin_ifaces; - char **telnet_stats_ifaces; -} ifaces_desc_t; - -#define MAX_IFACES 8 -#define MAX_ADMIN_LISTENERS 16 - -class ifaces_desc { - public: - PtrArray *ifaces; - ifaces_desc() { - ifaces=new PtrArray(); +void ProxySQL_Admin::flush_logs() { + if (GloMyLogger) { + GloMyLogger->flush_log(); } - bool add(const char *iface) { - for (unsigned int i=0; ilen; i++) { - if (strcmp((const char *)ifaces->index(i),iface)==0) { - return false; + this->flush_error_log(); + proxysql_keylog_close(); + char* ssl_keylog_file = this->get_variable((char*)"ssl_keylog_file"); + if (ssl_keylog_file != NULL) { + if (strlen(ssl_keylog_file) > 0) { + if (proxysql_keylog_open(ssl_keylog_file) == false) { + // re-opening file failed, setting ssl_keylog_enabled to false + GloVars.global.ssl_keylog_enabled = false; + proxy_warning("Cannot open SSLKEYLOGFILE '%s' for writing.\n", ssl_keylog_file); } } - ifaces->add(strdup(iface)); - return true; - } - ~ifaces_desc() { - while(ifaces->len) { - char *d=(char *)ifaces->remove_index_fast(0); - free(d); - } - delete ifaces; + free(ssl_keylog_file); } -}; - -class admin_main_loop_listeners { - private: - int version; -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_t rwlock; -#else - rwlock_t rwlock; -#endif +} - char ** reset_ifaces(char **ifaces) { - int i; - if (ifaces) { - for (i=0; i(MySQL_Session*, char const*, int, char const*); +template void ProxySQL_Admin::send_ok_msg_to_client(PgSQL_Session*, char const*, int, char const*); +template void ProxySQL_Admin::send_error_msg_to_client(MySQL_Session*, char const*, unsigned short); +template void ProxySQL_Admin::send_error_msg_to_client(PgSQL_Session*, char const*, unsigned short); +template int ProxySQL_Admin::FlushDigestTableToDisk<(SERVER_TYPE)0>(SQLite3DB*); +template int ProxySQL_Admin::FlushDigestTableToDisk<(SERVER_TYPE)1>(SQLite3DB*); - public: - int nfds; - struct pollfd *fds; - int *callback_func; - int get_version() { return version; } - void wrlock() { -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_wrlock(&rwlock); -#else - spin_wrlock(&rwlock); -#endif - } - void wrunlock() { -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_unlock(&rwlock); -#else - spin_wrunlock(&rwlock); -#endif - } - ifaces_desc *ifaces_mysql; - ifaces_desc *ifaces_telnet_admin; - ifaces_desc *ifaces_telnet_stats; - ifaces_desc_t descriptor_new; - admin_main_loop_listeners() { -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_init(&rwlock, NULL); -#else - spinlock_rwlock_init(&rwlock); -#endif - ifaces_mysql=new ifaces_desc(); - ifaces_telnet_admin=new ifaces_desc(); - ifaces_telnet_stats=new ifaces_desc(); - version=0; - descriptor_new.mysql_ifaces=NULL; - descriptor_new.telnet_admin_ifaces=NULL; - descriptor_new.telnet_stats_ifaces=NULL; - } - - - void update_ifaces(char *list, ifaces_desc **ifd) { - wrlock(); - delete *ifd; - *ifd=new ifaces_desc(); - int i=0; - tokenizer_t tok; - tokenizer( &tok, list, ";", TOKENIZER_NO_EMPTIES ); - const char* token; - for ( token = tokenize( &tok ) ; token && i < MAX_IFACES ; token = tokenize( &tok ) ) { - (*ifd)->add(token); - i++; - } - free_tokenizer( &tok ); - version++; - wrunlock(); - } - - - bool update_ifaces(char *list, char ***_ifaces) { - wrlock(); - int i; - char **ifaces=*_ifaces; - tokenizer_t tok; - tokenizer( &tok, list, ";", TOKENIZER_NO_EMPTIES ); - const char* token; - ifaces=reset_ifaces(ifaces); - i=0; - for ( token = tokenize( &tok ) ; token && i < MAX_IFACES ; token = tokenize( &tok ) ) { - ifaces[i]=(char *)malloc(strlen(token)+1); - strcpy(ifaces[i],token); - i++; - } - free_tokenizer( &tok ); - version++; - wrunlock(); - return true; - } -}; - -static admin_main_loop_listeners S_amll; +void ProxySQL_Admin::flush_configdb() { // see #923 + wrlock(); + admindb->execute((char *)"DETACH DATABASE disk"); + delete configdb; + configdb=new SQLite3DB(); + configdb->open((char *)GloVars.admindb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + __attach_db(admindb, configdb, (char *)"disk"); + // Fully synchronous is not required. See to #1055 + // https://sqlite.org/pragma.html#pragma_synchronous + configdb->execute("PRAGMA synchronous=0"); + wrunlock(); +} +bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsigned int query_no_space_length, bool admin) { + bool ret=false; + bool refresh=false; + bool stats_mysql_processlist=false; + bool stats_pgsql_processlist=false; + bool stats_mysql_free_connections=false; + bool stats_pgsql_free_connections=false; + bool stats_mysql_connection_pool=false; + bool stats_mysql_connection_pool_reset=false; + bool stats_mysql_query_digest=false; + bool stats_mysql_query_digest_reset=false; + bool stats_mysql_errors=false; + bool stats_mysql_errors_reset=false; + bool stats_pgsql_errors = false; + bool stats_pgsql_errors_reset = false; + bool stats_mysql_global=false; + bool stats_memory_metrics=false; + bool stats_mysql_commands_counters=false; + bool stats_mysql_query_rules=false; + bool stats_mysql_users=false; + bool stats_pgsql_users = false; + bool stats_mysql_gtid_executed=false; + bool stats_mysql_client_host_cache=false; + bool stats_mysql_client_host_cache_reset=false; + bool stats_pgsql_client_host_cache = false; + bool stats_pgsql_client_host_cache_reset = false; + bool dump_global_variables=false; + bool runtime_scheduler=false; + bool runtime_restapi_routes=false; + bool runtime_mysql_users=false; + bool runtime_mysql_firewall=false; + bool runtime_mysql_ldap_mapping=false; + bool runtime_mysql_servers=false; + bool runtime_mysql_query_rules=false; + bool runtime_mysql_query_rules_fast_routing=false; -bool admin_handler_command_kill_connection(char *query_no_space, unsigned int query_no_space_length, MySQL_Session *sess, ProxySQL_Admin *pa) { - uint32_t id=atoi(query_no_space+16); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Trying to kill session %u\n", id); - bool rc=GloMTH->kill_session(id); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if (rc) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - } else { - char buf[1024]; - sprintf(buf,"Unknown thread id: %u", id); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, buf); - } - return false; -} + bool runtime_pgsql_users = false; + bool runtime_pgsql_firewall = false; + bool runtime_pgsql_ldap_mapping = false; + bool runtime_pgsql_servers = false; + bool runtime_pgsql_query_rules = false; + bool runtime_pgsql_query_rules_fast_routing = false; -static void flush_logs_handler() { - GloAdmin->flush_logs(); -} + bool stats_pgsql_global = false; + bool stats_pgsql_connection_pool = false; + bool stats_pgsql_connection_pool_reset = false; -void ProxySQL_Admin::flush_logs() { - if (GloMyLogger) { - GloMyLogger->flush_log(); - } - this->flush_error_log(); - proxysql_keylog_close(); - char* ssl_keylog_file = this->get_variable((char*)"ssl_keylog_file"); - if (ssl_keylog_file != NULL) { - if (strlen(ssl_keylog_file) > 0) { - if (proxysql_keylog_open(ssl_keylog_file) == false) { - // re-opening file failed, setting ssl_keylog_enabled to false - GloVars.global.ssl_keylog_enabled = false; - proxy_warning("Cannot open SSLKEYLOGFILE '%s' for writing.\n", ssl_keylog_file); - } - } - free(ssl_keylog_file); - } -} + bool runtime_proxysql_servers=false; + bool runtime_checksums_values=false; -/* - * returns false if the command is a valid one and is processed - * return true if the command is not a valid one and needs to be executed by SQLite (that will return an error) - */ -bool admin_handler_command_proxysql(char *query_no_space, unsigned int query_no_space_length, MySQL_Session *sess, ProxySQL_Admin *pa) { + bool runtime_coredump_filters=false; -#if (defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__) || defined(__mips__)) && defined(__linux) - // currently only support x86-32, x86-64, ARM, and MIPS on Linux - if (!(strncasecmp("PROXYSQL COREDUMP", query_no_space, strlen("PROXYSQL COREDUMP")))) { - string filename = "core"; - if (query_no_space_length > strlen("PROXYSQL COREDUMP")) { - if (query_no_space[strlen("PROXYSQL COREDUMP")] == ' ') { - filename = string(query_no_space+strlen("PROXYSQL COREDUMP ")); - } else { - filename = ""; - } - } - if (filename == "") { - proxy_error("Received incorrect PROXYSQL COREDUMP command: %s\n", query_no_space); - } else { - proxy_info("Received PROXYSQL COREDUMP command: %s\n", query_no_space); - // generates a core dump - WriteCoreDump(filename.c_str()); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - string msg = "Coredump: " + filename; - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)msg.c_str()); - return false; - } - } - if (!(strncasecmp("PROXYSQL COMPRESSEDCOREDUMP", query_no_space, strlen("PROXYSQL COMPRESSEDCOREDUMP")))) { - string filename = "core"; - if (query_no_space_length > strlen("PROXYSQL COMPRESSEDCOREDUMP")) { - if (query_no_space[strlen("PROXYSQL COMPRESSEDCOREDUMP")] == ' ') { - filename = string(query_no_space+strlen("PROXYSQL COMPRESSEDCOREDUMP ")); - } else { - filename = ""; - } - } - if (filename == "") { - proxy_error("Received incorrect PROXYSQL COMPRESSEDCOREDUMP command: %s\n", query_no_space); - } else { - proxy_info("Received PROXYSQL COMPRESSEDCOREDUMP command: %s\n", query_no_space); - // generates a compressed core dump - WriteCompressedCoreDump(filename.c_str(), SIZE_MAX, COREDUMPER_COMPRESSED, NULL); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - string msg = "Coredump: " + filename; - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)msg.c_str()); - return false; - } - } -#endif + bool stats_mysql_prepared_statements_info = false; - if (!(strncasecmp("PROXYSQL CLUSTER_NODE_UUID ", query_no_space, strlen("PROXYSQL CLUSTER_NODE_UUID ")))) { - int l = strlen("PROXYSQL CLUSTER_NODE_UUID "); - if (sess->client_myds->addr.port == 0) { - proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID not from TCP socket. Exiting client\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Received PROXYSQL CLUSTER_NODE_UUID not from TCP socket"); - sess->client_myds->shut_soft(); - return false; - } - if (query_no_space_length >= (unsigned int)l+36+2) { - uuid_t uu; - char *A_uuid = NULL; - char *B_interface = NULL; - c_split_2(query_no_space+l, " ", &A_uuid, &B_interface); // we split the value - if (uuid_parse(A_uuid, uu)==0 && B_interface && strlen(B_interface)) { - proxy_info("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d : %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); - if (sess->proxysql_node_address==NULL) { - sess->proxysql_node_address = new ProxySQL_Node_Address(sess->client_myds->addr.addr, sess->client_myds->addr.port); - sess->proxysql_node_address->uuid = strdup(A_uuid); - if (sess->proxysql_node_address->admin_mysql_ifaces) { - free(sess->proxysql_node_address->admin_mysql_ifaces); - } - sess->proxysql_node_address->admin_mysql_ifaces = strdup(B_interface); - proxy_info("Created new link with Cluster node %s:%d : %s at interface %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid, B_interface); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - free(A_uuid); - free(B_interface); - return false; - } else { - if (strcmp(A_uuid, sess->proxysql_node_address->uuid)) { - proxy_error("Cluster node %s:%d is sending a new UUID : %s . Former UUID : %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid, sess->proxysql_node_address->uuid); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with a new UUID not matching the previous one"); - sess->client_myds->shut_soft(); - free(A_uuid); - free(B_interface); - return false; - } else { - proxy_info("Cluster node %s:%d is sending again its UUID : %s\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, A_uuid); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - free(A_uuid); - free(B_interface); - return false; - } - } - free(A_uuid); - free(B_interface); - return false; - } else { - proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d with invalid format: %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with invalid format"); - sess->client_myds->shut_soft(); - return false; - } - } else { - proxy_warning("Received PROXYSQL CLUSTER_NODE_UUID from %s:%d with invalid format: %s . Exiting client\n", sess->client_myds->addr.addr, sess->client_myds->addr.port, query_no_space+l); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Received PROXYSQL CLUSTER_NODE_UUID with invalid format"); - sess->client_myds->shut_soft(); - return false; - } - } - if (query_no_space_length==strlen("PROXYSQL READONLY") && !strncasecmp("PROXYSQL READONLY",query_no_space, query_no_space_length)) { - // this command enables admin_read_only , so the admin module is in read_only mode - proxy_info("Received PROXYSQL READONLY command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->set_read_only(true); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - if (query_no_space_length==strlen("PROXYSQL READWRITE") && !strncasecmp("PROXYSQL READWRITE",query_no_space, query_no_space_length)) { - // this command disables admin_read_only , so the admin module won't be in read_only mode - proxy_info("Received PROXYSQL WRITE command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->set_read_only(false); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - if (query_no_space_length==strlen("PROXYSQL START") && !strncasecmp("PROXYSQL START",query_no_space, query_no_space_length)) { - proxy_info("Received PROXYSQL START command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - bool rc=false; - if (nostart_) { - rc=__sync_bool_compare_and_swap(&GloVars.global.nostart,1,0); - } - if (rc) { - // Set the status variable 'threads_initialized' to 0 because it's initialized back - // in main 'init_phase3'. After GloMTH have been initialized again. - __sync_bool_compare_and_swap(&GloMTH->status_variables.threads_initialized, 1, 0); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Starting ProxySQL following PROXYSQL START command\n"); - while(__sync_fetch_and_add(&GloMTH->status_variables.threads_initialized, 0) == 1) { - usleep(1000); - } - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - } else { - proxy_warning("ProxySQL was already started when received PROXYSQL START command\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"ProxySQL already started"); - } - return false; - } +#ifdef PROXYSQLCLICKHOUSE + bool runtime_clickhouse_users = false; +#endif /* PROXYSQLCLICKHOUSE */ - if (query_no_space_length==strlen("PROXYSQL RESTART") && !strncasecmp("PROXYSQL RESTART",query_no_space, query_no_space_length)) { - proxy_info("Received PROXYSQL RESTART command\n"); - // This function was introduced into 'prometheus::Registry' for being - // able to do a complete reset of all the 'prometheus counters'. It - // shall only be used during ProxySQL shutdown phases. - GloVars.prometheus_registry->ResetCounters(); - __sync_bool_compare_and_swap(&glovars.shutdown,0,1); - glovars.reload=1; - return false; - } + bool monitor_mysql_server_group_replication_log=false; - if (query_no_space_length==strlen("PROXYSQL STOP") && !strncasecmp("PROXYSQL STOP",query_no_space, query_no_space_length)) { - proxy_info("Received PROXYSQL STOP command\n"); - // to speed up this process we first change wait_timeout to 0 - // MySQL_thread will call poll() with a maximum timeout of 100ms - old_wait_timeout=GloMTH->get_variable_int((char *)"wait_timeout"); - GloMTH->set_variable((char *)"wait_timeout",(char *)"0"); - GloMTH->commit(); - GloMTH->signal_all_threads(0); - GloMTH->stop_listeners(); - char buf[32]; - sprintf(buf,"%d",old_wait_timeout); - GloMTH->set_variable((char *)"wait_timeout",buf); - GloMTH->commit(); - glovars.reload=2; - // This function was introduced into 'prometheus::Registry' for being - // able to do a complete reset of all the 'prometheus counters'. It - // shall only be used during ProxySQL shutdown phases. - GloVars.prometheus_registry->ResetCounters(); - __sync_bool_compare_and_swap(&glovars.shutdown,0,1); - // After setting the shutdown flag, we should wake all threads and wait for - // the shutdown phase to complete. - GloMTH->signal_all_threads(0); - while (__sync_fetch_and_add(&glovars.shutdown,0)==1) { - usleep(1000); - } - // After shutdown phase is completed, we must to send a 'OK' to the - // mysql client, otherwise, since this session might not be drop due - // to the waiting condition, the client wont disconnect and will - // keep forever waiting for acknowledgement. - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } + bool monitor_mysql_server_galera_log=false; - if (query_no_space_length==strlen("PROXYSQL PAUSE") && !strncasecmp("PROXYSQL PAUSE",query_no_space, query_no_space_length)) { - proxy_info("Received PROXYSQL PAUSE command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if (nostart_) { - if (__sync_fetch_and_add((uint8_t *)(&GloVars.global.nostart),0)) { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"ProxySQL MySQL module not running, impossible to pause"); - return false; - } - } - if (proxysql_mysql_paused==false) { - // to speed up this process we first change poll_timeout to 10 - // MySQL_thread will call poll() with a maximum timeout of 10ms - old_wait_timeout=GloMTH->get_variable_int((char *)"poll_timeout"); - GloMTH->set_variable((char *)"poll_timeout",(char *)"10"); - GloMTH->commit(); - GloMTH->signal_all_threads(0); - GloMTH->stop_listeners(); - proxysql_mysql_paused=true; - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - // we now rollback poll_timeout - char buf[32]; - sprintf(buf,"%d",old_wait_timeout); - GloMTH->set_variable((char *)"poll_timeout",buf); - GloMTH->commit(); - } else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"ProxySQL MySQL module is already paused, impossible to pause"); - } - return false; - } + bool monitor_mysql_server_aws_aurora_log=false; + bool monitor_mysql_server_aws_aurora_check_status=false; - if (query_no_space_length==strlen("PROXYSQL RESUME") && !strncasecmp("PROXYSQL RESUME",query_no_space, query_no_space_length)) { - proxy_info("Received PROXYSQL RESUME command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if (nostart_) { - if (__sync_fetch_and_add((uint8_t *)(&GloVars.global.nostart),0)) { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"ProxySQL MySQL module not running, impossible to resume"); - return false; - } - } - if (proxysql_mysql_paused==true) { - // to speed up this process we first change poll_timeout to 10 - // MySQL_thread will call poll() with a maximum timeout of 10ms - old_wait_timeout=GloMTH->get_variable_int((char *)"poll_timeout"); - GloMTH->set_variable((char *)"poll_timeout",(char *)"10"); - GloMTH->commit(); - GloMTH->signal_all_threads(0); - GloMTH->start_listeners(); - //char buf[32]; - //sprintf(buf,"%d",old_wait_timeout); - //GloMTH->set_variable((char *)"poll_timeout",buf); - //GloMTH->commit(); - proxysql_mysql_paused=false; - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - // we now rollback poll_timeout - char buf[32]; - sprintf(buf,"%d",old_wait_timeout); - GloMTH->set_variable((char *)"poll_timeout",buf); - GloMTH->commit(); - } else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"ProxySQL MySQL module is not paused, impossible to resume"); - } - return false; - } + bool stats_proxysql_servers_checksums = false; + bool stats_proxysql_servers_metrics = false; + bool stats_proxysql_message_metrics = false; + bool stats_proxysql_message_metrics_reset = false; - if (query_no_space_length==strlen("PROXYSQL SHUTDOWN SLOW") && !strncasecmp("PROXYSQL SHUTDOWN SLOW",query_no_space, query_no_space_length)) { - glovars.proxy_restart_on_error=false; - glovars.reload=0; - proxy_info("Received PROXYSQL SHUTDOWN SLOW command\n"); - __sync_bool_compare_and_swap(&glovars.shutdown,0,1); - return false; - } + //bool stats_proxysql_servers_status = false; // temporary disabled because not implemented - if (query_no_space_length==strlen("PROXYSQL FLUSH LOGS") && !strncasecmp("PROXYSQL FLUSH LOGS",query_no_space, query_no_space_length)) { - proxy_info("Received PROXYSQL FLUSH LOGS command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_logs(); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + if (strcasestr(query_no_space, "pgsql processlist") || + strcasestr(query_no_space, "stats_pgsql_processlist")) + // This will match the following usecases: + // SHOW PGSQL PROCESSLIST + // SHOW FULL PGSQL PROCESSLIST + // SELECT * FROM stats_pgsql_processlist + { + stats_pgsql_processlist = true; refresh = true; + } else if (strcasestr(query_no_space,"processlist")) + // This will match the following usecases: + // SHOW PROCESSLIST + // SHOW FULL PROCESSLIST + // SELECT * FROM stats_mysql_processlist + { + stats_mysql_processlist=true; refresh=true; } - - if (query_no_space_length==strlen("PROXYSQL FLUSH QUERY CACHE") && !strncasecmp("PROXYSQL FLUSH QUERY CACHE",query_no_space, query_no_space_length)) { - proxy_info("Received PROXYSQL FLUSH QUERY CACHE command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if (GloQC) { - GloQC->flush(); + if (strstr(query_no_space,"stats_mysql_query_digest")) + { stats_mysql_query_digest=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_query_digest_reset")) + { stats_mysql_query_digest_reset=true; refresh=true; } + if (stats_mysql_query_digest_reset == true && stats_mysql_query_digest == true) { + int nd = 0; + int ndr= 0; + char *c = NULL; + char *_ret = NULL; + c = (char *)query_no_space; + _ret = NULL; + while ((_ret = strstr(c,"stats_mysql_query_digest_reset"))) { + ndr++; + c = _ret + strlen("stats_mysql_query_digest_reset"); } - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if (!strcasecmp("PROXYSQL FLUSH MYSQL CLIENT HOSTS", query_no_space)) { - proxy_info("Received PROXYSQL FLUSH MYSQL CLIENT HOSTS command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if (GloMTH) { - GloMTH->flush_client_host_cache(); + c = (char *)query_no_space; + _ret = NULL; + while ((_ret = strstr(c,"stats_mysql_query_digest"))) { + nd++; + c = _ret + strlen("stats_mysql_query_digest"); } - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("PROXYSQL FLUSH CONFIGDB") && !strncasecmp("PROXYSQL FLUSH CONFIGDB",query_no_space, query_no_space_length)) // see #923 - ) { - proxy_info("Received %s command\n", query_no_space); - proxy_warning("A misconfigured configdb will cause undefined behaviors\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_configdb(); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if (strcasecmp("PROXYSQL RELOAD TLS",query_no_space) == 0) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - std::string s; - int rc = ProxySQL_create_or_load_TLS(false, s); - if (rc == 0) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, s.length() ? (char *)s.c_str() : NULL); - } else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, s.length() ? (char *)s.c_str() : (char *)"RELOAD TLS failed"); + if (nd == ndr) { + stats_mysql_query_digest = false; } - return false; - } - -#ifndef NOJEM - if (query_no_space_length==strlen("PROXYSQL MEMPROFILE START") && !strncasecmp("PROXYSQL MEMPROFILE START",query_no_space, query_no_space_length)) { - bool en=true; - mallctl("prof.active", NULL, NULL, &en, sizeof(bool)); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - if (query_no_space_length==strlen("PROXYSQL MEMPROFILE STOP") && !strncasecmp("PROXYSQL MEMPROFILE STOP",query_no_space, query_no_space_length)) { - bool en=false; - mallctl("prof.active", NULL, NULL, &en, sizeof(bool)); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } -#endif - -#ifdef WITHGCOV - if (query_no_space_length==strlen("PROXYSQL GCOV DUMP") && !strncasecmp("PROXYSQL GCOV DUMP",query_no_space, query_no_space_length)) { - proxy_info("Received %s command\n", query_no_space); - __gcov_dump(); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - if (query_no_space_length==strlen("PROXYSQL GCOV RESET") && !strncasecmp("PROXYSQL GCOV RESET",query_no_space, query_no_space_length)) { - proxy_info("Received %s command\n", query_no_space); - __gcov_reset(); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; } -#endif - - if (query_no_space_length==strlen("PROXYSQL KILL") && !strncasecmp("PROXYSQL KILL",query_no_space, query_no_space_length)) { - proxy_info("Received PROXYSQL KILL command\n"); - exit(EXIT_SUCCESS); - } - - if (query_no_space_length==strlen("PROXYSQL SHUTDOWN") && !strncasecmp("PROXYSQL SHUTDOWN",query_no_space, query_no_space_length)) { - // in 2.1 , PROXYSQL SHUTDOWN behaves like PROXYSQL KILL : quick exit - // the former PROXYQL SHUTDOWN is now replaced with PROXYSQL SHUTDOWN SLOW - proxy_info("Received PROXYSQL SHUTDOWN command\n"); - exit(EXIT_SUCCESS); - } - - return true; -} - -// Returns true if the given name is either a know mysql or admin global variable. -bool is_valid_global_variable(const char *var_name) { - if (strlen(var_name) > 6 && !strncmp(var_name, "mysql-", 6) && GloMTH->has_variable(var_name + 6)) { - return true; - } else if (strlen(var_name) > 6 && !strncmp(var_name, "admin-", 6) && SPA->has_variable(var_name + 6)) { - return true; - } else if (strlen(var_name) > 5 && !strncmp(var_name, "ldap-", 5) && GloMyLdapAuth && GloMyLdapAuth->has_variable(var_name + 5)) { - return true; - } else if (strlen(var_name) > 13 && !strncmp(var_name, "sqliteserver-", 13) && GloSQLite3Server && GloSQLite3Server->has_variable(var_name + 13)) { - return true; -#ifdef PROXYSQLCLICKHOUSE - } else if (strlen(var_name) > 11 && !strncmp(var_name, "clickhouse-", 11) && GloClickHouseServer && GloClickHouseServer->has_variable(var_name + 11)) { - return true; -#endif /* PROXYSQLCLICKHOUSE */ + if (strstr(query_no_space,"stats_mysql_errors")) + { stats_mysql_errors=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_errors_reset")) + { stats_mysql_errors_reset=true; refresh=true; } + if (strstr(query_no_space, "stats_pgsql_errors")) + { stats_pgsql_errors = true; refresh = true; } + if (strstr(query_no_space, "stats_pgsql_errors_reset")) + { stats_pgsql_errors_reset = true; refresh = true; } + if (strstr(query_no_space,"stats_mysql_global")) + { stats_mysql_global=true; refresh=true; } + if (strstr(query_no_space, "stats_pgsql_global")) + { stats_pgsql_global = true; refresh = true; } + if (strstr(query_no_space,"stats_memory_metrics")) + { stats_memory_metrics=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_connection_pool_reset")) { + stats_mysql_connection_pool_reset=true; refresh=true; } else { - return false; + if (strstr(query_no_space,"stats_mysql_connection_pool")) + { stats_mysql_connection_pool=true; refresh=true; } } -} - -// This method translates a 'SET variable=value' command into an equivalent UPDATE. It doesn't yes support setting -// multiple variables at once. -// -// It modifies the original query. -bool admin_handler_command_set(char *query_no_space, unsigned int query_no_space_length, MySQL_Session *sess, ProxySQL_Admin *pa, char **q, unsigned int *ql) { - if (!strstr(query_no_space,(char *)"password")) { // issue #599 - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received command %s\n", query_no_space); - if (strncasecmp(query_no_space,(char *)"set autocommit",strlen((char *)"set autocommit"))) { - if (strncasecmp(query_no_space,(char *)"SET @@session.autocommit",strlen((char *)"SET @@session.autocommit"))) { - proxy_info("Received command %s\n", query_no_space); - } - } - } - // Get a pointer to the beginnig of var=value entry and split to get var name and value - char *set_entry = query_no_space + strlen("SET "); - char *untrimmed_var_name=NULL; - char *var_value=NULL; - c_split_2(set_entry, "=", &untrimmed_var_name, &var_value); - - // Trim spaces from var name to allow writing like 'var = value' - char *var_name = trim_spaces_in_place(untrimmed_var_name); - - if (strstr(var_name,(char *)"password") || strcmp(var_name,(char *)"mysql-default_authentication_plugin")==0) { - proxy_info("Received SET command for %s\n", var_name); - } - - bool run_query = false; - // Check if the command tries to set a non-existing variable. - if (strcmp(var_name,"mysql-init_connect")==0) { - char *err_msg_fmt = (char *) "ERROR: Global variable '%s' is not configurable using SET command. You must run UPDATE global_variables"; - size_t buff_len = strlen(err_msg_fmt) + strlen(var_name) + 1; - char *buff = (char *) malloc(buff_len); - snprintf(buff, buff_len, err_msg_fmt, var_name); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, buff); - free(buff); - run_query = false; + if (strstr(query_no_space, "stats_pgsql_connection_pool_reset")) { + stats_pgsql_connection_pool_reset = true; refresh = true; } else { - if (!is_valid_global_variable(var_name)) { - char *err_msg_fmt = (char *) "ERROR: Unknown global variable: '%s'."; - size_t buff_len = strlen(err_msg_fmt) + strlen(var_name) + 1; - char *buff = (char *) malloc(buff_len); - snprintf(buff, buff_len, err_msg_fmt, var_name); - SPA->send_MySQL_OK(&sess->client_myds->myprot, buff); - free(buff); - run_query = false; - } else { - const char *update_format = (char *)"UPDATE global_variables SET variable_value=%s WHERE variable_name='%s'"; - // Computed length is more than needed since it also counts the format modifiers (%s). - size_t query_len = strlen(update_format) + strlen(var_name) + strlen(var_value) + 1; - char *query = (char *)l_alloc(query_len); - snprintf(query, query_len, update_format, var_value, var_name); - - run_query = true; - l_free(*ql,*q); - *q = query; - *ql = strlen(*q) + 1; + if (strstr(query_no_space, "stats_pgsql_connection_pool")) { + stats_pgsql_connection_pool = true; refresh = true; } } - free(untrimmed_var_name); - free(var_value); - return run_query; -} - -/* Note: - * This function can modify the original query - */ -bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query_no_space_length, MySQL_Session *sess, ProxySQL_Admin *pa, char **q, unsigned int *ql) { - proxy_debug(PROXY_DEBUG_ADMIN, 5, "Received command %s\n", query_no_space); - -#ifdef DEBUG - if ((query_no_space_length>11) && ( (!strncasecmp("SAVE DEBUG ", query_no_space, 11)) || (!strncasecmp("LOAD DEBUG ", query_no_space, 11))) ) { - if ( - (query_no_space_length==strlen("LOAD DEBUG TO MEMORY") && !strncasecmp("LOAD DEBUG TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD DEBUG TO MEM") && !strncasecmp("LOAD DEBUG TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD DEBUG FROM DISK") && !strncasecmp("LOAD DEBUG FROM DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - // we are now copying the data from memory to disk - // tables involved are: - // * debug_levels - // * debug_filters - // We only delete from filters and not from levels because the - // levels are hardcoded and fixed in number, while filters can - // be arbitrary - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->admindb->execute("DELETE FROM main.debug_filters"); - SPA->admindb->execute("INSERT OR REPLACE INTO main.debug_levels SELECT * FROM disk.debug_levels"); - SPA->admindb->execute("INSERT INTO main.debug_filters SELECT * FROM disk.debug_filters"); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded debug levels/filters to MEMORY\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("SAVE DEBUG FROM MEMORY") && !strncasecmp("SAVE DEBUG FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE DEBUG FROM MEM") && !strncasecmp("SAVE DEBUG FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE DEBUG TO DISK") && !strncasecmp("SAVE DEBUG TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - // we are now copying the data from disk to memory - // tables involved are: - // * debug_levels - // * debug_filters - // We only delete from filters and not from levels because the - // levels are hardcoded and fixed in number, while filters can - // be arbitrary - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->admindb->execute("DELETE FROM disk.debug_filters"); - SPA->admindb->execute("INSERT OR REPLACE INTO disk.debug_levels SELECT * FROM main.debug_levels"); - SPA->admindb->execute("INSERT INTO disk.debug_filters SELECT * FROM main.debug_filters"); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved debug levels/filters to DISK\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("LOAD DEBUG FROM MEMORY") && !strncasecmp("LOAD DEBUG FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD DEBUG FROM MEM") && !strncasecmp("LOAD DEBUG FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD DEBUG TO RUNTIME") && !strncasecmp("LOAD DEBUG TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD DEBUG TO RUN") && !strncasecmp("LOAD DEBUG TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - int rc=SPA->load_debug_to_runtime(); - if (rc) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded debug levels/filters to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 1, "Error while loading debug levels/filters to RUNTIME\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Error while loading debug levels/filters to RUNTIME"); - } - return false; - } - - if ( - (query_no_space_length==strlen("SAVE DEBUG TO MEMORY") && !strncasecmp("SAVE DEBUG TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE DEBUG TO MEM") && !strncasecmp("SAVE DEBUG TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE DEBUG FROM RUNTIME") && !strncasecmp("SAVE DEBUG FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE DEBUG FROM RUN") && !strncasecmp("SAVE DEBUG FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_debug_from_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved debug levels/filters from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - } -#endif /* DEBUG */ - - if ((query_no_space_length>13) && ( (!strncasecmp("SAVE RESTAPI ", query_no_space, 13)) || (!strncasecmp("LOAD RESTAPI ", query_no_space, 13))) ) { - - if (FlushCommandWrapper(sess, "restapi", query_no_space, query_no_space_length) == true) - return false; - - if ( - (query_no_space_length==strlen("LOAD RESTAPI FROM MEMORY") && !strncasecmp("LOAD RESTAPI FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD RESTAPI FROM MEM") && !strncasecmp("LOAD RESTAPI FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD RESTAPI TO RUNTIME") && !strncasecmp("LOAD RESTAPI TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD RESTAPI TO RUN") && !strncasecmp("LOAD RESTAPI TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->proxysql_restapi().load_restapi_to_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded restapito RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("LOAD RESTAPI FROM CONFIG") && !strncasecmp("LOAD RESTAPI FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - int rows=0; - rows=SPA->proxysql_config().Read_Restapi_from_configfile(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded restapi from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); - } - return false; - } - - if ( - (query_no_space_length==strlen("SAVE RESTAPI TO MEMORY") && !strncasecmp("SAVE RESTAPI TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE RESTAPI TO MEM") && !strncasecmp("SAVE RESTAPI TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE RESTAPI FROM RUNTIME") && !strncasecmp("SAVE RESTAPI FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE RESTAPI FROM RUN") && !strncasecmp("SAVE RESTAPI FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_scheduler_runtime_to_database(false); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved scheduler from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - } - - if ((query_no_space_length>15) && ( (!strncasecmp("SAVE SCHEDULER ", query_no_space, 15)) || (!strncasecmp("LOAD SCHEDULER ", query_no_space, 15))) ) { - - if (FlushCommandWrapper(sess, "scheduler", query_no_space, query_no_space_length) == true) - return false; - - if ( - (query_no_space_length==strlen("LOAD SCHEDULER FROM MEMORY") && !strncasecmp("LOAD SCHEDULER FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SCHEDULER FROM MEM") && !strncasecmp("LOAD SCHEDULER FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SCHEDULER TO RUNTIME") && !strncasecmp("LOAD SCHEDULER TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SCHEDULER TO RUN") && !strncasecmp("LOAD SCHEDULER TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->load_scheduler_to_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded scheduler to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("LOAD SCHEDULER FROM CONFIG") && !strncasecmp("LOAD SCHEDULER FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - int rows=0; - rows=SPA->proxysql_config().Read_Scheduler_from_configfile(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded scheduler from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); - } - return false; - } - - if ( - (query_no_space_length==strlen("SAVE SCHEDULER TO MEMORY") && !strncasecmp("SAVE SCHEDULER TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SCHEDULER TO MEM") && !strncasecmp("SAVE SCHEDULER TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SCHEDULER FROM RUNTIME") && !strncasecmp("SAVE SCHEDULER FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SCHEDULER FROM RUN") && !strncasecmp("SAVE SCHEDULER FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_scheduler_runtime_to_database(false); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved scheduler from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - } - if ((query_no_space_length>16) && (!strncasecmp("LOAD MYSQL USER ", query_no_space, 16)) ) { - if (query_no_space_length>27) { - if (!strncasecmp(" TO RUNTIME", query_no_space+query_no_space_length-11, 11)) { - char *name=(char *)malloc(query_no_space_length-27+1); - strncpy(name,query_no_space+16,query_no_space_length-27); - name[query_no_space_length-27]=0; - int i=0; - int s=strlen(name); - bool legitname=true; - for (i=0; i= 'a' && c <= 'z') || - (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') || - ( (c == '-') || (c == '+') || (c == '_')) - ) { - v=true; - } - if (v==false) { - legitname=false; - } - } - if (legitname) { - proxy_info("Loading user %s\n", name); - pthread_mutex_lock(&users_mutex); - SPA->public_add_active_users(USERNAME_BACKEND, name); - SPA->public_add_active_users(USERNAME_FRONTEND, name); - pthread_mutex_unlock(&users_mutex); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - } else { - proxy_info("Tried to load invalid user %s\n", name); - char *s=(char *)"Invalid name %s"; - char *m=(char *)malloc(strlen(s)+strlen(name)+1); - sprintf(m,s,name); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - free(name); - return false; - } - } - } -#ifdef PROXYSQLCLICKHOUSE - if ( ( GloVars.global.clickhouse_server == true ) && (query_no_space_length>22) && ( (!strncasecmp("SAVE CLICKHOUSE USERS ", query_no_space, 22)) || (!strncasecmp("LOAD CLICKHOUSE USERS ", query_no_space, 22))) ) { - if ( - (query_no_space_length==strlen("LOAD CLICKHOUSE USERS TO MEMORY") && !strncasecmp("LOAD CLICKHOUSE USERS TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE USERS TO MEM") && !strncasecmp("LOAD CLICKHOUSE USERS TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE USERS FROM DISK") && !strncasecmp("LOAD CLICKHOUSE USERS FROM DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_clickhouse_users__from_disk_to_memory(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading clickhouse users to MEMORY\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("SAVE CLICKHOUSE USERS FROM MEMORY") && !strncasecmp("SAVE CLICKHOUSE USERS FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE USERS FROM MEM") && !strncasecmp("SAVE CLICKHOUSE USERS FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE USERS TO DISK") && !strncasecmp("SAVE CLICKHOUSE USERS TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_clickhouse_users__from_memory_to_disk(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving clickhouse users to DISK\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("LOAD CLICKHOUSE USERS FROM MEMORY") && !strncasecmp("LOAD CLICKHOUSE USERS FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE USERS FROM MEM") && !strncasecmp("LOAD CLICKHOUSE USERS FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE USERS TO RUNTIME") && !strncasecmp("LOAD CLICKHOUSE USERS TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE USERS TO RUN") && !strncasecmp("LOAD CLICKHOUSE USERS TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->init_clickhouse_users(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded clickhouse users to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("SAVE CLICKHOUSE USERS TO MEMORY") && !strncasecmp("SAVE CLICKHOUSE USERS TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE USERS TO MEM") && !strncasecmp("SAVE CLICKHOUSE USERS TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE USERS FROM RUNTIME") && !strncasecmp("SAVE CLICKHOUSE USERS FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE USERS FROM RUN") && !strncasecmp("SAVE CLICKHOUSE USERS FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_clickhouse_users_runtime_to_database(false); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved clickhouse users from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - } -#endif /* PROXYSQLCLICKHOUSE */ - - if ((query_no_space_length>17) && ( (!strcasecmp("SAVE MYSQL DIGEST TO DISK", query_no_space) ) )) { - proxy_info("Received %s command\n", query_no_space); - unsigned long long curtime1=monotonic_time(); - int r1 = SPA->FlushDigestTableToDisk(SPA->statsdb_disk); - unsigned long long curtime2=monotonic_time(); - curtime1 = curtime1/1000; - curtime2 = curtime2/1000; - proxy_info("Saved stats_mysql_query_digest to disk: %llums to write %u entries\n", curtime2-curtime1, r1); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - return false; - } - if ((query_no_space_length>17) && ( (!strncasecmp("SAVE MYSQL USERS ", query_no_space, 17)) || (!strncasecmp("LOAD MYSQL USERS ", query_no_space, 17))) ) { - - string modname = "mysql_users"; - tuple, vector>& t = load_save_disk_commands[modname]; - if ( is_admin_command_or_alias(get<1>(t), query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_mysql_users__from_disk_to_memory(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading %s to MEMORY\n", modname.c_str()); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - if ( is_admin_command_or_alias(get<2>(t), query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->flush_mysql_users__from_memory_to_disk(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving %s to DISK\n", modname.c_str()); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - if ( is_admin_command_or_alias(LOAD_MYSQL_USERS_FROM_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->init_users(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql users to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("LOAD MYSQL USERS FROM CONFIG") && !strncasecmp("LOAD MYSQL USERS FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - int rows=0; - rows=SPA->proxysql_config().Read_MySQL_Users_from_configfile(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql users from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); - } - return false; - } - - if ( is_admin_command_or_alias(SAVE_MYSQL_USERS_TO_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_mysql_users_runtime_to_database(false); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql users from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - } - if ((query_no_space_length>28) && ( (!strncasecmp("SAVE SQLITESERVER VARIABLES ", query_no_space, 28)) || (!strncasecmp("LOAD SQLITESERVER VARIABLES ", query_no_space, 28))) ) { - - if ( - (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES TO MEMORY") && !strncasecmp("LOAD SQLITESERVER VARIABLES TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES TO MEM") && !strncasecmp("LOAD SQLITESERVER VARIABLES TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES FROM DISK") && !strncasecmp("LOAD SQLITESERVER VARIABLES FROM DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'sqliteserver-%'"); - *ql=strlen(*q)+1; - return true; - } - - if ( - (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES FROM MEMORY") && !strncasecmp("SAVE SQLITESERVER VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES FROM MEM") && !strncasecmp("SAVE SQLITESERVER VARIABLES FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES TO DISK") && !strncasecmp("SAVE SQLITESERVER VARIABLES TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'sqliteserver-%'"); - *ql=strlen(*q)+1; - return true; - } - - if ( - (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES FROM MEMORY") && !strncasecmp("LOAD SQLITESERVER VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES FROM MEM") && !strncasecmp("LOAD SQLITESERVER VARIABLES FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES TO RUNTIME") && !strncasecmp("LOAD SQLITESERVER VARIABLES TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD SQLITESERVER VARIABLES TO RUN") && !strncasecmp("LOAD SQLITESERVER VARIABLES TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->load_sqliteserver_variables_to_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded SQLiteServer variables to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES TO MEMORY") && !strncasecmp("SAVE SQLITESERVER VARIABLES TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES TO MEM") && !strncasecmp("SAVE SQLITESERVER VARIABLES TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES FROM RUNTIME") && !strncasecmp("SAVE SQLITESERVER VARIABLES FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE SQLITESERVER VARIABLES FROM RUN") && !strncasecmp("SAVE SQLITESERVER VARIABLES FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_sqliteserver_variables_from_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved SQLiteServer variables from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - } -#ifdef PROXYSQLCLICKHOUSE - if ((query_no_space_length>26) && ( (!strncasecmp("SAVE CLICKHOUSE VARIABLES ", query_no_space, 26)) || (!strncasecmp("LOAD CLICKHOUSE VARIABLES ", query_no_space, 26))) ) { - - if ( - (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES TO MEMORY") && !strncasecmp("LOAD CLICKHOUSE VARIABLES TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES TO MEM") && !strncasecmp("LOAD CLICKHOUSE VARIABLES TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES FROM DISK") && !strncasecmp("LOAD CLICKHOUSE VARIABLES FROM DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'clickhouse-%'"); - *ql=strlen(*q)+1; - return true; - } - - if ( - (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES FROM MEMORY") && !strncasecmp("SAVE CLICKHOUSE VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES FROM MEM") && !strncasecmp("SAVE CLICKHOUSE VARIABLES FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES TO DISK") && !strncasecmp("SAVE CLICKHOUSE VARIABLES TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'clickhouse-%'"); - *ql=strlen(*q)+1; - return true; - } - - if ( - (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES FROM MEMORY") && !strncasecmp("LOAD CLICKHOUSE VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES FROM MEM") && !strncasecmp("LOAD CLICKHOUSE VARIABLES FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES TO RUNTIME") && !strncasecmp("LOAD CLICKHOUSE VARIABLES TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD CLICKHOUSE VARIABLES TO RUN") && !strncasecmp("LOAD CLICKHOUSE VARIABLES TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->load_clickhouse_variables_to_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded clickhouse variables to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } + if (strstr(query_no_space,"stats_mysql_free_connections")) + { stats_mysql_free_connections=true; refresh=true; } + if (strstr(query_no_space, "stats_pgsql_free_connections")) + { stats_pgsql_free_connections=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_commands_counters")) + { stats_mysql_commands_counters=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_query_rules")) + { stats_mysql_query_rules=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_users")) + { stats_mysql_users=true; refresh=true; } + if (strstr(query_no_space,"stats_pgsql_users")) + { stats_pgsql_users = true; refresh = true; } + if (strstr(query_no_space,"stats_mysql_gtid_executed")) + { stats_mysql_gtid_executed=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_client_host_cache")) + { stats_mysql_client_host_cache=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_client_host_cache_reset")) + { stats_mysql_client_host_cache_reset=true; refresh=true; } + if (strstr(query_no_space, "stats_pgsql_client_host_cache")) + { stats_pgsql_client_host_cache = true; refresh = true; } + if (strstr(query_no_space, "stats_pgsql_client_host_cache_reset")) + { stats_pgsql_client_host_cache_reset = true; refresh = true; } + if (strstr(query_no_space,"stats_proxysql_servers_checksums")) + { stats_proxysql_servers_checksums = true; refresh = true; } + if (strstr(query_no_space,"stats_proxysql_servers_metrics")) + { stats_proxysql_servers_metrics = true; refresh = true; } + if (strstr(query_no_space,"stats_proxysql_message_metrics")) + { stats_proxysql_message_metrics=true; refresh=true; } + if (strstr(query_no_space,"stats_proxysql_message_metrics_reset")) + { stats_proxysql_message_metrics_reset=true; refresh=true; } - if ( - (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES TO MEMORY") && !strncasecmp("SAVE CLICKHOUSE VARIABLES TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES TO MEM") && !strncasecmp("SAVE CLICKHOUSE VARIABLES TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES FROM RUNTIME") && !strncasecmp("SAVE CLICKHOUSE VARIABLES FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE CLICKHOUSE VARIABLES FROM RUN") && !strncasecmp("SAVE CLICKHOUSE VARIABLES FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_clickhouse_variables_from_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved clickhouse variables from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } + // temporary disabled because not implemented +/* + if (strstr(query_no_space,"stats_proxysql_servers_status")) + { stats_proxysql_servers_status = true; refresh = true; } +*/ + if (strstr(query_no_space,"stats_mysql_prepared_statements_info")) { + stats_mysql_prepared_statements_info=true; refresh=true; } -#endif /* PROXYSQLCLICKHOUSE */ - - if (GloMyLdapAuth) { - if ((query_no_space_length>20) && ( (!strncasecmp("SAVE LDAP VARIABLES ", query_no_space, 20)) || (!strncasecmp("LOAD LDAP VARIABLES ", query_no_space, 20))) ) { - + if (admin) { + if (strstr(query_no_space,"global_variables")) + { dump_global_variables=true; refresh=true; } + if (strstr(query_no_space,"runtime_")) { if ( - (query_no_space_length==strlen("LOAD LDAP VARIABLES TO MEMORY") && !strncasecmp("LOAD LDAP VARIABLES TO MEMORY",query_no_space, query_no_space_length)) + strstr(query_no_space,"runtime_mysql_servers") + || + strstr(query_no_space,"runtime_mysql_replication_hostgroups") + || + strstr(query_no_space,"runtime_mysql_group_replication_hostgroups") + || + strstr(query_no_space,"runtime_mysql_galera_hostgroups") + || + strstr(query_no_space,"runtime_mysql_aws_aurora_hostgroups") || - (query_no_space_length==strlen("LOAD LDAP VARIABLES TO MEM") && !strncasecmp("LOAD LDAP VARIABLES TO MEM",query_no_space, query_no_space_length)) + strstr(query_no_space,"runtime_mysql_hostgroup_attributes") || - (query_no_space_length==strlen("LOAD LDAP VARIABLES FROM DISK") && !strncasecmp("LOAD LDAP VARIABLES FROM DISK",query_no_space, query_no_space_length)) + strstr(query_no_space,"runtime_mysql_servers_ssl_params") ) { - proxy_info("Received %s command\n", query_no_space); - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'ldap-%'"); - *ql=strlen(*q)+1; - return true; + runtime_mysql_servers=true; refresh=true; } - if ( - (query_no_space_length==strlen("SAVE LDAP VARIABLES FROM MEMORY") && !strncasecmp("SAVE LDAP VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) + strstr(query_no_space, "runtime_pgsql_servers") || - (query_no_space_length==strlen("SAVE LDAP VARIABLES FROM MEM") && !strncasecmp("SAVE LDAP VARIABLES FROM MEM",query_no_space, query_no_space_length)) + strstr(query_no_space, "runtime_pgsql_replication_hostgroups") || - (query_no_space_length==strlen("SAVE LDAP VARIABLES TO DISK") && !strncasecmp("SAVE LDAP VARIABLES TO DISK",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'ldap-%'"); - *ql=strlen(*q)+1; - return true; + strstr(query_no_space, "runtime_pgsql_hostgroup_attributes") + ) { + runtime_pgsql_servers = true; refresh = true; } - if ( - (query_no_space_length==strlen("LOAD LDAP VARIABLES FROM MEMORY") && !strncasecmp("LOAD LDAP VARIABLES FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD LDAP VARIABLES FROM MEM") && !strncasecmp("LOAD LDAP VARIABLES FROM MEM",query_no_space, query_no_space_length)) + strstr(query_no_space,"runtime_mysql_firewall_whitelist_rules") || - (query_no_space_length==strlen("LOAD LDAP VARIABLES TO RUNTIME") && !strncasecmp("LOAD LDAP VARIABLES TO RUNTIME",query_no_space, query_no_space_length)) + strstr(query_no_space,"runtime_mysql_firewall_whitelist_users") || - (query_no_space_length==strlen("LOAD LDAP VARIABLES TO RUN") && !strncasecmp("LOAD LDAP VARIABLES TO RUN",query_no_space, query_no_space_length)) + strstr(query_no_space,"runtime_mysql_firewall_whitelist_sqli_fingerprints") ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->load_ldap_variables_to_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded ldap variables to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + runtime_mysql_firewall=true; refresh=true; } - if ( - (query_no_space_length==strlen("SAVE LDAP VARIABLES TO MEMORY") && !strncasecmp("SAVE LDAP VARIABLES TO MEMORY",query_no_space, query_no_space_length)) + strstr(query_no_space, "runtime_pgsql_firewall_whitelist_rules") || - (query_no_space_length==strlen("SAVE LDAP VARIABLES TO MEM") && !strncasecmp("SAVE LDAP VARIABLES TO MEM",query_no_space, query_no_space_length)) + strstr(query_no_space, "runtime_pgsql_firewall_whitelist_users") || - (query_no_space_length==strlen("SAVE LDAP VARIABLES FROM RUNTIME") && !strncasecmp("SAVE LDAP VARIABLES FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE LDAP VARIABLES FROM RUN") && !strncasecmp("SAVE LDAP VARIABLES FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_ldap_variables_from_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved ldap variables from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + strstr(query_no_space, "runtime_pgsql_firewall_whitelist_sqli_fingerprints") + ) { + runtime_pgsql_firewall = true; refresh = true; } - } - } - - if ((query_no_space_length>21) && ( (!strncasecmp("SAVE MYSQL VARIABLES ", query_no_space, 21)) || (!strncasecmp("LOAD MYSQL VARIABLES ", query_no_space, 21))) ) { - - string modname = "mysql_variables"; - tuple, vector>& t = load_save_disk_commands[modname]; - if ( is_admin_command_or_alias(get<1>(t), query_no_space, query_no_space_length) ) { - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'mysql-%'"); - *ql=strlen(*q)+1; - return true; - } - - if ( is_admin_command_or_alias(get<2>(t), query_no_space, query_no_space_length) ) { - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'mysql-%'"); - *ql=strlen(*q)+1; - return true; - } - - if ( is_admin_command_or_alias(LOAD_MYSQL_VARIABLES_FROM_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->load_mysql_variables_to_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql variables to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("LOAD MYSQL VARIABLES FROM CONFIG") && !strncasecmp("LOAD MYSQL VARIABLES FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - int rows=0; - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - rows=SPA->proxysql_config().Read_Global_Variables_from_configfile("mysql"); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql variables from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); + if (strstr(query_no_space,"runtime_mysql_users")) { + runtime_mysql_users=true; refresh=true; } - return false; - } + if (strstr(query_no_space, "runtime_pgsql_users")) { + runtime_pgsql_users = true; refresh = true; + } + if (GloMyLdapAuth) { + if (strstr(query_no_space,"runtime_mysql_ldap_mapping")) { + runtime_mysql_ldap_mapping=true; refresh=true; + } + if (strstr(query_no_space, "runtime_pgsql_ldap_mapping")) { + runtime_mysql_ldap_mapping = true; refresh = true; + } + } + if (strstr(query_no_space,"runtime_mysql_query_rules")) { + runtime_mysql_query_rules=true; refresh=true; + } + if (strstr(query_no_space, "runtime_pgsql_query_rules")) { + runtime_pgsql_query_rules = true; refresh = true; + } + if (strstr(query_no_space,"runtime_mysql_query_rules_fast_routing")) { + runtime_mysql_query_rules_fast_routing=true; refresh=true; + } + if (strstr(query_no_space, "runtime_pgsql_query_rules_fast_routing")) { + runtime_pgsql_query_rules_fast_routing = true; refresh = true; + } + if (strstr(query_no_space,"runtime_scheduler")) { + runtime_scheduler=true; refresh=true; + } + if (strstr(query_no_space,"runtime_restapi_routes")) { + runtime_restapi_routes=true; refresh=true; + } + if (strstr(query_no_space,"runtime_proxysql_servers")) { + runtime_proxysql_servers=true; refresh=true; + } + if (strstr(query_no_space,"runtime_checksums_values")) { + runtime_checksums_values=true; refresh=true; + } + if (strstr(query_no_space,"runtime_coredump_filters")) { + runtime_coredump_filters=true; refresh=true; + } +#ifdef PROXYSQLCLICKHOUSE + if (( GloVars.global.clickhouse_server == true ) && strstr(query_no_space,"runtime_clickhouse_users")) { + runtime_clickhouse_users=true; refresh=true; + } +#endif /* PROXYSQLCLICKHOUSE */ - if ( is_admin_command_or_alias(SAVE_MYSQL_VARIABLES_TO_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_mysql_variables_from_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql variables from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; } - } - - if ((query_no_space_length > 14) && (!strncasecmp("LOAD COREDUMP ", query_no_space, 14))) { - - if ( is_admin_command_or_alias(LOAD_COREDUMP_FROM_MEMORY, query_no_space, query_no_space_length) ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin* SPA = (ProxySQL_Admin*)pa; - bool rc = SPA->load_coredump_to_runtime(); - if (rc) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded coredump filters to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 1, "Error while loading coredump filters to RUNTIME\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char*)"Error while loading coredump filters to RUNTIME"); - } - return false; - } + if (strstr(query_no_space,"mysql_server_group_replication_log")) { + monitor_mysql_server_group_replication_log=true; refresh=true; } - - if ((query_no_space_length>19) && ( (!strncasecmp("SAVE MYSQL SERVERS ", query_no_space, 19)) || (!strncasecmp("LOAD MYSQL SERVERS ", query_no_space, 19))) ) { - - if (FlushCommandWrapper(sess, "mysql_servers", query_no_space, query_no_space_length) == true) - return false; - - if ( is_admin_command_or_alias(LOAD_MYSQL_SERVERS_FROM_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->mysql_servers_wrlock(); - SPA->load_mysql_servers_to_runtime(); - SPA->mysql_servers_wrunlock(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql servers to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - if ( - (query_no_space_length==strlen("LOAD MYSQL SERVERS FROM CONFIG") && !strncasecmp("LOAD MYSQL SERVERS FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - int rows=0; - rows=SPA->proxysql_config().Read_MySQL_Servers_from_configfile(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql servers from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); + if (strstr(query_no_space,"mysql_server_galera_log")) { + monitor_mysql_server_galera_log=true; refresh=true; + } + if (strstr(query_no_space,"mysql_server_aws_aurora_log")) { + monitor_mysql_server_aws_aurora_log=true; refresh=true; + } + if (strstr(query_no_space,"mysql_server_aws_aurora_check_status")) { + monitor_mysql_server_aws_aurora_check_status=true; refresh=true; + } +// if (stats_mysql_processlist || stats_mysql_connection_pool || stats_mysql_query_digest || stats_mysql_query_digest_reset) { + if (refresh==true) { + //pthread_mutex_lock(&admin_mutex); + //ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (stats_mysql_processlist) + stats___mysql_processlist(); + if (stats_pgsql_processlist) + stats___pgsql_processlist(); + if (stats_mysql_query_digest_reset) { + stats___mysql_query_digests_v2(true, stats_mysql_query_digest, false); + } else { + if (stats_mysql_query_digest) { + stats___mysql_query_digests_v2(false, false, false); } - return false; } - - if ( is_admin_command_or_alias(SAVE_MYSQL_SERVERS_TO_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->mysql_servers_wrlock(); - SPA->save_mysql_servers_runtime_to_database(false); - SPA->mysql_servers_wrunlock(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql servers from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + if (stats_mysql_errors) + stats___mysql_errors(false); + if (stats_mysql_errors_reset) { + stats___mysql_errors(true); } - } - - if ((query_no_space_length>22) && ( (!strncasecmp("SAVE PROXYSQL SERVERS ", query_no_space, 22)) || (!strncasecmp("LOAD PROXYSQL SERVERS ", query_no_space, 22))) ) { - - if (FlushCommandWrapper(sess, "proxysql_servers", query_no_space, query_no_space_length) == true) - return false; -/* - string modname = "proxysql_servers"; - tuple, vector>& t = load_save_disk_commands[modname]; - if (FlushCommandWrapper(sess, get<1>(t), query_no_space, query_no_space_length, modname, "disk_to_memory") == true) - return false; - - if (FlushCommandWrapper(sess, get<2>(t), query_no_space, query_no_space_length, modname, "memory_to_disk") == true) - return false; -*/ - if ( - (query_no_space_length==strlen("LOAD PROXYSQL SERVERS FROM MEMORY") && !strncasecmp("LOAD PROXYSQL SERVERS FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD PROXYSQL SERVERS FROM MEM") && !strncasecmp("LOAD PROXYSQL SERVERS FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD PROXYSQL SERVERS TO RUNTIME") && !strncasecmp("LOAD PROXYSQL SERVERS TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD PROXYSQL SERVERS TO RUN") && !strncasecmp("LOAD PROXYSQL SERVERS TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - //SPA->mysql_servers_wrlock(); - // before calling load_proxysql_servers_to_runtime() we release - // sql_query_global_mutex to prevent a possible deadlock due to - // a race condition - // load_proxysql_servers_to_runtime() calls ProxySQL_Cluster::load_servers_list() - // that then calls ProxySQL_Cluster_Nodes::load_servers_list(), holding a mutex - pthread_mutex_unlock(&SPA->sql_query_global_mutex); - SPA->load_proxysql_servers_to_runtime(true); - // we re-acquired the mutex because it will be released by the calling function - pthread_mutex_lock(&SPA->sql_query_global_mutex); - //SPA->mysql_servers_wrunlock(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded ProxySQL servers to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + if (stats_pgsql_errors) { + stats___pgsql_errors(false); } - if ( - (query_no_space_length==strlen("SAVE PROXYSQL SERVERS TO MEMORY") && !strncasecmp("SAVE PROXYSQL SERVERS TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE PROXYSQL SERVERS TO MEM") && !strncasecmp("SAVE PROXYSQL SERVERS TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE PROXYSQL SERVERS FROM RUNTIME") && !strncasecmp("SAVE PROXYSQL SERVERS FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE PROXYSQL SERVERS FROM RUN") && !strncasecmp("SAVE PROXYSQL SERVERS FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - //SPA->mysql_servers_wrlock(); - // before save_proxysql_servers_runtime_to_database() we release - // sql_query_global_mutex to prevent a possible deadlock due to - // a race condition - // save_proxysql_servers_runtime_to_database() calls ProxySQL_Cluster::dump_table_proxysql_servers() - // that then holds a mutex - pthread_mutex_unlock(&SPA->sql_query_global_mutex); - SPA->save_proxysql_servers_runtime_to_database(false); - // we re-acquired the mutex because it will be released by the calling function - pthread_mutex_lock(&SPA->sql_query_global_mutex); - //SPA->mysql_servers_wrunlock(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved ProxySQL servers from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + if (stats_pgsql_errors_reset) { + stats___pgsql_errors(true); } - - if ( - (query_no_space_length==strlen("LOAD PROXYSQL SERVERS FROM CONFIG") && !strncasecmp("LOAD PROXYSQL SERVERS FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - int rows=0; - rows=SPA->proxysql_config().Read_ProxySQL_Servers_from_configfile(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded ProxySQL servers from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); - } - return false; + if (stats_mysql_connection_pool_reset) { + stats___mysql_connection_pool(true); + } else { + if (stats_mysql_connection_pool) + stats___mysql_connection_pool(false); } - - } - - if ((query_no_space_length>20) && ( (!strncasecmp("SAVE MYSQL FIREWALL ", query_no_space, 20)) || (!strncasecmp("LOAD MYSQL FIREWALL ", query_no_space, 20))) ) { - - if (FlushCommandWrapper(sess, "mysql_firewall", query_no_space, query_no_space_length) == true) - return false; - - if ( - (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM CONFIG") && !strncasecmp("LOAD MYSQL FIREWALL FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - int rows=0; - // FIXME: not implemented yet - //rows=SPA->Read_MySQL_Firewall_from_configfile(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql firewall from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); - } - return false; + if (stats_pgsql_connection_pool_reset) { + stats___pgsql_connection_pool(true); + } else { + if (stats_pgsql_connection_pool) + stats___pgsql_connection_pool(false); } + if (stats_mysql_free_connections) + stats___mysql_free_connections(); + if (stats_pgsql_free_connections) + stats___pgsql_free_connections(); + if (stats_mysql_global) + stats___mysql_global(); + if (stats_pgsql_global) + stats___pgsql_global(); + if (stats_memory_metrics) + stats___memory_metrics(); + if (stats_mysql_query_rules) + stats___mysql_query_rules(); + if (stats_mysql_commands_counters) + stats___mysql_commands_counters(); + if (stats_mysql_users) + stats___mysql_users(); + if (stats_pgsql_users) + stats___pgsql_users(); + if (stats_mysql_gtid_executed) + stats___mysql_gtid_executed(); - if ( - (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM MEMORY") && !strncasecmp("LOAD MYSQL FIREWALL FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL FIREWALL FROM MEM") && !strncasecmp("LOAD MYSQL FIREWALL FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL FIREWALL TO RUNTIME") && !strncasecmp("LOAD MYSQL FIREWALL TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL FIREWALL TO RUN") && !strncasecmp("LOAD MYSQL FIREWALL TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - char *err=SPA->load_mysql_firewall_to_runtime(); - if (err==NULL) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql firewall to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - } else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, err); - } - return false; + // cluster + if (stats_proxysql_servers_metrics) { + stats___proxysql_servers_metrics(); } - - if ( - (query_no_space_length==strlen("SAVE MYSQL FIREWALL TO MEMORY") && !strncasecmp("SAVE MYSQL FIREWALL TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL FIREWALL TO MEM") && !strncasecmp("SAVE MYSQL FIREWALL TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL FIREWALL FROM RUNTIME") && !strncasecmp("SAVE MYSQL FIREWALL FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL FIREWALL FROM RUN") && !strncasecmp("SAVE MYSQL FIREWALL FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_mysql_firewall_from_runtime(false); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql firewall from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + if (stats_proxysql_servers_checksums) { + stats___proxysql_servers_checksums(); } - } - - if ((query_no_space_length>23) && ( (!strncasecmp("SAVE MYSQL QUERY RULES ", query_no_space, 23)) || (!strncasecmp("LOAD MYSQL QUERY RULES ", query_no_space, 23))) ) { - - if (FlushCommandWrapper(sess, "mysql_query_rules", query_no_space, query_no_space_length) == true) - return false; - - if ( - (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM CONFIG") && !strncasecmp("LOAD MYSQL QUERY RULES FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - int rows=0; - rows=SPA->proxysql_config().Read_MySQL_Query_Rules_from_configfile(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql query rules from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); + if (stats_proxysql_message_metrics_reset) { + stats___proxysql_message_metrics(true); + } else { + if (stats_proxysql_message_metrics) { + stats___proxysql_message_metrics(false); } - return false; } - if ( - (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM MEMORY") && !strncasecmp("LOAD MYSQL QUERY RULES FROM MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL QUERY RULES FROM MEM") && !strncasecmp("LOAD MYSQL QUERY RULES FROM MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL QUERY RULES TO RUNTIME") && !strncasecmp("LOAD MYSQL QUERY RULES TO RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("LOAD MYSQL QUERY RULES TO RUN") && !strncasecmp("LOAD MYSQL QUERY RULES TO RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - char *err=SPA->load_mysql_query_rules_to_runtime(); - if (err==NULL) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql query rules to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - } else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, err); - } - return false; + // temporary disabled because not implemented +// if (stats_proxysql_servers_status) { +// stats___proxysql_servers_status(); +// } + if (stats_mysql_prepared_statements_info) { + stats___mysql_prepared_statements_info(); } - if ( - (query_no_space_length==strlen("SAVE MYSQL QUERY RULES TO MEMORY") && !strncasecmp("SAVE MYSQL QUERY RULES TO MEMORY",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL QUERY RULES TO MEM") && !strncasecmp("SAVE MYSQL QUERY RULES TO MEM",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL QUERY RULES FROM RUNTIME") && !strncasecmp("SAVE MYSQL QUERY RULES FROM RUNTIME",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SAVE MYSQL QUERY RULES FROM RUN") && !strncasecmp("SAVE MYSQL QUERY RULES FROM RUN",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_mysql_query_rules_from_runtime(false); - SPA->save_mysql_query_rules_fast_routing_from_runtime(false); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql query rules from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + if (stats_mysql_client_host_cache) { + stats___mysql_client_host_cache(false); } - } - - if ((query_no_space_length>21) && ( (!strncasecmp("SAVE ADMIN VARIABLES ", query_no_space, 21)) || (!strncasecmp("LOAD ADMIN VARIABLES ", query_no_space, 21))) ) { - - if ( is_admin_command_or_alias(LOAD_ADMIN_VARIABLES_TO_MEMORY, query_no_space, query_no_space_length) ) { - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables WHERE variable_name LIKE 'admin-%'"); - *ql=strlen(*q)+1; - return true; + if (stats_mysql_client_host_cache_reset) { + stats___mysql_client_host_cache(true); } - - if ( is_admin_command_or_alias(SAVE_ADMIN_VARIABLES_FROM_MEMORY, query_no_space, query_no_space_length) ) { - l_free(*ql,*q); - *q=l_strdup("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'admin-%'"); - *ql=strlen(*q)+1; - return true; + if (stats_pgsql_client_host_cache) { + stats___pgsql_client_host_cache(false); } - - if ( is_admin_command_or_alias(LOAD_ADMIN_VARIABLES_FROM_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->load_admin_variables_to_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded admin variables to RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; + if (stats_pgsql_client_host_cache_reset) { + stats___pgsql_client_host_cache(true); } - if ( - (query_no_space_length==strlen("LOAD ADMIN VARIABLES FROM CONFIG") && !strncasecmp("LOAD ADMIN VARIABLES FROM CONFIG",query_no_space, query_no_space_length)) - ) { - proxy_info("Received %s command\n", query_no_space); - if (GloVars.configfile_open) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); - if (GloVars.confFile->OpenFile(NULL)==true) { - int rows=0; - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - rows=SPA->proxysql_config().Read_Global_Variables_from_configfile("admin"); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded admin variables from CONFIG\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); - GloVars.confFile->CloseFile(); - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); - char *s=(char *)"Unable to open or parse config file %s"; - char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); - sprintf(m,s,GloVars.config_file); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); - free(m); - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); + if (admin) { + if (dump_global_variables) { + pthread_mutex_lock(&GloVars.checksum_mutex); + admindb->execute("DELETE FROM runtime_global_variables"); // extra + flush_admin_variables___runtime_to_database(admindb, false, false, false, true); + flush_mysql_variables___runtime_to_database(admindb, false, false, false, true); +#ifdef PROXYSQLCLICKHOUSE + flush_clickhouse_variables___runtime_to_database(admindb, false, false, false, true); +#endif /* PROXYSQLCLICKHOUSE */ + flush_sqliteserver_variables___runtime_to_database(admindb, false, false, false, true); + flush_ldap_variables___runtime_to_database(admindb, false, false, false, true); + flush_pgsql_variables___runtime_to_database(admindb, false, false, false, true); + pthread_mutex_unlock(&GloVars.checksum_mutex); } - return false; - } - - if ( is_admin_command_or_alias(SAVE_ADMIN_VARIABLES_TO_MEMORY, query_no_space, query_no_space_length) ) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_admin_variables_from_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved admin variables from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - return false; - } - - } - - if (!strncasecmp("SAVE CONFIG TO FILE", query_no_space, strlen("SAVE CONFIG TO FILE"))) { - std::string fileName = query_no_space + strlen("SAVE CONFIG TO FILE"); - - fileName.erase(0, fileName.find_first_not_of("\t\n\v\f\r ")); - fileName.erase(fileName.find_last_not_of("\t\n\v\f\r ") + 1); - if (fileName.size() == 0) { - proxy_error("ProxySQL Admin Error: empty file name\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"ProxySQL Admin Error: empty file name"); - return false; - } - std::string data; - data.reserve(100000); - data += config_header; - int rc = pa->proxysql_config().Write_Global_Variables_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Users_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); - rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); - rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); - if (rc) { - std::stringstream ss; - proxy_error("ProxySQL Admin Error: Cannot extract configuration\n"); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"ProxySQL Admin Error: Cannot extract configuration"); - return false; - } else { - std::ofstream out; - out.open(fileName); - if (out.is_open()) { - out << data; - out.close(); - if (!out) { - std::stringstream ss; - ss << "ProxySQL Admin Error: Error writing file " << fileName; - proxy_error("%s\n", ss.str().c_str()); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char*)ss.str().c_str()); - return false; - } else { - std::stringstream ss; - ss << "File " << fileName << " is saved."; - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char*)ss.str().c_str()); - return false; - } - } else { - std::stringstream ss; - ss << "ProxySQL Admin Error: Cannot open file " << fileName; - proxy_error("%s\n", ss.str().c_str()); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char*)ss.str().c_str()); - return false; + if (runtime_mysql_servers) { + int old_hostgroup_manager_verbose = mysql_thread___hostgroup_manager_verbose; + mysql_thread___hostgroup_manager_verbose = 0; + mysql_servers_wrlock(); + save_mysql_servers_runtime_to_database(true); + mysql_servers_wrunlock(); + mysql_thread___hostgroup_manager_verbose = old_hostgroup_manager_verbose; } - } - } - - return true; -} - -void ProxySQL_Admin::flush_configdb() { // see #923 - wrlock(); - admindb->execute((char *)"DETACH DATABASE disk"); - delete configdb; - configdb=new SQLite3DB(); - configdb->open((char *)GloVars.admindb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - __attach_db(admindb, configdb, (char *)"disk"); - // Fully synchronous is not required. See to #1055 - // https://sqlite.org/pragma.html#pragma_synchronous - configdb->execute("PRAGMA synchronous=0"); - wrunlock(); -} - -bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsigned int query_no_space_length, bool admin) { - bool ret=false; - bool refresh=false; - bool stats_mysql_processlist=false; - bool stats_mysql_free_connections=false; - bool stats_mysql_connection_pool=false; - bool stats_mysql_connection_pool_reset=false; - bool stats_mysql_query_digest=false; - bool stats_mysql_query_digest_reset=false; - bool stats_mysql_errors=false; - bool stats_mysql_errors_reset=false; - bool stats_mysql_global=false; - bool stats_memory_metrics=false; - bool stats_mysql_commands_counters=false; - bool stats_mysql_query_rules=false; - bool stats_mysql_users=false; - bool stats_mysql_gtid_executed=false; - bool stats_mysql_client_host_cache=false; - bool stats_mysql_client_host_cache_reset=false; - bool dump_global_variables=false; - - bool runtime_scheduler=false; - bool runtime_restapi_routes=false; - bool runtime_mysql_users=false; - bool runtime_mysql_firewall=false; - bool runtime_mysql_ldap_mapping=false; - bool runtime_mysql_servers=false; - bool runtime_mysql_query_rules=false; - bool runtime_mysql_query_rules_fast_routing=false; - - bool runtime_proxysql_servers=false; - bool runtime_checksums_values=false; - - bool runtime_coredump_filters=false; - - bool stats_mysql_prepared_statements_info = false; - -#ifdef PROXYSQLCLICKHOUSE - bool runtime_clickhouse_users = false; -#endif /* PROXYSQLCLICKHOUSE */ - - bool monitor_mysql_server_group_replication_log=false; - - bool monitor_mysql_server_galera_log=false; - - bool monitor_mysql_server_aws_aurora_log=false; - bool monitor_mysql_server_aws_aurora_check_status=false; - - bool stats_proxysql_servers_checksums = false; - bool stats_proxysql_servers_metrics = false; - bool stats_proxysql_message_metrics = false; - bool stats_proxysql_message_metrics_reset = false; - - //bool stats_proxysql_servers_status = false; // temporary disabled because not implemented - - if (strcasestr(query_no_space,"processlist")) - // This will match the following usecases: - // SHOW PROCESSLIST - // SHOW FULL PROCESSLIST - // SELECT * FROM stats_mysql_processlist - { stats_mysql_processlist=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_query_digest")) - { stats_mysql_query_digest=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_query_digest_reset")) - { stats_mysql_query_digest_reset=true; refresh=true; } - if (stats_mysql_query_digest_reset == true && stats_mysql_query_digest == true) { - int nd = 0; - int ndr= 0; - char *c = NULL; - char *_ret = NULL; - c = (char *)query_no_space; - _ret = NULL; - while ((_ret = strstr(c,"stats_mysql_query_digest_reset"))) { - ndr++; - c = _ret + strlen("stats_mysql_query_digest_reset"); - } - c = (char *)query_no_space; - _ret = NULL; - while ((_ret = strstr(c,"stats_mysql_query_digest"))) { - nd++; - c = _ret + strlen("stats_mysql_query_digest"); - } - if (nd == ndr) { - stats_mysql_query_digest = false; - } - } - if (strstr(query_no_space,"stats_mysql_errors")) - { stats_mysql_errors=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_errors_reset")) - { stats_mysql_errors_reset=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_global")) - { stats_mysql_global=true; refresh=true; } - if (strstr(query_no_space,"stats_memory_metrics")) - { stats_memory_metrics=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_connection_pool_reset")) - { - stats_mysql_connection_pool_reset=true; refresh=true; - } else { - if (strstr(query_no_space,"stats_mysql_connection_pool")) - { stats_mysql_connection_pool=true; refresh=true; } - } - if (strstr(query_no_space,"stats_mysql_free_connections")) - { stats_mysql_free_connections=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_commands_counters")) - { stats_mysql_commands_counters=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_query_rules")) - { stats_mysql_query_rules=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_users")) - { stats_mysql_users=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_gtid_executed")) - { stats_mysql_gtid_executed=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_client_host_cache")) - { stats_mysql_client_host_cache=true; refresh=true; } - if (strstr(query_no_space,"stats_mysql_client_host_cache_reset")) - { stats_mysql_client_host_cache_reset=true; refresh=true; } - - if (strstr(query_no_space,"stats_proxysql_servers_checksums")) - { stats_proxysql_servers_checksums = true; refresh = true; } - if (strstr(query_no_space,"stats_proxysql_servers_metrics")) - { stats_proxysql_servers_metrics = true; refresh = true; } - if (strstr(query_no_space,"stats_proxysql_message_metrics")) - { stats_proxysql_message_metrics=true; refresh=true; } - if (strstr(query_no_space,"stats_proxysql_message_metrics_reset")) - { stats_proxysql_message_metrics_reset=true; refresh=true; } - - // temporary disabled because not implemented -/* - if (strstr(query_no_space,"stats_proxysql_servers_status")) - { stats_proxysql_servers_status = true; refresh = true; } -*/ - if (strstr(query_no_space,"stats_mysql_prepared_statements_info")) { - stats_mysql_prepared_statements_info=true; refresh=true; - } - if (admin) { - if (strstr(query_no_space,"global_variables")) - { dump_global_variables=true; refresh=true; } - if (strstr(query_no_space,"runtime_")) { - if ( - strstr(query_no_space,"runtime_mysql_servers") - || - strstr(query_no_space,"runtime_mysql_replication_hostgroups") - || - strstr(query_no_space,"runtime_mysql_group_replication_hostgroups") - || - strstr(query_no_space,"runtime_mysql_galera_hostgroups") - || - strstr(query_no_space,"runtime_mysql_aws_aurora_hostgroups") - || - strstr(query_no_space,"runtime_mysql_hostgroup_attributes") - || - strstr(query_no_space,"runtime_mysql_servers_ssl_params") - ) { - runtime_mysql_servers=true; refresh=true; + if (runtime_pgsql_servers) { + int old_hostgroup_manager_verbose = pgsql_thread___hostgroup_manager_verbose; + pgsql_thread___hostgroup_manager_verbose = 0; + pgsql_servers_wrlock(); + save_pgsql_servers_runtime_to_database(true); + pgsql_servers_wrunlock(); + pgsql_thread___hostgroup_manager_verbose = old_hostgroup_manager_verbose; } - if ( - strstr(query_no_space,"runtime_mysql_firewall_whitelist_rules") - || - strstr(query_no_space,"runtime_mysql_firewall_whitelist_users") - || - strstr(query_no_space,"runtime_mysql_firewall_whitelist_sqli_fingerprints") - ) { - runtime_mysql_firewall=true; refresh=true; + if (runtime_proxysql_servers) { + //mysql_servers_wrlock(); + // before save_proxysql_servers_runtime_to_database() we release + // sql_query_global_mutex to prevent a possible deadlock due to + // a race condition + // save_proxysql_servers_runtime_to_database() calls ProxySQL_Cluster::dump_table_proxysql_servers() + pthread_mutex_unlock(&SPA->sql_query_global_mutex); + save_proxysql_servers_runtime_to_database(true); + pthread_mutex_lock(&SPA->sql_query_global_mutex); + //mysql_servers_wrunlock(); } - if (strstr(query_no_space,"runtime_mysql_users")) { - runtime_mysql_users=true; refresh=true; + if (runtime_mysql_users) { + save_mysql_users_runtime_to_database(true); } - if (GloMyLdapAuth) { - if (strstr(query_no_space,"runtime_mysql_ldap_mapping")) { - runtime_mysql_ldap_mapping=true; refresh=true; - } + if (runtime_pgsql_users) { + save_pgsql_users_runtime_to_database(true); } - if (strstr(query_no_space,"runtime_mysql_query_rules")) { - runtime_mysql_query_rules=true; refresh=true; + if (runtime_mysql_firewall) { + save_mysql_firewall_from_runtime(true); } - if (strstr(query_no_space,"runtime_mysql_query_rules_fast_routing")) { - runtime_mysql_query_rules_fast_routing=true; refresh=true; + if (runtime_pgsql_firewall) { + save_pgsql_firewall_from_runtime(true); } - if (strstr(query_no_space,"runtime_scheduler")) { - runtime_scheduler=true; refresh=true; + if (runtime_mysql_ldap_mapping) { + save_mysql_ldap_mapping_runtime_to_database(true); } - if (strstr(query_no_space,"runtime_restapi_routes")) { - runtime_restapi_routes=true; refresh=true; + if (runtime_pgsql_ldap_mapping) { + save_pgsql_ldap_mapping_runtime_to_database(true); } - if (strstr(query_no_space,"runtime_proxysql_servers")) { - runtime_proxysql_servers=true; refresh=true; + if (runtime_mysql_query_rules) { + save_mysql_query_rules_from_runtime(true); } - if (strstr(query_no_space,"runtime_checksums_values")) { - runtime_checksums_values=true; refresh=true; + if (runtime_pgsql_query_rules) { + save_pgsql_query_rules_from_runtime(true); } - if (strstr(query_no_space,"runtime_coredump_filters")) { - runtime_coredump_filters=true; refresh=true; + if (runtime_mysql_query_rules_fast_routing) { + save_mysql_query_rules_fast_routing_from_runtime(true); + } + if (runtime_pgsql_query_rules_fast_routing) { + save_pgsql_query_rules_fast_routing_from_runtime(true); + } + if (runtime_scheduler) { + save_scheduler_runtime_to_database(true); + } + if (runtime_restapi_routes) { + proxysql_restapi().save_restapi_runtime_to_database(true); + } + if (runtime_checksums_values) { + dump_checksums_values_table(); + } + if (runtime_coredump_filters) { + dump_coredump_filter_values_table(); } #ifdef PROXYSQLCLICKHOUSE - if (( GloVars.global.clickhouse_server == true ) && strstr(query_no_space,"runtime_clickhouse_users")) { - runtime_clickhouse_users=true; refresh=true; + if (runtime_clickhouse_users) { + save_clickhouse_users_runtime_to_database(true); } #endif /* PROXYSQLCLICKHOUSE */ } - } - if (strstr(query_no_space,"mysql_server_group_replication_log")) { - monitor_mysql_server_group_replication_log=true; refresh=true; - } - if (strstr(query_no_space,"mysql_server_galera_log")) { - monitor_mysql_server_galera_log=true; refresh=true; - } - if (strstr(query_no_space,"mysql_server_aws_aurora_log")) { - monitor_mysql_server_aws_aurora_log=true; refresh=true; - } - if (strstr(query_no_space,"mysql_server_aws_aurora_check_status")) { - monitor_mysql_server_aws_aurora_check_status=true; refresh=true; - } -// if (stats_mysql_processlist || stats_mysql_connection_pool || stats_mysql_query_digest || stats_mysql_query_digest_reset) { - if (refresh==true) { - //pthread_mutex_lock(&admin_mutex); - //ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if (stats_mysql_processlist) - stats___mysql_processlist(); - if (stats_mysql_query_digest_reset) { - stats___mysql_query_digests_v2(true, stats_mysql_query_digest, false); - } else { - if (stats_mysql_query_digest) { - stats___mysql_query_digests_v2(false, false, false); + if (monitor_mysql_server_group_replication_log) { + if (GloMyMon) { + GloMyMon->populate_monitor_mysql_server_group_replication_log(); } } - if (stats_mysql_errors) - stats___mysql_errors(false); - if (stats_mysql_errors_reset) { - stats___mysql_errors(true); - } - if (stats_mysql_connection_pool_reset) { - stats___mysql_connection_pool(true); - } else { - if (stats_mysql_connection_pool) - stats___mysql_connection_pool(false); - } - if (stats_mysql_free_connections) - stats___mysql_free_connections(); - if (stats_mysql_global) - stats___mysql_global(); - if (stats_memory_metrics) - stats___memory_metrics(); - if (stats_mysql_query_rules) - stats___mysql_query_rules(); - if (stats_mysql_commands_counters) - stats___mysql_commands_counters(); - if (stats_mysql_users) - stats___mysql_users(); - if (stats_mysql_gtid_executed) - stats___mysql_gtid_executed(); - - // cluster - if (stats_proxysql_servers_metrics) { - stats___proxysql_servers_metrics(); - } - if (stats_proxysql_servers_checksums) { - stats___proxysql_servers_checksums(); - } - if (stats_proxysql_message_metrics_reset) { - stats___proxysql_message_metrics(true); - } else { - if (stats_proxysql_message_metrics) { - stats___proxysql_message_metrics(false); - } - } - - // temporary disabled because not implemented -// if (stats_proxysql_servers_status) { -// stats___proxysql_servers_status(); -// } - if (stats_mysql_prepared_statements_info) { - stats___mysql_prepared_statements_info(); - } - - if (stats_mysql_client_host_cache) { - stats___mysql_client_host_cache(false); - } - if (stats_mysql_client_host_cache_reset) { - stats___mysql_client_host_cache(true); - } - - if (admin) { - if (dump_global_variables) { - pthread_mutex_lock(&GloVars.checksum_mutex); - admindb->execute("DELETE FROM runtime_global_variables"); // extra - flush_admin_variables___runtime_to_database(admindb, false, false, false, true); - flush_mysql_variables___runtime_to_database(admindb, false, false, false, true); -#ifdef PROXYSQLCLICKHOUSE - flush_clickhouse_variables___runtime_to_database(admindb, false, false, false, true); -#endif /* PROXYSQLCLICKHOUSE */ - flush_sqliteserver_variables___runtime_to_database(admindb, false, false, false, true); - flush_ldap_variables___runtime_to_database(admindb, false, false, false, true); - pthread_mutex_unlock(&GloVars.checksum_mutex); - } - if (runtime_mysql_servers) { - int old_hostgroup_manager_verbose = mysql_thread___hostgroup_manager_verbose; - mysql_thread___hostgroup_manager_verbose = 0; - mysql_servers_wrlock(); - save_mysql_servers_runtime_to_database(true); - mysql_servers_wrunlock(); - mysql_thread___hostgroup_manager_verbose = old_hostgroup_manager_verbose; - } - if (runtime_proxysql_servers) { - //mysql_servers_wrlock(); - // before save_proxysql_servers_runtime_to_database() we release - // sql_query_global_mutex to prevent a possible deadlock due to - // a race condition - // save_proxysql_servers_runtime_to_database() calls ProxySQL_Cluster::dump_table_proxysql_servers() - pthread_mutex_unlock(&SPA->sql_query_global_mutex); - save_proxysql_servers_runtime_to_database(true); - pthread_mutex_lock(&SPA->sql_query_global_mutex); - //mysql_servers_wrunlock(); - } - if (runtime_mysql_users) { - save_mysql_users_runtime_to_database(true); - } - if (runtime_mysql_firewall) { - save_mysql_firewall_from_runtime(true); - } - if (runtime_mysql_ldap_mapping) { - save_mysql_ldap_mapping_runtime_to_database(true); - } - if (runtime_mysql_query_rules) { - save_mysql_query_rules_from_runtime(true); - } - if (runtime_mysql_query_rules_fast_routing) { - save_mysql_query_rules_fast_routing_from_runtime(true); - } - if (runtime_scheduler) { - save_scheduler_runtime_to_database(true); - } - if (runtime_restapi_routes) { - proxysql_restapi().save_restapi_runtime_to_database(true); - } - if (runtime_checksums_values) { - dump_checksums_values_table(); - } - if (runtime_coredump_filters) { - dump_coredump_filter_values_table(); - } -#ifdef PROXYSQLCLICKHOUSE - if (runtime_clickhouse_users) { - save_clickhouse_users_runtime_to_database(true); - } -#endif /* PROXYSQLCLICKHOUSE */ - - } - if (monitor_mysql_server_group_replication_log) { - if (GloMyMon) { - GloMyMon->populate_monitor_mysql_server_group_replication_log(); - } - } - if (monitor_mysql_server_galera_log) { - if (GloMyMon) { - GloMyMon->populate_monitor_mysql_server_galera_log(); - } + if (monitor_mysql_server_galera_log) { + if (GloMyMon) { + GloMyMon->populate_monitor_mysql_server_galera_log(); + } } if (monitor_mysql_server_aws_aurora_log) { if (GloMyMon) { @@ -3526,7 +1638,10 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign stats_mysql_query_digest || stats_mysql_query_digest_reset || stats_mysql_errors || stats_mysql_errors_reset || stats_mysql_global || stats_memory_metrics || stats_mysql_commands_counters || stats_mysql_query_rules || stats_mysql_users || - stats_mysql_gtid_executed || stats_mysql_free_connections + stats_mysql_gtid_executed || stats_mysql_free_connections || + stats_pgsql_global || stats_pgsql_connection_pool || stats_pgsql_connection_pool_reset || + stats_pgsql_free_connections || stats_pgsql_users || stats_pgsql_processlist || + stats_pgsql_errors || stats_pgsql_errors_reset ) { ret = true; } @@ -3710,7013 +1825,2372 @@ SQLite3_result * ProxySQL_Admin::generate_show_table_status(const char *tablenam return result; } -/** - * @brief Helper function that converts the current timezone - * expressed in seconds into a string of the format: - * - '[-]HH:MM:00'. - * Following the same pattern as the possible values returned by the SQL query - * 'SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())' in a MySQL server. - * @return A string holding the specified representation of the - * supplied timezone. - */ -std::string timediff_timezone_offset() { - std::string time_zone_offset {}; - char result[8]; - time_t rawtime; - struct tm *info; - int offset; - - time(&rawtime); - info = localtime(&rawtime); - strftime(result, 8, "%z", info); - offset = (result[0] == '+') ? 1 : 0; - time_zone_offset = ((std::string)(result)).substr(offset, 3-offset) + ":" + ((std::string)(result)).substr(3, 2) + ":00"; - - return time_zone_offset; + +template +void admin_session_handler(S* sess, void *_pa, PtrSize_t *pkt); + +void ProxySQL_Admin::vacuum_stats(bool is_admin) { + if (variables.vacuum_stats==false) { + return; + } + const vector tablenames = { + "stats_mysql_commands_counters", + "stats_mysql_free_connections", + "stats_pgsql_free_connections", + "stats_mysql_connection_pool", + "stats_mysql_connection_pool_reset", + "stats_pgsql_connection_pool", + "stats_pgsql_connection_pool_reset", + "stats_mysql_prepared_statements_info", + "stats_mysql_processlist", + "stats_pgsql_processlist", + "stats_mysql_query_digest", + "stats_mysql_query_digest_reset", + "stats_mysql_query_rules", + "stats_mysql_users", + "stats_pgsql_users", + "stats_proxysql_servers_checksums", + "stats_proxysql_servers_metrics", + "stats_proxysql_servers_status", + }; + string s; + SQLite3DB *tmpdb = NULL; + if (is_admin == true) { + tmpdb = admindb; + } else { + tmpdb = statsdb; + } + for (auto it = tablenames.begin(); it != tablenames.end(); it++) { + s = "DELETE FROM "; + if (is_admin == true) s+= "stats."; + s += *it; + tmpdb->execute(s.c_str()); + } + s = "VACUUM"; + if (is_admin == true) + s+= " stats"; + tmpdb->execute(s.c_str()); } -void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { - ProxySQL_Admin *pa=(ProxySQL_Admin *)_pa; - bool needs_vacuum = false; - char *error=NULL; - int cols; - int affected_rows = 0; - bool run_query=true; - SQLite3_result *resultset=NULL; - char *strA=NULL; - char *strB=NULL; - int strAl, strBl; - char *query=NULL; - unsigned int query_length=pkt->size-sizeof(mysql_hdr); - query=(char *)l_alloc(query_length); - memcpy(query,(char *)pkt->ptr+sizeof(mysql_hdr)+1,query_length-1); - query[query_length-1]=0; - - char *query_no_space=(char *)l_alloc(query_length); - memcpy(query_no_space,query,query_length); - - unsigned int query_no_space_length=remove_spaces(query_no_space); - //fprintf(stderr,"%s----\n",query_no_space); - - if (query_no_space_length) { - // fix bug #925 - while (query_no_space[query_no_space_length-1]==';' || query_no_space[query_no_space_length-1]==' ') { - query_no_space_length--; - query_no_space[query_no_space_length]=0; - } - } - - // add global mutex, see bug #1188 - pthread_mutex_lock(&pa->sql_query_global_mutex); - - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - if (!strncasecmp("LOGENTRY ", query_no_space, strlen("LOGENTRY "))) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received command LOGENTRY: %s\n", query_no_space + strlen("LOGENTRY ")); - proxy_info("Received command LOGENTRY: %s\n", query_no_space + strlen("LOGENTRY ")); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, 0); - run_query=false; - goto __run_query; - } - } - - // handle special queries from Cluster - // for bug #1188 , ProxySQL Admin needs to know the exact query - - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - string tn = ""; - if (!strncasecmp(CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_RUNTIME_MYSQL_SERVERS))) { - tn = "cluster_mysql_servers"; - } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS))) { - tn = "mysql_replication_hostgroups"; - } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS))) { - tn = "mysql_group_replication_hostgroups"; - } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_GALERA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GALERA))) { - tn = "mysql_galera_hostgroups"; - } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_AURORA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_AURORA))) { - tn = "mysql_aws_aurora_hostgroups"; - } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES, query_no_space, strlen(CLUSTER_QUERY_MYSQL_HOSTGROUP_ATTRIBUTES))) { - tn = "mysql_hostgroup_attributes"; - } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_SSL_PARAMS))) { - tn = "mysql_servers_ssl_params"; - } else if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS_V2, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS_V2))) { - tn = "mysql_servers_v2"; - } - if (tn != "") { - GloAdmin->mysql_servers_wrlock(); - resultset = MyHGM->get_current_mysql_table(tn); - GloAdmin->mysql_servers_wrunlock(); - - if (resultset == nullptr) { - // 'mysql_servers_v2' is a virtual table that represents the latest 'main.mysql_servers' - // records promoted by the user. This section shouldn't be reached, since the initial resulset - // for this table ('MySQL_HostGroups_Manager::incoming_mysql_servers') is generated during - // initialization, and it's only updated in subsequent user config promotions. In case we - // reach here, an empty resultset should be replied, as it would mean that no user - // config has ever been promoted to runtime, and thus, this virtual table should remain empty. - if (tn == "mysql_servers_v2") { - const string query_empty_resultset { - string { MYHGM_GEN_CLUSTER_ADMIN_RUNTIME_SERVERS } + " LIMIT 0" - }; - - char *error=NULL; - int cols=0; - int affected_rows=0; - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); - GloAdmin->mysql_servers_wrlock(); - GloAdmin->admindb->execute_statement(query_empty_resultset.c_str(), &error, &cols, &affected_rows, &resultset); - GloAdmin->mysql_servers_wrunlock(); - } else { - resultset = MyHGM->dump_table_mysql(tn); - } +void *child_mysql(void *arg) { + if (GloMTH == nullptr) { return NULL; } - if (resultset) { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; - run_query=false; - goto __run_query; - } - } else { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - run_query=false; - goto __run_query; - } - + pthread_attr_t thread_attr; + size_t tmp_stack_size=0; + if (!pthread_attr_init(&thread_attr)) { + if (!pthread_attr_getstacksize(&thread_attr , &tmp_stack_size )) { + __sync_fetch_and_add(&GloVars.statuses.stack_memory_admin_threads,tmp_stack_size); } } - if (!strncasecmp(CLUSTER_QUERY_MYSQL_USERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_USERS))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { - pthread_mutex_lock(&users_mutex); - resultset = GloMyAuth->get_current_mysql_users(); - pthread_mutex_unlock(&users_mutex); - if (resultset != nullptr) { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - run_query=false; - goto __run_query; - } - } + arg_proxysql_adm*myarg = (arg_proxysql_adm*)arg; + int client = myarg->client_t; + + //struct sockaddr *addr = arg->addr; + //socklen_t addr_size; + + GloMTH->wrlock(); + { + char *s=GloMTH->get_variable((char *)"server_capabilities"); + mysql_thread___server_capabilities=atoi(s); + free(s); } + GloMTH->wrunlock(); - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - if (!strncasecmp(CLUSTER_QUERY_MYSQL_QUERY_RULES, query_no_space, strlen(CLUSTER_QUERY_MYSQL_QUERY_RULES))) { - GloQPro->wrlock(); - resultset = GloQPro->get_current_query_rules_inner(); - if (resultset == NULL) { - GloQPro->wrunlock(); // unlock first - resultset = GloQPro->get_current_query_rules(); - if (resultset) { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; - run_query=false; - goto __run_query; - } - } else { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - //delete resultset; // DO NOT DELETE . This is the inner resultset of Query_Processor - GloQPro->wrunlock(); - run_query=false; - goto __run_query; - } - } - if (!strncasecmp(CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING, query_no_space, strlen(CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING))) { - GloQPro->wrlock(); - resultset = GloQPro->get_current_query_rules_fast_routing_inner(); - if (resultset == NULL) { - GloQPro->wrunlock(); // unlock first - resultset = GloQPro->get_current_query_rules_fast_routing(); - if (resultset) { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; - run_query=false; - goto __run_query; - } - } else { - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - //delete resultset; // DO NOT DELETE . This is the inner resultset of Query_Processor - GloQPro->wrunlock(); - run_query=false; - goto __run_query; - } - } - } - - // if the client simply executes: - // SELECT COUNT(*) FROM runtime_mysql_query_rules_fast_routing - // we just return the count - if (strcmp("SELECT COUNT(*) FROM runtime_mysql_query_rules_fast_routing", query_no_space)==0) { - int cnt = GloQPro->get_current_query_rules_fast_routing_count(); - l_free(query_length,query); - char buf[256]; - sprintf(buf,"SELECT %d AS 'COUNT(*)'", cnt); - query=l_strdup(buf); - query_length=strlen(query)+1; - goto __run_query; - } - - if (!strncasecmp("TRUNCATE ", query_no_space, strlen("TRUNCATE "))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - if (strstr(query_no_space,"stats_mysql_query_digest")) { - bool truncate_digest_table = false; - static char * truncate_digest_table_queries[] = { - (char *)"TRUNCATE TABLE stats.stats_mysql_query_digest", - (char *)"TRUNCATE TABLE stats.stats_mysql_query_digest_reset", - (char *)"TRUNCATE TABLE stats_mysql_query_digest", - (char *)"TRUNCATE TABLE stats_mysql_query_digest_reset", - (char *)"TRUNCATE stats.stats_mysql_query_digest", - (char *)"TRUNCATE stats.stats_mysql_query_digest_reset", - (char *)"TRUNCATE stats_mysql_query_digest", - (char *)"TRUNCATE stats_mysql_query_digest_reset" - }; - size_t l=sizeof(truncate_digest_table_queries)/sizeof(char *); - unsigned int i; - for (i=0;iadmindb->execute("DELETE FROM stats.stats_mysql_query_digest"); - SPA->admindb->execute("DELETE FROM stats.stats_mysql_query_digest_reset"); - SPA->vacuum_stats(true); - // purge the digest map, asynchronously, in single thread - char *msg = NULL; - int r1 = ProxySQL_Test___PurgeDigestTable(true, false, &msg); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg, r1); - free(msg); - run_query=false; - goto __run_query; - } - } + struct pollfd fds[1]; + nfds_t nfds=1; + int rc; + pthread_mutex_unlock(&sock_mutex); + MySQL_Thread *mysql_thr=new MySQL_Thread(); + mysql_thr->curtime=monotonic_time(); + GloMyQPro->init_thread(); + mysql_thr->refresh_variables(); + MySQL_Session *sess=mysql_thr->create_new_session_and_client_data_stream(client); + sess->thread=mysql_thr; + sess->session_type = PROXYSQL_SESSION_ADMIN; + sess->handler_function=admin_session_handler; + MySQL_Data_Stream *myds=sess->client_myds; + sess->start_time=mysql_thr->curtime; + + sess->client_myds->client_addrlen=myarg->addr_size; + sess->client_myds->client_addr=myarg->addr; + + switch (sess->client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)sess->client_myds->client_addr; + char buf[INET_ADDRSTRLEN]; + inet_ntop(sess->client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + sess->client_myds->addr.addr = strdup(buf); + sess->client_myds->addr.port = htons(ipv4->sin_port); + break; + } + case AF_INET6: { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)sess->client_myds->client_addr; + char buf[INET6_ADDRSTRLEN]; + inet_ntop(sess->client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + sess->client_myds->addr.addr = strdup(buf); + sess->client_myds->addr.port = htons(ipv6->sin6_port); + break; } + default: + sess->client_myds->addr.addr = strdup("localhost"); + break; } -#ifdef DEBUG - /** - * @brief Handles the 'PROXYSQL_SIMULATOR' command. Performing the operation specified in the payload - * format. - * @details The 'PROXYSQL_SIMULATOR' command is specified the following format. Allowing to perform a - * certain internal state changing operation. Payload spec: - * ``` - * PROXYSQL_SIMULATOR ${operation} ${hg} ${address}:${port} ${operation_params} - * ``` - * - * Supported operations include: - * - mysql_error: Find the server specified by 'hostname:port' in the specified hostgroup and calls - * 'MySrvC::connect_error()' with the provider 'error_code'. - * - * Payload example: - * ``` - * PROXYSQL_SIMULATOR mysql_error 1 127.0.0.1 3306 1234 - * ``` - */ - if (!strncasecmp("PROXYSQL_SIMULATOR ", query_no_space, strlen("PROXYSQL_SIMULATOR "))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - proxy_warning("Received PROXYSQL_SIMULATOR command: %s\n", query_no_space); - - re2::RE2::Options opts = re2::RE2::Options(RE2::Quiet); - re2::RE2 pattern("\\s*(\\w+) (\\d+) (\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+) (\\d+)\\s*\\;*", opts); - re2::StringPiece input(query_no_space + strlen("PROXYSQL_SIMULATOR")); - - std::string command, s_hg, srv_addr, s_port, s_errcode {}; - bool c_res = re2::RE2::Consume(&input, pattern, &command, &s_hg, &srv_addr, &s_port, &s_errcode); - - long i_hg = 0; - long i_port = 0; - long i_errcode = 0; - - if (c_res == true) { - char* endptr = nullptr; - i_hg = std::strtol(s_hg.c_str(), &endptr, 10); - if (errno == ERANGE || errno == EINVAL) i_hg = LONG_MIN; - i_port = std::strtol(s_port.c_str(), &endptr, 10); - if (errno == ERANGE || errno == EINVAL) i_port = LONG_MIN; - i_errcode = std::strtol(s_errcode.c_str(), &endptr, 10); - if (errno == ERANGE || errno == EINVAL) i_errcode = LONG_MIN; - } - - if (c_res == true && i_hg != LONG_MIN && i_port != LONG_MIN && i_errcode != LONG_MIN) { - MyHGM->wrlock(); - - MySrvC* mysrvc = MyHGM->find_server_in_hg(i_hg, srv_addr, i_port); - if (mysrvc != nullptr) { - int backup_mysql_thread___shun_on_failures = mysql_thread___shun_on_failures; - mysql_thread___shun_on_failures = 1; - - // Set the error twice to surpass 'mysql_thread___shun_on_failures' value. - mysrvc->connect_error(i_errcode, false); - mysrvc->connect_error(i_errcode, false); - - mysql_thread___shun_on_failures = backup_mysql_thread___shun_on_failures; - - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - } else { - std::string t_err_msg { "Supplied server '%s:%d' wasn't found in hg '%d'" }; - std::string err_msg {}; - string_format(t_err_msg, err_msg, srv_addr.c_str(), i_port, i_hg); + fds[0].fd=client; + fds[0].revents=0; + fds[0].events=POLLIN|POLLOUT; + //free(arg->addr); // do not free + free(arg); - proxy_info("%s\n", err_msg.c_str()); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, const_cast(err_msg.c_str())); - } + sess->client_myds->myprot.generate_pkt_initial_handshake(true,NULL,NULL, &sess->thread_session_id, false); - MyHGM->wrunlock(); + while (__sync_fetch_and_add(&glovars.shutdown,0)==0) { + if (myds->available_data_out()) { + fds[0].events=POLLIN|POLLOUT; + } else { + fds[0].events=POLLIN; + } + fds[0].revents=0; + rc=poll(fds,nfds,__sync_fetch_and_add(&__admin_refresh_interval,0)); + if (rc == -1) { + if (errno == EINTR) { + continue; } else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char*)"Invalid arguments supplied with query 'PROXYSQL_SIMULATOR'"); + goto __exit_child_mysql; } - - run_query=false; - goto __run_query; } - } -#endif // DEBUG - if (!strncasecmp("PROXYSQLTEST ", query_no_space, strlen("PROXYSQLTEST "))) { - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - int test_n = 0; - int test_arg1 = 0; - int test_arg2 = 0; - int test_arg3 = -1; - int test_arg4 = -1; - int r1 = 0; - proxy_warning("Received PROXYSQLTEST command: %s\n", query_no_space); - char *msg = NULL; - sscanf(query_no_space+strlen("PROXYSQLTEST "),"%d %d %d %d %d", &test_n, &test_arg1, &test_arg2, &test_arg3, &test_arg4); - if (test_n) { - switch (test_n) { - case 1: - // generate test_arg1*1000 entries in digest map - if (test_arg1==0) { - test_arg1=1; - } - r1 = ProxySQL_Test___GenerateRandomQueryInDigestTable(test_arg1); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 2: - // get all the entries from the digest map, but without writing to DB - // it uses multiple threads - r1 = ProxySQL_Test___GetDigestTable(false, false); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 3: - // get all the entries from the digest map and reset, but without writing to DB - // it uses multiple threads - r1 = ProxySQL_Test___GetDigestTable(true, false); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 4: - // purge the digest map, synchronously, in single thread - r1 = ProxySQL_Test___PurgeDigestTable(false, false, NULL); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 5: - // purge the digest map, synchronously, in multiple threads - r1 = ProxySQL_Test___PurgeDigestTable(false, true, NULL); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 6: - // purge the digest map, asynchronously, in single thread - r1 = ProxySQL_Test___PurgeDigestTable(true, false, &msg); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg, r1); - free(msg); - run_query=false; - break; - case 7: - // get all the entries from the digest map and reset, but without writing to DB - // it uses multiple threads - // it locks for a very short time and doesn't use SQLite3_result, but swap - r1 = ProxySQL_Test___GetDigestTable(true, true); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 8: - // get all the entries from the digest map and reset, AND write to DB - r1 = SPA->FlushDigestTableToDisk(SPA->statsdb_disk); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 11: // generate username - case 15: // no username, empty string - // generate random mysql_query_rules_fast_routing - if (test_arg1==0) { - test_arg1=10000; - } - if (test_n==15) { - r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, true); - } else { - r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, false); - } - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)"Generated new mysql_query_rules_fast_routing table", r1); - run_query=false; - break; - case 12: // generate username - case 16: // no username, empty string - // generate random mysql_query_rules_fast_routing and LOAD TO RUNTIME - if (test_arg1==0) { - test_arg1=10000; - } - if (test_n==16) { - r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, true); - } else { - r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, false); - } - msg = SPA->load_mysql_query_rules_to_runtime(); - if (msg==NULL) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)"Generated new mysql_query_rules_fast_routing table and loaded to runtime", r1); - } else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, msg); - } - run_query=false; - break; - case 13: - // LOAD MYSQL QUERY RULES TO RUNTIME for N times - if (test_arg1==0) { - test_arg1=1; - } - for (int i=0; iload_mysql_query_rules_to_runtime(); - } - msg = (char *)malloc(128); - sprintf(msg,"Loaded mysql_query_rules_fast_routing to runtime %d times",test_arg1); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg); - run_query=false; - free(msg); - break; - case 14: // old algorithm - case 17: // perform dual lookup, with and without username - // Allows to verify and benchmark 'mysql_query_rules_fast_routing'. Every options - // verifies all 'mysql_query_rules_fast_routing' rules: - // - Test num: 14 old algorithm, 17 perform a dual lookup. - // - arg1: 1-N Number of times the computation should be repeated. - // - arg2: 1-N Number of parallel threads for the test. - // - arg3: 1-0 Wether or not to acquire a read_lock before searching in the hashmap. - // - arg4: 1-0 Wether or not to create thread specific hashmaps for the search. - if (test_arg1==0) { - test_arg1=1; - } - // To preserve classic mode - if (test_arg3 == -1) { - test_arg3 = 1; - } - if (test_arg4 == -1) { - test_arg4 = 0; - } - { - int ret1, ret2; - bool bret = SPA->ProxySQL_Test___Verify_mysql_query_rules_fast_routing( - &ret1, &ret2, test_arg1, (test_n==14 ? 0 : 1), test_arg2, test_arg3, test_arg4 - ); - if (bret) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)"Verified all rules in mysql_query_rules_fast_routing", ret1); - } else { - if (ret1==-1) { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Severe error in verifying rules in mysql_query_rules_fast_routing"); - } else { - msg = (char *)malloc(256); - sprintf(msg,"Error verifying mysql_query_rules_fast_routing. Found %d rows out of %d", ret1, ret2); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, msg); - free(msg); - } - } - } - run_query=false; - break; - case 21: - // refresh mysql variables N*1000 times - if (test_arg1==0) { - test_arg1=1; - } - test_arg1 *= 1000; - ProxySQL_Test___Refresh_MySQL_Variables(test_arg1); - msg = (char *)malloc(128); - sprintf(msg,"Refreshed MySQL Variables %d times",test_arg1); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg); - run_query=false; - free(msg); - break; - case 22: - // get all the entries from the digest map, but WRITING to DB - // it uses multiple threads - // It locks the maps while generating the resultset - r1 = SPA->stats___mysql_query_digests(false, true); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 23: - // get all the entries from the digest map, but WRITING to DB - // it uses multiple threads for creating the resultset - r1 = SPA->stats___mysql_query_digests_v2(false, false, true); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 24: - // get all the entries from the digest map, but WRITING to DB - // Do not create a resultset, uses the digest_umap - r1 = SPA->stats___mysql_query_digests_v2(false, false, false); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 25: - // get all the entries from the digest map AND RESET, but WRITING to DB - // it uses multiple threads - // It locks the maps while generating the resultset - r1 = SPA->stats___mysql_query_digests(true, true); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 26: - // get all the entries from the digest map AND RESET, but WRITING to DB - // it uses multiple threads for creating the resultset - r1 = SPA->stats___mysql_query_digests_v2(true, true, true); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 27: - // get all the entries from the digest map AND RESET, but WRITING to DB - // Do not create a resultset, uses the digest_umap - r1 = SPA->stats___mysql_query_digests_v2(true, true, false); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); - run_query=false; - break; - case 31: - { - if (test_arg1==0) { - test_arg1=1; - } - if (test_arg1 > 4) { - test_arg1=1; - } -/* - if (test_arg1 == 2 || test_arg1 == 3) { - if (test_arg2 == 0) { - test_arg2 = 1; - } - } -*/ - int ret1; - int ret2; - SPA->ProxySQL_Test___Load_MySQL_Whitelist(&ret1, &ret2, test_arg1, test_arg2); - if (test_arg1==1 || test_arg1==4) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)"Processed all rows from firewall whitelist", ret1); - } else if (test_arg1==2 || test_arg1==3) { - if (ret1 == ret2) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)"Verified all rows from firewall whitelist", ret1); - } else { - msg = (char *)malloc(256); - sprintf(msg,"Error verifying firewall whitelist. Found %d entries out of %d", ret2, ret1); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, msg); - free(msg); - } - } - run_query=false; - } - break; - case 41: - { - char msg[256]; - unsigned long long d = SPA->ProxySQL_Test___MySQL_HostGroups_Manager_read_only_action(); - sprintf(msg, "Tested in %llums\n", d); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg); - run_query=false; - } - break; -#ifdef DEBUG - case 51: - { - char msg[256]; - unsigned long long d = SPA->ProxySQL_Test___MySQL_HostGroups_Manager_HG_lookup(); - sprintf(msg, "Tested in %llums\n", d); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg); - run_query=false; - } - break; - case 52: - { - char msg[256]; - SPA->mysql_servers_wrlock(); - SPA->admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id=5211"); - SPA->admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, port, weight) VALUES (5211,'127.0.0.2',3306,10000)"); - SPA->admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, port, weight) VALUES (5211,'127.0.0.3',3306,8000)"); - SPA->admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, port, weight) VALUES (5211,'127.0.0.4',3306,8000)"); - SPA->admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, port, weight) VALUES (5211,'127.0.0.5',3306,7000)"); - SPA->load_mysql_servers_to_runtime(); - SPA->mysql_servers_wrunlock(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql servers to RUNTIME\n"); - unsigned long long d = SPA->ProxySQL_Test___MySQL_HostGroups_Manager_Balancing_HG5211(); - sprintf(msg, "Tested in %llums\n", d); - SPA->mysql_servers_wrlock(); - SPA->admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id=5211"); - SPA->load_mysql_servers_to_runtime(); - SPA->mysql_servers_wrunlock(); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg); - run_query=false; - } - break; - case 53: - { - // Test monitor tasks timeout - // test_arg1: 1 = ON, 0 = OFF - char msg[256]; - GloMyMon->proxytest_forced_timeout = (test_arg1) ? true : false; - sprintf(msg, "Monitor task timeout flag is:%s\n", GloMyMon->proxytest_forced_timeout ? "ON" : "OFF"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg); - run_query = false; - } - break; - case 54: - { - run_query = false; - if (test_arg1 == 0) { - test_arg1 = 1000; - } - if (GloMTH->variables.ssl_p2s_ca == NULL && - GloMTH->variables.ssl_p2s_capath == NULL) { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"'mysql-ssl_p2s_ca' and 'mysql-ssl_p2s_capath' have not been configured"); - break; - } - char msg[256]; - uint64_t duration = 0ULL; - if (SPA->ProxySQL_Test___CA_Certificate_Load_And_Verify(&duration, test_arg1, GloMTH->variables.ssl_p2s_ca, - GloMTH->variables.ssl_p2s_capath)) { - sprintf(msg, "Took %lums in loading and verifying CA Certificate for %d times\n", duration, test_arg1); - SPA->send_MySQL_OK(&sess->client_myds->myprot, msg); - } - else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Unable to verify CA Certificate"); - } - } - break; -#endif // DEBUG - default: - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Invalid test"); - run_query=false; - break; - } - } else { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Invalid test"); + mysql_thr->curtime = monotonic_time(); + myds->revents=fds[0].revents; + int rb = 0; + rb = myds->read_from_net(); + if (myds->net_failure) goto __exit_child_mysql; + myds->read_pkts(); + if (myds->encrypted == true) { + // PMC-10004 + // we probably should use SSL_pending() and/or SSL_has_pending() to determine + // if there is more data to be read, but it doesn't seem to be working. + // Therefore we try to call read_from_net() again as long as there is data. + // Previously we hardcoded 16KB but it seems that it can return in smaller + // chunks of 4KB. + // We finally removed the chunk size as it seems that any size is possible. + while (rb > 0) { + rb = myds->read_from_net(); + if (myds->net_failure) goto __exit_child_mysql; + myds->read_pkts(); } - goto __run_query; } + sess->to_process=1; + int rc=sess->handler(); + if (rc==-1) goto __exit_child_mysql; } - { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - needs_vacuum = SPA->GenericRefreshStatistics(query_no_space,query_no_space_length, ( sess->session_type == PROXYSQL_SESSION_ADMIN ? true : false ) ); - } - - - if (!strncasecmp("SHOW GLOBAL VARIABLES LIKE 'read_only'", query_no_space, strlen("SHOW GLOBAL VARIABLES LIKE 'read_only'"))) { - l_free(query_length,query); - char *q=(char *)"SELECT 'read_only' Variable_name, '%s' Value FROM global_variables WHERE Variable_name='admin-read_only'"; - query_length=strlen(q)+5; - query=(char *)l_alloc(query_length); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - bool ro=SPA->get_read_only(); - //sprintf(query,q,( ro ? "ON" : "OFF")); - PtrSize_t pkt_2; - if (ro) { - pkt_2.size=110; - pkt_2.ptr=l_alloc(pkt_2.size); - memcpy(pkt_2.ptr,READ_ONLY_ON,pkt_2.size); - } else { - pkt_2.size=111; - pkt_2.ptr=l_alloc(pkt_2.size); - memcpy(pkt_2.ptr,READ_ONLY_OFF,pkt_2.size); - } - sess->status=WAITING_CLIENT_DATA; - sess->client_myds->DSS=STATE_SLEEP; - sess->client_myds->PSarrayOUT->add(pkt_2.ptr,pkt_2.size); - run_query=false; - goto __run_query; - } - - if (!strncasecmp("SELECT @@global.read_only", query_no_space, strlen("SELECT @@global.read_only"))) { - l_free(query_length,query); - char *q=(char *)"SELECT 'read_only' Variable_name, '%s' Value FROM global_variables WHERE Variable_name='admin-read_only'"; - query_length=strlen(q)+5; - query=(char *)l_alloc(query_length); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - bool ro=SPA->get_read_only(); - //sprintf(query,q,( ro ? "ON" : "OFF")); - PtrSize_t pkt_2; - if (ro) { - pkt_2.size=73; - pkt_2.ptr=l_alloc(pkt_2.size); - memcpy(pkt_2.ptr,READ_ONLY_1,pkt_2.size); - } else { - pkt_2.size=73; - pkt_2.ptr=l_alloc(pkt_2.size); - memcpy(pkt_2.ptr,READ_ONLY_0,pkt_2.size); - } - sess->status=WAITING_CLIENT_DATA; - sess->client_myds->DSS=STATE_SLEEP; - sess->client_myds->PSarrayOUT->add(pkt_2.ptr,pkt_2.size); - run_query=false; - goto __run_query; - } - - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - if ((query_no_space_length>13) && (!strncasecmp("PULL VERSION ", query_no_space, 13))) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received PULL command\n"); - if ((query_no_space_length>27) && (!strncasecmp("PULL VERSION MYSQL SERVERS ", query_no_space, 27))) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received PULL VERSION MYSQL SERVERS command\n"); - unsigned int wait_mysql_servers_version = 0; - unsigned int wait_timeout = 0; - int rc = sscanf(query_no_space+27,"%u %u",&wait_mysql_servers_version, &wait_timeout); - if (rc < 2) { - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Invalid argument"); - run_query=false; - goto __run_query; - } else { - MyHGM->wait_servers_table_version(wait_mysql_servers_version, wait_timeout); - l_free(query_length,query); - unsigned int curver = MyHGM->get_servers_table_version(); - char buf[256]; - sprintf(buf,"SELECT %u AS 'version'", curver); - query=l_strdup(buf); - query_length=strlen(query)+1; - //SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - //run_query=false; - goto __run_query; - } - } - } +__exit_child_mysql: + delete mysql_thr; - if ((query_no_space_length == strlen("SELECT GLOBAL_CHECKSUM()")) && (!strncasecmp("SELECT GLOBAL_CHECKSUM()", query_no_space, strlen("SELECT GLOBAL_CHECKSUM()")))) { - char buf[32]; - pthread_mutex_lock(&GloVars.checksum_mutex); - sprintf(buf,"%lu",GloVars.checksums_values.global_checksum); - pthread_mutex_unlock(&GloVars.checksum_mutex); - uint16_t setStatus = 0; - MySQL_Data_Stream *myds=sess->client_myds; - MySQL_Protocol *myprot=&sess->client_myds->myprot; - myds->DSS=STATE_QUERY_SENT_DS; - int sid=1; - myprot->generate_pkt_column_count(true,NULL,NULL,sid,1); sid++; - myprot->generate_pkt_field(true,NULL,NULL,sid,(char *)"",(char *)"",(char *)"",(char *)"CHECKSUM",(char *)"",63,31,MYSQL_TYPE_LONGLONG,161,0,false,0,NULL); sid++; - myds->DSS=STATE_COLUMN_DEFINITION; - myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; - char **p=(char **)malloc(sizeof(char*)*1); - unsigned long *l=(unsigned long *)malloc(sizeof(unsigned long *)*1); - l[0]=strlen(buf);; - p[0]=buf; - myprot->generate_pkt_row(true,NULL,NULL,sid,1,l,p); sid++; - myds->DSS=STATE_ROW; - myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; - myds->DSS=STATE_SLEEP; - run_query=false; - free(l); - free(p); - goto __run_query; - } - - - if ((query_no_space_length>8) && (!strncasecmp("PROXYSQL ", query_no_space, 8))) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received PROXYSQL command\n"); - //pthread_mutex_lock(&admin_mutex); - run_query=admin_handler_command_proxysql(query_no_space, query_no_space_length, sess, pa); - //pthread_mutex_unlock(&admin_mutex); - goto __run_query; - } - if ((query_no_space_length>5) && ( (!strncasecmp("SAVE ", query_no_space, 5)) || (!strncasecmp("LOAD ", query_no_space, 5))) ) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received LOAD or SAVE command\n"); - run_query=admin_handler_command_load_or_save(query_no_space, query_no_space_length, sess, pa, &query, &query_length); - goto __run_query; - } - if ((query_no_space_length>16) && ( (!strncasecmp("KILL CONNECTION ", query_no_space, 16)) || (!strncasecmp("KILL CONNECTION ", query_no_space, 16))) ) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received KILL CONNECTION command\n"); - run_query=admin_handler_command_kill_connection(query_no_space, query_no_space_length, sess, pa); - goto __run_query; - } - - - // queries generated by mysqldump - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if ( - !strncmp("/*!40014 SET ", query_no_space, 13) || - !strncmp("/*!40101 SET ", query_no_space, 13) || - !strncmp("/*!40103 SET ", query_no_space, 13) || - !strncmp("/*!40111 SET ", query_no_space, 13) || - !strncmp("/*!80000 SET ", query_no_space, 13) || - !strncmp("/*!50503 SET ", query_no_space, 13) || - !strncmp("/*!50717 SET ", query_no_space, 13) || - !strncmp("/*!50717 SELECT ", query_no_space, strlen("/*!50717 SELECT ")) || - !strncmp("/*!50717 PREPARE ", query_no_space, strlen("/*!50717 PREPARE ")) || - !strncmp("/*!50717 EXECUTE ", query_no_space, strlen("/*!50717 EXECUTE ")) || - !strncmp("/*!50717 DEALLOCATE ", query_no_space, strlen("/*!50717 DEALLOCATE ")) || - !strncmp("/*!50112 SET ", query_no_space, strlen("/*!50112 SET ")) || - !strncmp("/*!50112 PREPARE ", query_no_space, strlen("/*!50112 PREPARE ")) || - !strncmp("/*!50112 EXECUTE ", query_no_space, strlen("/*!50112 EXECUTE ")) || - !strncmp("/*!50112 DEALLOCATE ", query_no_space, strlen("/*!50112 DEALLOCATE ")) || - !strncmp("/*!40000 ALTER TABLE", query_no_space, strlen("/*!40000 ALTER TABLE")) - || - !strncmp("/*!40100 SET @@SQL_MODE='' */", query_no_space, strlen("/*!40100 SET @@SQL_MODE='' */")) - || - !strncmp("/*!40103 SET TIME_ZONE=", query_no_space, strlen("/*!40103 SET TIME_ZONE=")) - || - !strncmp("LOCK TABLES", query_no_space, strlen("LOCK TABLES")) - || - !strncmp("UNLOCK TABLES", query_no_space, strlen("UNLOCK TABLES")) - || - !strncmp("SET SQL_QUOTE_SHOW_CREATE=1", query_no_space, strlen("SET SQL_QUOTE_SHOW_CREATE=1")) - || - !strncmp("SET SESSION character_set_results", query_no_space, strlen("SET SESSION character_set_results")) - || - !strncasecmp("USE ", query_no_space, strlen("USE ")) // this applies to all clients, not only mysqldump - ) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - run_query=false; - goto __run_query; - } - - if (!strncmp("SHOW STATUS LIKE 'binlog_snapshot_gtid_executed'", query_no_space, strlen("SHOW STATUS LIKE 'binlog_snapshot_gtid_executed'"))) { - l_free(query_length, query); - query = l_strdup("SELECT variable_name AS Variable_name, Variable_value AS Value FROM global_variables WHERE 1=0"); - query_length = strlen(query)+1; - goto __run_query; - } - if (!strncmp("SELECT COLUMN_NAME, JSON_EXTRACT(HISTOGRAM, '$.\"number-of-buckets-specified\"') FROM information_schema.COLUMN_STATISTICS", query_no_space, strlen("SELECT COLUMN_NAME, JSON_EXTRACT(HISTOGRAM, '$.\"number-of-buckets-specified\"') FROM information_schema.COLUMN_STATISTICS"))) { - l_free(query_length, query); - query = l_strdup("SELECT variable_name AS COLUMN_NAME, Variable_value AS 'JSON_EXTRACT(HISTOGRAM, ''$.\"number-of-buckets-specified\"'')' FROM global_variables WHERE 1=0"); - query_length = strlen(query)+1; - goto __run_query; - } - if (!strncmp("SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'performance_schema' AND table_name = 'session_variables'", query_no_space, strlen("SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = 'performance_schema' AND table_name = 'session_variables'"))) { - l_free(query_length,query); - query=l_strdup("SELECT 0 as 'COUNT(*)'"); - query_length=strlen(query)+1; - goto __run_query; - } - if (!strncmp("SHOW VARIABLES LIKE 'gtid\\_mode'", query_no_space, strlen("SHOW VARIABLES LIKE 'gtid\\_mode'"))) { - l_free(query_length,query); - query=l_strdup("SELECT variable_name Variable_name, Variable_value Value FROM global_variables WHERE Variable_name='gtid_mode'"); - query_length=strlen(query)+1; - goto __run_query; - } - if (!strncmp("select @@collation_database", query_no_space, strlen("select @@collation_database"))) { - l_free(query_length,query); - query=l_strdup("SELECT Collation '@@collation_database' FROM mysql_collations WHERE Collation='utf8_general_ci' LIMIT 1"); - query_length=strlen(query)+1; - goto __run_query; - } - if (!strncmp("SHOW VARIABLES LIKE 'ndbinfo\\_version'", query_no_space, strlen("SHOW VARIABLES LIKE 'ndbinfo\\_version'"))) { - l_free(query_length,query); - query=l_strdup("SELECT variable_name Variable_name, Variable_value Value FROM global_variables WHERE Variable_name='ndbinfo_version'"); - query_length=strlen(query)+1; - goto __run_query; - } - if (!strncasecmp("show table status like '", query_no_space, strlen("show table status like '"))) { - char *strA=query_no_space+24; - int strAl=strlen(strA); - if (strAl<2) { // error - goto __run_query; - } - char *err=NULL; - SQLite3_result *resultset=SPA->generate_show_table_status(strA, &err); - sess->SQLite3_to_MySQL(resultset, err, 0, &sess->client_myds->myprot); - if (resultset) delete resultset; - if (err) free(err); - run_query=false; - goto __run_query; - } - if (!strncasecmp("show fields from ", query_no_space, strlen("show fields from "))) { - char *strA=query_no_space+17; - int strAl=strlen(strA); - if (strAl==0) { // error - goto __run_query; - } - if (strA[0]=='`') { - strA++; - strAl--; - } - if (strAl<2) { // error - goto __run_query; - } - char *err=NULL; - SQLite3_result *resultset=SPA->generate_show_fields_from(strA, &err); - sess->SQLite3_to_MySQL(resultset, err, 0, &sess->client_myds->myprot); - if (resultset) delete resultset; - if (err) free(err); - run_query=false; - goto __run_query; - } - } - - // FIXME: this should be removed, it is just a POC for issue #253 . What is important is the call to GloMTH->signal_all_threads(); - if (!strncasecmp("SIGNAL MYSQL THREADS", query_no_space, strlen("SIGNAL MYSQL THREADS"))) { - GloMTH->signal_all_threads(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received %s command\n", query_no_space); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_admin_variables_from_runtime(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Sent signal to all mysql threads\n"); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - run_query=false; - goto __run_query; - } - - // fix bug #442 - if (!strncmp("SET SQL_SAFE_UPDATES=1", query_no_space, strlen("SET SQL_SAFE_UPDATES=1"))) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - run_query=false; - goto __run_query; - } - - // fix bug #1047 - if ( - (!strncasecmp("BEGIN", query_no_space, strlen("BEGIN"))) - || - (!strncasecmp("START TRANSACTION", query_no_space, strlen("START TRANSACTION"))) - || - (!strncasecmp("COMMIT", query_no_space, strlen("COMMIT"))) - || - (!strncasecmp("ROLLBACK", query_no_space, strlen("ROLLBACK"))) - || - (!strncasecmp("SET character_set_results", query_no_space, strlen("SET character_set_results"))) - || - (!strncasecmp("SET SQL_AUTO_IS_NULL", query_no_space, strlen("SET SQL_AUTO_IS_NULL"))) - || - (!strncasecmp("SET NAMES", query_no_space, strlen("SET NAMES"))) - || - (!strncasecmp("SET AUTOCOMMIT", query_no_space, strlen("SET AUTOCOMMIT"))) - ) { - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); - run_query=false; - goto __run_query; - } + __sync_fetch_and_sub(&GloVars.statuses.stack_memory_admin_threads,tmp_stack_size); - // MySQL client check command for dollars quote support, starting at version '8.1.0'. See #4300. - if (!strncasecmp("SELECT $$", query_no_space, strlen("SELECT $$"))) { - pair err_info { get_dollar_quote_error(mysql_thread___server_version) }; - SPA->send_MySQL_ERR(&sess->client_myds->myprot, const_cast(err_info.second), err_info.first); - run_query=false; - goto __run_query; - } + return NULL; +} + +void* child_postgres(void* arg) { + if (GloPTH == nullptr) { return NULL; } - if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { - if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT '(ProxySQL Admin Module)'"); - query_length=strlen(query)+1; - goto __run_query; + pthread_attr_t thread_attr; + size_t tmp_stack_size = 0; + if (!pthread_attr_init(&thread_attr)) { + if (!pthread_attr_getstacksize(&thread_attr, &tmp_stack_size)) { + __sync_fetch_and_add(&GloVars.statuses.stack_memory_admin_threads, tmp_stack_size); } } - if (!strncasecmp("select concat(@@version, ' ', @@version_comment)", query_no_space, strlen("select concat(@@version, ' ', @@version_comment)"))) { - l_free(query_length,query); - char *q = const_cast("SELECT '%s Admin Module'"); - query_length = strlen(q) + strlen(PROXYSQL_VERSION) + 1; - query = static_cast(l_alloc(query_length)); - sprintf(query, q, PROXYSQL_VERSION); - goto __run_query; - } + arg_proxysql_adm* myarg = (arg_proxysql_adm*)arg; + int client = myarg->client_t; - // add support for SELECT current_user() and SELECT user() - // see https://github.com/sysown/proxysql/issues/1105#issuecomment-990940585 - if ( - (strcasecmp("SELECT current_user()", query_no_space) == 0) - || - (strcasecmp("SELECT user()", query_no_space) == 0) - ) { - bool current = false; - if (strcasestr(query_no_space, "current") != NULL) - current = true; - l_free(query_length,query); - std::string s = "SELECT '"; - s += sess->client_myds->myconn->userinfo->username ; - if (strlen(sess->client_myds->addr.addr) > 0) { - s += "@"; - s += sess->client_myds->addr.addr; - } - s += "' AS '"; - if (current == true) { - s+= "current_"; - } - s += "user()'"; - query=l_strdup(s.c_str()); - query_length=strlen(query)+1; - goto __run_query; - } - - if (!strncasecmp("select @@sql_mode", query_no_space, strlen("select @@sql_mode"))) { - l_free(query_length,query); - char *q = const_cast("SELECT \"\" as \"@@sql_mode\""); - query_length = strlen(q) + strlen(PROXYSQL_VERSION) + 1; - query = static_cast(l_alloc(query_length)); - sprintf(query, q, PROXYSQL_VERSION); - goto __run_query; - } - - // trivial implementation for 'connection_id()' to support 'mycli'. See #3247 - if (!strncasecmp("select connection_id()", query_no_space, strlen("select connection_id()"))) { - l_free(query_length,query); - // 'connection_id()' is always forced to be '0' - query=l_strdup("SELECT 0 AS 'CONNECTION_ID()'"); - query_length=strlen(query)+1; - goto __run_query; - } - - // implementation for 'SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())' in order to support'csharp' connector. See #2543 - if (!strncasecmp("SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())", query_no_space, strlen("SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP())"))) { - l_free(query_length,query); - char *query1=(char*)"SELECT '%s' as 'TIMEDIFF(NOW(), UTC_TIMESTAMP()'"; - - // compute the timezone diff - std::string timezone_offset_str = timediff_timezone_offset(); - char *query2=(char *)malloc(strlen(query1) + strlen(timezone_offset_str.c_str()) + 1); - - // format the query - sprintf(query2, query1, timezone_offset_str.c_str()); - - // copy the resulting query - query=l_strdup(query2); - query_length=strlen(query2) + 1; - - // free the buffer used to format - free(query2); - goto __run_query; - } - - // implementation for '"select @@max_allowed_packet, @@character_set_client, @@character_set_connection, @@license, @@sql_mode, @@lower_case_table_names"' - // in order to support 'csharp' connector. See #2543 - if ( - !strncasecmp( - "select @@max_allowed_packet, @@character_set_client, @@character_set_connection, @@license, @@sql_mode, @@lower_case_table_names", - query_no_space, - strlen("select @@max_allowed_packet, @@character_set_client, @@character_set_connection, @@license, @@sql_mode, @@lower_case_table_names") - ) - ) { - l_free(query_length,query); - char *query1= - const_cast( - "select '67108864' as '@@max_allowed_packet', 'utf8' as '@@character_set_client', 'utf8' as '@@character_set_connection', '' as '@@license', '' as '@@sql_mode', '' as '@@lower_case_table_names'" - ); - query=l_strdup(query1); - query_length=strlen(query1)+1; - goto __run_query; - } + //struct sockaddr *addr = arg->addr; + //socklen_t addr_size; - if (query_no_space_length==SELECT_DB_USER_LEN) { - if (!strncasecmp(SELECT_DB_USER, query_no_space, query_no_space_length)) { - l_free(query_length,query); - char *query1=(char *)"SELECT \"admin\" AS 'DATABASE()', \"%s\" AS 'USER()'"; - char *query2=(char *)malloc(strlen(query1)+strlen(sess->client_myds->myconn->userinfo->username)+10); - sprintf(query2,query1,sess->client_myds->myconn->userinfo->username); - query=l_strdup(query2); - query_length=strlen(query2)+1; - free(query2); - goto __run_query; - } + GloPTH->wrlock(); + { + char* s = GloPTH->get_variable((char*)"server_capabilities"); + mysql_thread___server_capabilities = atoi(s); + free(s); } + GloPTH->wrunlock(); - if (query_no_space_length==SELECT_CHARSET_VARIOUS_LEN) { - if (!strncasecmp(SELECT_CHARSET_VARIOUS, query_no_space, query_no_space_length)) { - l_free(query_length,query); - char *query1=(char *)"select 'utf8' as '@@character_set_client', 'utf8' as '@@character_set_connection', 'utf8' as '@@character_set_server', 'utf8' as '@@character_set_database' limit 1"; - query=l_strdup(query1); - query_length=strlen(query1)+1; - goto __run_query; - } - } + struct pollfd fds[1]; + nfds_t nfds = 1; + int rc; + pthread_mutex_unlock(&sock_mutex); + PgSQL_Thread* pgsql_thr = new PgSQL_Thread(); + pgsql_thr->curtime = monotonic_time(); + GloPgQPro->init_thread(); + pgsql_thr->refresh_variables(); + PgSQL_Session* sess = pgsql_thr->create_new_session_and_client_data_stream(client); + sess->thread = pgsql_thr; + sess->session_type = PROXYSQL_SESSION_ADMIN; + sess->handler_function=admin_session_handler; + PgSQL_Data_Stream* myds = sess->client_myds; + sess->start_time = pgsql_thr->curtime; - if (!strncasecmp("SELECT @@version", query_no_space, strlen("SELECT @@version"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS '@@version'"; - if (GloMyLdapAuth == nullptr) { - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); - } else { - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen("-Enterprise"); - } - query=(char *)l_alloc(query_length); - if (GloMyLdapAuth == nullptr) { - sprintf(query, q, PROXYSQL_VERSION); - } else { - sprintf(query, q, PROXYSQL_VERSION"-Enterprise"); - } - goto __run_query; - } + sess->client_myds->client_addrlen = myarg->addr_size; + sess->client_myds->client_addr = myarg->addr; - if (!strncasecmp("SELECT version()", query_no_space, strlen("SELECT version()"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS 'version()'"; - if (GloMyLdapAuth == nullptr) { - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); - } else { - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION)+strlen("-Enterprise"); - } - query=(char *)l_alloc(query_length); - if (GloMyLdapAuth == nullptr) { - sprintf(query, q, PROXYSQL_VERSION); - } else { - sprintf(query, q, PROXYSQL_VERSION"-Enterprise"); - } - goto __run_query; - } - if (!strncasecmp("SHOW VARIABLES WHERE Variable_name in", query_no_space, strlen("SHOW VARIABLES WHERE Variable_name in"))) { - // Allow MariaDB ConnectorJ to connect to Admin #743 - if (!strncasecmp("SHOW VARIABLES WHERE Variable_name in ('max_allowed_packet','system_time_zone','time_zone','sql_mode')", query_no_space, strlen("SHOW VARIABLES WHERE Variable_name in ('max_allowed_packet','system_time_zone','time_zone','sql_mode')"))) { - l_free(query_length,query); - char *q=(char *)"SELECT 'max_allowed_packet' Variable_name,'4194304' Value UNION ALL SELECT 'sql_mode', 'STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION' UNION ALL SELECT 'system_time_zone', 'UTC' UNION ALL SELECT 'time_zone','SYSTEM'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - sprintf(query,q,PROXYSQL_VERSION); - goto __run_query; - } - // Allow MariaDB ConnectorJ 2.4.1 to connect to Admin #2009 - if (!strncasecmp("SHOW VARIABLES WHERE Variable_name in ('max_allowed_packet','system_time_zone','time_zone','auto_increment_increment')", query_no_space, strlen("SHOW VARIABLES WHERE Variable_name in ('max_allowed_packet','system_time_zone','time_zone','auto_increment_increment')"))) { - l_free(query_length,query); - char *q=(char *)"SELECT 'max_allowed_packet' Variable_name,'4194304' Value UNION ALL SELECT 'auto_increment_increment', '1' UNION ALL SELECT 'system_time_zone', 'UTC' UNION ALL SELECT 'time_zone','SYSTEM'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - sprintf(query,q,PROXYSQL_VERSION); - goto __run_query; - } - } - { - bool rc; - rc=RE2::PartialMatch(query_no_space,*(RE2 *)(pa->match_regexes.re[0])); - if (rc) { - string *new_query=new std::string(query_no_space); - RE2::Replace(new_query,(char *)"^(\\w+)\\s+@@(\\w+)\\s*",(char *)"SELECT variable_value AS '@@max_allowed_packet' FROM global_variables WHERE variable_name='mysql-max_allowed_packet'"); - free(query); - query_length=new_query->length()+1; - query=(char *)malloc(query_length); - memcpy(query,new_query->c_str(),query_length-1); - query[query_length-1]='\0'; - delete new_query; - goto __run_query; - } - } - { - bool rc; - rc=RE2::PartialMatch(query_no_space,*(RE2 *)(pa->match_regexes.re[1])); - if (rc) { - string *new_query=new std::string(query_no_space); - RE2::Replace(new_query,(char *)"^(\\w+) *@@([0-9A-Za-z_-]+) *",(char *)"SELECT variable_value AS '@@\\2' FROM global_variables WHERE variable_name='\\2' COLLATE NOCASE UNION ALL SELECT variable_value AS '@@\\2' FROM stats.stats_mysql_global WHERE variable_name='\\2' COLLATE NOCASE"); - free(query); - query_length=new_query->length()+1; - query=(char *)malloc(query_length); - memcpy(query,new_query->c_str(),query_length-1); - query[query_length-1]='\0'; - GloAdmin->stats___mysql_global(); - delete new_query; - goto __run_query; - } - } - { - bool rc; - rc=RE2::PartialMatch(query_no_space,*(RE2 *)(pa->match_regexes.re[2])); - if (rc) { - string *new_query=new std::string(query_no_space); - RE2::Replace(new_query,(char *)"([Ss][Hh][Oo][Ww]\\s+[Vv][Aa][Rr][Ii][Aa][Bb][Ll][Ee][Ss]\\s+[Ww][Hh][Ee][Rr][Ee])",(char *)"SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables WHERE"); - free(query); - query_length=new_query->length()+1; - query=(char *)malloc(query_length); - memcpy(query,new_query->c_str(),query_length-1); - query[query_length-1]='\0'; - delete new_query; - goto __run_query; - } - } - { - bool rc; - rc=RE2::PartialMatch(query_no_space,*(RE2 *)(pa->match_regexes.re[3])); - if (rc) { - string *new_query=new std::string(query_no_space); - RE2::Replace(new_query,(char *)"([Ss][Hh][Oo][Ww]\\s+[Vv][Aa][Rr][Ii][Aa][Bb][Ll][Ee][Ss]\\s+[Ll][Ii][Kk][Ee])",(char *)"SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables WHERE variable_name LIKE"); - free(query); - query_length=new_query->length()+1; - query=(char *)malloc(query_length); - memcpy(query,new_query->c_str(),query_length-1); - query[query_length-1]='\0'; - delete new_query; - goto __run_query; - } - } + switch (sess->client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in* ipv4 = (struct sockaddr_in*)sess->client_myds->client_addr; + char buf[INET_ADDRSTRLEN]; + inet_ntop(sess->client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + sess->client_myds->addr.addr = strdup(buf); + sess->client_myds->addr.port = htons(ipv4->sin_port); + break; + } + case AF_INET6: { + struct sockaddr_in6* ipv6 = (struct sockaddr_in6*)sess->client_myds->client_addr; + char buf[INET6_ADDRSTRLEN]; + inet_ntop(sess->client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + sess->client_myds->addr.addr = strdup(buf); + sess->client_myds->addr.port = htons(ipv6->sin6_port); + break; + } + default: + sess->client_myds->addr.addr = strdup("localhost"); + break; + } + + fds[0].fd = client; + fds[0].revents = 0; + fds[0].events = POLLIN | POLLOUT; + //free(arg->addr); // do not free + free(arg); - if (!strncasecmp("SET ", query_no_space, 4)) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received SET\n"); - run_query = admin_handler_command_set(query_no_space, query_no_space_length, sess, pa, &query, &query_length); - goto __run_query; - } + myds->DSS = STATE_SERVER_HANDSHAKE; + sess->status = CONNECTING_CLIENT; - if(!strncasecmp("CHECKSUM ", query_no_space, 9)){ - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received CHECKSUM command\n"); - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SQLite3_result *resultset=NULL; - char *tablename=NULL; - char *error=NULL; - int affected_rows=0; - int cols=0; - if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL SERVERS") && !strncasecmp("CHECKSUM DISK MYSQL SERVERS", query_no_space, strlen(query_no_space))){ - char *q=(char *)"SELECT * FROM mysql_servers ORDER BY hostgroup_id, hostname, port"; - tablename=(char *)"MYSQL SERVERS"; - SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL USERS") && !strncasecmp("CHECKSUM DISK MYSQL USERS", query_no_space, strlen(query_no_space))){ - char *q=(char *)"SELECT * FROM mysql_users ORDER BY username"; - tablename=(char *)"MYSQL USERS"; - SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL QUERY RULES") && !strncasecmp("CHECKSUM DISK MYSQL QUERY RULES", query_no_space, strlen(query_no_space))){ - char *q=(char *)"SELECT * FROM mysql_query_rules ORDER BY rule_id"; - tablename=(char *)"MYSQL QUERY RULES"; - SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL VARIABLES") && !strncasecmp("CHECKSUM DISK MYSQL VARIABLES", query_no_space, strlen(query_no_space))){ - char *q=(char *)"SELECT * FROM global_variables WHERE variable_name LIKE 'mysql-%' ORDER BY variable_name"; - tablename=(char *)"MYSQL VARIABLES"; - SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if (strlen(query_no_space)==strlen("CHECKSUM DISK MYSQL REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM DISK MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))){ - char *q=(char *)"SELECT * FROM mysql_replication_hostgroups ORDER BY writer_hostgroup"; - tablename=(char *)"MYSQL REPLICATION HOSTGROUPS"; - SPA->configdb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL SERVERS") && !strncasecmp("CHECKSUM MEMORY MYSQL SERVERS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL SERVERS") && !strncasecmp("CHECKSUM MEM MYSQL SERVERS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL SERVERS") && !strncasecmp("CHECKSUM MYSQL SERVERS", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_servers ORDER BY hostgroup_id, hostname, port"; - tablename=(char *)"MYSQL SERVERS"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL USERS") && !strncasecmp("CHECKSUM MEMORY MYSQL USERS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL USERS") && !strncasecmp("CHECKSUM MEM MYSQL USERS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL USERS") && !strncasecmp("CHECKSUM MYSQL USERS", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_users ORDER BY username"; - tablename=(char *)"MYSQL USERS"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL QUERY RULES") && !strncasecmp("CHECKSUM MEMORY MYSQL QUERY RULES", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL QUERY RULES") && !strncasecmp("CHECKSUM MEM MYSQL QUERY RULES", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL QUERY RULES") && !strncasecmp("CHECKSUM MYSQL QUERY RULES", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_query_rules ORDER BY rule_id"; - tablename=(char *)"MYSQL QUERY RULES"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL VARIABLES") && !strncasecmp("CHECKSUM MEMORY MYSQL VARIABLES", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL VARIABLES") && !strncasecmp("CHECKSUM MEM MYSQL VARIABLES", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL VARIABLES") && !strncasecmp("CHECKSUM MYSQL VARIABLES", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM global_variables WHERE variable_name LIKE 'mysql-%' ORDER BY variable_name"; - tablename=(char *)"MYSQL VARIABLES"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_replication_hostgroups ORDER BY writer_hostgroup"; - tablename=(char *)"MYSQL REPLICATION HOSTGROUPS"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL GROUP REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL GROUP REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL GROUP REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL GROUP REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL GROUP REPLICATION HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL GROUP REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_group_replication_hostgroups ORDER BY writer_hostgroup"; - tablename=(char *)"MYSQL GROUP REPLICATION HOSTGROUPS"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL GALERA HOSTGROUPS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL GALERA HOSTGROUPS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL GALERA HOSTGROUPS", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_galera_hostgroups ORDER BY writer_hostgroup"; - tablename=(char *)"MYSQL GALERA HOSTGROUPS"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL AURORA HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY MYSQL AURORA HOSTGROUPS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL AURORA HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL AURORA HOSTGROUPS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL AURORA HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL AURORA HOSTGROUPS", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"; - tablename=(char *)"MYSQL AURORA HOSTGROUPS"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEMORY MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MEM MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL HOSTGROUP ATTRIBUTES") && !strncasecmp("CHECKSUM MYSQL HOSTGROUP ATTRIBUTES", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_hostgroup_attributes ORDER BY hostgroup_id"; - tablename=(char *)"MYSQL HOSTGROUP ATTRIBUTES"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); - } - if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL SERVERS SSL PARAMS") && !strncasecmp("CHECKSUM MEMORY MYSQL SERVERS SSL PARAMS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL SERVERS SSL PARAMS") && !strncasecmp("CHECKSUM MEM MYSQL SERVERS SSL PARAMS", query_no_space, strlen(query_no_space))) - || - (strlen(query_no_space)==strlen("CHECKSUM MYSQL SERVERS SSL PARAMS") && !strncasecmp("CHECKSUM MYSQL SERVERS SSL PARAMS", query_no_space, strlen(query_no_space)))){ - char *q=(char *)"SELECT * FROM mysql_servers_ssl_params ORDER BY hostname, port, username"; - tablename=(char *)"MYSQL HOSTGROUP ATTRIBUTES"; - SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + while (__sync_fetch_and_add(&glovars.shutdown, 0) == 0) { + if (myds->available_data_out()) { + fds[0].events = POLLIN | POLLOUT; } - - if (error) { - proxy_error("Error: %s\n", error); - char buf[1024]; - sprintf(buf,"%s", error); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, buf); - run_query=false; - } else if (resultset) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS 'table', '%s' AS 'checksum'"; - char *checksum=(char *)resultset->checksum(); - query=(char *)malloc(strlen(q)+strlen(tablename)+strlen(checksum)+1); - sprintf(query,q,tablename,checksum); - query_length = strlen(query); - free(checksum); - delete resultset; + else { + fds[0].events = POLLIN; } - goto __run_query; - } - - if (!strncasecmp("SELECT CONFIG INTO OUTFILE", query_no_space, strlen("SELECT CONFIG INTO OUTFILE"))) { - std::string fileName = query_no_space + strlen("SELECT CONFIG INTO OUTFILE"); - fileName.erase(0, fileName.find_first_not_of("\t\n\v\f\r ")); - fileName.erase(fileName.find_last_not_of("\t\n\v\f\r ") + 1); - if (fileName.size() == 0) { - std::stringstream ss; - ss << "ProxySQL Admin Error: empty file name"; - sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); - } - std::string data; - data.reserve(100000); - data += config_header; - int rc = pa->proxysql_config().Write_Global_Variables_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Users_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); - rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); - rc = pa->proxysql_config().Write_Restapi_to_configfile(data); - rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); - if (rc) { - std::stringstream ss; - ss << "ProxySQL Admin Error: Cannot extract configuration"; - sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); - } else { - std::ofstream out; - out.open(fileName); - if (out.is_open()) { - out << data; - out.close(); - if (!out) { - std::stringstream ss; - ss << "ProxySQL Admin Error: Error writing file " << fileName; - sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); - } else { - std::stringstream ss; - ss << "File " << fileName << " is saved."; - SPA->send_MySQL_OK(&sess->client_myds->myprot, (char*)ss.str().c_str(), data.size()); - } - } else { - std::stringstream ss; - ss << "ProxySQL Admin Error: Cannot open file " << fileName; - sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); - } - } - run_query = false; - goto __run_query; - } - - if (query_no_space_length==strlen("SELECT CONFIG FILE") && !strncasecmp("SELECT CONFIG FILE", query_no_space, query_no_space_length)) { - std::string data; - data.reserve(100000); - data += config_header; - int rc = pa->proxysql_config().Write_Global_Variables_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Users_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); - rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); - rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); - rc = pa->proxysql_config().Write_Restapi_to_configfile(data); - rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); - if (rc) { - std::stringstream ss; - ss << "ProxySQL Admin Error: Cannot write proxysql.cnf"; - sess->SQLite3_to_MySQL(resultset, (char*)ss.str().c_str(), affected_rows, &sess->client_myds->myprot); - } else { - char *pta[1]; - pta[0]=NULL; - pta[0]=(char*)data.c_str(); - SQLite3_result* resultset = new SQLite3_result(1); - resultset->add_column_definition(SQLITE_TEXT,"Data"); - resultset->add_row(pta); - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; + fds[0].revents = 0; + rc = poll(fds, nfds, __sync_fetch_and_add(&__admin_refresh_interval, 0)); + if (rc == -1) { + if (errno == EINTR) { + continue; + } + else { + goto __exit_child_postgres; + } + } + pgsql_thr->curtime = monotonic_time(); + myds->revents = fds[0].revents; + int rb = 0; + rb = myds->read_from_net(); + if (myds->net_failure) goto __exit_child_postgres; + myds->read_pkts(); + if (myds->encrypted == true) { + // PMC-10004 + // we probably should use SSL_pending() and/or SSL_has_pending() to determine + // if there is more data to be read, but it doesn't seem to be working. + // Therefore we try to call read_from_net() again as long as there is data. + // Previously we hardcoded 16KB but it seems that it can return in smaller + // chunks of 4KB. + // We finally removed the chunk size as it seems that any size is possible. + while (rb > 0) { + rb = myds->read_from_net(); + if (myds->net_failure) goto __exit_child_postgres; + myds->read_pkts(); + } } - run_query = false; - goto __run_query; + sess->to_process = 1; + int rc = sess->handler(); + if (rc == -1) goto __exit_child_postgres; } - if (strncasecmp("SHOW ", query_no_space, 5)) { - goto __end_show_commands; // in the next block there are only SHOW commands - } - if (!strncasecmp("SHOW PROMETHEUS METRICS", query_no_space, strlen("SHOW PROMETHEUS METRICS"))) { - char* pta[1]; - pta[0] = NULL; - SQLite3_result* resultset = new SQLite3_result(1); - resultset->add_column_definition(SQLITE_TEXT,"Data"); + +__exit_child_postgres: + delete pgsql_thr; - if (__sync_fetch_and_add(&GloMTH->status_variables.threads_initialized, 0) == 1) { - auto result = pa->serial_exposer({}); - pta[0] = (char*)result.second.c_str(); - resultset->add_row(pta); - } else { - resultset->add_row(pta); - } + __sync_fetch_and_sub(&GloVars.statuses.stack_memory_admin_threads, tmp_stack_size); - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - delete resultset; - run_query = false; - - goto __run_query; - } - - if (!strncasecmp("SHOW GLOBAL VARIABLES LIKE 'version'", query_no_space, strlen("SHOW GLOBAL VARIABLES LIKE 'version'"))) { - l_free(query_length,query); - char *q=(char *)"SELECT 'version' Variable_name, '%s' Value FROM global_variables WHERE Variable_name='admin-version'"; - query_length=strlen(q)+20+strlen(PROXYSQL_VERSION); - query=(char *)l_alloc(query_length); - sprintf(query,q,PROXYSQL_VERSION); - goto __run_query; - } - - - if (query_no_space_length==strlen("SHOW TABLES") && !strncasecmp("SHOW TABLES",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT name AS tables FROM sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"); - query_length=strlen(query)+1; - goto __run_query; - } - - if (query_no_space_length==strlen("SHOW CHARSET") && !strncasecmp("SHOW CHARSET",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT Charset, Collation AS 'Default collation' FROM mysql_collations WHERE `Default`='Yes'"); - query_length=strlen(query)+1; - goto __run_query; - } - - if (query_no_space_length==strlen("SHOW COLLATION") && !strncasecmp("SHOW COLLATION",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT * FROM mysql_collations"); - query_length=strlen(query)+1; - goto __run_query; - } - - if ((query_no_space_length>15) && (!strncasecmp("SHOW TABLES IN ", query_no_space, 15))) { - strA=query_no_space+15; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"; - strBl=strlen(strB); - int l=strBl+strAl-2; - char *b=(char *)l_alloc(l+1); - snprintf(b,l+1,strB,strA); - b[l]=0; - l_free(query_length,query); - query=b; - query_length=l+1; - goto __run_query; - } - - if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES FROM ", query_no_space, 17))) { - strA=query_no_space+17; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"; - strBl=strlen(strB); - int l=strBl+strAl-2; - char *b=(char *)l_alloc(l+1); - snprintf(b,l+1,strB,strA); - b[l]=0; - l_free(query_length,query); - query=b; - query_length=l+1; - goto __run_query; - } - - if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES LIKE ", query_no_space, 17))) { - strA=query_no_space+17; - strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM sqlite_master WHERE type='table' AND name LIKE '%s'"; - strBl=strlen(strB); - char *tn=NULL; // tablename - tn=(char *)malloc(strAl+1); - unsigned int i=0, j=0; - while (i<(unsigned int)strAl) { - if (strA[i]!='\\' && strA[i]!='`' && strA[i]!='\'') { - tn[j]=strA[i]; - j++; - } - i++; - } - tn[j]=0; - int l=strBl+strlen(tn)-2; - char *b=(char *)l_alloc(l+1); - snprintf(b,l+1,strB,tn); - b[l]=0; - free(tn); - l_free(query_length,query); - query=b; - query_length=l+1; - goto __run_query; - } + return NULL; +} - if (query_no_space_length==strlen("SHOW MYSQL USERS") && !strncasecmp("SHOW MYSQL USERS",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT * FROM mysql_users ORDER BY username, active DESC, username ASC"); - query_length=strlen(query)+1; - goto __run_query; +void* child_telnet(void* arg) +{ + int bytes_read; + char line[LINESIZE+1]; + int client = *(int *)arg; + free(arg); + pthread_mutex_unlock(&sock_mutex); + memset(line,0,LINESIZE+1); + while ((strncmp(line, "quit", 4) != 0) && glovars.shutdown==0) { + bytes_read = recv(client, line, LINESIZE, 0); + if (bytes_read==-1) { + break; + } + char *eow = strchr(line, '\n'); + if (eow) *eow=0; + //SPA->is_command(line); + if (strncmp(line,"shutdown",8)==0) glovars.shutdown=1; + if (send(client, line, strlen(line), MSG_NOSIGNAL)==-1) break; + if (send(client, "\nOK\n", 4, MSG_NOSIGNAL)==-1) break; } + shutdown(client,SHUT_RDWR); + close(client); + return arg; +} - if (query_no_space_length==strlen("SHOW MYSQL SERVERS") && !strncasecmp("SHOW MYSQL SERVERS",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT * FROM mysql_servers ORDER BY hostgroup_id, hostname, port"); - query_length=strlen(query)+1; - goto __run_query; +/* +void* child_telnet_also(void* arg) +{ + int bytes_read; + char line[LINESIZE+1]; + int client = *(int *)arg; + free(arg); + pthread_mutex_unlock(&sock_mutex); + memset(line,0,LINESIZE+1); + while ((strncmp(line, "quit", 4) != 0) && glovars.shutdown==0) { + bytes_read = recv(client, line, LINESIZE, 0); + if (bytes_read==-1) { + break; + } + char *eow = strchr(line, '\n'); + if (eow) *eow=0; + if (strncmp(line,"shutdown",8)==0) glovars.shutdown=1; + if (send(client, line, strlen(line), MSG_NOSIGNAL)==-1) break; + if (send(client, "\nNOT OK\n", 8, MSG_NOSIGNAL)==-1) break; } + shutdown(client,SHUT_RDWR); + close(client); + return arg; +} +*/ - if ( - (query_no_space_length==strlen("SHOW GLOBAL VARIABLES") && !strncasecmp("SHOW GLOBAL VARIABLES",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SHOW ALL VARIABLES") && !strncasecmp("SHOW ALL VARIABLES",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SHOW VARIABLES") && !strncasecmp("SHOW VARIABLES",query_no_space, query_no_space_length)) - ) { - l_free(query_length,query); - query=l_strdup("SELECT variable_name AS Variable_name, variable_value AS Value FROM global_variables ORDER BY variable_name"); - query_length=strlen(query)+1; - goto __run_query; - } +void * admin_main_loop(void *arg) { + int i; + int rc; + int version=0; + struct pollfd *fds=((struct _main_args *)arg)->fds; + int nfds=((struct _main_args *)arg)->nfds; + int *callback_func=((struct _main_args *)arg)->callback_func; + volatile int *shutdown=((struct _main_args *)arg)->shutdown; + char *socket_names[MAX_ADMIN_LISTENERS]; + set_thread_name("Admin"); + for (i=0;istats___mysql_global(); - goto __run_query; - } - - strA=(char *)"SHOW CREATE TABLE "; - strB=(char *)"SELECT name AS 'table' , REPLACE(REPLACE(sql,' , ', X'2C0A20202020'),'CREATE TABLE %s (','CREATE TABLE %s ('||X'0A20202020') AS 'Create Table' FROM %s.sqlite_master WHERE type='table' AND name='%s'"; - strAl=strlen(strA); - if (strncasecmp("SHOW CREATE TABLE ", query_no_space, strAl)==0) { - strBl=strlen(strB); - char *dbh=NULL; - char *tbh=NULL; - c_split_2(query_no_space+strAl,".",&dbh,&tbh); - - if (strlen(tbh)==0) { - free(tbh); - tbh=dbh; - dbh=strdup("main"); - } - if (strlen(tbh)>=3 && tbh[0]=='`' && tbh[strlen(tbh)-1]=='`') { // tablename is quoted - char *tbh_tmp=(char *)malloc(strlen(tbh)-1); - strncpy(tbh_tmp,tbh+1,strlen(tbh)-2); - tbh_tmp[strlen(tbh)-2]=0; - free(tbh); - tbh=tbh_tmp; - } - int l=strBl+strlen(tbh)*3+strlen(dbh)-8; - char *buff=(char *)l_alloc(l+1); - snprintf(buff,l+1,strB,tbh,tbh,dbh,tbh); - buff[l]=0; - free(tbh); - free(dbh); - l_free(query_length,query); - query=buff; - query_length=l+1; - goto __run_query; - } - - if ( - (query_no_space_length==strlen("SHOW DATABASES") && !strncasecmp("SHOW DATABASES",query_no_space, query_no_space_length)) - || - (query_no_space_length==strlen("SHOW SCHEMAS") && !strncasecmp("SHOW SCHEMAS",query_no_space, query_no_space_length)) - ) { - l_free(query_length,query); - query=l_strdup("PRAGMA DATABASE_LIST"); - query_length=strlen(query)+1; - goto __run_query; + if(GloVars.global.nostart) { + admin_nostart_=true; + pthread_mutex_lock(&GloVars.global.start_mutex); } + __sync_fetch_and_add(&admin_load_main_,1); + while (glovars.shutdown==0 && *shutdown==0) + { + //int *client; + //int client_t; + //socklen_t addr_size = sizeof(addr); + pthread_t child; + size_t stacks; + unsigned long long curtime=monotonic_time(); + unsigned long long next_run=GloAdmin->scheduler_run_once(); + unsigned long long poll_wait=500000; + if (next_run < curtime + 500000) { + poll_wait=next_run-curtime; + } + if (poll_wait > 500000) { + poll_wait=500000; + } + poll_wait=poll_wait/1000; // conversion to millisecond + rc=poll(fds,nfds,poll_wait); + if ((admin_nostart_ && __sync_val_compare_and_swap(&GloVars.global.nostart,0,1)==0) || __sync_fetch_and_add(&glovars.shutdown,0)==1) { + admin_nostart_=false; + pthread_mutex_unlock(&GloVars.global.start_mutex); + } + if ((rc == -1 && errno == EINTR) || rc==0) { + // poll() timeout, try again + goto __end_while_pool; + } + for (i=1;iaddr=(struct sockaddr *)malloc(sizeof(custom_sockaddr)); + passarg->addr_size = sizeof(custom_sockaddr); + memset(passarg->addr, 0, sizeof(custom_sockaddr)); + passarg->client_t = accept(fds[i].fd, (struct sockaddr*)passarg->addr, &passarg->addr_size); +// printf("Connected: %s:%d sock=%d\n", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port), client_t); + pthread_attr_getstacksize (&attr, &stacks); +// printf("Default stack size = %d\n", stacks); + pthread_mutex_lock (&sock_mutex); + //client=(int *)malloc(sizeof(int)); + //*client= client_t; + //if ( pthread_create(&child, &attr, child_func[callback_func[i]], client) != 0 ) { + if ( pthread_create(&child, &attr, child_func[callback_func[i]], passarg) != 0 ) { + // LCOV_EXCL_START + perror("pthread_create"); + proxy_error("Thread creation\n"); + assert(0); + // LCOV_EXCL_STOP + } + } + fds[i].revents=0; + } +__end_while_pool: + { + if (GloProxyStats->MySQL_Threads_Handler_timetoget(curtime)) { + if (GloMTH) { + SQLite3_result * resultset=GloMTH->SQL3_GlobalStatus(false); + if (resultset) { + GloProxyStats->MySQL_Threads_Handler_sets(resultset); + delete resultset; + } + } + if (MyHGM) { + SQLite3_result * resultset=MyHGM->SQL3_Get_ConnPool_Stats(); + if (resultset) { + SQLite3_result * resultset2 = NULL; - if (query_no_space_length==strlen("SHOW FULL PROCESSLIST") && !strncasecmp("SHOW FULL PROCESSLIST",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT * FROM stats_mysql_processlist"); - query_length=strlen(query)+1; - goto __run_query; - } + // In debug, run the code to generate metrics so that it can be tested even if the web interface plugin isn't loaded. + #ifdef DEBUG + if (true) { + #else + if (GloVars.web_interface_plugin) { + #endif + resultset2 = MyHGM->SQL3_Connection_Pool(false); + } + GloProxyStats->MyHGM_Handler_sets(resultset, resultset2); + delete resultset; + if (resultset2) { + delete resultset2; + } + } + } + } + if (GloProxyStats->MySQL_Query_Cache_timetoget(curtime)) { + if (GloQC) { + SQLite3_result * resultset=GloQC->SQL3_getStats(); + if (resultset) { + GloProxyStats->MySQL_Query_Cache_sets(resultset); + delete resultset; + } + } + } + if (GloProxyStats->mysql_query_digest_to_disk_timetoget(curtime)) { + unsigned long long curtime1=monotonic_time(); + int r1 = SPA->FlushDigestTableToDisk(SPA->statsdb_disk); + unsigned long long curtime2=monotonic_time(); + curtime1 = curtime1/1000; + curtime2 = curtime2/1000; + proxy_info("Automatically saved stats_mysql_query_digest to disk: %llums to write %d entries\n", curtime2-curtime1, r1); + } + if (GloProxyStats->system_cpu_timetoget(curtime)) { + GloProxyStats->system_cpu_sets(); + } +#ifndef NOJEM + if (GloProxyStats->system_memory_timetoget(curtime)) { + GloProxyStats->system_memory_sets(); + } +#endif + } + if (S_amll.get_version()!=version) { + S_amll.wrlock(); + version=S_amll.get_version(); + for (i=1; ipipefd[0]; + fds[nfds].events=POLLIN; + fds[nfds].revents=0; + nfds++; + unsigned int j; + i=0; j=0; + for (j=0; jifaces->len; j++) { + char *add=NULL; char *port=NULL; char *sn=(char *)S_amll.ifaces_mysql->ifaces->index(j); + bool is_ipv6 = false; + char *h = NULL; + if (*sn == '[') { + is_ipv6 = true; + char *p = strchr(sn, ']'); + if (p == NULL) + proxy_error("Invalid IPv6 address: %s\n", sn); - if (query_no_space_length==strlen("SHOW PROCESSLIST") && !strncasecmp("SHOW PROCESSLIST",query_no_space, query_no_space_length)) { - l_free(query_length,query); - query=l_strdup("SELECT SessionID, user, db, hostgroup, command, time_ms, SUBSTR(info,0,100) info FROM stats_mysql_processlist"); - query_length=strlen(query)+1; - goto __run_query; - } + h = ++sn; // remove first '[' + *p = '\0'; + sn = p++; // remove last ']' + add = h; + port = ++p; // remove ':' + } else { + c_split_2(sn, ":" , &add, &port); + } + +#ifdef SO_REUSEPORT + int s = ( atoi(port) ? listen_on_port(add, atoi(port), 128, true) : listen_on_unix(add, 128)); +#else + int s = ( atoi(port) ? listen_on_port(add, atoi(port), 128) : listen_on_unix(add, 128)); +#endif + //if (s>0) { fds[nfds].fd=s; fds[nfds].events=POLLIN; fds[nfds].revents=0; callback_func[nfds]=0; socket_names[nfds]=strdup(sn); nfds++; } + if (s > 0) { + fds[nfds].fd = s; + fds[nfds].events = POLLIN; + fds[nfds].revents = 0; + callback_func[nfds] = 0; + socket_names[nfds] = strdup(sn); + nfds++; + } + if (is_ipv6 == false) { + if (add) free(add); + if (port) free(port); + } + } -__end_show_commands: + i = 0; j = 0; + for (; j < S_amll.ifaces_pgsql->ifaces->len; j++) { + char* add = NULL; char* port = NULL; char* sn = (char*)S_amll.ifaces_pgsql->ifaces->index(j); + bool is_ipv6 = false; + char* h = NULL; + if (*sn == '[') { + is_ipv6 = true; + char* p = strchr(sn, ']'); + if (p == NULL) + proxy_error("Invalid IPv6 address: %s\n", sn); - if (query_no_space_length==strlen("SELECT DATABASE()") && !strncasecmp("SELECT DATABASE()",query_no_space, query_no_space_length)) { - l_free(query_length,query); - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - query=l_strdup("SELECT \"admin\" AS 'DATABASE()'"); - } else { - query=l_strdup("SELECT \"stats\" AS 'DATABASE()'"); - } - query_length=strlen(query)+1; - goto __run_query; - } + h = ++sn; // remove first '[' + *p = '\0'; + sn = p++; // remove last ']' + add = h; + port = ++p; // remove ':' + } + else { + c_split_2(sn, ":", &add, &port); + } - // see issue #1022 - if (query_no_space_length==strlen("SELECT DATABASE() AS name") && !strncasecmp("SELECT DATABASE() AS name",query_no_space, query_no_space_length)) { - l_free(query_length,query); - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - query=l_strdup("SELECT \"admin\" AS 'name'"); - } else { - query=l_strdup("SELECT \"stats\" AS 'name'"); +#ifdef SO_REUSEPORT + int s = (atoi(port) ? listen_on_port(add, atoi(port), 128, true) : listen_on_unix(add, 128)); +#else + int s = (atoi(port) ? listen_on_port(add, atoi(port), 128) : listen_on_unix(add, 128)); +#endif + //if (s>0) { fds[nfds].fd=s; fds[nfds].events=POLLIN; fds[nfds].revents=0; callback_func[nfds]=0; socket_names[nfds]=strdup(sn); nfds++; } + if (s > 0) { + fds[nfds].fd = s; + fds[nfds].events = POLLIN; + fds[nfds].revents = 0; + callback_func[nfds] = 2; + socket_names[nfds] = strdup(sn); + nfds++; + } + if (is_ipv6 == false) { + if (add) free(add); + if (port) free(port); + } + } + S_amll.wrunlock(); } - query_length=strlen(query)+1; - goto __run_query; - } - if (sess->session_type == PROXYSQL_SESSION_STATS) { // no admin - if ( - (strncasecmp("PRAGMA",query_no_space,6)==0) - || - (strncasecmp("ATTACH",query_no_space,6)==0) - ) { - proxy_error("[WARNING]: Commands executed from stats interface in Admin Module: \"%s\"\n", query_no_space); - SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Command not allowed"); - run_query=false; - } - } - -__run_query: - if (sess->proxysql_node_address && (__sync_fetch_and_add(&glovars.shutdown,0)==0)) { - if (sess->client_myds->active) { - const string uuid { sess->proxysql_node_address->uuid }; - const string hostname { sess->proxysql_node_address->hostname }; - const string port { std::to_string(sess->proxysql_node_address->port) }; - const string mysql_ifaces { sess->proxysql_node_address->admin_mysql_ifaces }; - - time_t now = time(NULL); - string q = "INSERT OR REPLACE INTO stats_proxysql_servers_clients_status (uuid, hostname, port, admin_mysql_ifaces, last_seen_at) VALUES (\""; - q += uuid; - q += "\",\""; - q += hostname; - q += "\","; - q += port; - q += ",\""; - q += mysql_ifaces; - q += "\","; - q += std::to_string(now) + ")"; - SPA->statsdb->execute(q.c_str()); - - std::map m_labels { { "uuid", uuid }, { "hostname", hostname }, { "port", port } }; - const string m_id { uuid + ":" + hostname + ":" + port }; - - p_update_map_gauge( - SPA->metrics.p_proxysql_servers_clients_status_map, - SPA->metrics.p_dyn_gauge_array[p_admin_dyn_gauge::proxysql_servers_clients_status_last_seen_at], - m_id, m_labels, now - ); - } } - if (run_query) { - ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats - if (SPA->get_read_only()) { // disable writes if the admin interface is in read_only mode - SPA->admindb->execute("PRAGMA query_only = ON"); - SPA->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - SPA->admindb->execute("PRAGMA query_only = OFF"); - } else { - SPA->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - } - if (needs_vacuum) { - SPA->vacuum_stats(true); - } - } else { - SPA->statsdb->execute("PRAGMA query_only = ON"); - SPA->statsdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - SPA->statsdb->execute("PRAGMA query_only = OFF"); - if (needs_vacuum) { - SPA->vacuum_stats(false); + //if (__sync_add_and_fetch(shutdown,0)==0) __sync_add_and_fetch(shutdown,1); + for (i=0; iSQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); - } else { - char *a = (char *)"ProxySQL Admin Error: "; - char *new_msg = (char *)malloc(strlen(error)+strlen(a)+1); - sprintf(new_msg, "%s%s", a, error); - sess->SQLite3_to_MySQL(resultset, new_msg, affected_rows, &sess->client_myds->myprot); - free(new_msg); - free(error); - } - delete resultset; - } - if (run_query == true) { - pthread_mutex_unlock(&pa->sql_query_global_mutex); - } else { - // The admin module may have already been freed in case of "PROXYSQL STOP" - if (strcasecmp("PROXYSQL STOP",query_no_space)) - pthread_mutex_unlock(&pa->sql_query_global_mutex); + if (socket_names[i]) free(socket_names[i]); + if (add) free(add); + if (port) free(port); } - l_free(pkt->size-sizeof(mysql_hdr),query_no_space); // it is always freed here - l_free(query_length,query); + free(arg); + return NULL; } -void ProxySQL_Admin::vacuum_stats(bool is_admin) { - if (variables.vacuum_stats==false) { - return; +#ifdef DEBUG +#define DEB "_DEBUG" +#else +#define DEB "" +#endif /* DEBUG */ +#define PROXYSQL_ADMIN_VERSION "2.0.6.0805" DEB + +/** + * @brief Routine to be called before each scrape from prometheus. + */ +void update_modules_metrics() { + // Update mysql_threads_handler metrics + if (GloMTH) { + GloMTH->p_update_metrics(); } - const vector tablenames = { - "stats_mysql_commands_counters", - "stats_mysql_free_connections", - "stats_mysql_connection_pool", - "stats_mysql_connection_pool_reset", - "stats_mysql_prepared_statements_info", - "stats_mysql_processlist", - "stats_mysql_query_digest", - "stats_mysql_query_digest_reset", - "stats_mysql_query_rules", - "stats_mysql_users", - "stats_proxysql_servers_checksums", - "stats_proxysql_servers_metrics", - "stats_proxysql_servers_status", - }; - string s; - SQLite3DB *tmpdb = NULL; - if (is_admin == true) { - tmpdb = admindb; - } else { - tmpdb = statsdb; + // Update pgsql_threads_handler metrics + if (GloPTH) { + GloPTH->p_update_metrics(); } - for (auto it = tablenames.begin(); it != tablenames.end(); it++) { - s = "DELETE FROM "; - if (is_admin == true) s+= "stats."; - s += *it; - tmpdb->execute(s.c_str()); + // Update mysql_hostgroups_manager metrics + if (MyHGM) { + MyHGM->p_update_metrics(); } - s = "VACUUM"; - if (is_admin == true) - s+= " stats"; - tmpdb->execute(s.c_str()); -} - - -void *child_mysql(void *arg) { - if (GloMTH == nullptr) { return NULL; } - - pthread_attr_t thread_attr; - size_t tmp_stack_size=0; - if (!pthread_attr_init(&thread_attr)) { - if (!pthread_attr_getstacksize(&thread_attr , &tmp_stack_size )) { - __sync_fetch_and_add(&GloVars.statuses.stack_memory_admin_threads,tmp_stack_size); - } + // Update monitor metrics + if (GloMyMon) { + GloMyMon->p_update_metrics(); } - - arg_mysql_adm *myarg = (arg_mysql_adm *)arg; - int client = myarg->client_t; - - //struct sockaddr *addr = arg->addr; - //socklen_t addr_size; - - GloMTH->wrlock(); - { - char *s=GloMTH->get_variable((char *)"server_capabilities"); - mysql_thread___server_capabilities=atoi(s); - free(s); + // Update query_cache metrics + if (GloQC) { + GloQC->p_update_metrics(); + } + // Update cluster metrics + if (GloProxyCluster) { + GloProxyCluster->p_update_metrics(); } - GloMTH->wrunlock(); - - struct pollfd fds[1]; - nfds_t nfds=1; - int rc; - pthread_mutex_unlock(&sock_mutex); - MySQL_Thread *mysql_thr=new MySQL_Thread(); - mysql_thr->curtime=monotonic_time(); - GloQPro->init_thread(); - mysql_thr->refresh_variables(); - MySQL_Session *sess=mysql_thr->create_new_session_and_client_data_stream(client); - sess->thread=mysql_thr; - sess->session_type = PROXYSQL_SESSION_ADMIN; - sess->handler_function=admin_session_handler; - MySQL_Data_Stream *myds=sess->client_myds; - - sess->start_time=mysql_thr->curtime; - sess->client_myds->client_addrlen=myarg->addr_size; - sess->client_myds->client_addr=myarg->addr; + // Update admin metrics + GloAdmin->p_update_metrics(); +} - switch (sess->client_myds->client_addr->sa_family) { - case AF_INET: { - struct sockaddr_in *ipv4 = (struct sockaddr_in *)sess->client_myds->client_addr; - char buf[INET_ADDRSTRLEN]; - inet_ntop(sess->client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); - sess->client_myds->addr.addr = strdup(buf); - sess->client_myds->addr.port = htons(ipv4->sin_port); - break; - } - case AF_INET6: { - struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)sess->client_myds->client_addr; - char buf[INET6_ADDRSTRLEN]; - inet_ntop(sess->client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); - sess->client_myds->addr.addr = strdup(buf); - sess->client_myds->addr.port = htons(ipv6->sin6_port); - break; +ProxySQL_Admin::ProxySQL_Admin() : + serial_exposer(std::function { update_modules_metrics }) +{ +#ifdef DEBUG + debugdb_disk = NULL; + if (glovars.has_debug==false) { +#else + if (glovars.has_debug==true) { +#endif /* DEBUG */ + perror("Incompatible debugging version"); + exit(EXIT_FAILURE); } - default: - sess->client_myds->addr.addr = strdup("localhost"); - break; - } - fds[0].fd=client; - fds[0].revents=0; - fds[0].events=POLLIN|POLLOUT; - //free(arg->addr); // do not free - free(arg); - sess->client_myds->myprot.generate_pkt_initial_handshake(true,NULL,NULL, &sess->thread_session_id, false); - while (__sync_fetch_and_add(&glovars.shutdown,0)==0) { - if (myds->available_data_out()) { - fds[0].events=POLLIN|POLLOUT; - } else { - fds[0].events=POLLIN; - } - fds[0].revents=0; - rc=poll(fds,nfds,__sync_fetch_and_add(&__admin_refresh_interval,0)); - if (rc == -1) { - if (errno == EINTR) { - continue; - } else { - goto __exit_child_mysql; - } - } - mysql_thr->curtime = monotonic_time(); - myds->revents=fds[0].revents; - int rb = 0; - rb = myds->read_from_net(); - if (myds->net_failure) goto __exit_child_mysql; - myds->read_pkts(); - if (myds->encrypted == true) { - // PMC-10004 - // we probably should use SSL_pending() and/or SSL_has_pending() to determine - // if there is more data to be read, but it doesn't seem to be working. - // Therefore we try to call read_from_net() again as long as there is data. - // Previously we hardcoded 16KB but it seems that it can return in smaller - // chunks of 4KB. - // We finally removed the chunk size as it seems that any size is possible. - while (rb > 0) { - rb = myds->read_from_net(); - if (myds->net_failure) goto __exit_child_mysql; - myds->read_pkts(); - } - } - sess->to_process=1; - int rc=sess->handler(); - if (rc==-1) goto __exit_child_mysql; + if (proxysql_version == NULL) { + proxysql_version = strdup(PROXYSQL_VERSION); } + SPA=this; -__exit_child_mysql: - delete mysql_thr; + //Initialize locker +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_init(&rwlock,NULL); +#else + spinlock_rwlock_init(&rwlock); +#endif - __sync_fetch_and_sub(&GloVars.statuses.stack_memory_admin_threads,tmp_stack_size); +#ifdef PA_PTHREAD_MUTEX + pthread_mutex_init(&mysql_servers_lock, NULL); +#else + spinlock_rwlock_init(&mysql_servers_rwlock); +#endif - return NULL; -} +#ifdef PA_PTHREAD_MUTEX + pthread_mutex_init(&pgsql_servers_lock, NULL); +#else + spinlock_rwlock_init(&pgsql_servers_rwlock); +#endif -void* child_telnet(void* arg) -{ - int bytes_read; - char line[LINESIZE+1]; - int client = *(int *)arg; - free(arg); - pthread_mutex_unlock(&sock_mutex); - memset(line,0,LINESIZE+1); - while ((strncmp(line, "quit", 4) != 0) && glovars.shutdown==0) { - bytes_read = recv(client, line, LINESIZE, 0); - if (bytes_read==-1) { - break; - } - char *eow = strchr(line, '\n'); - if (eow) *eow=0; - //SPA->is_command(line); - if (strncmp(line,"shutdown",8)==0) glovars.shutdown=1; - if (send(client, line, strlen(line), MSG_NOSIGNAL)==-1) break; - if (send(client, "\nOK\n", 4, MSG_NOSIGNAL)==-1) break; - } - shutdown(client,SHUT_RDWR); - close(client); - return arg; -} + pthread_mutex_init(&sql_query_global_mutex, NULL); -void* child_telnet_also(void* arg) -{ - int bytes_read; - char line[LINESIZE+1]; - int client = *(int *)arg; - free(arg); - pthread_mutex_unlock(&sock_mutex); - memset(line,0,LINESIZE+1); - while ((strncmp(line, "quit", 4) != 0) && glovars.shutdown==0) { - bytes_read = recv(client, line, LINESIZE, 0); - if (bytes_read==-1) { - break; - } - char *eow = strchr(line, '\n'); - if (eow) *eow=0; - if (strncmp(line,"shutdown",8)==0) glovars.shutdown=1; - if (send(client, line, strlen(line), MSG_NOSIGNAL)==-1) break; - if (send(client, "\nNOT OK\n", 8, MSG_NOSIGNAL)==-1) break; + generate_load_save_disk_commands("mysql_firewall", "MYSQL FIREWALL"); + generate_load_save_disk_commands("mysql_query_rules", "MYSQL QUERY RULES"); + generate_load_save_disk_commands("mysql_users", "MYSQL USERS"); + generate_load_save_disk_commands("mysql_servers", "MYSQL SERVERS"); + generate_load_save_disk_commands("mysql_variables", "MYSQL VARIABLES"); + generate_load_save_disk_commands("pgsql_firewall", "PGSQL FIREWALL"); + generate_load_save_disk_commands("pgsql_query_rules", "PGSQL QUERY RULES"); + generate_load_save_disk_commands("pgsql_users", "PGSQL USERS"); + generate_load_save_disk_commands("pgsql_servers", "PGSQL SERVERS"); + generate_load_save_disk_commands("pgsql_variables", "PGSQL VARIABLES"); + generate_load_save_disk_commands("scheduler", "SCHEDULER"); + generate_load_save_disk_commands("restapi", "RESTAPI"); + generate_load_save_disk_commands("proxysql_servers", "PROXYSQL SERVERS"); + + { + // we perform some sanity check + assert(load_save_disk_commands.size() > 0); + for (auto it = load_save_disk_commands.begin(); it != load_save_disk_commands.end(); it++) { + vector& vec1 = get<1>(it->second); + assert(vec1.size() == 3); + vector& vec2 = get<2>(it->second); + assert(vec2.size() == 3); + } } - shutdown(client,SHUT_RDWR); - close(client); - return arg; -} -static void * admin_main_loop(void *arg) -{ - int i; - int rc; - int version=0; - struct pollfd *fds=((struct _main_args *)arg)->fds; - int nfds=((struct _main_args *)arg)->nfds; - int *callback_func=((struct _main_args *)arg)->callback_func; - volatile int *shutdown=((struct _main_args *)arg)->shutdown; - char *socket_names[MAX_ADMIN_LISTENERS]; - set_thread_name("Admin"); - for (i=0;ischeduler_run_once(); - unsigned long long poll_wait=500000; - if (next_run < curtime + 500000) { - poll_wait=next_run-curtime; - } - if (poll_wait > 500000) { - poll_wait=500000; - } - poll_wait=poll_wait/1000; // conversion to millisecond - rc=poll(fds,nfds,poll_wait); - if ((nostart_ && __sync_val_compare_and_swap(&GloVars.global.nostart,0,1)==0) || __sync_fetch_and_add(&glovars.shutdown,0)==1) { - nostart_=false; - pthread_mutex_unlock(&GloVars.global.start_mutex); - } - if ((rc == -1 && errno == EINTR) || rc==0) { - // poll() timeout, try again - goto __end_while_pool; - } - for (i=1;iaddr=(struct sockaddr *)malloc(sizeof(custom_sockaddr)); - passarg->addr_size = sizeof(custom_sockaddr); - memset(passarg->addr, 0, sizeof(custom_sockaddr)); - passarg->client_t = accept(fds[i].fd, (struct sockaddr*)passarg->addr, &passarg->addr_size); -// printf("Connected: %s:%d sock=%d\n", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port), client_t); - pthread_attr_getstacksize (&attr, &stacks); -// printf("Default stack size = %d\n", stacks); - pthread_mutex_lock (&sock_mutex); - //client=(int *)malloc(sizeof(int)); - //*client= client_t; - //if ( pthread_create(&child, &attr, child_func[callback_func[i]], client) != 0 ) { - if ( pthread_create(&child, &attr, child_func[callback_func[i]], passarg) != 0 ) { - // LCOV_EXCL_START - perror("pthread_create"); - proxy_error("Thread creation\n"); - assert(0); - // LCOV_EXCL_STOP - } - } - fds[i].revents=0; - } -__end_while_pool: - { - if (GloProxyStats->MySQL_Threads_Handler_timetoget(curtime)) { - if (GloMTH) { - SQLite3_result * resultset=GloMTH->SQL3_GlobalStatus(false); - if (resultset) { - GloProxyStats->MySQL_Threads_Handler_sets(resultset); - delete resultset; - } - } - if (MyHGM) { - SQLite3_result * resultset=MyHGM->SQL3_Get_ConnPool_Stats(); - if (resultset) { - SQLite3_result * resultset2 = NULL; - - // In debug, run the code to generate metrics so that it can be tested even if the web interface plugin isn't loaded. - #ifdef DEBUG - if (true) { - #else - if (GloVars.web_interface_plugin) { - #endif - resultset2 = MyHGM->SQL3_Connection_Pool(false); - } - GloProxyStats->MyHGM_Handler_sets(resultset, resultset2); - delete resultset; - if (resultset2) { - delete resultset2; - } - } - } - } - if (GloProxyStats->MySQL_Query_Cache_timetoget(curtime)) { - if (GloQC) { - SQLite3_result * resultset=GloQC->SQL3_getStats(); - if (resultset) { - GloProxyStats->MySQL_Query_Cache_sets(resultset); - delete resultset; - } - } - } - if (GloProxyStats->mysql_query_digest_to_disk_timetoget(curtime)) { - unsigned long long curtime1=monotonic_time(); - int r1 = SPA->FlushDigestTableToDisk(SPA->statsdb_disk); - unsigned long long curtime2=monotonic_time(); - curtime1 = curtime1/1000; - curtime2 = curtime2/1000; - proxy_info("Automatically saved stats_mysql_query_digest to disk: %llums to write %d entries\n", curtime2-curtime1, r1); - } - if (GloProxyStats->system_cpu_timetoget(curtime)) { - GloProxyStats->system_cpu_sets(); - } + variables.pgsql_ifaces= strdup("0.0.0.0:6132"); + variables.telnet_admin_ifaces=NULL; + variables.telnet_stats_ifaces=NULL; + variables.refresh_interval=2000; + variables.mysql_show_processlist_extended = false; + variables.pgsql_show_processlist_extended = false; + //variables.hash_passwords=true; // issue #676 + variables.vacuum_stats=true; // issue #1011 + variables.admin_read_only=false; // by default, the admin interface accepts writes + variables.admin_version=(char *)PROXYSQL_VERSION; + variables.cluster_username=strdup((char *)""); + variables.cluster_password=strdup((char *)""); + variables.cluster_check_interval_ms=1000; + variables.cluster_check_status_frequency=10; + variables.cluster_mysql_query_rules_diffs_before_sync = 3; + variables.cluster_mysql_servers_diffs_before_sync = 3; + variables.cluster_mysql_users_diffs_before_sync = 3; + variables.cluster_proxysql_servers_diffs_before_sync = 3; + variables.cluster_mysql_variables_diffs_before_sync = 3; + variables.cluster_admin_variables_diffs_before_sync = 3; + variables.cluster_ldap_variables_diffs_before_sync = 3; + variables.cluster_mysql_servers_sync_algorithm = 1; + checksum_variables.checksum_mysql_query_rules = true; + checksum_variables.checksum_mysql_servers = true; + checksum_variables.checksum_mysql_users = true; + checksum_variables.checksum_mysql_variables = true; + checksum_variables.checksum_admin_variables = true; + checksum_variables.checksum_ldap_variables = true; + variables.cluster_mysql_query_rules_save_to_disk = true; + variables.cluster_mysql_servers_save_to_disk = true; + variables.cluster_mysql_users_save_to_disk = true; + variables.cluster_proxysql_servers_save_to_disk = true; + variables.cluster_mysql_variables_save_to_disk = true; + variables.cluster_admin_variables_save_to_disk = true; + variables.cluster_ldap_variables_save_to_disk = true; + variables.stats_mysql_connection_pool = 60; + variables.stats_mysql_connections = 60; + variables.stats_mysql_query_cache = 60; + variables.stats_mysql_query_digest_to_disk = 0; + variables.stats_system_cpu = 60; + variables.stats_system_memory = 60; + GloProxyStats->variables.stats_mysql_connection_pool = 60; + GloProxyStats->variables.stats_mysql_connections = 60; + GloProxyStats->variables.stats_mysql_query_cache = 60; + GloProxyStats->variables.stats_mysql_query_digest_to_disk = 0; + GloProxyStats->variables.stats_system_cpu = 60; #ifndef NOJEM - if (GloProxyStats->system_memory_timetoget(curtime)) { - GloProxyStats->system_memory_sets(); - } -#endif - } - if (S_amll.get_version()!=version) { - S_amll.wrlock(); - version=S_amll.get_version(); - for (i=1; ipipefd[0]; - fds[nfds].events=POLLIN; - fds[nfds].revents=0; - nfds++; - unsigned int j; - i=0; j=0; - for (j=0; jifaces->len; j++) { - char *add=NULL; char *port=NULL; char *sn=(char *)S_amll.ifaces_mysql->ifaces->index(j); - bool is_ipv6 = false; - char *h = NULL; - if (*sn == '[') { - is_ipv6 = true; - char *p = strchr(sn, ']'); - if (p == NULL) - proxy_error("Invalid IPv6 address: %s\n", sn); - - h = ++sn; // remove first '[' - *p = '\0'; - sn = p++; // remove last ']' - add = h; - port = ++p; // remove ':' - } else { - c_split_2(sn, ":" , &add, &port); - } - -#ifdef SO_REUSEPORT - int s = ( atoi(port) ? listen_on_port(add, atoi(port), 128, true) : listen_on_unix(add, 128)); -#else - int s = ( atoi(port) ? listen_on_port(add, atoi(port), 128) : listen_on_unix(add, 128)); + GloProxyStats->variables.stats_system_memory = 60; #endif - if (s>0) { fds[nfds].fd=s; fds[nfds].events=POLLIN; fds[nfds].revents=0; callback_func[nfds]=0; socket_names[nfds]=strdup(sn); nfds++; } - if (is_ipv6 == false) { - if (add) free(add); - if (port) free(port); - } - } - S_amll.wrunlock(); - } - - } - //if (__sync_add_and_fetch(shutdown,0)==0) __sync_add_and_fetch(shutdown,1); - for (i=0; ip_update_metrics(); - } - // Update mysql_hostgroups_manager metrics - if (MyHGM) { - MyHGM->p_update_metrics(); - } - // Update monitor metrics - if (GloMyMon) { - GloMyMon->p_update_metrics(); - } - // Update query_cache metrics - if (GloQC) { - GloQC->p_update_metrics(); - } - // Update cluster metrics - if (GloProxyCluster) { - GloProxyCluster->p_update_metrics(); - } - - // Update admin metrics - GloAdmin->p_update_metrics(); -} - -ProxySQL_Admin::ProxySQL_Admin() : - serial_exposer(std::function { update_modules_metrics }) -{ -#ifdef DEBUG - debugdb_disk = NULL; - if (glovars.has_debug==false) { -#else - if (glovars.has_debug==true) { -#endif /* DEBUG */ - perror("Incompatible debugging version"); - exit(EXIT_FAILURE); - } - - if (proxysql_version == NULL) { - proxysql_version = strdup(PROXYSQL_VERSION); - } - SPA=this; - - //Initialize locker -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_init(&rwlock,NULL); -#else - spinlock_rwlock_init(&rwlock); -#endif - -#ifdef PA_PTHREAD_MUTEX - pthread_mutex_init(&mysql_servers_lock, NULL); -#else - spinlock_rwlock_init(&mysql_servers_rwlock); -#endif - - pthread_mutex_init(&sql_query_global_mutex, NULL); - - generate_load_save_disk_commands("mysql_firewall", "MYSQL FIREWALL"); - generate_load_save_disk_commands("mysql_query_rules", "MYSQL QUERY RULES"); - generate_load_save_disk_commands("mysql_users", "MYSQL USERS"); - generate_load_save_disk_commands("mysql_servers", "MYSQL SERVERS"); - generate_load_save_disk_commands("mysql_variables", "MYSQL VARIABLES"); - generate_load_save_disk_commands("scheduler", "SCHEDULER"); - generate_load_save_disk_commands("restapi", "RESTAPI"); - generate_load_save_disk_commands("proxysql_servers", "PROXYSQL SERVERS"); - - { - // we perform some sanity check - assert(load_save_disk_commands.size() > 0); - for (auto it = load_save_disk_commands.begin(); it != load_save_disk_commands.end(); it++) { - vector& vec1 = get<1>(it->second); - assert(vec1.size() == 3); - vector& vec2 = get<2>(it->second); - assert(vec2.size() == 3); - } - } - - - variables.admin_credentials=strdup("admin:admin"); - variables.stats_credentials=strdup("stats:stats"); - if (GloVars.__cmd_proxysql_admin_socket) { - variables.mysql_ifaces=strdup(GloVars.__cmd_proxysql_admin_socket); - } else { - variables.mysql_ifaces=strdup("0.0.0.0:6032"); // changed. See isseu #1103 - } - variables.telnet_admin_ifaces=NULL; - variables.telnet_stats_ifaces=NULL; - variables.refresh_interval=2000; - variables.mysql_show_processlist_extended = false; -// variables.hash_passwords=true; // issue #676 - variables.vacuum_stats=true; // issue #1011 - variables.admin_read_only=false; // by default, the admin interface accepts writes - variables.admin_version=(char *)PROXYSQL_VERSION; - variables.cluster_username=strdup((char *)""); - variables.cluster_password=strdup((char *)""); - variables.cluster_check_interval_ms=1000; - variables.cluster_check_status_frequency=10; - variables.cluster_mysql_query_rules_diffs_before_sync = 3; - variables.cluster_mysql_servers_diffs_before_sync = 3; - variables.cluster_mysql_users_diffs_before_sync = 3; - variables.cluster_proxysql_servers_diffs_before_sync = 3; - variables.cluster_mysql_variables_diffs_before_sync = 3; - variables.cluster_admin_variables_diffs_before_sync = 3; - variables.cluster_ldap_variables_diffs_before_sync = 3; - variables.cluster_mysql_servers_sync_algorithm = 1; - checksum_variables.checksum_mysql_query_rules = true; - checksum_variables.checksum_mysql_servers = true; - checksum_variables.checksum_mysql_users = true; - checksum_variables.checksum_mysql_variables = true; - checksum_variables.checksum_admin_variables = true; - checksum_variables.checksum_ldap_variables = true; - variables.cluster_mysql_query_rules_save_to_disk = true; - variables.cluster_mysql_servers_save_to_disk = true; - variables.cluster_mysql_users_save_to_disk = true; - variables.cluster_proxysql_servers_save_to_disk = true; - variables.cluster_mysql_variables_save_to_disk = true; - variables.cluster_admin_variables_save_to_disk = true; - variables.cluster_ldap_variables_save_to_disk = true; - variables.stats_mysql_connection_pool = 60; - variables.stats_mysql_connections = 60; - variables.stats_mysql_query_cache = 60; - variables.stats_mysql_query_digest_to_disk = 0; - variables.stats_system_cpu = 60; - variables.stats_system_memory = 60; - GloProxyStats->variables.stats_mysql_connection_pool = 60; - GloProxyStats->variables.stats_mysql_connections = 60; - GloProxyStats->variables.stats_mysql_query_cache = 60; - GloProxyStats->variables.stats_mysql_query_digest_to_disk = 0; - GloProxyStats->variables.stats_system_cpu = 60; -#ifndef NOJEM - GloProxyStats->variables.stats_system_memory = 60; -#endif - - variables.restapi_enabled = false; - variables.restapi_enabled_old = false; - variables.restapi_port = 6070; - variables.restapi_port_old = variables.restapi_port; - variables.web_enabled = false; - variables.web_enabled_old = false; - variables.web_port = 6080; - variables.web_port_old = variables.web_port; - variables.web_verbosity = 0; - variables.p_memory_metrics_interval = 61; - all_modules_started = false; -#ifdef DEBUG - variables.debug=GloVars.global.gdbg; - debug_output = 1; - proxysql_set_admin_debug_output(debug_output); -#endif /* DEBUG */ - variables.coredump_generation_interval_ms = 30000; - variables.coredump_generation_threshold = 10; - variables.ssl_keylog_file = strdup(""); - last_p_memory_metrics_ts = 0; - // create the scheduler - scheduler=new ProxySQL_External_Scheduler(); + variables.coredump_generation_interval_ms = 30000; + variables.coredump_generation_threshold = 10; + variables.ssl_keylog_file = strdup(""); + last_p_memory_metrics_ts = 0; + // create the scheduler + scheduler=new ProxySQL_External_Scheduler(); match_regexes.opt=(re2::RE2::Options *)new re2::RE2::Options(RE2::Quiet); re2::RE2::Options *opt2=(re2::RE2::Options *)match_regexes.opt; - opt2->set_case_sensitive(false); - match_regexes.re=(void **)malloc(sizeof(void *)*10); - match_regexes.re[0]=(RE2 *)new RE2("^SELECT\\s+@@max_allowed_packet\\s*", *opt2); - match_regexes.re[1]=(RE2 *)new RE2("^SELECT\\s+@@[0-9A-Za-z_-]+\\s*", *opt2); - match_regexes.re[2]=(RE2 *)new RE2("SHOW\\s+VARIABLES\\s+WHERE", *opt2); - match_regexes.re[3]=(RE2 *)new RE2("SHOW\\s+VARIABLES\\s+LIKE", *opt2); - static const char alphanum[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - rand_del[0] = '-'; - for (int i = 1; i < 4; i++) { - rand_del[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; - } - rand_del[4] = '-'; - rand_del[5] = 0; - - // Default initialize prometheus collectable flag - registered_prometheus_collectable = false; - - // Initialize prometheus metrics - init_prometheus_counter_array(admin_metrics_map, this->metrics.p_counter_array); - init_prometheus_gauge_array(admin_metrics_map, this->metrics.p_gauge_array); - init_prometheus_dyn_gauge_array(admin_metrics_map, this->metrics.p_dyn_gauge_array); - - // NOTE: Imposing fixed value to 'version_info' matching 'mysqld_exporter' - this->metrics.p_gauge_array[p_admin_gauge::version_info]->Set(1); -}; - -void ProxySQL_Admin::wrlock() { -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_wrlock(&rwlock); -#else - spin_wrlock(&rwlock); -#endif -}; - -void ProxySQL_Admin::wrunlock() { -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_unlock(&rwlock); -#else - spin_wrunlock(&rwlock); -#endif -}; - -void ProxySQL_Admin::mysql_servers_wrlock() { - #ifdef PA_PTHREAD_MUTEX - pthread_mutex_lock(&mysql_servers_lock); - #else - spin_wrlock(&mysql_servers_rwlock); - #endif -}; - -void ProxySQL_Admin::mysql_servers_wrunlock() { - #ifdef PA_PTHREAD_MUTEX - pthread_mutex_unlock(&mysql_servers_lock); - #else - spin_wrunlock(&mysql_servers_rwlock); - #endif -}; - -void ProxySQL_Admin::print_version() { - fprintf(stderr,"Standard ProxySQL Admin rev. %s -- %s -- %s\n", PROXYSQL_ADMIN_VERSION, __FILE__, __TIMESTAMP__); -}; - -void ProxySQL_Admin::init_ldap() { - if (GloMyLdapAuth) { - insert_into_tables_defs(tables_defs_admin,"mysql_ldap_mapping", ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_ldap_mapping", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_LDAP_MAPPING); - insert_into_tables_defs(tables_defs_config,"mysql_ldap_mapping", ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING); - } -} - -void ProxySQL_Admin::init_http_server() { - AdminHTTPServer = new ProxySQL_HTTP_Server(); - AdminHTTPServer->init(); - AdminHTTPServer->print_version(); -} - -struct boot_srv_info_t { - string member_id; - string member_host; - uint32_t member_port; - string member_state; - string member_role; - string member_version; -}; - -struct BOOT_SRV_INFO_T { - enum { - MEMBER_ID, - MEMBER_HOST, - MEMBER_PORT, - MEMBER_STATE, - MEMBER_ROLE, - MEMBER_VERSION - }; -}; - -struct boot_user_info_t { - string user; - string ssl_type; - string auth_string; - string auth_plugin; - bool password_expired; -}; - -struct BOOT_USER_INFO_T { - enum { - USER, - SSL_TYPE, - AUTH_STRING, - AUTH_PLUGIN, - PASSWORD_EXPIRED - }; -}; - -struct srv_defs_t { - int64_t weight; - int64_t max_conns; - int32_t use_ssl; -}; - -using boot_srv_cnf_t = pair; - -vector extract_boot_servers_info(MYSQL_RES* servers) { - vector servers_info {}; - - while (MYSQL_ROW row = mysql_fetch_row(servers)) { - servers_info.push_back({ - string { row[BOOT_SRV_INFO_T::MEMBER_ID] }, - string { row[BOOT_SRV_INFO_T::MEMBER_HOST] }, - static_cast(stoi(row[BOOT_SRV_INFO_T::MEMBER_PORT])), - string { row[BOOT_SRV_INFO_T::MEMBER_STATE] }, - string { row[BOOT_SRV_INFO_T::MEMBER_ROLE] }, - string { row[BOOT_SRV_INFO_T::MEMBER_VERSION] }, - }); - } - - return servers_info; -} - -string build_boot_servers_insert(const vector& srvs_info_defs) { - const string t_srvs_insert { - "INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,max_connections,use_ssl) VALUES " - }; - string t_srvs_values {}; - - for (const auto& info_defs : srvs_info_defs) { - const boot_srv_info_t& srv_info = info_defs.first; - const srv_defs_t& srv_defs = info_defs.second; - - const char t_values[] { "(%d, \"%s\", %d, \"%s\", %ld, %ld, %d)" }; - string srv_values = cstr_format( - t_values, - srv_info.member_role == "PRIMARY" ? 0 : 1, // HOSTGROUP_ID - srv_info.member_host.c_str(), // HOSTNAME - srv_info.member_port, // PORT - srv_info.member_state.c_str(), // STATUS - srv_defs.weight, // Weight - srv_defs.max_conns, // Max Connections - srv_defs.use_ssl // UseSSL - ).str; - - if (&info_defs != &srvs_info_defs.back()) { - srv_values += ","; - } - - t_srvs_values += srv_values; - } - - const string servers_insert { t_srvs_insert + t_srvs_values }; - - return servers_insert; -} - -string build_boot_users_insert(MYSQL_RES* users) { - vector users_info {}; - - while (MYSQL_ROW row = mysql_fetch_row(users)) { - users_info.push_back({ - string { row[BOOT_USER_INFO_T::USER] }, - string { row[BOOT_USER_INFO_T::SSL_TYPE] }, - string { row[BOOT_USER_INFO_T::AUTH_STRING] }, - string { row[BOOT_USER_INFO_T::AUTH_PLUGIN] }, - static_cast(atoi(row[BOOT_USER_INFO_T::PASSWORD_EXPIRED])) - }); - } - - // MySQL Users - const string t_users_insert { - "INSERT INTO mysql_users (username,password,active,use_ssl) VALUES " - }; - string t_users_values {}; - - for (const boot_user_info_t& user : users_info) { - uint32_t use_ssl = user.ssl_type.empty() ? 0 : 1; - const char t_values[] { "(\"%s\", \"%s\", %d, %d)" }; - - string srv_values = cstr_format( - t_values, - user.user.c_str(), // USERNAME - user.auth_string.c_str(), // HOSTNAME - 1, // ACTIVE: Always ON - use_ssl // USE_SSL: Dependent on backend user - ).str; - - if (&user != &users_info.back()) { - srv_values += ","; - } - - t_users_values += srv_values; - } - - const string users_insert { t_users_insert + t_users_values }; - - return users_insert; -} - -map get_cur_hg_attrs(SQLite3DB* admindb) { - map res {}; - - char* error = nullptr; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; - - admindb->execute_statement( - "SELECT hostgroup_id,servers_defaults FROM mysql_hostgroup_attributes", - &error, &cols, &affected_rows, &resultset - ); - - for (SQLite3_row* row : resultset->rows) { - const int32_t hid = atoi(row->fields[0]); - srv_defs_t srv_defs {}; - srv_defs.weight = 1; - srv_defs.max_conns = 512; - srv_defs.use_ssl = 1; - - nlohmann::json j_srv_defs = nlohmann::json::parse(row->fields[1]); - - const auto weight_check = [] (int64_t weight) -> bool { return weight >= 0; }; - srv_defs.weight = j_get_srv_default_int_val(j_srv_defs, hid, "weight", weight_check); - - const auto max_conns_check = [] (int64_t max_conns) -> bool { return max_conns >= 0; }; - srv_defs.max_conns = j_get_srv_default_int_val(j_srv_defs, hid, "max_connections", max_conns_check); - - const auto use_ssl_check = [] (int32_t use_ssl) -> bool { return use_ssl == 0 || use_ssl == 1; }; - srv_defs.use_ssl = j_get_srv_default_int_val(j_srv_defs, hid, "use_ssl", use_ssl_check); - - res.insert({ hid , srv_defs }); - } - - delete resultset; - - return res; -} - -vector build_srvs_info_with_defs( - const vector& srvs_info, - const map& hgid_defs, - const srv_defs_t global_defs -) { - vector res {}; - - for (const boot_srv_info_t& srv_info : srvs_info) { - if (srv_info.member_role == "PRIMARY") { - const auto hg_it = hgid_defs.find(0); - - if (hg_it != hgid_defs.end()) { - res.push_back({ srv_info, hg_it->second }); - } else { - res.push_back({ srv_info, global_defs }); - } - } else { - const auto hg_it = hgid_defs.find(1); - - if (hg_it != hgid_defs.end()) { - res.push_back({ srv_info, hg_it->second }); - } else { - res.push_back({ srv_info, global_defs }); - } - } - } - - return res; -} - -/** - * @brief Helper function used to check if tables are already filled with data. - * @details Handles the boilerplate operations of executing 'SELECT COUNT(*)' alike queries. - * @param admindb An already initialized instance of a SQLite3DB object to 'mem_admindb'. - * @param query The query to be executed, it's required to be 'SELECT COUNT(*)' alike. - * @return The resulting int of the 'COUNT(*)' in case of success, '-1' otherwise. In case of error, error - * cause are logged, and `assert` is called. - */ -int check_if_user_config(SQLite3DB* admindb, const char* query) { - char* error = nullptr; - int cols = 0; - int affected_rows = 0; - SQLite3_result* resultset = NULL; - - admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset); - if (error) { - proxy_error( - "Aborting due to failed query over SQLite3 - db: '%s', query: '%s', err: %s", admindb->get_url(), query, error - ); - assert(0); - } - - int count = -1; - - if (resultset != nullptr && !resultset->rows.empty() && resultset->rows[0]->cnt >= 0) { - const char* s_count = resultset->rows[0]->fields[0]; - char* p_end = nullptr; - - count = std::strtol(s_count, &p_end, 10); - - if (p_end == s_count || errno == ERANGE) { - proxy_error( - "Aborting due to invalid query output, expected single INT (E.g. 'COUNT(*)') - query: '%s'", query - ); - count = -1; - } - } - - if (count == -1) { - assert(0); - } - - delete resultset; - return count; -}; - -/** - * @brief Definition of an auxiliary table used to store bootstrap variables. - * @details Table is used only to store in configdb bootstrap variables that are required to persist between - * executions. - */ -#define ADMIN_SQLITE_TABLE_BOOTSTRAP_VARIABLES "CREATE TABLE IF NOT EXISTS bootstrap_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" - -bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { - cpu_timer cpt; - - if (flush_logs_function == NULL) { - flush_logs_function = flush_logs_handler; - } - - Admin_HTTP_Server = NULL; - AdminRestApiServer = NULL; - AdminHTTPServer = NULL; - -/* - AdminRestApiServer = new ProxySQL_RESTAPI_Server(); - AdminRestApiServer->print_version(); -*/ - - child_func[0]=child_mysql; - child_func[1]=child_telnet; - child_func[2]=child_telnet_also; - main_shutdown=0; - main_poll_nfds=0; - main_poll_fds=NULL; - main_callback_func=NULL; - - { - int rc=pipe(pipefd); - if (rc) { - perror("Call to pipe() failed"); - exit(EXIT_FAILURE); - } - } - - main_callback_func=(int *)malloc(sizeof(int)*MAX_ADMIN_LISTENERS); - main_poll_fds=(struct pollfd *)malloc(sizeof(struct pollfd)*MAX_ADMIN_LISTENERS); - main_poll_nfds=0; - - pthread_attr_t attr; - pthread_attr_init(&attr); - //pthread_attr_setstacksize (&attr, mystacksize); - - admindb=new SQLite3DB(); - admindb->open((char *)"file:mem_admindb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - admindb->execute("PRAGMA cache_size = -50000"); - //sqlite3_enable_load_extension(admindb->get_db(),1); - //sqlite3_auto_extension( (void(*)(void))sqlite3_json_init); - statsdb=new SQLite3DB(); - statsdb->open((char *)"file:mem_statsdb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - - // check if file exists , see #617 - bool admindb_file_exists=Proxy_file_exists(GloVars.admindb); - - configdb=new SQLite3DB(); - configdb->open((char *)GloVars.admindb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - // Fully synchronous is not required. See to #1055 - // https://sqlite.org/pragma.html#pragma_synchronous - configdb->execute("PRAGMA synchronous=0"); - - monitordb = new SQLite3DB(); - monitordb->open((char *)"file:mem_monitordb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - - statsdb_disk = new SQLite3DB(); - statsdb_disk->open((char *)GloVars.statsdb_disk, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); -// char *dbname = (char *)malloc(strlen(GloVars.statsdb_disk)+50); -// sprintf(dbname,"%s?mode=memory&cache=shared",GloVars.statsdb_disk); -// statsdb_disk->open(dbname, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_FULLMUTEX); -// free(dbname); - - statsdb_disk->execute("PRAGMA synchronous=0"); -// GloProxyStats->statsdb_disk = configdb; - GloProxyStats->init(); - - tables_defs_admin=new std::vector; - tables_defs_stats=new std::vector; - tables_defs_config=new std::vector; - - insert_into_tables_defs(tables_defs_admin,"mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_servers", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS); - insert_into_tables_defs(tables_defs_admin,"mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_users", ADMIN_SQLITE_RUNTIME_MYSQL_USERS); - insert_into_tables_defs(tables_defs_admin,"runtime_checksums_values", ADMIN_SQLITE_RUNTIME_CHECKSUMS_VALUES); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_AWS_AURORA_HOSTGROUPS); - insert_into_tables_defs(tables_defs_admin,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES); - insert_into_tables_defs(tables_defs_admin,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS_SSL_PARAMS); - insert_into_tables_defs(tables_defs_admin,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); - insert_into_tables_defs(tables_defs_admin,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_query_rules", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES_FAST_ROUTING); - insert_into_tables_defs(tables_defs_admin,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); - insert_into_tables_defs(tables_defs_admin,"runtime_global_variables", ADMIN_SQLITE_RUNTIME_GLOBAL_VARIABLES); - insert_into_tables_defs(tables_defs_admin,"mysql_collations", ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS); - insert_into_tables_defs(tables_defs_admin,"scheduler", ADMIN_SQLITE_TABLE_SCHEDULER); - insert_into_tables_defs(tables_defs_admin,"runtime_scheduler", ADMIN_SQLITE_TABLE_RUNTIME_SCHEDULER); - insert_into_tables_defs(tables_defs_admin,"mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_USERS); - insert_into_tables_defs(tables_defs_admin,"mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_RULES); - insert_into_tables_defs(tables_defs_admin,"mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); - insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); - insert_into_tables_defs(tables_defs_admin,"restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES); - insert_into_tables_defs(tables_defs_admin,"runtime_restapi_routes", ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES); - insert_into_tables_defs(tables_defs_admin,"coredump_filters", ADMIN_SQLITE_TABLE_COREDUMP_FILTERS); - insert_into_tables_defs(tables_defs_admin,"runtime_coredump_filters", ADMIN_SQLITE_RUNTIME_COREDUMP_FILTERS); -#ifdef DEBUG - insert_into_tables_defs(tables_defs_admin,"debug_levels", ADMIN_SQLITE_TABLE_DEBUG_LEVELS); - insert_into_tables_defs(tables_defs_admin,"debug_filters", ADMIN_SQLITE_TABLE_DEBUG_FILTERS); -#endif /* DEBUG */ -#ifdef PROXYSQLCLICKHOUSE - // ClickHouse - if (GloVars.global.clickhouse_server) { - insert_into_tables_defs(tables_defs_admin,"clickhouse_users", ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS); - insert_into_tables_defs(tables_defs_admin,"runtime_clickhouse_users", ADMIN_SQLITE_TABLE_RUNTIME_CLICKHOUSE_USERS); - } -#endif /* PROXYSQLCLICKHOUSE */ - - insert_into_tables_defs(tables_defs_config,"mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS); - insert_into_tables_defs(tables_defs_config,"mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS); - insert_into_tables_defs(tables_defs_config,"mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); - insert_into_tables_defs(tables_defs_config,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); - insert_into_tables_defs(tables_defs_config,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); - insert_into_tables_defs(tables_defs_config,"mysql_aws_aurora_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS); - insert_into_tables_defs(tables_defs_config,"mysql_hostgroup_attributes", ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES); - insert_into_tables_defs(tables_defs_config,"mysql_servers_ssl_params", ADMIN_SQLITE_TABLE_MYSQL_SERVERS_SSL_PARAMS); - insert_into_tables_defs(tables_defs_config,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); - insert_into_tables_defs(tables_defs_config,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); - insert_into_tables_defs(tables_defs_config,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); - insert_into_tables_defs(tables_defs_config,"global_settings", ADMIN_SQLITE_TABLE_GLOBAL_SETTINGS); - // the table is not required to be present on disk. Removing it due to #1055 - insert_into_tables_defs(tables_defs_config,"mysql_collations", ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS); - insert_into_tables_defs(tables_defs_config,"scheduler", ADMIN_SQLITE_TABLE_SCHEDULER); - insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS); - insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES); - insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); - insert_into_tables_defs(tables_defs_config, "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES); -#ifdef DEBUG - insert_into_tables_defs(tables_defs_config,"debug_levels", ADMIN_SQLITE_TABLE_DEBUG_LEVELS); - insert_into_tables_defs(tables_defs_config,"debug_filters", ADMIN_SQLITE_TABLE_DEBUG_FILTERS); -#endif /* DEBUG */ -#ifdef PROXYSQLCLICKHOUSE - // ClickHouse - if (GloVars.global.clickhouse_server) { - insert_into_tables_defs(tables_defs_config,"clickhouse_users", ADMIN_SQLITE_TABLE_CLICKHOUSE_USERS); - } -#endif /* PROXYSQLCLICKHOUSE */ - - insert_into_tables_defs(tables_defs_stats,"stats_mysql_query_rules", STATS_SQLITE_TABLE_MYSQL_QUERY_RULES); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_commands_counters", STATS_SQLITE_TABLE_MYSQL_COMMANDS_COUNTERS); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_processlist", STATS_SQLITE_TABLE_MYSQL_PROCESSLIST); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_connection_pool", STATS_SQLITE_TABLE_MYSQL_CONNECTION_POOL); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_connection_pool_reset", STATS_SQLITE_TABLE_MYSQL_CONNECTION_POOL_RESET); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_free_connections", STATS_SQLITE_TABLE_MYSQL_FREE_CONNECTIONS); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_query_digest", STATS_SQLITE_TABLE_MYSQL_QUERY_DIGEST); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_query_digest_reset", STATS_SQLITE_TABLE_MYSQL_QUERY_DIGEST_RESET); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_errors", STATS_SQLITE_TABLE_MYSQL_ERRORS); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_errors_reset", STATS_SQLITE_TABLE_MYSQL_ERRORS_RESET); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_global", STATS_SQLITE_TABLE_MYSQL_GLOBAL); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_gtid_executed", STATS_SQLITE_TABLE_MYSQL_GTID_EXECUTED); - insert_into_tables_defs(tables_defs_stats,"stats_memory_metrics", STATS_SQLITE_TABLE_MEMORY_METRICS); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_users", STATS_SQLITE_TABLE_MYSQL_USERS); - insert_into_tables_defs(tables_defs_stats,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); // workaround for issue #708 - insert_into_tables_defs(tables_defs_stats,"stats_mysql_prepared_statements_info", ADMIN_SQLITE_TABLE_STATS_MYSQL_PREPARED_STATEMENTS_INFO); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_client_host_cache", STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE); - insert_into_tables_defs(tables_defs_stats,"stats_mysql_client_host_cache_reset", STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE_RESET); - - // ProxySQL Cluster - insert_into_tables_defs(tables_defs_admin,"proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS); - insert_into_tables_defs(tables_defs_config,"proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS); - insert_into_tables_defs(tables_defs_admin,"runtime_proxysql_servers", ADMIN_SQLITE_TABLE_RUNTIME_PROXYSQL_SERVERS); - insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_checksums", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CHECKSUMS); - insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_metrics", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_METRICS); - insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_status", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_STATUS); - insert_into_tables_defs(tables_defs_stats,"stats_proxysql_servers_clients_status", STATS_SQLITE_TABLE_PROXYSQL_SERVERS_CLIENTS_STATUS); - insert_into_tables_defs(tables_defs_stats,"stats_proxysql_message_metrics", STATS_SQLITE_TABLE_PROXYSQL_MESSAGE_METRICS); - insert_into_tables_defs(tables_defs_stats,"stats_proxysql_message_metrics_reset", STATS_SQLITE_TABLE_PROXYSQL_MESSAGE_METRICS_RESET); - - // init ldap here - init_ldap(); - - // upgrade mysql_servers if needed (upgrade from previous version) - disk_upgrade_mysql_servers(); - - // upgrade mysql_users if needed (upgrade from previous version) - disk_upgrade_mysql_users(); - - // upgrade mysql_query_rules if needed (upgrade from previous version) - disk_upgrade_mysql_query_rules(); - - // upgrade scheduler if needed (upgrade from previous version) - disk_upgrade_scheduler(); - - // upgrade restapi_routes if needed (upgrade from previous version) - disk_upgrade_rest_api_routes(); - - check_and_build_standard_tables(admindb, tables_defs_admin); - check_and_build_standard_tables(configdb, tables_defs_config); - check_and_build_standard_tables(statsdb, tables_defs_stats); - - __attach_db(admindb, configdb, (char *)"disk"); - __attach_db(admindb, statsdb, (char *)"stats"); - __attach_db(admindb, monitordb, (char *)"monitor"); - __attach_db(statsdb, monitordb, (char *)"monitor"); - __attach_db(admindb, statsdb_disk, (char *)"stats_history"); - __attach_db(statsdb, statsdb_disk, (char *)"stats_history"); - - dump_mysql_collations(); - -#ifdef DEBUG - admindb->execute("ATTACH DATABASE 'file:mem_mydb?mode=memory&cache=shared' AS myhgm"); - admindb->execute("ATTACH DATABASE 'file:mem_monitor_internal_db?mode=memory&cache=shared' AS 'monitor_internal'"); - { - string debugdb_disk_path = string(GloVars.datadir) + "/" + "proxysql_debug.db"; - debugdb_disk = new SQLite3DB(); - debugdb_disk->open((char *)debugdb_disk_path.c_str(), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); - debugdb_disk->execute("CREATE TABLE IF NOT EXISTS debug_log (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , time INT NOT NULL , lapse INT NOT NULL , thread INT NOT NULL , file VARCHAR NOT NULL , line INT NOT NULL , funct VARCHAR NOT NULL , modnum INT NOT NULL , modname VARCHAR NOT NULL , verbosity INT NOT NULL , message VARCHAR , note VARCHAR , backtrace VARCHAR)"); -/* - // DO NOT CREATE INDEX. - // We can create index on a running instance or an archived DB if needed - debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_time ON debug_log (time)"); - debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_thread ON debug_log (thread)"); - debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_file ON debug_log (file)"); - debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_file_line ON debug_log (file,line)"); - debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_funct ON debug_log (funct)"); - debugdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_debug_log_modnum ON debug_log (modnum)"); -*/ - debugdb_disk->execute("PRAGMA synchronous=0"); - debugdb_disk->execute("PRAGMA journal_mode=OFF"); -/* - // DO NOT ATTACH DATABASE - // it seems sqlite starts randomly failing. For example these 2 TAP tests: - // - admin_show_fields_from-t - // - admin_show_table_status-t - string cmd = "ATTACH DATABASE '" + debugdb_disk_path + "' AS debugdb_disk"; - admindb->execute(cmd.c_str()); -*/ - proxysql_set_admin_debugdb_disk(debugdb_disk); - } -#endif /* DEBUG */ - -#ifdef DEBUG - flush_debug_levels_runtime_to_database(configdb, false); - flush_debug_levels_runtime_to_database(admindb, true); -#endif /* DEBUG */ - - // Set default values for the module variables in the target 'dbs' - flush_mysql_variables___runtime_to_database(configdb, false, false, false); - flush_mysql_variables___runtime_to_database(admindb, false, true, false); - - flush_admin_variables___runtime_to_database(configdb, false, false, false); - flush_admin_variables___runtime_to_database(admindb, false, true, false); - - load_or_update_global_settings(configdb); - - // Insert or update the configuration from 'disk' - __insert_or_replace_maintable_select_disktable(); - - // removing this line of code. It seems redundant - //flush_admin_variables___database_to_runtime(admindb,true); - - // workaround for issue #708 - statsdb->execute("INSERT OR IGNORE INTO global_variables VALUES('mysql-max_allowed_packet',4194304)"); - - -#ifdef DEBUG - if (GloVars.global.gdbg==false && GloVars.__cmd_proxysql_gdbg) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Enabling GloVars.global.gdbg because GloVars.__cmd_proxysql_gdbg==%d\n", GloVars.__cmd_proxysql_gdbg); - GloVars.global.gdbg=true; - } - load_debug_to_runtime(); -#endif /* DEBUG */ - - if (GloVars.__cmd_proxysql_reload || GloVars.__cmd_proxysql_initial || admindb_file_exists==false) { // see #617 - if (GloVars.configfile_open) { - proxysql_config().Read_MySQL_Servers_from_configfile(); - proxysql_config().Read_MySQL_Users_from_configfile(); - proxysql_config().Read_MySQL_Query_Rules_from_configfile(); - proxysql_config().Read_Global_Variables_from_configfile("admin"); - proxysql_config().Read_Global_Variables_from_configfile("mysql"); - proxysql_config().Read_Scheduler_from_configfile(); - proxysql_config().Read_Restapi_from_configfile(); - proxysql_config().Read_ProxySQL_Servers_from_configfile(); - __insert_or_replace_disktable_select_maintable(); - } - } - - /** - * @brief Inserts a default 'mysql_group_replication_hostgroup'. - * @details Uses the following defaults: - * - writer_hostgroup: 0 - * - reader_hostgroup: 1 - * - backup_writer_hostgroup: 2 - * - offline_hostgroup: 3 - * - max_writers: 9 - * - writer_is_also_reader: 0 -> Keep hostgroups separated - * - max_transactions_behind: 0 - * - * The number of writers in 'multi_primary_mode' wont be restricted, user should tune this value to - * convenience. By default 'max_writers' is set to 9, as is the current member limitation for Group - * Replication. - */ - const char insert_def_gr_hgs[] { - "INSERT INTO mysql_group_replication_hostgroups (" - "writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers," - "writer_is_also_reader" - ") VALUES (0,2,1,3,1,9,0)" - }; - vector servers_info {}; - - if (GloVars.global.gr_bootstrap_mode) { - // Check if user config is present for 'mysql_group_replication_hostgroups' - bool user_gr_hg_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_group_replication_hostgroups"); - if (user_gr_hg_cnf == false) { - admindb->execute(insert_def_gr_hgs); - } else { - proxy_info("Bootstrap config, found previous user 'mysql_group_replication_hostgroups' config, reusing...\n"); - } - - // Stores current user config for 'mysql_hostgroup_attributes::servers_defaults' - map hgid_defs {}; - // Check if user config is present for 'mysql_hostgroup_attributes' - bool user_gr_hg_attrs_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_hostgroup_attributes"); - int32_t have_ssl = 1; - - // SSL explicitly disabled by user for backend connections - if (GloVars.global.gr_bootstrap_ssl_mode) { - if (strcasecmp(GloVars.global.gr_bootstrap_ssl_mode, "DISABLED") == 0) { - have_ssl = 0; - } - } - - const int64_t DEF_GR_SRV_WEIGHT = 1; - const int64_t DEF_GR_SRV_MAX_CONNS = 512; - const int32_t DEF_GR_SRV_USE_SSL = have_ssl; - - // Update 'mysql_hostgroup_attributes' with sensible defaults for the new discovered instances - if (user_gr_hg_attrs_cnf == false) { - const nlohmann::json j_def_attrs { - { "weight", DEF_GR_SRV_WEIGHT }, - { "max_connections", DEF_GR_SRV_MAX_CONNS }, - { "use_ssl", DEF_GR_SRV_USE_SSL } - }; - const string str_def_attrs { j_def_attrs.dump() }; - const string insert_def_hg_attrs { - "INSERT INTO mysql_hostgroup_attributes (hostgroup_id, servers_defaults) VALUES" - " (0,'"+ str_def_attrs + "'), (1,'" + str_def_attrs + "')" - }; - admindb->execute(insert_def_hg_attrs.c_str()); - } else { - proxy_info("Bootstrap config, found previous user 'mysql_hostgroup_attributes' config, reusing...\n"); - hgid_defs = get_cur_hg_attrs(admindb); - } - - // Define the 'global defaults'. Either pure defaults, or user specified (argument). These values are - // supersede if previous user config is found for 'mysql_hostgroup_attributes::servers_defaults'. - srv_defs_t global_srvs_defs {}; - global_srvs_defs.weight = DEF_GR_SRV_WEIGHT; - global_srvs_defs.max_conns = DEF_GR_SRV_MAX_CONNS; - global_srvs_defs.use_ssl = DEF_GR_SRV_USE_SSL; - - servers_info = extract_boot_servers_info(bootstrap_info.servers); - auto full_srvs_info = build_srvs_info_with_defs(servers_info, hgid_defs, global_srvs_defs); - const string servers_insert { build_boot_servers_insert(full_srvs_info) }; - - admindb->execute("DELETE FROM mysql_servers"); - admindb->execute(servers_insert.c_str()); - - const string users_insert { build_boot_users_insert(bootstrap_info.users) }; - admindb->execute("DELETE FROM mysql_users"); - admindb->execute(users_insert.c_str()); - - // Make the configuration persistent - flush_GENERIC__from_to("mysql_servers", "memory_to_disk"); - flush_mysql_users__from_memory_to_disk(); - } - - // Admin variables 'bootstrap' modifications - if (GloVars.global.gr_bootstrap_mode) { - // TODO-NOTE: This MUST go away; 'admin-hash_passwords' will be deprecated - admindb->execute("UPDATE global_variables SET variable_value='false' WHERE variable_name='admin-hash_passwords'"); - } - flush_admin_variables___database_to_runtime(admindb,true); - - if (GloVars.global.gr_bootstrap_mode) { - flush_admin_variables___runtime_to_database(configdb, false, true, false); - } - - // MySQL variables / MySQL Query Rules 'bootstrap' modifications - if (GloVars.global.gr_bootstrap_mode && !servers_info.empty()) { - const uint64_t base_port { - GloVars.global.gr_bootstrap_conf_base_port == 0 ? 6446 : - GloVars.global.gr_bootstrap_conf_base_port - }; - const string bind_addr { - GloVars.global.gr_bootstrap_conf_bind_address == nullptr ? "0.0.0.0" : - string { GloVars.global.gr_bootstrap_conf_bind_address } - }; - const string s_rw_port { std::to_string(base_port) }; - const string s_ro_port { std::to_string(base_port + 1) }; - const string rw_addr { bind_addr + ":" + s_rw_port }; - const string ro_addr { bind_addr + ":" + s_ro_port }; - const string mysql_interfaces { rw_addr + ";" + ro_addr }; - - // Look for the default collation - const MARIADB_CHARSET_INFO* charset_info = proxysql_find_charset_nr(bootstrap_info.server_language); - const char* server_charset = charset_info == nullptr ? "" : charset_info->csname; - const char* server_collation = charset_info == nullptr ? "" : charset_info->name; - - // Holds user specified values, defaults, and implications of variables over others - const map bootstrap_mysql_vars { - { "mysql-server_version", bootstrap_info.server_version.c_str() }, - { "mysql-default_charset", server_charset }, - { "mysql-default_collation_connection", server_collation }, - { "mysql-interfaces", mysql_interfaces.c_str() }, - { "mysql-monitor_username", bootstrap_info.mon_user.c_str() }, - { "mysql-monitor_password", bootstrap_info.mon_pass.c_str() }, - { "mysql-have_ssl", "true" }, - { "mysql-ssl_p2s_ca", GloVars.global.gr_bootstrap_ssl_ca }, - { "mysql-ssl_p2s_capath", GloVars.global.gr_bootstrap_ssl_capath }, - { "mysql-ssl_p2s_cert", GloVars.global.gr_bootstrap_ssl_cert }, - { "mysql-ssl_p2s_cipher", GloVars.global.gr_bootstrap_ssl_cipher }, - { "mysql-ssl_p2s_crl", GloVars.global.gr_bootstrap_ssl_crl }, - { "mysql-ssl_p2s_crlpath", GloVars.global.gr_bootstrap_ssl_crlpath }, - { "mysql-ssl_p2s_key", GloVars.global.gr_bootstrap_ssl_key } - }; - - for (const pair& p_var_val : bootstrap_mysql_vars) { - if (p_var_val.second != nullptr) { - const string& name { p_var_val.first }; - const string& value { p_var_val.second }; - const string update_mysql_var { - "UPDATE global_variables SET variable_value='" + value + "' WHERE variable_name='" + name + "'" - }; - - admindb->execute(update_mysql_var.c_str()); - } - } - - // MySQL Query Rules - Port based RW split - { - // TODO: This should be able to contain in the future Unix socket based rules - const string insert_rw_split_rules { - "INSERT INTO mysql_query_rules (rule_id,active,proxy_port,destination_hostgroup,apply) VALUES " - " (0,1," + s_rw_port + ",0,1), (1,1," + s_ro_port + ",1,1)" - }; - - // Preserve previous user config targeting hostgroups 0/1 - bool user_qr_cnf = check_if_user_config(admindb, "SELECT COUNT(*) FROM mysql_query_rules"); - if (user_qr_cnf == false) { - admindb->execute(insert_rw_split_rules.c_str()); - } else { - proxy_info("Bootstrap config, found previous user 'mysql_query_rules' config, reusing...\n"); - } - - flush_GENERIC__from_to("mysql_query_rules", "memory_to_disk"); - } - - // Store the 'bootstrap_variables' - if (bootstrap_info.rand_gen_user) { - configdb->execute(ADMIN_SQLITE_TABLE_BOOTSTRAP_VARIABLES); - - const string insert_bootstrap_user { - "INSERT INTO bootstrap_variables (variable_name,variable_value) VALUES" - " ('bootstrap_username','" + string { bootstrap_info.mon_user } + "')" - }; - const string insert_bootstrap_pass { - "INSERT INTO bootstrap_variables (variable_name,variable_value) VALUES" - " ('bootstrap_password','" + string { bootstrap_info.mon_pass } + "')" - }; - - configdb->execute("DELETE FROM bootstrap_variables WHERE variable_name='bootstrap_username'"); - configdb->execute(insert_bootstrap_user.c_str()); - configdb->execute("DELETE FROM bootstrap_variables WHERE variable_name='bootstrap_password'"); - configdb->execute(insert_bootstrap_pass.c_str()); - } - } - flush_mysql_variables___database_to_runtime(admindb,true); - if (GloVars.global.gr_bootstrap_mode) { - flush_mysql_variables___runtime_to_database(configdb, false, true, false); - } -#ifdef PROXYSQLCLICKHOUSE - flush_clickhouse_variables___database_to_runtime(admindb,true); -#endif /* PROXYSQLCLICKHOUSE */ - flush_sqliteserver_variables___database_to_runtime(admindb,true); - - if (GloVars.__cmd_proxysql_admin_socket) { - set_variable((char *)"mysql_ifaces",GloVars.__cmd_proxysql_admin_socket); - } - - S_amll.update_ifaces(variables.mysql_ifaces, &S_amll.ifaces_mysql); - S_amll.update_ifaces(variables.telnet_admin_ifaces, &S_amll.ifaces_telnet_admin); - S_amll.update_ifaces(variables.telnet_stats_ifaces, &S_amll.ifaces_telnet_stats); - - - -// pthread_t admin_thr; - struct _main_args *arg=(struct _main_args *)malloc(sizeof(struct _main_args)); - arg->nfds=main_poll_nfds; - arg->fds=main_poll_fds; - arg->shutdown=&main_shutdown; - arg->callback_func=main_callback_func; - if (pthread_create(&admin_thr, &attr, admin_main_loop, (void *)arg) !=0 ) { - perror("Thread creation"); - exit(EXIT_FAILURE); - } - do { usleep(50); } while (__sync_fetch_and_sub(&load_main_,0)==0); - load_main_=0; - - // Register the global prometheus registry in the 'serial_exposer' - if (registered_prometheus_collectable == false) { - this->serial_exposer.RegisterCollectable(GloVars.prometheus_registry); - registered_prometheus_collectable = true; - } - -#ifdef DEBUG - std::cerr << "Admin initialized in "; -#endif -return true; -}; - -#ifdef PROXYSQLCLICKHOUSE -void ProxySQL_Admin::init_clickhouse_variables() { - flush_clickhouse_variables___runtime_to_database(configdb, false, false, false); - flush_clickhouse_variables___runtime_to_database(admindb, false, true, false); - flush_clickhouse_variables___database_to_runtime(admindb,true); -} -#endif /* CLICKHOUSE */ - -void ProxySQL_Admin::init_sqliteserver_variables() { - flush_sqliteserver_variables___runtime_to_database(configdb, false, false, false); - flush_sqliteserver_variables___runtime_to_database(admindb, false, true, false); - flush_sqliteserver_variables___database_to_runtime(admindb,true); -} - -void ProxySQL_Admin::init_ldap_variables() { -/* - if (variables.hash_passwords==true) { - proxy_info("Impossible to set admin-hash_passwords=true when LDAP is enabled. Reverting to false\n"); - variables.hash_passwords=false; - } -*/ - flush_ldap_variables___runtime_to_database(configdb, false, false, false); - flush_ldap_variables___runtime_to_database(admindb, false, true, false); - flush_ldap_variables___database_to_runtime(admindb,true); - admindb->execute((char *)"DETACH DATABASE disk"); - check_and_build_standard_tables(admindb, tables_defs_admin); - check_and_build_standard_tables(configdb, tables_defs_config); - __attach_db(admindb, configdb, (char *)"disk"); - admindb->execute("INSERT OR REPLACE INTO main.mysql_ldap_mapping SELECT * FROM disk.mysql_ldap_mapping"); -} - -void ProxySQL_Admin::admin_shutdown() { - int i; -// do { usleep(50); } while (main_shutdown==0); - if (Admin_HTTP_Server) { - if (variables.web_enabled) { - MHD_stop_daemon(Admin_HTTP_Server); - Admin_HTTP_Server = NULL; - } - } - delete AdminHTTPServer; - if (AdminRestApiServer) { - delete AdminRestApiServer; - AdminRestApiServer = NULL; - } - AdminHTTPServer = NULL; - pthread_join(admin_thr, NULL); - delete admindb; - delete statsdb; - delete configdb; - delete monitordb; - delete statsdb_disk; -#ifdef DEBUG - proxysql_set_admin_debugdb_disk(NULL); - delete debugdb_disk; -#endif - (*proxy_sqlite3_shutdown)(); - if (main_poll_fds) { - for (i=0;i::iterator it = map_test_mysql_firewall_whitelist_rules.begin(); it != map_test_mysql_firewall_whitelist_rules.end(); ++it) { - PtrArray* myptrarray = (PtrArray*)it->second; - delete myptrarray; - } - map_test_mysql_firewall_whitelist_rules.clear(); -}; - -// This function is used only used to export what collations are available -// it is mostly informative -void ProxySQL_Admin::dump_mysql_collations() { - const MARIADB_CHARSET_INFO * c = mariadb_compiled_charsets; - char buf[1024]; - char *query=(char *)"INSERT INTO mysql_collations VALUES (%d, \"%s\", \"%s\", \"\")"; - admindb->execute("DELETE FROM mysql_collations"); - do { - sprintf(buf,query,c->nr, c->name, c->csname); - admindb->execute(buf); - ++c; - } while (c[0].nr != 0); - admindb->execute("INSERT OR REPLACE INTO mysql_collations SELECT Id, Collation, Charset, 'Yes' FROM mysql_collations JOIN (SELECT MIN(Id) minid FROM mysql_collations GROUP BY Charset) t ON t.minid=mysql_collations.Id"); - // the table is not required to be present on disk. Removing it due to #1055 -// admindb->execute("DELETE FROM disk.mysql_collations"); -// admindb->execute("INSERT INTO disk.mysql_collations SELECT * FROM main.mysql_collations"); -} - -void ProxySQL_Admin::check_and_build_standard_tables(SQLite3DB *db, std::vector *tables_defs) { -// int i; - table_def_t *td; - db->execute("PRAGMA foreign_keys = OFF"); - for (std::vector::iterator it=tables_defs->begin(); it!=tables_defs->end(); ++it) { - td=*it; - db->check_and_build_table(td->table_name, td->table_def); - } - db->execute("PRAGMA foreign_keys = ON"); -}; - - - -void ProxySQL_Admin::insert_into_tables_defs(std::vector *tables_defs, const char *table_name, const char *table_def) { - table_def_t *td = new table_def_t; - td->table_name=strdup(table_name); - td->table_def=strdup(table_def); - tables_defs->push_back(td); -}; - -void ProxySQL_Admin::drop_tables_defs(std::vector *tables_defs) { - table_def_t *td; - while (!tables_defs->empty()) { - td=tables_defs->back(); - free(td->table_name); - td->table_name=NULL; - free(td->table_def); - td->table_def=NULL; - tables_defs->pop_back(); - delete td; - } -}; - -std::map request_headers(const httpserver::http_request& request) { - auto req_headers = request.get_headers(); - std::map result {}; - - for (const auto& header : req_headers) { - result.insert({header.first, header.second}); - } - - return result; -} - -std::shared_ptr make_response( - const std::pair, std::string>& res_data -) { - std::shared_ptr response = - std::make_shared(httpserver::string_response(res_data.second)); - - for (const auto& h_key_val : res_data.first) { - response->with_header(h_key_val.first, h_key_val.second); - } - - return response; -} - -/** - * @brief Checks if the supplied port is available. - * - * @param port_num The port number to check. - * @param free Output parameter. True if the port is free, false otherwise. - * - * @return Returns: - * - '-1' in case 'SO_REUSEADDR' fails to be set for the check. - * - '-2' in case of invalid arguments supplied. - * - '0' otherwise. - */ -int check_port_availability(int port_num, bool* port_free) { - int ecode = 0; - int sfd = 0; - int reuseaddr = 1; - struct sockaddr_in tmp_addr; - - if (port_num == 0 || port_free == nullptr) { - return -2; - } - - // set 'port_free' to false by default - *port_free = false; - - sfd = socket(AF_INET, SOCK_STREAM, 0); - memset(&tmp_addr, 0, sizeof(tmp_addr)); - tmp_addr.sin_family = AF_INET; - tmp_addr.sin_port = htons(port_num); - tmp_addr.sin_addr.s_addr = INADDR_ANY; - - if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuseaddr, sizeof(reuseaddr)) == -1) { - close(sfd); - ecode = -1; - } else { - if (::bind(sfd, (struct sockaddr*)&tmp_addr, sizeof(tmp_addr)) == -1) { - close(sfd); - } else { - *port_free = true; - close(sfd); - } - } - - return ecode; -} - -void ProxySQL_Admin::load_or_update_global_settings(SQLite3DB *db) { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT variable_name, variable_value FROM global_settings ORDER BY variable_name"; - db->execute_statement(q, &error , &cols , &affected_rows , &resultset); - if (error) { - proxy_error("Error on %s : %s\n", q, error); - } else { - // note: we don't lock, this is done only during bootstrap - { - char *uuid = NULL; - bool write_uuid = true; - // search for uuid - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - if (strcasecmp(r->fields[0],"uuid")==0) { - uuid = strdup(r->fields[1]); - uuid_t uu; - if (uuid) { - if (uuid_parse(uuid,uu)==0) { - // we successful read an UUID - } else { - proxy_error("Ignoring invalid UUID format in global_settings: %s\n", uuid); - free(uuid); - uuid = NULL; - } - } - } - } - if (uuid) { // we found an UUID in the DB - if (GloVars.uuid) { // an UUID is already defined - if (strcmp(uuid, GloVars.uuid)==0) { // the match - proxy_info("Using UUID: %s\n", uuid); - write_uuid = false; - } else { - // they do not match. The one on DB will be replaced - proxy_info("Using UUID: %s . Replacing UUID from database: %s\n", GloVars.uuid, uuid); - } - } else { - // the UUID already defined, so the one in the DB will be used - proxy_info("Using UUID from database: %s\n", uuid); - GloVars.uuid=strdup(uuid); - } - } else { - if (GloVars.uuid) { - // we will write the UUID in the DB - proxy_info("Using UUID: %s . Writing it to database\n", GloVars.uuid); - } else { - // UUID not defined anywhere, we will create a new one - uuid_t uu; - uuid_generate(uu); - char buf[40]; - uuid_unparse(uu, buf); - GloVars.uuid=strdup(buf); - proxy_info("Using UUID: %s , randomly generated. Writing it to database\n", GloVars.uuid); - } - } - if (write_uuid) { - std::string s = "INSERT OR REPLACE INTO global_settings VALUES (\"uuid\", \""; - s += GloVars.uuid; - s += "\")"; - db->execute(s.c_str()); - } - if (uuid) { - free(uuid); - uuid=NULL; - } - } - - if (resultset) { - delete resultset; - } - } -} - -void ProxySQL_Admin::load_restapi_server() { - if (!all_modules_started) { return; } - - std::function(const httpserver::http_request&)> prometheus_callback { - [this](const httpserver::http_request& request) { - auto headers = request_headers(request); - auto serial_response = this->serial_exposer(headers); - auto http_response = make_response(serial_response); - - return http_response; - } - }; - - bool free_restapi_port = false; - - // Helper lambda taking a boolean reference as a parameter to check if 'restapi_port' is available. - // In case of port not being free or error, logs an error 'ProxySQL_RestAPI_Server' isn't able to be started. - const auto check_restapi_port = [&](bool& restapi_port_free) -> void { - int e_port_check = check_port_availability(variables.restapi_port, &restapi_port_free); - - if (restapi_port_free == false) { - if (e_port_check == -1) { - proxy_error("Unable to start 'ProxySQL_RestAPI_Server', failed to set 'SO_REUSEADDR' to check port availability.\n"); - } else { - proxy_error( - "Unable to start 'ProxySQL_RestAPI_Server', port '%d' already in use.\n", - variables.restapi_port - ); - } - } - }; - - if (variables.restapi_enabled != variables.restapi_enabled_old) { - if (variables.restapi_enabled) { - check_restapi_port(free_restapi_port); - } - - if (variables.restapi_enabled && free_restapi_port) { - AdminRestApiServer = new ProxySQL_RESTAPI_Server( - variables.restapi_port, {{"/metrics", prometheus_callback}} - ); - } else { - delete AdminRestApiServer; - AdminRestApiServer = NULL; - } - variables.restapi_enabled_old = variables.restapi_enabled; - } else { - if (variables.restapi_port != variables.restapi_port_old) { - if (AdminRestApiServer) { - delete AdminRestApiServer; - AdminRestApiServer = NULL; - } - - if (variables.restapi_enabled) { - check_restapi_port(free_restapi_port); - } - - if (variables.restapi_enabled && free_restapi_port) { - AdminRestApiServer = new ProxySQL_RESTAPI_Server( - variables.restapi_port, {{"/metrics", prometheus_callback}} - ); - } - variables.restapi_port_old = variables.restapi_port; - } - } -} - -void ProxySQL_Admin::load_http_server() { - if (!all_modules_started) { return; } - - if (variables.web_enabled != variables.web_enabled_old) { - if (variables.web_enabled) { - if (GloVars.web_interface_plugin == NULL) { - char *key_pem; - char *cert_pem; - GloVars.get_SSL_pem_mem(&key_pem, &cert_pem); - Admin_HTTP_Server = MHD_start_daemon(MHD_USE_AUTO | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG | MHD_USE_SSL, - variables.web_port, - NULL, NULL, http_handler, NULL, - MHD_OPTION_CONNECTION_TIMEOUT, (unsigned int) 120, MHD_OPTION_STRICT_FOR_CLIENT, (int) 1, - MHD_OPTION_THREAD_POOL_SIZE, (unsigned int) 4, - MHD_OPTION_NONCE_NC_SIZE, (unsigned int) 300, - MHD_OPTION_HTTPS_MEM_KEY, key_pem, - MHD_OPTION_HTTPS_MEM_CERT, cert_pem, - MHD_OPTION_END); - free(key_pem); - free(cert_pem); - } else { - if (GloWebInterface) { - int sfd = 0; - int reuseaddr = 1; - struct sockaddr_in tmp_addr; - - sfd = socket(AF_INET, SOCK_STREAM, 0); - memset(&tmp_addr, 0, sizeof(tmp_addr)); - tmp_addr.sin_family = AF_INET; - tmp_addr.sin_port = htons(variables.web_port); - tmp_addr.sin_addr.s_addr = INADDR_ANY; - - if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuseaddr, sizeof(reuseaddr)) == -1) { - close(sfd); - proxy_error( - "Unable to start WebInterfacePlugin, failed to set 'SO_REUSEADDR' to check port '%d' availability.\n", - variables.web_port - ); - } else { - if (::bind(sfd, (struct sockaddr*)&tmp_addr, (socklen_t)sizeof(tmp_addr)) == -1) { - close(sfd); - proxy_error( - "Unable to start WebInterfacePlugin, port '%d' already in use.\n", - variables.web_port - ); - } else { - close(sfd); - GloWebInterface->start(variables.web_port); - } - } - } - } - } else { - if (GloVars.web_interface_plugin == NULL) { - MHD_stop_daemon(Admin_HTTP_Server); - Admin_HTTP_Server = NULL; - } else { - if (GloWebInterface) { - GloWebInterface->stop(); - } - } - } - variables.web_enabled_old = variables.web_enabled; - } else { - if (variables.web_port != variables.web_port_old) { - if (variables.web_enabled) { - if (GloVars.web_interface_plugin == NULL) { - MHD_stop_daemon(Admin_HTTP_Server); - Admin_HTTP_Server = NULL; - char *key_pem; - char *cert_pem; - GloVars.get_SSL_pem_mem(&key_pem, &cert_pem); - Admin_HTTP_Server = MHD_start_daemon(MHD_USE_AUTO | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG | MHD_USE_SSL, - variables.web_port, - NULL, NULL, http_handler, NULL, - MHD_OPTION_CONNECTION_TIMEOUT, (unsigned int) 120, MHD_OPTION_STRICT_FOR_CLIENT, (int) 1, - MHD_OPTION_THREAD_POOL_SIZE, (unsigned int) 4, - MHD_OPTION_NONCE_NC_SIZE, (unsigned int) 300, - MHD_OPTION_HTTPS_MEM_KEY, key_pem, - MHD_OPTION_HTTPS_MEM_CERT, cert_pem, - MHD_OPTION_END); - free(key_pem); - free(cert_pem); - } else { - if (GloWebInterface) { - GloWebInterface->start(variables.web_port); - } - } - } - variables.web_port_old = variables.web_port; - } - } -} - - -bool ProxySQL_Admin::flush_GENERIC_variables__retrieve__database_to_runtime(const std::string& modname, char* &error, int& cols, int& affected_rows, SQLite3_result* &resultset) { - string q = "SELECT substr(variable_name," + to_string(modname.length()+2) + ") vn, variable_value FROM global_variables WHERE variable_name LIKE '" + modname + "-%'"; - admindb->execute_statement(q.c_str(), &error , &cols , &affected_rows , &resultset); - if (error) { - proxy_error("Error on %s : %s\n", q.c_str(), error); - free(error); - return false; - } - return true; -} - -void ProxySQL_Admin::flush_GENERIC_variables__process__database_to_runtime( - const string& modname, SQLite3DB *db, SQLite3_result* resultset, - const bool& lock, const bool& replace, - const std::unordered_set& variables_read_only, - const std::unordered_set& variables_to_delete_silently, - const std::unordered_set& variables_deprecated, - const std::unordered_set& variables_special_values, - std::function special_variable_action -) { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - bool rc = false; - if (modname == "admin") { - rc = set_variable(r->fields[0],r->fields[1], lock); - } else if (modname == "mysql") { - rc = GloMTH->set_variable(r->fields[0],r->fields[1]); - } else if (modname == "sqliteserver") { - rc = GloSQLite3Server->set_variable(r->fields[0],r->fields[1]); -#ifdef PROXYSQLCLICKHOUSE - } else if (modname == "clickhouse") { - rc = GloClickHouseServer->set_variable(r->fields[0],r->fields[1]); -#endif // PROXYSQLCLICKHOUSE - } else if (modname == "ldap") { - rc = GloMyLdapAuth->set_variable(r->fields[0],r->fields[1]); - } - const string v = string(r->fields[0]); - if (rc==false) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Impossible to set variable %s with value \"%s\"\n", r->fields[0],r->fields[1]); - if (replace) { - char *val = NULL; - if (modname == "admin") { - val = get_variable(r->fields[0]); - } else if (modname == "mysql") { - val = GloMTH->get_variable(r->fields[0]); - } else if (modname == "sqliteserver") { - val = GloSQLite3Server->get_variable(r->fields[0]); -#ifdef PROXYSQLCLICKHOUSE - } else if (modname == "clickhouse") { - val = GloClickHouseServer->get_variable(r->fields[0]); -#endif // PROXYSQLCLICKHOUSE - } else if (modname == "ldap") { - val = GloMyLdapAuth->get_variable(r->fields[0]); - } - char q[1000]; - if (val) { - if (variables_read_only.count(v) > 0) { - proxy_warning("Impossible to set read-only variable %s with value \"%s\". Resetting to current \"%s\".\n", r->fields[0],r->fields[1], val); - } else { - proxy_warning("Impossible to set variable %s with value \"%s\". Resetting to current \"%s\".\n", r->fields[0],r->fields[1], val); - } - sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"%s-%s\",\"%s\")", modname.c_str(), r->fields[0],val); - db->execute(q); - free(val); - } else { - if (variables_to_delete_silently.count(v) > 0) { - sprintf(q,"DELETE FROM disk.global_variables WHERE variable_name=\"%s-%s\"", modname.c_str(), r->fields[0]); - db->execute(q); - } else if (variables_deprecated.count(v) > 0) { - proxy_error("Global variable %s-%s is deprecated.\n", modname.c_str(), r->fields[0]); - sprintf(q,"DELETE FROM disk.global_variables WHERE variable_name=\"%s-%s\"", modname.c_str(), r->fields[0]); - db->execute(q); - } else { - proxy_warning("Impossible to set not existing variable %s with value \"%s\". Deleting. If the variable name is correct, this version doesn't support it\n", r->fields[0],r->fields[1]); - } - sprintf(q,"DELETE FROM global_variables WHERE variable_name=\"%s-%s\"", modname.c_str(), r->fields[0]); - db->execute(q); - } - } - } else { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Set variable %s with value \"%s\"\n", r->fields[0],r->fields[1]); - if (variables_special_values.count(v) > 0) { - if (special_variable_action != nullptr) { - special_variable_action(v, r->fields[1], db); - } - } - } - } -} - -void ProxySQL_Admin::flush_admin_variables___database_to_runtime( - SQLite3DB *db, bool replace, const string& checksum, const time_t epoch, bool lock -) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ADMIN variables. Replace:%d\n", replace); - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - if (flush_GENERIC_variables__retrieve__database_to_runtime("admin", error, cols, affected_rows, resultset) == true) { - wrlock(); - flush_GENERIC_variables__process__database_to_runtime("admin", db, resultset, lock, replace, {"version"}, {"debug"}, {}, {}); - //commit(); NOT IMPLEMENTED - - // Checksums are always generated - 'admin-checksum_*' deprecated - - { - // generate checksum for cluster - pthread_mutex_lock(&GloVars.checksum_mutex); - flush_admin_variables___runtime_to_database(admindb, false, false, false, true); - flush_GENERIC_variables__checksum__database_to_runtime("admin", checksum, epoch); - pthread_mutex_unlock(&GloVars.checksum_mutex); - } - wrunlock(); - { - load_http_server(); - load_restapi_server(); - // Update the admin variable for 'web_verbosity' - admin___web_verbosity = variables.web_verbosity; - } - } - if (resultset) delete resultset; -} - -void ProxySQL_Admin::flush_GENERIC_variables__checksum__database_to_runtime(const string& modname, const string& checksum, const time_t epoch) { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - std::string q; - q="SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE '" + modname + "-\%' "; - if (modname == "mysql") { - q += " AND variable_name NOT IN ('mysql-threads')"; - if (GloVars.cluster_sync_interfaces == false) { - q += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_MYSQL); - } - } else if (modname == "admin") { - if (GloVars.cluster_sync_interfaces == false) { - q += " AND variable_name NOT IN " + string(CLUSTER_SYNC_INTERFACES_ADMIN); - } - } - q += " ORDER BY variable_name"; - admindb->execute_statement(q.c_str(), &error , &cols , &affected_rows , &resultset); - uint64_t hash1 = resultset->raw_checksum(); - uint32_t d32[2]; - char buf[20]; - memcpy(&d32, &hash1, sizeof(hash1)); - sprintf(buf,"0x%0X%0X", d32[0], d32[1]); - ProxySQL_Checksum_Value *checkvar = NULL; - if (modname == "admin") { - checkvar = &GloVars.checksums_values.admin_variables; - } else if (modname == "mysql") { - checkvar = &GloVars.checksums_values.mysql_variables; - } else if (modname == "ldap") { - checkvar = &GloVars.checksums_values.ldap_variables; - } - assert(checkvar != NULL); - checkvar->set_checksum(buf); - checkvar->version++; - time_t t = time(NULL); - if (epoch != 0 && checksum != "" && checkvar->checksum == checksum) { - checkvar->epoch = epoch; - } else { - checkvar->epoch = t; - } - GloVars.epoch_version = t; - GloVars.generate_global_checksum(); - GloVars.checksums_values.updates_cnt++; - string modnameupper = modname; - for (char &c : modnameupper) { c = std::toupper(c); } - proxy_info( - "Computed checksum for 'LOAD %s VARIABLES TO RUNTIME' was '%s', with epoch '%llu'\n", - modnameupper.c_str(), checkvar->checksum, checkvar->epoch - ); - delete resultset; -} - -void ProxySQL_Admin::flush_mysql_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum, const time_t epoch) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing MySQL variables. Replace:%d\n", replace); - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - if (flush_GENERIC_variables__retrieve__database_to_runtime("mysql", error, cols, affected_rows, resultset) == true) { - GloMTH->wrlock(); - char * previous_default_charset = GloMTH->get_variable_string((char *)"default_charset"); - char * previous_default_collation_connection = GloMTH->get_variable_string((char *)"default_collation_connection"); - assert(previous_default_charset); - assert(previous_default_collation_connection); - flush_GENERIC_variables__process__database_to_runtime("mysql", db, resultset, false, replace, {}, {"session_debug"}, {"forward_autocommit"}, - {"default_collation_connection", "default_charset", "show_processlist_extended"}, - [](const std::string& varname, const char *varvalue, SQLite3DB* db) { - if (varname == "default_collation_connection" || varname == "default_charset") { - char *val=GloMTH->get_variable((char *)varname.c_str()); - if (val) { - if (strcmp(val,varvalue)) { - char q[1000]; - proxy_warning("Variable %s with value \"%s\" is being replaced with value \"%s\".\n", varname.c_str(), varvalue, val); - sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-%s\",\"%s\")", varname.c_str() ,val); - db->execute(q); - } - free(val); - } - } else if (varname == "show_processlist_extended") { - GloAdmin->variables.mysql_show_processlist_extended = atoi(varvalue); - } - } - ); - char q[1000]; - char * default_charset = GloMTH->get_variable_string((char *)"default_charset"); - char * default_collation_connection = GloMTH->get_variable_string((char *)"default_collation_connection"); - assert(default_charset); - assert(default_collation_connection); - MARIADB_CHARSET_INFO * ci = NULL; - ci = proxysql_find_charset_name(default_charset); - if (ci == NULL) { - // invalid charset - proxy_error("Found an incorrect value for mysql-default_charset: %s\n", default_charset); - // let's try to get a charset from collation connection - ci = proxysql_find_charset_collate(default_collation_connection); - if (ci == NULL) { - proxy_error("Found an incorrect value for mysql-default_collation_connection: %s\n", default_collation_connection); - const char *p = mysql_tracked_variables[SQL_CHARACTER_SET].default_value; - ci = proxysql_find_charset_name(p); - assert(ci); - proxy_info("Resetting mysql-default_charset to hardcoded default value: %s\n", ci->csname); - sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_charset\",\"%s\")", ci->csname); - db->execute(q); - GloMTH->set_variable((char *)"default_charset",ci->csname); - proxy_info("Resetting mysql-default_collation_connection to hardcoded default value: %s\n", ci->name); - sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_collation_connection\",\"%s\")", ci->name); - db->execute(q); - GloMTH->set_variable((char *)"default_collation_connection",ci->name); - } else { - proxy_info("Changing mysql-default_charset to %s using configured mysql-default_collation_connection %s\n", ci->csname, ci->name); - sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_charset\",\"%s\")", ci->csname); - db->execute(q); - GloMTH->set_variable((char *)"default_charset",ci->csname); - } - } else { - MARIADB_CHARSET_INFO * cic = NULL; - cic = proxysql_find_charset_collate(default_collation_connection); - if (cic == NULL) { - proxy_error("Found an incorrect value for mysql-default_collation_connection: %s\n", default_collation_connection); - proxy_info("Changing mysql-default_collation_connection to %s using configured mysql-default_charset: %s\n", ci->name, ci->csname); - sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_collation_connection\",\"%s\")", ci->name); - db->execute(q); - GloMTH->set_variable((char *)"default_collation_connection",ci->name); - } else { - if (strcmp(cic->csname,ci->csname)==0) { - // mysql-default_collation_connection and mysql-default_charset are compatible - } else { - proxy_error("Found incompatible values for mysql-default_charset (%s) and mysql-default_collation_connection (%s)\n", default_charset, default_collation_connection); - bool use_collation = true; - if (strcmp(default_charset, previous_default_charset)) { // charset changed - if (strcmp(default_collation_connection, previous_default_collation_connection)==0) { // collation didn't change - // the user has changed the charset but not the collation - // we use charset as source of truth - use_collation = false; - } - } - if (use_collation) { - proxy_info("Changing mysql-default_charset to %s using configured mysql-default_collation_connection %s\n", cic->csname, cic->name); - sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_charset\",\"%s\")", cic->csname); - db->execute(q); - GloMTH->set_variable((char *)"default_charset",cic->csname); - } else { - proxy_info("Changing mysql-default_collation_connection to %s using configured mysql-default_charset: %s\n", ci->name, ci->csname); - sprintf(q,"INSERT OR REPLACE INTO global_variables VALUES(\"mysql-default_collation_connection\",\"%s\")", ci->name); - db->execute(q); - GloMTH->set_variable((char *)"default_collation_connection",ci->name); - } - } - } - } - free(default_charset); - free(default_collation_connection); - free(previous_default_charset); - free(previous_default_collation_connection); - GloMTH->commit(); - GloMTH->wrunlock(); - - { - // NOTE: 'GloMTH->wrunlock()' should have been called before this point to avoid possible - // deadlocks. See issue #3847. - pthread_mutex_lock(&GloVars.checksum_mutex); - // generate checksum for cluster - flush_mysql_variables___runtime_to_database(admindb, false, false, false, true, true); - flush_GENERIC_variables__checksum__database_to_runtime("mysql", checksum, epoch); - pthread_mutex_unlock(&GloVars.checksum_mutex); - } - } - if (resultset) delete resultset; -} - -void ProxySQL_Admin::flush_sqliteserver_variables___database_to_runtime(SQLite3DB *db, bool replace) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing SQLiteServer variables. Replace:%d\n", replace); - if ( - (GloVars.global.sqlite3_server == false) - || - ( GloSQLite3Server == NULL ) - ) { - return; - } - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - if (flush_GENERIC_variables__retrieve__database_to_runtime("sqliteserver", error, cols, affected_rows, resultset) == true) { - GloSQLite3Server->wrlock(); - flush_GENERIC_variables__process__database_to_runtime("sqliteserver", db, resultset, false, replace, {}, {"session_debug"}, {}, {}); - //GloClickHouse->commit(); - GloSQLite3Server->wrunlock(); - } - if (resultset) delete resultset; -} - -void ProxySQL_Admin::flush_sqliteserver_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ClickHouse variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); - if (GloVars.global.sqlite3_server == false) { - return; - } - if (onlyifempty) { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'sqliteserver-%'"; - db->execute_statement(q, &error , &cols , &affected_rows , &resultset); - int matching_rows=0; - if (error) { - proxy_error("Error on %s : %s\n", q, error); - return; - } else { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - matching_rows+=atoi(r->fields[0]); - } - } - if (resultset) delete resultset; - if (matching_rows) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has ClickHouse variables - skipping\n"); - return; - } - } - if (del) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting ClickHouse variables from global_variables\n"); - db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'sqliteserver-%'"); - } - if (runtime) { - db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'sqliteserver-%'"); - } - char *a; - char *b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(\"sqliteserver-%s\",\"%s\")"; - if (replace) { - a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(\"sqliteserver-%s\",\"%s\")"; - } else { - a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(\"sqliteserver-%s\",\"%s\")"; - } - int l=strlen(a)+200; - GloSQLite3Server->wrlock(); - char **varnames=GloSQLite3Server->get_variables_list(); - for (int i=0; varnames[i]; i++) { - char *val=GloSQLite3Server->get_variable(varnames[i]); - l+=( varnames[i] ? strlen(varnames[i]) : 6); - l+=( val ? strlen(val) : 6); - char *query=(char *)malloc(l); - sprintf(query, a, varnames[i], val); - if (runtime) { - db->execute(query); - sprintf(query, b, varnames[i], val); - } - db->execute(query); - if (val) - free(val); - free(query); - } - GloSQLite3Server->wrunlock(); - for (int i=0; varnames[i]; i++) { - free(varnames[i]); - } - free(varnames); -} - - -#ifdef PROXYSQLCLICKHOUSE -void ProxySQL_Admin::flush_clickhouse_variables___database_to_runtime(SQLite3DB *db, bool replace) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ClickHouse variables. Replace:%d\n", replace); - if ( - (GloVars.global.clickhouse_server == false) - || - ( GloClickHouseServer == NULL ) - ) { - return; - } - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - if (flush_GENERIC_variables__retrieve__database_to_runtime("clickhouse", error, cols, affected_rows, resultset) == true) { - GloClickHouseServer->wrlock(); - flush_GENERIC_variables__process__database_to_runtime("clickhouse", db, resultset, false, replace, {}, {"session_debug"}, {}, {}); - //GloClickHouse->commit(); - GloClickHouseServer->wrunlock(); - } - if (resultset) delete resultset; -} - -void ProxySQL_Admin::flush_clickhouse_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ClickHouse variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); - if ( - (GloVars.global.clickhouse_server == false) - || - ( GloClickHouseServer == NULL ) - ) { - return; - } - if (onlyifempty) { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'clickhouse-%'"; - db->execute_statement(q, &error , &cols , &affected_rows , &resultset); - int matching_rows=0; - if (error) { - proxy_error("Error on %s : %s\n", q, error); - return; - } else { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - matching_rows+=atoi(r->fields[0]); - } - } - if (resultset) delete resultset; - if (matching_rows) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has ClickHouse variables - skipping\n"); - return; - } - } - if (del) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting ClickHouse variables from global_variables\n"); - db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'clickhouse-%'"); - } - if (runtime) { - db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'clickhouse-%'"); - } - char *a; - char *b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(\"clickhouse-%s\",\"%s\")"; - if (replace) { - a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(\"clickhouse-%s\",\"%s\")"; - } else { - a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(\"clickhouse-%s\",\"%s\")"; - } - int l=strlen(a)+200; - GloClickHouseServer->wrlock(); - char **varnames=GloClickHouseServer->get_variables_list(); - for (int i=0; varnames[i]; i++) { - char *val=GloClickHouseServer->get_variable(varnames[i]); - l+=( varnames[i] ? strlen(varnames[i]) : 6); - l+=( val ? strlen(val) : 6); - char *query=(char *)malloc(l); - sprintf(query, a, varnames[i], val); - if (runtime) { - db->execute(query); - sprintf(query, b, varnames[i], val); - } - db->execute(query); - if (val) - free(val); - free(query); - } - GloClickHouseServer->wrunlock(); - for (int i=0; varnames[i]; i++) { - free(varnames[i]); - } - free(varnames); -} -#endif /* PROXYSQLCLICKHOUSE */ - -bool ProxySQL_Admin::ProxySQL_Test___Load_MySQL_Whitelist(int *ret1, int *ret2, int cmd, int loops) { - // cmd == 1 : populate the structure with a global mutex - // cmd == 2 : perform lookup with a global mutex - // cmd == 3 : perform lookup with a mutex for each call - // cmd == 4 : populate the structure with a global mutex , but without cleaning up - // all accept an extra argument that is the number of loops - char *q = (char *)"SELECT * FROM mysql_firewall_whitelist_rules ORDER BY RANDOM()"; - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - bool ret = true; - int _ret1 = 0; - // cleanup - if (cmd == 1 || cmd == 2 || cmd == 4) { - pthread_mutex_lock(&test_mysql_firewall_whitelist_mutex); - } - admindb->execute_statement(q, &error , &cols , &affected_rows , &resultset); - if (error) { - proxy_error("Error on %s : %s\n", q, error); - return false; - } else { - *ret1 = resultset->rows_count; - int loop = 0; - //if (cmd == 1) { - // loop = loops -1; - //} - for ( ; loop < loops ; loop++) { - _ret1 = 0; - if (cmd == 1) { - for (std::unordered_map::iterator it = map_test_mysql_firewall_whitelist_rules.begin() ; it != map_test_mysql_firewall_whitelist_rules.end(); ++it) { - PtrArray * myptrarray = (PtrArray *)it->second; - delete myptrarray; - } - map_test_mysql_firewall_whitelist_rules.clear(); - } - if (cmd == 4) { - for (std::unordered_map::iterator it = map_test_mysql_firewall_whitelist_rules.begin() ; it != map_test_mysql_firewall_whitelist_rules.end(); ++it) { - PtrArray * myptrarray = (PtrArray *)it->second; - myptrarray->reset(); - } - } - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int active = atoi(r->fields[0]); - if (active == 0) { - continue; - } - char * username = r->fields[1]; - char * client_address = r->fields[2]; - char * schemaname = r->fields[3]; - char * flagIN = r->fields[4]; - char * digest_hex = r->fields[5]; - unsigned long long digest_num = strtoull(digest_hex,NULL,0); - string s = username; - s += rand_del; - s += client_address; - s += rand_del; - s += schemaname; - s += rand_del; - s += flagIN; - std::unordered_map:: iterator it2; - if (cmd == 1 || cmd == 4) { - it2 = map_test_mysql_firewall_whitelist_rules.find(s); - if (it2 != map_test_mysql_firewall_whitelist_rules.end()) { - PtrArray * myptrarray = (PtrArray *)it2->second; - myptrarray->add((void *)digest_num); - } else { - PtrArray * myptrarray = new PtrArray(); - myptrarray->add((void *)digest_num); - map_test_mysql_firewall_whitelist_rules[s] = (void *)myptrarray; - //proxy_info("Inserted key: %s\n" , s.c_str()); - } - } else if (cmd == 2 || cmd == 3) { - if (cmd == 3) { - pthread_mutex_lock(&test_mysql_firewall_whitelist_mutex); - } - it2 = map_test_mysql_firewall_whitelist_rules.find(s); - if (it2 != map_test_mysql_firewall_whitelist_rules.end()) { - PtrArray * myptrarray = (PtrArray *)it2->second; - void * r = bsearch(&digest_num, myptrarray->pdata, myptrarray->len, sizeof(unsigned long long), int_cmp); - if (r) _ret1++; - } else { - //proxy_error("Not found: %s %s %s %s\n", username, client_address, schemaname, flagIN); - proxy_error("Not found: %s\n", s.c_str()); - } - if (cmd == 3) { - pthread_mutex_unlock(&test_mysql_firewall_whitelist_mutex); - } - } - } - if (cmd == 1 || cmd == 4) { - std::unordered_map::iterator it = map_test_mysql_firewall_whitelist_rules.begin(); - while (it != map_test_mysql_firewall_whitelist_rules.end()) { - PtrArray * myptrarray = (PtrArray *)it->second; - switch (cmd) { - case 1: - qsort(myptrarray->pdata, myptrarray->len, sizeof(unsigned long long), int_cmp); - it++; - break; - case 4: - if (myptrarray->len) { - qsort(myptrarray->pdata, myptrarray->len, sizeof(unsigned long long), int_cmp); - it++; - } else { - delete myptrarray; - it = map_test_mysql_firewall_whitelist_rules.erase(it); - } - break; - default: - break; - } - } - } - } - } - if (cmd == 2 || cmd == 3) { - *ret2 = _ret1; - } - if (resultset) delete resultset; - if (cmd == 1 || cmd == 2 || cmd == 4) { - pthread_mutex_unlock(&test_mysql_firewall_whitelist_mutex); - } - return ret; -} - -// if dual is not 0 , we call the new search algorithm -bool ProxySQL_Admin::ProxySQL_Test___Verify_mysql_query_rules_fast_routing( - int *ret1, int *ret2, int cnt, int dual, int ths, bool lock, bool maps_per_thread -) { - // A thread param of '0' is equivalent to not testing - if (ths == 0) { ths = 1; } - char *q = (char *)"SELECT username, schemaname, flagIN, destination_hostgroup FROM mysql_query_rules_fast_routing ORDER BY RANDOM()"; - - bool ret = true; - int matching_rows = 0; - - SQLite3_result *resultset=NULL; - { - char *error=NULL; - int cols=0; - int affected_rows=0; - admindb->execute_statement(q, &error , &cols , &affected_rows , &resultset); - - if (error) { - proxy_error("Error on %s : %s\n", q, error); - *ret1 = -1; - return false; - } - } - *ret2 = resultset->rows_count; - - char *query2=(char *)"SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM main.mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN"; - SQLite3_result* resultset2 = nullptr; - - if (maps_per_thread) { - char* error2 = nullptr; - int cols2 = 0; - int affected_rows2 = 0; - admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); - - if (error2) { - proxy_error("Error on %s : %s\n", query2, error2); - return false; - } - } - - vector results(ths, 0); - vector th_hashmaps {}; - - if (maps_per_thread) { - for (uint32_t i = 0; i < static_cast(ths); i++) { - th_hashmaps.push_back(GloQPro->create_fast_routing_hashmap(resultset2)); - } - } - - const auto perform_searches = - [&results,&dual](khash_t(khStrInt)* hashmap, SQLite3_result* resultset, uint32_t pos, bool lock) -> void - { - uint32_t matching_rows = 0; - - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int dest_HG = atoi(r->fields[3]); - int ret_HG = -1; - if (dual) { - ret_HG = GloQPro->testing___find_HG_in_mysql_query_rules_fast_routing_dual( - hashmap, r->fields[0], r->fields[1], atoi(r->fields[2]), lock - ); - } else { - ret_HG = GloQPro->testing___find_HG_in_mysql_query_rules_fast_routing( - r->fields[0], r->fields[1], atoi(r->fields[2]) - ); - } - - if (dest_HG == ret_HG) { - matching_rows++; - } - } - - results[pos] = matching_rows; - }; - - proxy_info("Test with params - cnt: %d, threads: %d, lock: %d, maps_per_thread: %d\n", cnt, ths, lock, maps_per_thread); - - unsigned long long curtime1 = monotonic_time() / 1000; - std::vector workers {}; - - for (int i = 0; i < ths; i++) { - khash_t(khStrInt)* hashmap = maps_per_thread ? th_hashmaps[i].rules_fast_routing : nullptr; - workers.push_back(std::thread(perform_searches, hashmap, resultset, i, lock)); - } - - for (std::thread& w : workers) { - w.join(); - } - - matching_rows = results[0]; - if (matching_rows != resultset->rows_count) { - ret = false; - } - *ret1 = matching_rows; - - if (ret == true) { - if (cnt > 1) { - for (int i=1 ; i < cnt; i++) { - std::vector workers {}; - - for (int i = 0; i < ths; i++) { - khash_t(khStrInt)* hashmap = maps_per_thread ? th_hashmaps[i].rules_fast_routing : nullptr; - workers.push_back(std::thread(perform_searches, hashmap, resultset, i, lock)); - } - - for (std::thread& w : workers) { - w.join(); - } - } - } - } - - unsigned long long curtime2 = monotonic_time() / 1000; - uint32_t total_maps_size = 0; - - for (const fast_routing_hashmap_t& hashmap : th_hashmaps) { - total_maps_size += hashmap.rules_fast_routing___keys_values___size; - total_maps_size += kh_size(hashmap.rules_fast_routing) * ((sizeof(int) + sizeof(char *) + 4)); - - kh_destroy(khStrInt, hashmap.rules_fast_routing); - free(hashmap.rules_fast_routing___keys_values); - } - - proxy_info("Test took %llums\n", curtime2 - curtime1); - proxy_info("Verified rows %d\n", results[0]); - proxy_info("Total maps size %dkb\n", total_maps_size / 1024); - - if (resultset) delete resultset; - if (resultset2) delete resultset2; - - return ret; -} - -unsigned int ProxySQL_Admin::ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(unsigned int cnt, bool empty) { - char *a = (char *)"INSERT OR IGNORE INTO mysql_query_rules_fast_routing VALUES (?1, ?2, ?3, ?4, '')"; - int rc; - sqlite3_stmt *statement1=NULL; - rc=admindb->prepare_v2(a, &statement1); - ASSERT_SQLITE_OK(rc, admindb); - admindb->execute("DELETE FROM mysql_query_rules_fast_routing"); - char * username_buf = (char *)malloc(128); - char * schemaname_buf = (char *)malloc(256); - - if (empty==false) { - strcpy(username_buf,"user_name_"); - } else { - strcpy(username_buf,""); - } - strcpy(schemaname_buf,"shard_name_"); - int _k; - for (unsigned int i=0; iget_db())==0) { - i--; - } - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); - } - (*proxy_sqlite3_finalize)(statement1); - free(username_buf); - free(schemaname_buf); - return cnt; -} - -void ProxySQL_Admin::flush_mysql_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime, bool use_lock) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing MySQL variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); - if (onlyifempty) { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'mysql-%'"; - db->execute_statement(q, &error , &cols , &affected_rows , &resultset); - int matching_rows=0; - if (error) { - proxy_error("Error on %s : %s\n", q, error); - return; - } else { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - matching_rows+=atoi(r->fields[0]); - } - } - if (resultset) delete resultset; - if (matching_rows) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has MySQL variables - skipping\n"); - return; - } - } - if (del) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting MySQL variables from global_variables\n"); - db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'mysql-%'"); - } - static char *a; - static char *b; - if (replace) { - a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(?1, ?2)"; - } else { - a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(?1, ?2)"; - } - int rc; - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement2=NULL; - - rc=db->prepare_v2(a, &statement1); - ASSERT_SQLITE_OK(rc, db); - if (runtime) { - db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'mysql-%'"); - b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(?1, ?2)"; - - rc=db->prepare_v2(b, &statement2); - ASSERT_SQLITE_OK(rc, db); - } - if (use_lock) { - GloMTH->wrlock(); - db->execute("BEGIN"); - } - char **varnames=GloMTH->get_variables_list(); - for (int i=0; varnames[i]; i++) { - char *val=GloMTH->get_variable(varnames[i]); - char *qualified_name=(char *)malloc(strlen(varnames[i])+7); - sprintf(qualified_name, "mysql-%s", varnames[i]); - rc=(*proxy_sqlite3_bind_text)(statement1, 1, qualified_name, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(statement1, 2, (val ? val : (char *)""), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, db); - if (runtime) { - rc=(*proxy_sqlite3_bind_text)(statement2, 1, qualified_name, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_bind_text)(statement2, 2, (val ? val : (char *)""), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, db); - SAFE_SQLITE3_STEP2(statement2); - rc=(*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, db); - rc=(*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, db); - } - if (val) - free(val); - free(qualified_name); - } - if (use_lock) { - db->execute("COMMIT"); - GloMTH->wrunlock(); - } - (*proxy_sqlite3_finalize)(statement1); - if (runtime) - (*proxy_sqlite3_finalize)(statement2); - for (int i=0; varnames[i]; i++) { - free(varnames[i]); - } - free(varnames); -} - -void ProxySQL_Admin::flush_ldap_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum, const time_t epoch) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing LDAP variables. Replace:%d\n", replace); - if (GloMyLdapAuth == NULL) { - return; - } - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - if (flush_GENERIC_variables__retrieve__database_to_runtime("ldap", error, cols, affected_rows, resultset) == true) { - GloMyLdapAuth->wrlock(); - flush_GENERIC_variables__process__database_to_runtime("admin", db, resultset, false, replace, {}, {}, {}, {}); - GloMyLdapAuth->wrunlock(); - - // Checksums are always generated - 'admin-checksum_*' deprecated - { - pthread_mutex_lock(&GloVars.checksum_mutex); - // generate checksum for cluster - flush_ldap_variables___runtime_to_database(admindb, false, false, false, true); - flush_GENERIC_variables__checksum__database_to_runtime("ldap", checksum, epoch); - pthread_mutex_unlock(&GloVars.checksum_mutex); - } - } - if (resultset) delete resultset; -} - -void ProxySQL_Admin::flush_ldap_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing LDAP variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); - if (GloMyLdapAuth == NULL) { - return; - } - if (onlyifempty) { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'ldap-%'"; - db->execute_statement(q, &error , &cols , &affected_rows , &resultset); - int matching_rows=0; - if (error) { - proxy_error("Error on %s : %s\n", q, error); - return; - } else { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - matching_rows+=atoi(r->fields[0]); - } - } - if (resultset) delete resultset; - if (matching_rows) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has LDAP variables - skipping\n"); - return; - } - } - if (del) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting LDAP variables from global_variables\n"); - db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'ldap-%'"); - } - if (runtime) { - db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'ldap-%'"); - } - char *a; - char *b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(\"ldap-%s\",\"%s\")"; - if (replace) { - a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(\"ldap-%s\",\"%s\")"; - } else { - a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(\"ldap-%s\",\"%s\")"; - } - int l=strlen(a)+200; - GloMyLdapAuth->wrlock(); - char **varnames=GloMyLdapAuth->get_variables_list(); - for (int i=0; varnames[i]; i++) { - char *val=GloMyLdapAuth->get_variable(varnames[i]); - l+=( varnames[i] ? strlen(varnames[i]) : 6); - l+=( val ? strlen(val) : 6); - char *query=(char *)malloc(l); - sprintf(query, a, varnames[i], val); - if (runtime) { - db->execute(query); - sprintf(query, b, varnames[i], val); - } - db->execute(query); - if (val) - free(val); - free(query); - } - GloMyLdapAuth->wrunlock(); - for (int i=0; varnames[i]; i++) { - free(varnames[i]); - } - free(varnames); -} - -char **ProxySQL_Admin::get_variables_list() { - size_t l=sizeof(admin_variables_names)/sizeof(char *); - unsigned int i; - char **ret=(char **)malloc(sizeof(char *)*l); - for (i=0;iset_case_sensitive(false); + match_regexes.re=(void **)malloc(sizeof(void *)*10); + match_regexes.re[0]=(RE2 *)new RE2("^SELECT\\s+@@max_allowed_packet\\s*", *opt2); + match_regexes.re[1]=(RE2 *)new RE2("^SELECT\\s+@@[0-9A-Za-z_-]+\\s*", *opt2); + match_regexes.re[2]=(RE2 *)new RE2("SHOW\\s+VARIABLES\\s+WHERE", *opt2); + match_regexes.re[3]=(RE2 *)new RE2("SHOW\\s+VARIABLES\\s+LIKE", *opt2); + + // Default initialize prometheus collectable flag + registered_prometheus_collectable = false; + + // Initialize prometheus metrics + init_prometheus_counter_array(admin_metrics_map, this->metrics.p_counter_array); + init_prometheus_gauge_array(admin_metrics_map, this->metrics.p_gauge_array); + init_prometheus_dyn_gauge_array(admin_metrics_map, this->metrics.p_dyn_gauge_array); + + // NOTE: Imposing fixed value to 'version_info' matching 'mysqld_exporter' + this->metrics.p_gauge_array[p_admin_gauge::version_info]->Set(1); +}; + +void ProxySQL_Admin::wrlock() { +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_wrlock(&rwlock); +#else + spin_wrlock(&rwlock); +#endif +}; + +void ProxySQL_Admin::wrunlock() { +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_unlock(&rwlock); +#else + spin_wrunlock(&rwlock); +#endif +}; + +void ProxySQL_Admin::mysql_servers_wrlock() { + #ifdef PA_PTHREAD_MUTEX + pthread_mutex_lock(&mysql_servers_lock); + #else + spin_wrlock(&mysql_servers_rwlock); + #endif +}; + +void ProxySQL_Admin::mysql_servers_wrunlock() { + #ifdef PA_PTHREAD_MUTEX + pthread_mutex_unlock(&mysql_servers_lock); + #else + spin_wrunlock(&mysql_servers_rwlock); + #endif +}; + +void ProxySQL_Admin::pgsql_servers_wrlock() { +#ifdef PA_PTHREAD_MUTEX + pthread_mutex_lock(&pgsql_servers_lock); +#else + spin_wrlock(&pgsql_servers_rwlock); +#endif +}; + +void ProxySQL_Admin::pgsql_servers_wrunlock() { +#ifdef PA_PTHREAD_MUTEX + pthread_mutex_unlock(&pgsql_servers_lock); +#else + spin_wrunlock(&pgsql_servers_rwlock); +#endif +}; + +void ProxySQL_Admin::print_version() { + fprintf(stderr,"Standard ProxySQL Admin rev. %s -- %s -- %s\n", PROXYSQL_ADMIN_VERSION, __FILE__, __TIMESTAMP__); +}; + +void ProxySQL_Admin::init_ldap() { + if (GloMyLdapAuth) { + insert_into_tables_defs(tables_defs_admin,"mysql_ldap_mapping", ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_ldap_mapping", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_LDAP_MAPPING); + insert_into_tables_defs(tables_defs_config,"mysql_ldap_mapping", ADMIN_SQLITE_TABLE_MYSQL_LDAP_MAPPING); } +} + +void ProxySQL_Admin::init_http_server() { + AdminHTTPServer = new ProxySQL_HTTP_Server(); + AdminHTTPServer->init(); + AdminHTTPServer->print_version(); +} + +#ifdef PROXYSQLCLICKHOUSE +void ProxySQL_Admin::init_clickhouse_variables() { + flush_clickhouse_variables___runtime_to_database(configdb, false, false, false); + flush_clickhouse_variables___runtime_to_database(admindb, false, true, false); + flush_clickhouse_variables___database_to_runtime(admindb,true); +} +#endif /* CLICKHOUSE */ + +void ProxySQL_Admin::init_sqliteserver_variables() { + flush_sqliteserver_variables___runtime_to_database(configdb, false, false, false); + flush_sqliteserver_variables___runtime_to_database(admindb, false, true, false); + flush_sqliteserver_variables___database_to_runtime(admindb,true); +} + +void ProxySQL_Admin::init_ldap_variables() { /* - if (!strcasecmp(name,"hash_passwords")) { - return strdup((variables.hash_passwords ? "true" : "false")); + if (variables.hash_passwords==true) { + proxy_info("Impossible to set admin-hash_passwords=true when LDAP is enabled. Reverting to false\n"); + variables.hash_passwords=false; } */ - if (!strcasecmp(name,"vacuum_stats")) { - return strdup((variables.vacuum_stats ? "true" : "false")); - } - if (!strcasecmp(name,"checksum_mysql_query_rules")) { - return strdup((checksum_variables.checksum_mysql_query_rules ? "true" : "false")); - } - if (!strcasecmp(name,"checksum_mysql_servers")) { - return strdup((checksum_variables.checksum_mysql_servers ? "true" : "false")); - } - if (!strcasecmp(name,"checksum_mysql_users")) { - return strdup((checksum_variables.checksum_mysql_users ? "true" : "false")); + flush_ldap_variables___runtime_to_database(configdb, false, false, false); + flush_ldap_variables___runtime_to_database(admindb, false, true, false); + flush_ldap_variables___database_to_runtime(admindb,true); + admindb->execute((char *)"DETACH DATABASE disk"); + check_and_build_standard_tables(admindb, tables_defs_admin); + check_and_build_standard_tables(configdb, tables_defs_config); + __attach_db(admindb, configdb, (char *)"disk"); + admindb->execute("INSERT OR REPLACE INTO main.mysql_ldap_mapping SELECT * FROM disk.mysql_ldap_mapping"); +} + +void ProxySQL_Admin::init_pgsql_variables() { + flush_pgsql_variables___runtime_to_database(configdb, false, false, false); + flush_pgsql_variables___runtime_to_database(admindb, false, true, false); + flush_pgsql_variables___database_to_runtime(admindb, true); +} + +void ProxySQL_Admin::admin_shutdown() { + int i; +// do { usleep(50); } while (main_shutdown==0); + if (Admin_HTTP_Server) { + if (variables.web_enabled) { + MHD_stop_daemon(Admin_HTTP_Server); + Admin_HTTP_Server = NULL; + } } - if (!strcasecmp(name,"checksum_mysql_variables")) { - return strdup((checksum_variables.checksum_mysql_variables ? "true" : "false")); + delete AdminHTTPServer; + if (AdminRestApiServer) { + delete AdminRestApiServer; + AdminRestApiServer = NULL; } - if (!strcasecmp(name,"checksum_admin_variables")) { - return strdup((checksum_variables.checksum_admin_variables ? "true" : "false")); + AdminHTTPServer = NULL; + pthread_join(admin_thr, NULL); + delete admindb; + delete statsdb; + delete configdb; + delete monitordb; + delete statsdb_disk; +#ifdef DEBUG + proxysql_set_admin_debugdb_disk(NULL); + delete debugdb_disk; +#endif + (*proxy_sqlite3_shutdown)(); + if (main_poll_fds) { + for (i=0;iexecute("DELETE FROM mysql_collations"); + do { + sprintf(buf,query,c->nr, c->name, c->csname); + admindb->execute(buf); + ++c; + } while (c[0].nr != 0); + admindb->execute("INSERT OR REPLACE INTO mysql_collations SELECT Id, Collation, Charset, 'Yes' FROM mysql_collations JOIN (SELECT MIN(Id) minid FROM mysql_collations GROUP BY Charset) t ON t.minid=mysql_collations.Id"); + // the table is not required to be present on disk. Removing it due to #1055 +// admindb->execute("DELETE FROM disk.mysql_collations"); +// admindb->execute("INSERT INTO disk.mysql_collations SELECT * FROM main.mysql_collations"); +} + +void ProxySQL_Admin::check_and_build_standard_tables(SQLite3DB *db, std::vector *tables_defs) { +// int i; + table_def_t *td; + db->execute("PRAGMA foreign_keys = OFF"); + for (std::vector::iterator it=tables_defs->begin(); it!=tables_defs->end(); ++it) { + td=*it; + db->check_and_build_table(td->table_name, td->table_def); } - if (!strcasecmp(name, "ssl_keylog_file")) { - char* ssl_keylog_file = s_strdup(variables.ssl_keylog_file); - if (ssl_keylog_file != NULL && strlen(ssl_keylog_file) > 0) { - if ((ssl_keylog_file[0] != '/')) { // relative path - char* tmp_ssl_keylog_file = (char*)malloc(strlen(GloVars.datadir) + strlen(ssl_keylog_file) + 2); - sprintf(tmp_ssl_keylog_file, "%s/%s", GloVars.datadir, ssl_keylog_file); - free(ssl_keylog_file); - ssl_keylog_file = tmp_ssl_keylog_file; - } - } - return ssl_keylog_file; + db->execute("PRAGMA foreign_keys = ON"); +}; + + + +void ProxySQL_Admin::insert_into_tables_defs(std::vector *tables_defs, const char *table_name, const char *table_def) { + table_def_t *td = new table_def_t; + td->table_name=strdup(table_name); + td->table_def=strdup(table_def); + tables_defs->push_back(td); +}; + +void ProxySQL_Admin::drop_tables_defs(std::vector *tables_defs) { + table_def_t *td; + while (!tables_defs->empty()) { + td=tables_defs->back(); + free(td->table_name); + td->table_name=NULL; + free(td->table_def); + td->table_def=NULL; + tables_defs->pop_back(); + delete td; } - return NULL; -} +}; +std::map request_headers(const httpserver::http_request& request) { + auto req_headers = request.get_headers(); + std::map result {}; -#ifdef DEBUG -void ProxySQL_Admin::add_credentials(char *type, char *credentials, int hostgroup_id) { -#else -void ProxySQL_Admin::add_credentials(char *credentials, int hostgroup_id) { -#endif /* DEBUG */ - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Adding %s credentials: %s\n", type, credentials); - tokenizer_t tok; - tokenizer( &tok, credentials, ";", TOKENIZER_NO_EMPTIES ); - const char* token; - for (token = tokenize( &tok ); token; token = tokenize( &tok )) { - char *user=NULL; - char *pass=NULL; - c_split_2(token, ":", &user, &pass); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Adding %s credential: \"%s\", user:%s, pass:%s\n", type, token, user, pass); - if (GloMyAuth) { // this check if required if GloMyAuth doesn't exist yet - GloMyAuth->add(user,pass,USERNAME_FRONTEND,0,hostgroup_id,(char *)"main",0,0,0,1000,(char*)"",(char *)""); - } - free(user); - free(pass); + for (const auto& header : req_headers) { + result.insert({header.first, header.second}); } - free_tokenizer( &tok ); + + return result; } -#ifdef DEBUG -void ProxySQL_Admin::delete_credentials(char *type, char *credentials) { -#else -void ProxySQL_Admin::delete_credentials(char *credentials) { -#endif /* DEBUG */ - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Removing old %s credentials: %s\n", type, credentials); - tokenizer_t tok; - tokenizer( &tok, credentials, ";", TOKENIZER_NO_EMPTIES ); - const char* token; - for (token = tokenize( &tok ); token; token = tokenize( &tok )) { - char *user=NULL; - char *pass=NULL; - c_split_2(token, ":", &user, &pass); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Removing %s credential: \"%s\", user:%s, pass:%s\n", type, token, user, pass); - if (GloMyAuth) { // this check if required if GloMyAuth doesn't exist yet - GloMyAuth->del(user,USERNAME_FRONTEND); - } - free(user); - free(pass); +std::shared_ptr make_response( + const std::pair, std::string>& res_data +) { + std::shared_ptr response = + std::make_shared(httpserver::string_response(res_data.second)); + + for (const auto& h_key_val : res_data.first) { + response->with_header(h_key_val.first, h_key_val.second); } - free_tokenizer( &tok ); + + return response; } -bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this is the public function, accessible from admin - size_t vallen=strlen(value); +/** + * @brief Checks if the supplied port is available. + * + * @param port_num The port number to check. + * @param free Output parameter. True if the port is free, false otherwise. + * + * @return Returns: + * - '-1' in case 'SO_REUSEADDR' fails to be set for the check. + * - '-2' in case of invalid arguments supplied. + * - '0' otherwise. + */ +int check_port_availability(int port_num, bool* port_free) { + int ecode = 0; + int sfd = 0; + int reuseaddr = 1; + struct sockaddr_in tmp_addr; - if (!strcasecmp(name,"admin_credentials")) { - if (vallen) { - bool update_creds=false; - if ((variables.admin_credentials==NULL) || strcasecmp(variables.admin_credentials,value) ) update_creds=true; - if (update_creds && variables.admin_credentials) { -#ifdef DEBUG - delete_credentials((char *)"admin",variables.admin_credentials); -#else - delete_credentials(variables.admin_credentials); -#endif /* DEBUG */ - } - free(variables.admin_credentials); - variables.admin_credentials=strdup(value); - if (update_creds && variables.admin_credentials) { -#ifdef DEBUG - add_credentials((char *)"admin",variables.admin_credentials, ADMIN_HOSTGROUP); -#else - add_credentials(variables.admin_credentials, ADMIN_HOSTGROUP); -#endif /* DEBUG */ - } - return true; - } else { - return false; - } + if (port_num == 0 || port_free == nullptr) { + return -2; } - if (!strcasecmp(name,"stats_credentials")) { - if (vallen) { - bool update_creds=false; - if ((variables.stats_credentials==NULL) || strcasecmp(variables.stats_credentials,value) ) update_creds=true; - if (update_creds && variables.stats_credentials) { -#ifdef DEBUG - delete_credentials((char *)"stats",variables.stats_credentials); -#else - delete_credentials(variables.stats_credentials); -#endif /* DEBUG */ - } - free(variables.stats_credentials); - variables.stats_credentials=strdup(value); - if (update_creds && variables.stats_credentials) { -#ifdef DEBUG - add_credentials((char *)"admin",variables.stats_credentials, STATS_HOSTGROUP); -#else - add_credentials(variables.stats_credentials, STATS_HOSTGROUP); -#endif /* DEBUG */ - } - return true; + + // set 'port_free' to false by default + *port_free = false; + + sfd = socket(AF_INET, SOCK_STREAM, 0); + memset(&tmp_addr, 0, sizeof(tmp_addr)); + tmp_addr.sin_family = AF_INET; + tmp_addr.sin_port = htons(port_num); + tmp_addr.sin_addr.s_addr = INADDR_ANY; + + if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuseaddr, sizeof(reuseaddr)) == -1) { + close(sfd); + ecode = -1; + } else { + if (::bind(sfd, (struct sockaddr*)&tmp_addr, sizeof(tmp_addr)) == -1) { + close(sfd); } else { - return false; + *port_free = true; + close(sfd); } } - if (!strncasecmp(name,"stats_",strlen("stats_"))) { - if (!strcasecmp(name,"stats_mysql_connection_pool")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 300) { - intv = round_intv_to_time_interval(name, intv); - variables.stats_mysql_connection_pool=intv; - GloProxyStats->variables.stats_mysql_connection_pool=intv; - return true; - } else { - return false; - } - } - if (!strcasecmp(name,"stats_mysql_connections")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 300) { - intv = round_intv_to_time_interval(name, intv); - variables.stats_mysql_connections=intv; - GloProxyStats->variables.stats_mysql_connections=intv; - return true; - } else { - return false; - } - } - if (!strcasecmp(name,"stats_mysql_query_cache")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 300) { - intv = round_intv_to_time_interval(name, intv); - variables.stats_mysql_query_cache=intv; - GloProxyStats->variables.stats_mysql_query_cache=intv; - return true; - } else { - return false; - } - } - if (!strcasecmp(name,"stats_mysql_query_digest_to_disk")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 24*3600) { - variables.stats_mysql_query_digest_to_disk=intv; - GloProxyStats->variables.stats_mysql_query_digest_to_disk=intv; - return true; - } else { - return false; + + return ecode; +} + +void ProxySQL_Admin::load_or_update_global_settings(SQLite3DB *db) { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *q=(char *)"SELECT variable_name, variable_value FROM global_settings ORDER BY variable_name"; + db->execute_statement(q, &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", q, error); + } else { + // note: we don't lock, this is done only during bootstrap + { + char *uuid = NULL; + bool write_uuid = true; + // search for uuid + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + if (strcasecmp(r->fields[0],"uuid")==0) { + uuid = strdup(r->fields[1]); + uuid_t uu; + if (uuid) { + if (uuid_parse(uuid,uu)==0) { + // we successful read an UUID + } else { + proxy_error("Ignoring invalid UUID format in global_settings: %s\n", uuid); + free(uuid); + uuid = NULL; + } + } + } } - } - if (!strcasecmp(name,"stats_system_cpu")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 600) { - intv = round_intv_to_time_interval(name, intv); - variables.stats_system_cpu=intv; - GloProxyStats->variables.stats_system_cpu=intv; - return true; + if (uuid) { // we found an UUID in the DB + if (GloVars.uuid) { // an UUID is already defined + if (strcmp(uuid, GloVars.uuid)==0) { // the match + proxy_info("Using UUID: %s\n", uuid); + write_uuid = false; + } else { + // they do not match. The one on DB will be replaced + proxy_info("Using UUID: %s . Replacing UUID from database: %s\n", GloVars.uuid, uuid); + } + } else { + // the UUID already defined, so the one in the DB will be used + proxy_info("Using UUID from database: %s\n", uuid); + GloVars.uuid=strdup(uuid); + } } else { - return false; + if (GloVars.uuid) { + // we will write the UUID in the DB + proxy_info("Using UUID: %s . Writing it to database\n", GloVars.uuid); + } else { + // UUID not defined anywhere, we will create a new one + uuid_t uu; + uuid_generate(uu); + char buf[40]; + uuid_unparse(uu, buf); + GloVars.uuid=strdup(buf); + proxy_info("Using UUID: %s , randomly generated. Writing it to database\n", GloVars.uuid); + } } - } -#ifndef NOJEM - if (!strcasecmp(name,"stats_system_memory")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 600) { - intv = round_intv_to_time_interval(name, intv); - variables.stats_system_memory=intv; - GloProxyStats->variables.stats_system_memory=intv; - return true; - } else { - return false; + if (write_uuid) { + std::string s = "INSERT OR REPLACE INTO global_settings VALUES (\"uuid\", \""; + s += GloVars.uuid; + s += "\")"; + db->execute(s.c_str()); } - } -#endif - } - if (!strcasecmp(name,"mysql_ifaces")) { - if (vallen) { - bool update_creds=false; - if ((variables.mysql_ifaces==NULL) || strcasecmp(variables.mysql_ifaces,value) ) update_creds=true; - if (variables.mysql_ifaces) - free(variables.mysql_ifaces); - variables.mysql_ifaces=strdup(value); - if (update_creds && variables.mysql_ifaces) { - S_amll.update_ifaces(variables.mysql_ifaces, &S_amll.ifaces_mysql); + if (uuid) { + free(uuid); + uuid=NULL; } - GloProxyCluster->set_admin_mysql_ifaces(value); - return true; - } else { - return false; - } - } - if (!strcasecmp(name,"cluster_username")) { - if (vallen) { - free(variables.cluster_username); - variables.cluster_username=strdup(value); - GloProxyCluster->set_username(variables.cluster_username); - return true; - } else { - return true; } - } - if (!strcasecmp(name,"cluster_password")) { - if (vallen) { - free(variables.cluster_password); - variables.cluster_password=strdup(value); - GloProxyCluster->set_password(variables.cluster_password); - return true; - } else { - return true; + + if (resultset) { + delete resultset; } } - if (!strcasecmp(name,"telnet_admin_ifaces")) { - if (vallen) { - bool update_creds=false; - if ((variables.telnet_admin_ifaces==NULL) || strcasecmp(variables.telnet_admin_ifaces,value) ) update_creds=true; - if (variables.telnet_admin_ifaces) - free(variables.telnet_admin_ifaces); - variables.telnet_admin_ifaces=strdup(value); - if (update_creds && variables.telnet_admin_ifaces) { - S_amll.update_ifaces(variables.telnet_admin_ifaces, &S_amll.ifaces_telnet_admin); - } - return true; - } else { - return false; +} + +void ProxySQL_Admin::load_restapi_server() { + if (!all_modules_started) { return; } + + std::function(const httpserver::http_request&)> prometheus_callback { + [this](const httpserver::http_request& request) { + auto headers = request_headers(request); + auto serial_response = this->serial_exposer(headers); + auto http_response = make_response(serial_response); + + return http_response; } - } - if (!strcasecmp(name,"telnet_stats_ifaces")) { - if (vallen) { - bool update_creds=false; - if ((variables.telnet_stats_ifaces==NULL) || strcasecmp(variables.telnet_stats_ifaces,value) ) update_creds=true; - if (variables.telnet_stats_ifaces) - free(variables.telnet_stats_ifaces); - variables.telnet_stats_ifaces=strdup(value); - if (update_creds && variables.telnet_stats_ifaces) { - S_amll.update_ifaces(variables.telnet_stats_ifaces, &S_amll.ifaces_telnet_stats); + }; + + bool free_restapi_port = false; + + // Helper lambda taking a boolean reference as a parameter to check if 'restapi_port' is available. + // In case of port not being free or error, logs an error 'ProxySQL_RestAPI_Server' isn't able to be started. + const auto check_restapi_port = [&](bool& restapi_port_free) -> void { + int e_port_check = check_port_availability(variables.restapi_port, &restapi_port_free); + + if (restapi_port_free == false) { + if (e_port_check == -1) { + proxy_error("Unable to start 'ProxySQL_RestAPI_Server', failed to set 'SO_REUSEADDR' to check port availability.\n"); + } else { + proxy_error( + "Unable to start 'ProxySQL_RestAPI_Server', port '%d' already in use.\n", + variables.restapi_port + ); } - return true; - } else { - return false; - } - } - if (!strcasecmp(name,"refresh_interval")) { - int intv=atoi(value); - if (intv > 100 && intv < 100000) { - variables.refresh_interval=intv; - __admin_refresh_interval=intv; - return true; - } else { - return false; } - } - if (!strcasecmp(name,"cluster_check_interval_ms")) { - int intv=atoi(value); - if (intv >= 10 && intv <= 300000) { - variables.cluster_check_interval_ms=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_check_interval_ms, intv); - return true; - } else { - return false; + }; + + if (variables.restapi_enabled != variables.restapi_enabled_old) { + if (variables.restapi_enabled) { + check_restapi_port(free_restapi_port); } - } - if (!strcasecmp(name,"cluster_check_status_frequency")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 10000) { - variables.cluster_check_status_frequency=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_check_status_frequency, intv); - return true; + + if (variables.restapi_enabled && free_restapi_port) { + AdminRestApiServer = new ProxySQL_RESTAPI_Server( + variables.restapi_port, {{"/metrics", prometheus_callback}} + ); } else { - return false; + delete AdminRestApiServer; + AdminRestApiServer = NULL; } - } - if (!strcasecmp(name,"cluster_mysql_query_rules_diffs_before_sync")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 1000) { - intv = checksum_variables.checksum_mysql_query_rules ? intv : 0; - if (variables.cluster_mysql_query_rules_diffs_before_sync == 0 && intv != 0) { - proxy_info("Re-enabled previously disabled 'admin-cluster_admin_variables_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); - GloProxyCluster->Reset_Global_Checksums(lock); + variables.restapi_enabled_old = variables.restapi_enabled; + } else { + if (variables.restapi_port != variables.restapi_port_old) { + if (AdminRestApiServer) { + delete AdminRestApiServer; + AdminRestApiServer = NULL; } - variables.cluster_mysql_query_rules_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync, intv); - return true; - } else { - return false; - } - } - if (!strcasecmp(name,"cluster_mysql_servers_diffs_before_sync")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 1000) { - intv = checksum_variables.checksum_mysql_servers ? intv : 0; - if (variables.cluster_mysql_servers_diffs_before_sync == 0 && intv != 0) { - proxy_info("Re-enabled previously disabled 'admin-cluster_mysql_servers_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); - GloProxyCluster->Reset_Global_Checksums(lock); + + if (variables.restapi_enabled) { + check_restapi_port(free_restapi_port); } - variables.cluster_mysql_servers_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, intv); - return true; - } else { - return false; - } - } - if (!strcasecmp(name,"cluster_mysql_users_diffs_before_sync")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 1000) { - intv = checksum_variables.checksum_mysql_users ? intv : 0; - if (variables.cluster_mysql_users_diffs_before_sync == 0 && intv != 0) { - proxy_info("Re-enabled previously disabled 'admin-cluster_mysql_users_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); - GloProxyCluster->Reset_Global_Checksums(lock); + + if (variables.restapi_enabled && free_restapi_port) { + AdminRestApiServer = new ProxySQL_RESTAPI_Server( + variables.restapi_port, {{"/metrics", prometheus_callback}} + ); } - variables.cluster_mysql_users_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_diffs_before_sync, intv); - return true; - } else { - return false; + variables.restapi_port_old = variables.restapi_port; } } - if (!strcasecmp(name,"cluster_proxysql_servers_diffs_before_sync")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 1000) { - if (variables.cluster_proxysql_servers_diffs_before_sync == 0 && intv != 0) { - proxy_info("Re-enabled previously disabled 'admin-cluster_proxysql_servers_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); - GloProxyCluster->Reset_Global_Checksums(lock); +} + +void ProxySQL_Admin::load_http_server() { + if (!all_modules_started) { return; } + + if (variables.web_enabled != variables.web_enabled_old) { + if (variables.web_enabled) { + if (GloVars.web_interface_plugin == NULL) { + char *key_pem; + char *cert_pem; + GloVars.get_SSL_pem_mem(&key_pem, &cert_pem); + Admin_HTTP_Server = MHD_start_daemon(MHD_USE_AUTO | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG | MHD_USE_SSL, + variables.web_port, + NULL, NULL, http_handler, NULL, + MHD_OPTION_CONNECTION_TIMEOUT, (unsigned int) 120, MHD_OPTION_STRICT_FOR_CLIENT, (int) 1, + MHD_OPTION_THREAD_POOL_SIZE, (unsigned int) 4, + MHD_OPTION_NONCE_NC_SIZE, (unsigned int) 300, + MHD_OPTION_HTTPS_MEM_KEY, key_pem, + MHD_OPTION_HTTPS_MEM_CERT, cert_pem, + MHD_OPTION_END); + free(key_pem); + free(cert_pem); + } else { + if (GloWebInterface) { + int sfd = 0; + int reuseaddr = 1; + struct sockaddr_in tmp_addr; + + sfd = socket(AF_INET, SOCK_STREAM, 0); + memset(&tmp_addr, 0, sizeof(tmp_addr)); + tmp_addr.sin_family = AF_INET; + tmp_addr.sin_port = htons(variables.web_port); + tmp_addr.sin_addr.s_addr = INADDR_ANY; + + if (setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR, (char *)&reuseaddr, sizeof(reuseaddr)) == -1) { + close(sfd); + proxy_error( + "Unable to start WebInterfacePlugin, failed to set 'SO_REUSEADDR' to check port '%d' availability.\n", + variables.web_port + ); + } else { + if (::bind(sfd, (struct sockaddr*)&tmp_addr, (socklen_t)sizeof(tmp_addr)) == -1) { + close(sfd); + proxy_error( + "Unable to start WebInterfacePlugin, port '%d' already in use.\n", + variables.web_port + ); + } else { + close(sfd); + GloWebInterface->start(variables.web_port); + } + } + } } - variables.cluster_proxysql_servers_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_proxysql_servers_diffs_before_sync, intv); - return true; } else { - return false; - } - } - if (!strcasecmp(name,"cluster_mysql_variables_diffs_before_sync")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 1000) { - intv = checksum_variables.checksum_mysql_variables ? intv : 0; - if (variables.cluster_mysql_variables_diffs_before_sync == 0 && intv != 0) { - proxy_info("Re-enabled previously disabled 'admin-cluster_mysql_variables_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); - GloProxyCluster->Reset_Global_Checksums(lock); + if (GloVars.web_interface_plugin == NULL) { + MHD_stop_daemon(Admin_HTTP_Server); + Admin_HTTP_Server = NULL; + } else { + if (GloWebInterface) { + GloWebInterface->stop(); + } } - variables.cluster_mysql_variables_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync, intv); - return true; - } else { - return false; } - } - if (!strcasecmp(name,"cluster_admin_variables_diffs_before_sync")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 1000) { - intv = checksum_variables.checksum_admin_variables ? intv : 0; - if (variables.cluster_admin_variables_diffs_before_sync == 0 && intv != 0) { - proxy_info("Re-enabled previously disabled 'admin-cluster_admin_variables_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); - GloProxyCluster->Reset_Global_Checksums(lock); + variables.web_enabled_old = variables.web_enabled; + } else { + if (variables.web_port != variables.web_port_old) { + if (variables.web_enabled) { + if (GloVars.web_interface_plugin == NULL) { + MHD_stop_daemon(Admin_HTTP_Server); + Admin_HTTP_Server = NULL; + char *key_pem; + char *cert_pem; + GloVars.get_SSL_pem_mem(&key_pem, &cert_pem); + Admin_HTTP_Server = MHD_start_daemon(MHD_USE_AUTO | MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_ERROR_LOG | MHD_USE_SSL, + variables.web_port, + NULL, NULL, http_handler, NULL, + MHD_OPTION_CONNECTION_TIMEOUT, (unsigned int) 120, MHD_OPTION_STRICT_FOR_CLIENT, (int) 1, + MHD_OPTION_THREAD_POOL_SIZE, (unsigned int) 4, + MHD_OPTION_NONCE_NC_SIZE, (unsigned int) 300, + MHD_OPTION_HTTPS_MEM_KEY, key_pem, + MHD_OPTION_HTTPS_MEM_CERT, cert_pem, + MHD_OPTION_END); + free(key_pem); + free(cert_pem); + } else { + if (GloWebInterface) { + GloWebInterface->start(variables.web_port); + } + } } - variables.cluster_admin_variables_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_diffs_before_sync, intv); - return true; - } else { - return false; + variables.web_port_old = variables.web_port; } } - if (!strcasecmp(name,"cluster_ldap_variables_diffs_before_sync")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 1000) { - intv = checksum_variables.checksum_ldap_variables ? intv : 0; - if (variables.cluster_ldap_variables_diffs_before_sync == 0 && intv != 0) { - proxy_info("Re-enabled previously disabled 'admin-cluster_ldap_variables_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); - GloProxyCluster->Reset_Global_Checksums(lock); - } - variables.cluster_ldap_variables_diffs_before_sync=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync, intv); - return true; - } else { - return false; - } +} + +char **ProxySQL_Admin::get_variables_list() { + size_t l=sizeof(admin_variables_names)/sizeof(char *); + unsigned int i; + char **ret=(char **)malloc(sizeof(char *)*l); + for (i=0;i= 1 && intv <= 3) { + return ret; +} - if (variables.cluster_mysql_servers_sync_algorithm != intv) { - proxy_info("'cluster_mysql_servers_sync_algorithm' updated. Resetting global checksums to force Cluster re-sync.\n"); - GloProxyCluster->Reset_Global_Checksums(lock); - } - variables.cluster_mysql_servers_sync_algorithm=intv; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_sync_algorithm, intv); - return true; - } else { - return false; - } - } - if (!strcasecmp(name,"version")) { - if (strcasecmp(value,(char *)PROXYSQL_VERSION)==0) { +// Returns true if the given name is the name of an existing admin variable +bool ProxySQL_Admin::has_variable(const char *name) { + size_t no_vars = sizeof(admin_variables_names) / sizeof(char *); + for (unsigned int i = 0; i < no_vars-1 ; ++i) { + size_t var_len = strlen(admin_variables_names[i]); + if (strlen(name) == var_len && !strncmp(name, admin_variables_names[i], var_len)) { return true; - } else { - return false; } } - if (!strcasecmp(name,"hash_passwords")) { - proxy_warning("Variable admin-hash_passwords is now deprecated and removed. See github issue #4218\n"); -/* - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.hash_passwords=true; - if (GloMyLdapAuth) { - proxy_info("Impossible to set admin-hash_passwords=true when LDAP is enabled. Reverting to false\n"); - variables.hash_passwords=false; - } - return true; + return false; +} + +char * ProxySQL_Admin::get_variable(char *name) { +#define INTBUFSIZE 4096 + char intbuf[INTBUFSIZE]; + if (!strcasecmp(name,"version")) return s_strdup(variables.admin_version); + if (!strcasecmp(name,"cluster_username")) return s_strdup(variables.cluster_username); + if (!strcasecmp(name,"cluster_password")) return s_strdup(variables.cluster_password); + if (!strncasecmp(name,"stats_",strlen("stats_"))) { + if (!strcasecmp(name,"stats_credentials")) + return s_strdup(variables.stats_credentials); + if (!strcasecmp(name,"stats_mysql_connection_pool")) { + sprintf(intbuf,"%d",variables.stats_mysql_connection_pool); + return strdup(intbuf); } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.hash_passwords=false; - return true; + if (!strcasecmp(name,"stats_mysql_connections")) { + sprintf(intbuf,"%d",variables.stats_mysql_connections); + return strdup(intbuf); } - return false; -*/ - } - if (!strcasecmp(name,"vacuum_stats")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.vacuum_stats=true; - return true; + if (!strcasecmp(name,"stats_mysql_query_cache")) { + sprintf(intbuf,"%d",variables.stats_mysql_query_cache); + return strdup(intbuf); } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.vacuum_stats=false; - return true; + if (!strcasecmp(name,"stats_mysql_query_digest_to_disk")) { + sprintf(intbuf,"%d",variables.stats_mysql_query_digest_to_disk); + return strdup(intbuf); } - return false; - } - if (!strcasecmp(name,"restapi_enabled")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.restapi_enabled=true; - return true; + if (!strcasecmp(name,"stats_system_cpu")) { + sprintf(intbuf,"%d",variables.stats_system_cpu); + return strdup(intbuf); } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.restapi_enabled=false; - return true; + if (!strcasecmp(name,"stats_system_memory")) { + sprintf(intbuf,"%d",variables.stats_system_memory); + return strdup(intbuf); } - return false; } - if (!strcasecmp(name,"restapi_port")) { - int intv=atoi(value); - if (intv > 0 && intv < 65535) { - variables.restapi_port=intv; - return true; - } else { - return false; - } + if (!strcasecmp(name,"admin_credentials")) return s_strdup(variables.admin_credentials); + if (!strcasecmp(name,"mysql_ifaces")) return s_strdup(variables.mysql_ifaces); + if (!strcasecmp(name,"pgsql_ifaces")) return s_strdup(variables.pgsql_ifaces); + if (!strcasecmp(name,"telnet_admin_ifaces")) return s_strdup(variables.telnet_admin_ifaces); + if (!strcasecmp(name,"telnet_stats_ifaces")) return s_strdup(variables.telnet_stats_ifaces); + if (!strcasecmp(name,"cluster_check_interval_ms")) { + sprintf(intbuf,"%d",variables.cluster_check_interval_ms); + return strdup(intbuf); } - if (!strcasecmp(name,"web_enabled")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.web_enabled=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.web_enabled=false; - return true; - } - return false; + if (!strcasecmp(name,"cluster_check_status_frequency")) { + sprintf(intbuf,"%d",variables.cluster_check_status_frequency); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_mysql_query_rules_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_mysql_query_rules_diffs_before_sync); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_mysql_servers_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_mysql_servers_diffs_before_sync); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_mysql_users_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_mysql_users_diffs_before_sync); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_proxysql_servers_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_proxysql_servers_diffs_before_sync); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_mysql_variables_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_mysql_variables_diffs_before_sync); + return strdup(intbuf); + } + if (!strcasecmp(name,"cluster_admin_variables_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_admin_variables_diffs_before_sync); + return strdup(intbuf); } - if (!strcasecmp(name,"web_port")) { - int intv=atoi(value); - if (intv > 0 && intv < 65535) { - variables.web_port=intv; - return true; - } else { - return false; - } + if (!strcasecmp(name,"cluster_ldap_variables_diffs_before_sync")) { + sprintf(intbuf,"%d",variables.cluster_ldap_variables_diffs_before_sync); + return strdup(intbuf); } - if (!strcasecmp(name,"web_verbosity")) { - int intv=atoi(value); - if (intv >= 0 && intv <= 10) { - variables.web_verbosity=intv; - return true; - } else { - return false; - } + if (!strcasecmp(name,"cluster_mysql_servers_sync_algorithm")) { + sprintf(intbuf, "%d", variables.cluster_mysql_servers_sync_algorithm); + return strdup(intbuf); } if (!strcasecmp(name,"cluster_mysql_query_rules_save_to_disk")) { - bool rt = false; - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.cluster_mysql_query_rules_save_to_disk=true; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_save_to_disk, true); - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.cluster_mysql_query_rules_save_to_disk=false; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_save_to_disk, false); - return true; - } - return rt; + return strdup((variables.cluster_mysql_query_rules_save_to_disk ? "true" : "false")); } if (!strcasecmp(name,"cluster_mysql_servers_save_to_disk")) { - bool rt = false; - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.cluster_mysql_servers_save_to_disk=true; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_save_to_disk, true); - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.cluster_mysql_servers_save_to_disk=false; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_save_to_disk, false); - return true; - } - return rt; + return strdup((variables.cluster_mysql_servers_save_to_disk ? "true" : "false")); } if (!strcasecmp(name,"cluster_mysql_users_save_to_disk")) { - bool rt = false; - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.cluster_mysql_users_save_to_disk=true; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_save_to_disk, true); - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.cluster_mysql_users_save_to_disk=false; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_save_to_disk, false); - return true; - } - return rt; + return strdup((variables.cluster_mysql_users_save_to_disk ? "true" : "false")); } if (!strcasecmp(name,"cluster_proxysql_servers_save_to_disk")) { - bool rt = false; - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.cluster_proxysql_servers_save_to_disk=true; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_proxysql_servers_save_to_disk, true); - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.cluster_proxysql_servers_save_to_disk=false; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_proxysql_servers_save_to_disk, false); - return true; - } - return rt; + return strdup((variables.cluster_proxysql_servers_save_to_disk ? "true" : "false")); } if (!strcasecmp(name,"cluster_mysql_variables_save_to_disk")) { - bool rt = false; - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.cluster_mysql_variables_save_to_disk=true; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_save_to_disk, true); - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.cluster_mysql_variables_save_to_disk=false; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_save_to_disk, false); - return true; - } - return rt; + return strdup((variables.cluster_mysql_variables_save_to_disk ? "true" : "false")); } if (!strcasecmp(name,"cluster_admin_variables_save_to_disk")) { - bool rt = false; - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.cluster_admin_variables_save_to_disk=true; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_save_to_disk, true); - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.cluster_admin_variables_save_to_disk=false; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_save_to_disk, false); - return true; - } - return rt; + return strdup((variables.cluster_admin_variables_save_to_disk ? "true" : "false")); } if (!strcasecmp(name,"cluster_ldap_variables_save_to_disk")) { - bool rt = false; - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.cluster_ldap_variables_save_to_disk=true; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_save_to_disk, true); - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.cluster_ldap_variables_save_to_disk=false; - rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_save_to_disk, false); - return true; - } - return rt; + return strdup((variables.cluster_ldap_variables_save_to_disk ? "true" : "false")); + } + if (!strcasecmp(name,"refresh_interval")) { + sprintf(intbuf,"%d",variables.refresh_interval); + return strdup(intbuf); + } + if (!strcasecmp(name,"read_only")) { + return strdup((variables.admin_read_only ? "true" : "false")); + } +/* + if (!strcasecmp(name,"hash_passwords")) { + return strdup((variables.hash_passwords ? "true" : "false")); + } +*/ + if (!strcasecmp(name,"vacuum_stats")) { + return strdup((variables.vacuum_stats ? "true" : "false")); } if (!strcasecmp(name,"checksum_mysql_query_rules")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - checksum_variables.checksum_mysql_query_rules=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - checksum_variables.checksum_mysql_query_rules=false; - variables.cluster_mysql_query_rules_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync, 0); - proxy_warning("Disabling deprecated 'admin-checksum_mysql_query_rules', setting 'admin-cluster_mysql_query_rules_diffs_before_sync=0'\n"); - return true; - } - return false; + return strdup((checksum_variables.checksum_mysql_query_rules ? "true" : "false")); } if (!strcasecmp(name,"checksum_mysql_servers")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - checksum_variables.checksum_mysql_servers=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - checksum_variables.checksum_mysql_servers=false; - variables.cluster_mysql_servers_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, 0); - proxy_warning("Disabling deprecated 'admin-checksum_mysql_servers', setting 'admin-cluster_mysql_servers_diffs_before_sync=0'\n"); - return true; - } - - return false; + return strdup((checksum_variables.checksum_mysql_servers ? "true" : "false")); } if (!strcasecmp(name,"checksum_mysql_users")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - checksum_variables.checksum_mysql_users=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - checksum_variables.checksum_mysql_users=false; - variables.cluster_mysql_users_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_diffs_before_sync, 0); - proxy_warning("Disabling deprecated 'admin-checksum_mysql_users', setting 'admin-cluster_mysql_users_diffs_before_sync=0'\n"); - return true; - } - return false; + return strdup((checksum_variables.checksum_mysql_users ? "true" : "false")); } if (!strcasecmp(name,"checksum_mysql_variables")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - checksum_variables.checksum_mysql_variables=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - checksum_variables.checksum_mysql_variables=false; - variables.cluster_mysql_variables_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync, 0); - proxy_warning("Disabling deprecated 'admin-checksum_mysql_variables', setting 'admin-cluster_mysql_variables_diffs_before_sync=0'\n"); - return true; - } - return false; + return strdup((checksum_variables.checksum_mysql_variables ? "true" : "false")); } if (!strcasecmp(name,"checksum_admin_variables")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - checksum_variables.checksum_admin_variables=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - checksum_variables.checksum_admin_variables=false; - variables.cluster_admin_variables_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_diffs_before_sync, 0); - proxy_warning("Disabling deprecated 'admin-checksum_admin_variables', setting 'admin-cluster_admin_variables_diffs_before_sync=0'\n"); - return true; - } - return false; + return strdup((checksum_variables.checksum_admin_variables ? "true" : "false")); } if (!strcasecmp(name,"checksum_ldap_variables")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - checksum_variables.checksum_ldap_variables=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - checksum_variables.checksum_ldap_variables=false; - variables.cluster_ldap_variables_diffs_before_sync = 0; - __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync, 0); - proxy_warning("Disabling deprecated 'admin-checksum_ldap_variables', setting 'admin-cluster_ldap_variables_diffs_before_sync=0'\n"); - return true; - } - return false; + return strdup((checksum_variables.checksum_ldap_variables ? "true" : "false")); } - if (!strcasecmp(name,"read_only")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.admin_read_only=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.admin_read_only=false; - return true; - } - return false; + if (!strcasecmp(name,"restapi_enabled")) { + return strdup((variables.restapi_enabled ? "true" : "false")); + } + if (!strcasecmp(name,"restapi_port")) { + sprintf(intbuf,"%d",variables.restapi_port); + return strdup(intbuf); + } + if (!strcasecmp(name,"web_enabled")) { + return strdup((variables.web_enabled ? "true" : "false")); + } + if (!strcasecmp(name,"web_verbosity")) { + sprintf(intbuf, "%d", variables.web_verbosity); + return strdup(intbuf); + } + if (!strcasecmp(name,"web_port")) { + sprintf(intbuf,"%d",variables.web_port); + return strdup(intbuf); } if (!strcasecmp(name,"prometheus_memory_metrics_interval")) { - const auto fval = atoi(value); - if (fval > 0 && fval < 7*24*3600) { - variables.p_memory_metrics_interval = fval; - return true; - } else { - return false; - } + sprintf(intbuf, "%d", variables.p_memory_metrics_interval); + return strdup(intbuf); } #ifdef DEBUG if (!strcasecmp(name,"debug")) { - if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { - variables.debug=true; - GloVars.global.gdbg=true; - return true; - } - if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { - variables.debug=false; - GloVars.global.gdbg=false; - return true; - } - return false; + return strdup((variables.debug ? "true" : "false")); } if (!strcasecmp(name,"debug_output")) { - const auto fval = atoi(value); - if (fval > 0 && fval <= 3) { - debug_output = fval; - proxysql_set_admin_debug_output(debug_output); - return true; - } else { - return false; - } - return false; + sprintf(intbuf, "%d", debug_output); + return strdup(intbuf); } #endif /* DEBUG */ if (!strcasecmp(name,"coredump_generation_interval_ms")) { - int intv=atoi(value); - if (intv >= 0 && intv < INT_MAX) { - variables.coredump_generation_interval_ms=intv; - coredump_generation_interval_ms=intv; - return true; - } else { - return false; - } + sprintf(intbuf,"%d",variables.coredump_generation_interval_ms); + return strdup(intbuf); } if (!strcasecmp(name,"coredump_generation_threshold")) { - int intv=atoi(value); - if (intv > 0 && intv <= 500) { - variables.coredump_generation_threshold=intv; - coredump_generation_threshold=intv; - return true; - } else { - return false; - } + sprintf(intbuf,"%d",variables.coredump_generation_threshold); + return strdup(intbuf); } if (!strcasecmp(name, "ssl_keylog_file")) { - if (strcmp(variables.ssl_keylog_file, value)) { - if (vallen == 0 || strcmp(value, "(null)") == 0) { - proxysql_keylog_close(); - free(variables.ssl_keylog_file); - variables.ssl_keylog_file = strdup(""); - GloVars.global.ssl_keylog_enabled = false; - } else { - char* sslkeylogfile = NULL; - const bool is_absolute_path = (value[0] == '/'); - if (is_absolute_path) { // absolute path - sslkeylogfile = strdup(value); - } else { // relative path - sslkeylogfile = (char*)malloc(strlen(GloVars.datadir) + strlen(value) + 2); - sprintf(sslkeylogfile, "%s/%s", GloVars.datadir, value); - } - if (proxysql_keylog_open(sslkeylogfile) == false) { - free(sslkeylogfile); - proxy_warning("Cannot open SSLKEYLOGFILE '%s' for writing.\n", value); - return false; - } - free(variables.ssl_keylog_file); - if (is_absolute_path) { - variables.ssl_keylog_file = sslkeylogfile; - sslkeylogfile = NULL; - } else { - variables.ssl_keylog_file = strdup(value); - } - if (sslkeylogfile) - free(sslkeylogfile); - GloVars.global.ssl_keylog_enabled = true; + char* ssl_keylog_file = s_strdup(variables.ssl_keylog_file); + if (ssl_keylog_file != NULL && strlen(ssl_keylog_file) > 0) { + if ((ssl_keylog_file[0] != '/')) { // relative path + char* tmp_ssl_keylog_file = (char*)malloc(strlen(GloVars.datadir) + strlen(ssl_keylog_file) + 2); + sprintf(tmp_ssl_keylog_file, "%s/%s", GloVars.datadir, ssl_keylog_file); + free(ssl_keylog_file); + ssl_keylog_file = tmp_ssl_keylog_file; } } - return true; + return ssl_keylog_file; } - return false; + return NULL; } -void ProxySQL_Admin::p_update_metrics() { - // Update proxysql_uptime - auto t1 = monotonic_time(); - auto new_uptime = (t1 - GloVars.global.start_time)/1000/1000; - auto cur_uptime = this->metrics.p_counter_array[p_admin_counter::uptime]->Value(); - this->metrics.p_counter_array[p_admin_counter::uptime]->Increment(new_uptime - cur_uptime); - - // Update memory metrics - this->p_stats___memory_metrics(); - // Update stmt metrics - this->p_update_stmt_metrics(); - - // updated mysql_listener_paused - int st = ( proxysql_mysql_paused == true ? 1 : 0); - this->metrics.p_gauge_array[p_admin_gauge::mysql_listener_paused]->Set(st); +template +#ifdef DEBUG +void ProxySQL_Admin::add_credentials(char *type, char *credentials, int hostgroup_id) { +#else +void ProxySQL_Admin::add_credentials(char *credentials, int hostgroup_id) { +#endif /* DEBUG */ + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Adding %s credentials: %s\n", type, credentials); + tokenizer_t tok; + tokenizer( &tok, credentials, ";", TOKENIZER_NO_EMPTIES ); + const char* token; + for (token = tokenize( &tok ); token; token = tokenize( &tok )) { + char *user=NULL; + char *pass=NULL; + c_split_2(token, ":", &user, &pass); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Adding %s credential: \"%s\", user:%s, pass:%s\n", type, token, user, pass); + + if constexpr (pt == SERVER_TYPE_MYSQL) { + if (GloMyAuth) { // this check if required if GloMyAuth doesn't exist yet + GloMyAuth->add(user, pass, USERNAME_FRONTEND, 0, hostgroup_id, (char*)"main", 0, 0, 0, 1000, (char*)"", (char*)""); + } + } else if constexpr (pt == SERVER_TYPE_PGSQL) { + if (GloPgAuth) { // this check if required if GloPgAuth doesn't exist yet + GloPgAuth->add(user, pass, USERNAME_FRONTEND, 0, hostgroup_id, 0, 0, 1000, (char*)"", (char*)""); + } + } + free(user); + free(pass); + } + free_tokenizer( &tok ); } -/** - * @brief Gets the number of currently opened file descriptors. In case of error '-1' is - * returned and error is logged. - * @return On success, the number of currently opened file descriptors, '-1' otherwise. - */ -int32_t get_open_fds() { -#if defined(__FreeBSD__) || defined(__APPLE__) - DIR* dir = opendir("/dev/fd"); +template +#ifdef DEBUG +void ProxySQL_Admin::delete_credentials(char *type, char *credentials) { #else - DIR* dir = opendir("/proc/self/fd"); -#endif - if (dir == NULL) { - proxy_error("'opendir()' failed with error: '%d'\n", errno); - return -1; - } - - struct dirent* dp = nullptr; - int32_t count = -3; +void ProxySQL_Admin::delete_credentials(char *credentials) { +#endif /* DEBUG */ + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Removing old %s credentials: %s\n", type, credentials); + tokenizer_t tok; + tokenizer( &tok, credentials, ";", TOKENIZER_NO_EMPTIES ); + const char* token; + for (token = tokenize( &tok ); token; token = tokenize( &tok )) { + char *user=NULL; + char *pass=NULL; + c_split_2(token, ":", &user, &pass); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Removing %s credential: \"%s\", user:%s, pass:%s\n", type, token, user, pass); - while ((dp = readdir(dir)) != NULL) { - count++; + if constexpr (pt == SERVER_TYPE_MYSQL) { + if (GloMyAuth) { // this check if required if GloMyAuth doesn't exist yet + GloMyAuth->del(user, USERNAME_FRONTEND); + } + } + else if constexpr (pt == SERVER_TYPE_PGSQL) { + if (GloPgAuth) { // this check if required if GloPgAuth doesn't exist yet + GloPgAuth->del(user, USERNAME_FRONTEND); + } + } + free(user); + free(pass); } - - closedir(dir); - - return count; + free_tokenizer( &tok ); } -void ProxySQL_Admin::p_stats___memory_metrics() { - if (!GloMTH) return; +bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this is the public function, accessible from admin + size_t vallen=strlen(value); - // Check that last execution is older than the specified interval - unsigned long long new_ts = monotonic_time() / 1000 / 1000; - if (new_ts < last_p_memory_metrics_ts + variables.p_memory_metrics_interval) { - return; + if (!strcasecmp(name,"admin_credentials")) { + if (vallen) { + bool update_creds=false; + if ((variables.admin_credentials==NULL) || strcasecmp(variables.admin_credentials,value) ) update_creds=true; + if (update_creds && variables.admin_credentials) { +#ifdef DEBUG + delete_credentials((char*)"admin", variables.admin_credentials); + delete_credentials((char *)"admin",variables.admin_credentials); +#else + delete_credentials(variables.admin_credentials); + delete_credentials(variables.admin_credentials); +#endif /* DEBUG */ + } + free(variables.admin_credentials); + variables.admin_credentials=strdup(value); + if (update_creds && variables.admin_credentials) { +#ifdef DEBUG + add_credentials((char *)"admin",variables.admin_credentials, ADMIN_HOSTGROUP); + add_credentials((char*)"admin", variables.admin_credentials, ADMIN_HOSTGROUP); +#else + add_credentials(variables.admin_credentials, ADMIN_HOSTGROUP); + add_credentials(variables.admin_credentials, ADMIN_HOSTGROUP); +#endif /* DEBUG */ + } + return true; + } else { + return false; + } } - // Update the 'memory_metrics' last exec timestamp - last_p_memory_metrics_ts = new_ts; - - // proxysql_connpool_memory_bytes metric - const auto connpool_mem = MyHGM->Get_Memory_Stats(); - this->metrics.p_gauge_array[p_admin_gauge::connpool_memory_bytes]->Set(connpool_mem); - - // proxysql_sqlite3_memory_bytes metric - int highwater = 0; - int current = 0; - (*proxy_sqlite3_status)(SQLITE_STATUS_MEMORY_USED, ¤t, &highwater, 0); - this->metrics.p_gauge_array[p_admin_gauge::sqlite3_memory_bytes]->Set(current); - - // proxysql_jemalloc_* memory metrics - // =============================================================== - size_t - allocated = 0, - resident = 0, - active = 0, - mapped = 0, - metadata = 0, - retained = 0, - sz = sizeof(size_t); - -#ifndef NOJEM - mallctl("stats.resident", &resident, &sz, NULL, 0); - mallctl("stats.active", &active, &sz, NULL, 0); - mallctl("stats.allocated", &allocated, &sz, NULL, 0); - mallctl("stats.mapped", &mapped, &sz, NULL, 0); - mallctl("stats.metadata", &metadata, &sz, NULL, 0); - mallctl("stats.retained", &retained, &sz, NULL, 0); -#endif // NOJEM - - this->metrics.p_gauge_array[p_admin_gauge::jemalloc_resident]->Set(resident); - this->metrics.p_gauge_array[p_admin_gauge::jemalloc_active]->Set(active); - const auto cur_allocated = this->metrics.p_counter_array[p_admin_counter::jemalloc_allocated]->Value(); - this->metrics.p_counter_array[p_admin_counter::jemalloc_allocated]->Increment(allocated - cur_allocated); - this->metrics.p_gauge_array[p_admin_gauge::jemalloc_mapped]->Set(mapped); - this->metrics.p_gauge_array[p_admin_gauge::jemalloc_metadata]->Set(metadata); - this->metrics.p_gauge_array[p_admin_gauge::jemalloc_retained]->Set(retained); - - // =============================================================== - - // proxysql_auth_memory metric - unsigned long mu = GloMyAuth->memory_usage(); - this->metrics.p_gauge_array[p_admin_gauge::auth_memory_bytes]->Set(mu); - - // proxysql_query_digest_memory metric - const auto& query_digest_t_size = GloQPro->get_query_digests_total_size(); - this->metrics.p_gauge_array[p_admin_gauge::query_digest_memory_bytes]->Set(query_digest_t_size); - - // mysql_query_rules_memory metric - const auto& rules_mem_used = GloQPro->get_rules_mem_used(); - this->metrics.p_gauge_array[p_admin_gauge::mysql_query_rules_memory_bytes]->Set(rules_mem_used); - - // mysql_firewall_users_table metric - const auto& firewall_users_table = GloQPro->get_mysql_firewall_memory_users_table(); - this->metrics.p_gauge_array[p_admin_gauge::mysql_firewall_users_table]->Set(firewall_users_table); - - // mysql_firewall_users_config metric - const auto& firewall_users_config = GloQPro->get_mysql_firewall_memory_users_config(); - this->metrics.p_gauge_array[p_admin_gauge::mysql_firewall_users_config]->Set(firewall_users_config); - - // mysql_firewall_rules_table metric - const auto& firewall_rules_table = GloQPro->get_mysql_firewall_memory_rules_table(); - this->metrics.p_gauge_array[p_admin_gauge::mysql_firewall_rules_table]->Set(firewall_rules_table); - - // mysql_firewall_rules_table metric - const auto& firewall_rules_config = GloQPro->get_mysql_firewall_memory_rules_config(); - this->metrics.p_gauge_array[p_admin_gauge::mysql_firewall_rules_config]->Set(firewall_rules_config); - - // proxysql_stack_memory_mysql_threads - const auto& stack_memory_mysql_threads = - __sync_fetch_and_add(&GloVars.statuses.stack_memory_mysql_threads, 0); - this->metrics.p_gauge_array[p_admin_gauge::stack_memory_mysql_threads]->Set(stack_memory_mysql_threads); - - // proxysql_stack_memory_admin_threads - const auto& stack_memory_admin_threads = - __sync_fetch_and_add(&GloVars.statuses.stack_memory_admin_threads, 0); - this->metrics.p_gauge_array[p_admin_gauge::stack_memory_admin_threads]->Set(stack_memory_admin_threads); - - // proxysql_stack_memory_cluster_threads - const auto& stack_memory_cluster_threads = - __sync_fetch_and_add(&GloVars.statuses.stack_memory_cluster_threads, 0); - this->metrics.p_gauge_array[p_admin_gauge::stack_memory_cluster_threads]->Set(stack_memory_cluster_threads); - - // proxysql_prepare_statement_memory metric - uint64_t prepare_stmt_metadata_mem_used; - uint64_t prepare_stmt_backend_mem_used; - GloMyStmt->get_memory_usage(prepare_stmt_metadata_mem_used, prepare_stmt_backend_mem_used); - this->metrics.p_gauge_array[p_admin_gauge::prepare_stmt_metadata_memory_bytes]->Set(prepare_stmt_metadata_mem_used); - this->metrics.p_gauge_array[p_admin_gauge::prepare_stmt_backend_memory_bytes]->Set(prepare_stmt_backend_mem_used); - - // Update opened file descriptors - int32_t cur_fds = get_open_fds(); - if (cur_fds != -1) { - this->metrics.p_gauge_array[p_admin_gauge::fds_in_use]->Set(cur_fds); + if (!strcasecmp(name,"stats_credentials")) { + if (vallen) { + bool update_creds=false; + if ((variables.stats_credentials==NULL) || strcasecmp(variables.stats_credentials,value) ) update_creds=true; + if (update_creds && variables.stats_credentials) { +#ifdef DEBUG + delete_credentials((char *)"stats",variables.stats_credentials); + delete_credentials((char*)"stats", variables.stats_credentials); +#else + delete_credentials(variables.stats_credentials); + delete_credentials(variables.stats_credentials); +#endif /* DEBUG */ + } + free(variables.stats_credentials); + variables.stats_credentials=strdup(value); + if (update_creds && variables.stats_credentials) { +#ifdef DEBUG + add_credentials((char *)"admin",variables.stats_credentials, STATS_HOSTGROUP); + add_credentials((char*)"admin", variables.stats_credentials, STATS_HOSTGROUP); +#else + add_credentials(variables.stats_credentials, STATS_HOSTGROUP); + add_credentials(variables.stats_credentials, STATS_HOSTGROUP); +#endif /* DEBUG */ + } + return true; + } else { + return false; + } } -} - -void ProxySQL_Admin::stats___memory_metrics() { - if (!GloMTH) return; - SQLite3_result * resultset = NULL; - - int highwater; - int current; - char bu[32]; - char *vn=NULL; - char *query=NULL; - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_memory_metrics"); - char *a=(char *)"INSERT INTO stats_memory_metrics VALUES (\"%s\",\"%s\")"; - if (resultset) { - delete resultset; - resultset=NULL; - } - (*proxy_sqlite3_status)(SQLITE_STATUS_MEMORY_USED, ¤t, &highwater, 0); - vn=(char *)"SQLite3_memory_bytes"; - sprintf(bu,"%d",current); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); + if (!strncasecmp(name,"stats_",strlen("stats_"))) { + if (!strcasecmp(name,"stats_mysql_connection_pool")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 300) { + intv = round_intv_to_time_interval(name, intv); + variables.stats_mysql_connection_pool=intv; + GloProxyStats->variables.stats_mysql_connection_pool=intv; + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"stats_mysql_connections")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 300) { + intv = round_intv_to_time_interval(name, intv); + variables.stats_mysql_connections=intv; + GloProxyStats->variables.stats_mysql_connections=intv; + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"stats_mysql_query_cache")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 300) { + intv = round_intv_to_time_interval(name, intv); + variables.stats_mysql_query_cache=intv; + GloProxyStats->variables.stats_mysql_query_cache=intv; + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"stats_mysql_query_digest_to_disk")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 24*3600) { + variables.stats_mysql_query_digest_to_disk=intv; + GloProxyStats->variables.stats_mysql_query_digest_to_disk=intv; + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"stats_system_cpu")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 600) { + intv = round_intv_to_time_interval(name, intv); + variables.stats_system_cpu=intv; + GloProxyStats->variables.stats_system_cpu=intv; + return true; + } else { + return false; + } + } #ifndef NOJEM - { - uint64_t epoch = 1; - size_t allocated = 0, resident = 0, active = 0, mapped = 0 , metadata = 0, retained = 0 , sz = sizeof(size_t); - mallctl("epoch", &epoch, &sz, &epoch, sz); - mallctl("stats.resident", &resident, &sz, NULL, 0); - mallctl("stats.active", &active, &sz, NULL, 0); - mallctl("stats.allocated", &allocated, &sz, NULL, 0); - mallctl("stats.mapped", &mapped, &sz, NULL, 0); - mallctl("stats.metadata", &metadata, &sz, NULL, 0); - mallctl("stats.retained", &retained, &sz, NULL, 0); -// float frag_pct = ((float)active / allocated)*100 - 100; -// size_t frag_bytes = active - allocated; -// float rss_pct = ((float)resident / allocated)*100 - 100; -// size_t rss_bytes = resident - allocated; -// float metadata_pct = ((float)metadata / resident)*100; - vn=(char *)"jemalloc_resident"; - sprintf(bu,"%lu",resident); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"jemalloc_active"; - sprintf(bu,"%lu",active); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"jemalloc_allocated"; - sprintf(bu,"%lu",allocated); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"jemalloc_mapped"; - sprintf(bu,"%lu",mapped); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"jemalloc_metadata"; - sprintf(bu,"%lu",metadata); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"jemalloc_retained"; - sprintf(bu,"%lu",retained); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - } + if (!strcasecmp(name,"stats_system_memory")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 600) { + intv = round_intv_to_time_interval(name, intv); + variables.stats_system_memory=intv; + GloProxyStats->variables.stats_system_memory=intv; + return true; + } else { + return false; + } + } #endif - { - if (GloMyAuth) { - unsigned long mu = GloMyAuth->memory_usage(); - vn=(char *)"Auth_memory"; - sprintf(bu,"%lu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); + } + if (!strcasecmp(name,"mysql_ifaces")) { + if (vallen) { + bool update_creds=false; + if ((variables.mysql_ifaces==NULL) || strcasecmp(variables.mysql_ifaces,value) ) update_creds=true; + if (variables.mysql_ifaces) + free(variables.mysql_ifaces); + variables.mysql_ifaces=strdup(value); + if (update_creds && variables.mysql_ifaces) { + S_amll.update_ifaces(variables.mysql_ifaces, &S_amll.ifaces_mysql); + } + GloProxyCluster->set_admin_mysql_ifaces(value); + return true; + } else { + return false; } } - { - if (GloQPro) { - unsigned long long mu = GloQPro->get_query_digests_total_size(); - vn=(char *)"query_digest_memory"; - sprintf(bu,"%llu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - } - if (GloQPro) { - unsigned long long mu = GloQPro->get_rules_mem_used(); - vn=(char *)"mysql_query_rules_memory"; - sprintf(bu,"%llu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - } - if (GloMyStmt) { - uint64_t prep_stmt_metadata_mem_usage; - uint64_t prep_stmt_backend_mem_usage; - GloMyStmt->get_memory_usage(prep_stmt_metadata_mem_usage, prep_stmt_backend_mem_usage); - vn = (char*)"prepare_statement_metadata_memory"; - sprintf(bu, "%lu", prep_stmt_metadata_mem_usage); - query=(char*)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query, a, vn, bu); - statsdb->execute(query); - free(query); - vn = (char*)"prepare_statement_backend_memory"; - sprintf(bu, "%lu", prep_stmt_backend_mem_usage); - query=(char*)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query, a, vn, bu); - statsdb->execute(query); - free(query); + if (!strcasecmp(name, "pgsql_ifaces")) { + if (vallen) { + bool update_creds = false; + if ((variables.pgsql_ifaces == NULL) || strcasecmp(variables.pgsql_ifaces, value)) update_creds = true; + if (variables.pgsql_ifaces) + free(variables.pgsql_ifaces); + variables.pgsql_ifaces = strdup(value); + if (update_creds && variables.pgsql_ifaces) { + S_amll.update_ifaces(variables.pgsql_ifaces, &S_amll.ifaces_pgsql); + } + //GloProxyCluster->set_admin_pgsql_ifaces(value); + return true; } - if (GloQPro) { - unsigned long long mu = 0; - mu = GloQPro->get_mysql_firewall_memory_users_table(); - vn=(char *)"mysql_firewall_users_table"; - sprintf(bu,"%llu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - mu = GloQPro->get_mysql_firewall_memory_users_config(); - vn=(char *)"mysql_firewall_users_config"; - sprintf(bu,"%llu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - mu = GloQPro->get_mysql_firewall_memory_rules_table(); - vn=(char *)"mysql_firewall_rules_table"; - sprintf(bu,"%llu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - mu = GloQPro->get_mysql_firewall_memory_rules_config(); - vn=(char *)"mysql_firewall_rules_config"; - sprintf(bu,"%llu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); + else { + return false; } } - { - unsigned long mu; - mu = __sync_fetch_and_add(&GloVars.statuses.stack_memory_mysql_threads,0); - vn=(char *)"stack_memory_mysql_threads"; - sprintf(bu,"%lu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - mu = __sync_fetch_and_add(&GloVars.statuses.stack_memory_admin_threads,0); - vn=(char *)"stack_memory_admin_threads"; - sprintf(bu,"%lu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - mu = __sync_fetch_and_add(&GloVars.statuses.stack_memory_cluster_threads,0); - vn=(char *)"stack_memory_cluster_threads"; - sprintf(bu,"%lu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - } - statsdb->execute("COMMIT"); -} - -void ProxySQL_Admin::p_update_stmt_metrics() { - if (GloMyStmt) { - uint64_t stmt_client_active_unique { 0 }; - uint64_t stmt_client_active_total { 0 }; - uint64_t stmt_max_stmt_id { 0 }; - uint64_t stmt_cached { 0 }; - uint64_t stmt_server_active_unique { 0 }; - uint64_t stmt_server_active_total { 0 }; - GloMyStmt->get_metrics( - &stmt_client_active_unique, - &stmt_client_active_total, - &stmt_max_stmt_id, - &stmt_cached, - &stmt_server_active_unique, - &stmt_server_active_total - ); - - this->metrics.p_gauge_array[p_admin_gauge::stmt_client_active_total]->Set(stmt_client_active_total); - this->metrics.p_gauge_array[p_admin_gauge::stmt_client_active_unique]->Set(stmt_client_active_unique); - - this->metrics.p_gauge_array[p_admin_gauge::stmt_server_active_total]->Set(stmt_server_active_total); - this->metrics.p_gauge_array[p_admin_gauge::stmt_server_active_unique]->Set(stmt_server_active_unique); - - this->metrics.p_gauge_array[p_admin_gauge::stmt_max_stmt_id]->Set(stmt_max_stmt_id); - this->metrics.p_gauge_array[p_admin_gauge::stmt_cached]->Set(stmt_cached); + if (!strcasecmp(name,"cluster_username")) { + if (vallen) { + free(variables.cluster_username); + variables.cluster_username=strdup(value); + GloProxyCluster->set_username(variables.cluster_username); + return true; + } else { + return true; + } } -} - -void ProxySQL_Admin::stats___mysql_global() { - if (!GloMTH) return; - SQLite3_result * resultset=GloMTH->SQL3_GlobalStatus(true); - if (resultset==NULL) return; - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_mysql_global"); - char *a=(char *)"INSERT INTO stats_mysql_global VALUES (\"%s\",\"%s\")"; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int arg_len=0; - for (int i=0; i<2; i++) { - arg_len+=strlen(r->fields[i]); + if (!strcasecmp(name,"cluster_password")) { + if (vallen) { + free(variables.cluster_password); + variables.cluster_password=strdup(value); + GloProxyCluster->set_password(variables.cluster_password); + return true; + } else { + return true; } - char *query=(char *)malloc(strlen(a)+arg_len+32); - sprintf(query,a,r->fields[0],r->fields[1]); - statsdb->execute(query); - free(query); } - delete resultset; - resultset=NULL; - - resultset=MyHGM->SQL3_Get_ConnPool_Stats(); - if (resultset) { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int arg_len=0; - for (int i=0; i<2; i++) { - arg_len+=strlen(r->fields[i]); + if (!strcasecmp(name,"telnet_admin_ifaces")) { + if (vallen) { + bool update_creds=false; + if ((variables.telnet_admin_ifaces==NULL) || strcasecmp(variables.telnet_admin_ifaces,value) ) update_creds=true; + if (variables.telnet_admin_ifaces) + free(variables.telnet_admin_ifaces); + variables.telnet_admin_ifaces=strdup(value); + if (update_creds && variables.telnet_admin_ifaces) { + S_amll.update_ifaces(variables.telnet_admin_ifaces, &S_amll.ifaces_telnet_admin); } - char *query=(char *)malloc(strlen(a)+arg_len+32); - sprintf(query,a,r->fields[0],r->fields[1]); - statsdb->execute(query); - free(query); + return true; + } else { + return false; } - delete resultset; - resultset=NULL; } - - int highwater; - int current; - (*proxy_sqlite3_status)(SQLITE_STATUS_MEMORY_USED, ¤t, &highwater, 0); - char bu[32]; - char *vn=NULL; - char *query=NULL; - vn=(char *)"SQLite3_memory_bytes"; - sprintf(bu,"%d",current); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - - unsigned long long connpool_mem=MyHGM->Get_Memory_Stats(); - vn=(char *)"ConnPool_memory_bytes"; - sprintf(bu,"%llu",connpool_mem); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - - if (GloMyStmt) { - uint64_t stmt_client_active_unique = 0; - uint64_t stmt_client_active_total = 0; - uint64_t stmt_max_stmt_id = 0; - uint64_t stmt_cached = 0; - uint64_t stmt_server_active_unique = 0; - uint64_t stmt_server_active_total = 0; - GloMyStmt->get_metrics(&stmt_client_active_unique,&stmt_client_active_total,&stmt_max_stmt_id,&stmt_cached,&stmt_server_active_unique,&stmt_server_active_total); - vn=(char *)"Stmt_Client_Active_Total"; - sprintf(bu,"%lu",stmt_client_active_total); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"Stmt_Client_Active_Unique"; - sprintf(bu,"%lu",stmt_client_active_unique); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"Stmt_Server_Active_Total"; - sprintf(bu,"%lu",stmt_server_active_total); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"Stmt_Server_Active_Unique"; - sprintf(bu,"%lu",stmt_server_active_unique); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"Stmt_Max_Stmt_id"; - sprintf(bu,"%lu",stmt_max_stmt_id); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - vn=(char *)"Stmt_Cached"; - sprintf(bu,"%lu",stmt_cached); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); - } - - if (GloQC && (resultset=GloQC->SQL3_getStats())) { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int arg_len=0; - for (int i=0; i<2; i++) { - arg_len+=strlen(r->fields[i]); + if (!strcasecmp(name,"telnet_stats_ifaces")) { + if (vallen) { + bool update_creds=false; + if ((variables.telnet_stats_ifaces==NULL) || strcasecmp(variables.telnet_stats_ifaces,value) ) update_creds=true; + if (variables.telnet_stats_ifaces) + free(variables.telnet_stats_ifaces); + variables.telnet_stats_ifaces=strdup(value); + if (update_creds && variables.telnet_stats_ifaces) { + S_amll.update_ifaces(variables.telnet_stats_ifaces, &S_amll.ifaces_telnet_stats); } - char *query=(char *)malloc(strlen(a)+arg_len+32); - sprintf(query,a,r->fields[0],r->fields[1]); - statsdb->execute(query); - free(query); + return true; + } else { + return false; } - delete resultset; - resultset=NULL; } - - if (GloMyLdapAuth) { - resultset=GloMyLdapAuth->SQL3_getStats(); - if (resultset) { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int arg_len=0; - for (int i=0; i<2; i++) { - arg_len+=strlen(r->fields[i]); - } - char *query=(char *)malloc(strlen(a)+arg_len+32); - sprintf(query,a,r->fields[0],r->fields[1]); - statsdb->execute(query); - free(query); - } - delete resultset; - resultset=NULL; + if (!strcasecmp(name,"refresh_interval")) { + int intv=atoi(value); + if (intv > 100 && intv < 100000) { + variables.refresh_interval=intv; + __admin_refresh_interval=intv; + return true; + } else { + return false; } } - - if (GloQPro) { - unsigned long long mu = GloQPro->get_new_req_conns_count(); - vn=(char *)"new_req_conns_count"; - sprintf(bu,"%llu",mu); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); + if (!strcasecmp(name,"cluster_check_interval_ms")) { + int intv=atoi(value); + if (intv >= 10 && intv <= 300000) { + variables.cluster_check_interval_ms=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_check_interval_ms, intv); + return true; + } else { + return false; + } } - { - vn=(char *)"mysql_listener_paused"; - sprintf(bu, "%s", ( proxysql_mysql_paused==true ? "true" : "false") ); - query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); - sprintf(query,a,vn,bu); - statsdb->execute(query); - free(query); + if (!strcasecmp(name,"cluster_check_status_frequency")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 10000) { + variables.cluster_check_status_frequency=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_check_status_frequency, intv); + return true; + } else { + return false; + } } - statsdb->execute("COMMIT"); -} - -void ProxySQL_Admin::stats___mysql_processlist() { - int rc; - if (!GloMTH) return; - mysql_thread___show_processlist_extended = variables.mysql_show_processlist_extended; - SQLite3_result * resultset=GloMTH->SQL3_Processlist(); - if (resultset==NULL) return; - - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement32=NULL; - - char *query1=NULL; - char *query32=NULL; - std::string query32s = ""; - - query1 = (char *)"INSERT OR IGNORE INTO stats_mysql_processlist VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)"; - query32s = "INSERT OR IGNORE INTO stats_mysql_processlist VALUES " + generate_multi_rows_query(32,16); - query32 = (char *)query32s.c_str(); - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - rc = statsdb->prepare_v2(query32, &statement32); - ASSERT_SQLITE_OK(rc, statsdb); - - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_mysql_processlist"); - - int row_idx=0; - int max_bulk_row_idx=resultset->rows_count/32; - max_bulk_row_idx=max_bulk_row_idx*32; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r1=*it; - int idx=row_idx%32; - if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, statsdb); // ThreadID - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // SessionID - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // db - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // cli_host - if (r1->fields[5]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // cli_port - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+6); ASSERT_SQLITE_OK(rc, statsdb); - } - if (r1->fields[6]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+8); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_host - if (r1->fields[8]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_port - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+9); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+10, r1->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host - if (r1->fields[10]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+11); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // command - if (r1->fields[12]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // time_ms - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+13); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+14, r1->fields[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // info - if (r1->fields[14]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+15, atoll(r1->fields[14])); ASSERT_SQLITE_OK(rc, statsdb); // status_flags - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+15); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+16, r1->fields[15], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // extended_info - if (idx==31) { - SAFE_SQLITE3_STEP2(statement32); - rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + if (!strcasecmp(name,"cluster_mysql_query_rules_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_mysql_query_rules ? intv : 0; + if (variables.cluster_mysql_query_rules_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_admin_variables_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); } - } else { // single row - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // ThreadID - rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // SessionID - rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user - rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // db - rc=(*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // cli_host - if (r1->fields[5]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // cli_port - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 6); ASSERT_SQLITE_OK(rc, statsdb); + variables.cluster_mysql_query_rules_diffs_before_sync=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync, intv); + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_mysql_servers_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_mysql_servers ? intv : 0; + if (variables.cluster_mysql_servers_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_mysql_servers_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); } - if (r1->fields[6]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 8); ASSERT_SQLITE_OK(rc, statsdb); + variables.cluster_mysql_servers_diffs_before_sync=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, intv); + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_mysql_users_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_mysql_users ? intv : 0; + if (variables.cluster_mysql_users_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_mysql_users_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); } - rc=(*proxy_sqlite3_bind_text)(statement1, 8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_host - if (r1->fields[8]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_port - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 9); ASSERT_SQLITE_OK(rc, statsdb); + variables.cluster_mysql_users_diffs_before_sync=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_diffs_before_sync, intv); + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_proxysql_servers_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + if (variables.cluster_proxysql_servers_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_proxysql_servers_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); } - rc=(*proxy_sqlite3_bind_text)(statement1, 10, r1->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host - if (r1->fields[10]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 11); ASSERT_SQLITE_OK(rc, statsdb); + variables.cluster_proxysql_servers_diffs_before_sync=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_proxysql_servers_diffs_before_sync, intv); + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_mysql_variables_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_mysql_variables ? intv : 0; + if (variables.cluster_mysql_variables_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_mysql_variables_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); } - rc=(*proxy_sqlite3_bind_text)(statement1, 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // command - if (r1->fields[12]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // time_ms - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 13); ASSERT_SQLITE_OK(rc, statsdb); + variables.cluster_mysql_variables_diffs_before_sync=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync, intv); + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_admin_variables_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_admin_variables ? intv : 0; + if (variables.cluster_admin_variables_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_admin_variables_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); } - rc=(*proxy_sqlite3_bind_text)(statement1, 14, r1->fields[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // info - if (r1->fields[14]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 15, atoll(r1->fields[14])); ASSERT_SQLITE_OK(rc, statsdb); // status_flags - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 15); ASSERT_SQLITE_OK(rc, statsdb); + variables.cluster_admin_variables_diffs_before_sync=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_diffs_before_sync, intv); + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"cluster_ldap_variables_diffs_before_sync")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1000) { + intv = checksum_variables.checksum_ldap_variables ? intv : 0; + if (variables.cluster_ldap_variables_diffs_before_sync == 0 && intv != 0) { + proxy_info("Re-enabled previously disabled 'admin-cluster_ldap_variables_diffs_before_sync'. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); } - rc=(*proxy_sqlite3_bind_text)(statement1, 16, r1->fields[15], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // extended_info - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + variables.cluster_ldap_variables_diffs_before_sync=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync, intv); + return true; + } else { + return false; } - row_idx++; } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement32); - statsdb->execute("COMMIT"); - delete resultset; -} + if (!strcasecmp(name,"cluster_mysql_servers_sync_algorithm")) { + int intv=atoi(value); + if (intv >= 1 && intv <= 3) { -void ProxySQL_Admin::stats___mysql_connection_pool(bool _reset) { + if (variables.cluster_mysql_servers_sync_algorithm != intv) { + proxy_info("'cluster_mysql_servers_sync_algorithm' updated. Resetting global checksums to force Cluster re-sync.\n"); + GloProxyCluster->Reset_Global_Checksums(lock); + } - if (!MyHGM) return; - SQLite3_result * resultset=MyHGM->SQL3_Connection_Pool(_reset); - if (resultset==NULL) return; - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_mysql_connection_pool"); - char *a=(char *)"INSERT INTO stats_mysql_connection_pool VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int arg_len=0; - for (int i=0; i<14; i++) { - arg_len+=strlen(r->fields[i]); + variables.cluster_mysql_servers_sync_algorithm=intv; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_sync_algorithm, intv); + return true; + } else { + return false; } - char *query=(char *)malloc(strlen(a)+arg_len+32); - sprintf(query,a,r->fields[0],r->fields[1],r->fields[2],r->fields[3],r->fields[4],r->fields[5],r->fields[6],r->fields[7],r->fields[8],r->fields[9],r->fields[10],r->fields[11],r->fields[12],r->fields[13]); - statsdb->execute(query); - free(query); } - if (_reset) { - statsdb->execute("DELETE FROM stats_mysql_connection_pool_reset"); - statsdb->execute("INSERT INTO stats_mysql_connection_pool_reset SELECT * FROM stats_mysql_connection_pool"); + if (!strcasecmp(name,"version")) { + if (strcasecmp(value,(char *)PROXYSQL_VERSION)==0) { + return true; + } else { + return false; + } } - statsdb->execute("COMMIT"); - delete resultset; -} - -void ProxySQL_Admin::stats___mysql_free_connections() { - int rc; - if (!MyHGM) return; - SQLite3_result * resultset=MyHGM->SQL3_Free_Connections(); - if (resultset==NULL) return; - - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement32=NULL; - - char *query1=NULL; - char *query32=NULL; - std::string query32s = ""; - - query1 = (char *)"INSERT INTO stats_mysql_free_connections VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)"; - query32s = "INSERT INTO stats_mysql_free_connections VALUES " + generate_multi_rows_query(32,13); - query32 = (char *)query32s.c_str(); - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - rc = statsdb->prepare_v2(query32, &statement32); - ASSERT_SQLITE_OK(rc, statsdb); - - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_mysql_free_connections"); - - int row_idx=0; - int max_bulk_row_idx=resultset->rows_count/32; - max_bulk_row_idx=max_bulk_row_idx*32; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r1=*it; - int idx=row_idx%32; - if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, statsdb); // FD - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*13)+2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host - if (r1->fields[3]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*13)+4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*13)+4); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // db - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // init_connect - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // time_zone - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+9, r1->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // sql_mode - if (r1->fields[9]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*13)+10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); // autocommit - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*13)+10); ASSERT_SQLITE_OK(rc, statsdb); - } - if (r1->fields[10]) { - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*13)+11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // idle_ms - } else { - rc = (*proxy_sqlite3_bind_null)(statement32, (idx*13)+11); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // statistics - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+13, r1->fields[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // mysql_info - if (idx==31) { - SAFE_SQLITE3_STEP2(statement32); - rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - } - } else { // single row - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // FD - rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup - rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host - if (r1->fields[3]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 4); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user - rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // db - rc=(*proxy_sqlite3_bind_text)(statement1, 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // init_connect - rc=(*proxy_sqlite3_bind_text)(statement1, 8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // time_zone - rc=(*proxy_sqlite3_bind_text)(statement1, 9, r1->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // sql_mode - if (r1->fields[9]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); // autocommit - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 10); ASSERT_SQLITE_OK(rc, statsdb); - } - if (r1->fields[10]) { - rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // idle_ms - } else { - rc = (*proxy_sqlite3_bind_null)(statement1, 11); ASSERT_SQLITE_OK(rc, statsdb); + if (!strcasecmp(name,"hash_passwords")) { + proxy_warning("Variable admin-hash_passwords is now deprecated and removed. See github issue #4218\n"); +/* + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.hash_passwords=true; + if (GloMyLdapAuth) { + proxy_info("Impossible to set admin-hash_passwords=true when LDAP is enabled. Reverting to false\n"); + variables.hash_passwords=false; } - rc=(*proxy_sqlite3_bind_text)(statement1, 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // statistics - rc=(*proxy_sqlite3_bind_text)(statement1, 13, r1->fields[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // mysql_info - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + return true; } - row_idx++; + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.hash_passwords=false; + return true; + } + return false; +*/ } - statsdb->execute("COMMIT"); - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement32); - delete resultset; -} - -void ProxySQL_Admin::stats___mysql_commands_counters() { - if (!GloQPro) return; - SQLite3_result * resultset=GloQPro->get_stats_commands_counters(); - if (resultset==NULL) return; - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_mysql_commands_counters"); - char *a=(char *)"INSERT INTO stats_mysql_commands_counters VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int arg_len=0; - for (int i=0; i<15; i++) { - arg_len+=strlen(r->fields[i]); + if (!strcasecmp(name,"vacuum_stats")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.vacuum_stats=true; + return true; } - char *query=(char *)malloc(strlen(a)+arg_len+32); - sprintf(query,a,r->fields[0],r->fields[1],r->fields[2],r->fields[3],r->fields[4],r->fields[5],r->fields[6],r->fields[7],r->fields[8],r->fields[9],r->fields[10],r->fields[11],r->fields[12],r->fields[13],r->fields[14]); - statsdb->execute(query); - free(query); + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.vacuum_stats=false; + return true; + } + return false; } - statsdb->execute("COMMIT"); - delete resultset; -} - -void ProxySQL_Admin::stats___mysql_query_rules() { - if (!GloQPro) return; - SQLite3_result * resultset=GloQPro->get_stats_query_rules(); - if (resultset==NULL) return; - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_mysql_query_rules"); - char *a=(char *)"INSERT INTO stats_mysql_query_rules VALUES (\"%s\",\"%s\")"; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int arg_len=0; - for (int i=0; i<2; i++) { - arg_len+=strlen(r->fields[i]); + if (!strcasecmp(name,"restapi_enabled")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.restapi_enabled=true; + return true; } - char *query=(char *)malloc(strlen(a)+arg_len+32); - sprintf(query,a,r->fields[0],r->fields[1]); - statsdb->execute(query); - free(query); + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.restapi_enabled=false; + return true; + } + return false; } - statsdb->execute("COMMIT"); - delete resultset; -} - -void ProxySQL_Admin::stats___proxysql_servers_checksums() { - // NOTE: This mutex unlock is required due to a race condition created when: - // - One Admin session has the following callstack: - // + admin_session_handler -> locks on 'sql_query_global_mutex' - // | GenericRefreshStatistics - // | stats___proxysql_servers_checksums - // | get_stats_proxysql_servers_checksums - // + stats_proxysql_servers_checksums -> tries to lock on 'ProxySQL_Cluster_Nodes::mutex' - // - One ProxySQL_Cluster thread has the following callstack: - // + ProxySQL_Cluster::Update_Node_Checksums - // + ProxySQL_Cluster_Nodes::Update_Node_Checksums -> locks on 'ProxySQL_Cluster_Nodes::mutex' - // | ProxySQL_Node_Entry::set_checksums - // + ProxySQL_Cluster::pull_mysql_query_rules_from_peer -> tries to lock on 'sql_query_global_mutex' - // Producing a deadlock scenario between the two threads. - pthread_mutex_unlock(&this->sql_query_global_mutex); - SQLite3_result* resultset = GloProxyCluster->get_stats_proxysql_servers_checksums(); - pthread_mutex_lock(&this->sql_query_global_mutex); - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_proxysql_servers_checksums"); - if (resultset) { - int rc; - sqlite3_stmt *statement1=NULL; - char *query1=NULL; - query1=(char *)"INSERT INTO stats_proxysql_servers_checksums VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r1=*it; - rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + if (!strcasecmp(name,"restapi_port")) { + int intv=atoi(value); + if (intv > 0 && intv < 65535) { + variables.restapi_port=intv; + return true; + } else { + return false; } - (*proxy_sqlite3_finalize)(statement1); } - statsdb->execute("COMMIT"); - delete resultset; -} - -void ProxySQL_Admin::stats___proxysql_servers_metrics() { - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_proxysql_servers_metrics"); - SQLite3_result *resultset=NULL; - resultset=GloProxyCluster->get_stats_proxysql_servers_metrics(); - if (resultset) { - int rc; - sqlite3_stmt *statement1=NULL; - char *query1=NULL; - query1=(char *)"INSERT INTO stats_proxysql_servers_metrics VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r1=*it; - rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoi(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + if (!strcasecmp(name,"web_enabled")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.web_enabled=true; + return true; } - (*proxy_sqlite3_finalize)(statement1); + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.web_enabled=false; + return true; + } + return false; } - statsdb->execute("COMMIT"); - delete resultset; -} - -void ProxySQL_Admin::stats___proxysql_message_metrics(bool reset) { - SQLite3_result* resultset = proxysql_get_message_stats(reset); - if (resultset == NULL) return; - - statsdb->execute("BEGIN"); - if (reset) { - statsdb->execute("DELETE FROM stats_proxysql_message_metrics_reset"); - } else { - statsdb->execute("DELETE FROM stats_proxysql_message_metrics"); + if (!strcasecmp(name,"web_port")) { + int intv=atoi(value); + if (intv > 0 && intv < 65535) { + variables.web_port=intv; + return true; + } else { + return false; + } } - - char* query1 = nullptr; - char* query32 = nullptr; - std::string query32s = ""; - - if (reset) { - query1=(char*)"INSERT INTO stats_proxysql_message_metrics_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"; - query32s = "INSERT INTO stats_proxysql_message_metrics_reset VALUES " + generate_multi_rows_query(32,7); - query32 = (char *)query32s.c_str(); - } else { - query1=(char*)"INSERT INTO stats_proxysql_message_metrics VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"; - query32s = "INSERT INTO stats_proxysql_message_metrics VALUES " + generate_multi_rows_query(32,7); - query32 = (char *)query32s.c_str(); + if (!strcasecmp(name,"web_verbosity")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 10) { + variables.web_verbosity=intv; + return true; + } else { + return false; + } } - - sqlite3_stmt* statement1 = nullptr; - sqlite3_stmt* statement32 = nullptr; - int rc = 0; - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - rc = statsdb->prepare_v2(query32, &statement32); - ASSERT_SQLITE_OK(rc, statsdb); - - int row_idx = 0; - int max_bulk_row_idx = resultset->rows_count/32; - max_bulk_row_idx = max_bulk_row_idx*32; - - for (SQLite3_row* r1 : resultset->rows) { - int idx=row_idx%32; - - if (row_idxfields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // message_id - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*7)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // filename - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*7)+3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); // line - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*7)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // func - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*7)+5, atoll(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); // count_star - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*7)+6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // first_seen - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*7)+7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // last_seen - - if (idx==31) { - SAFE_SQLITE3_STEP2(statement32); - rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - } - } else { // single row - rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // message_id - rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // filename - rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); // line - rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // func - rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); // count_star - rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // first_seen - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // last_seen - - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + if (!strcasecmp(name,"cluster_mysql_query_rules_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_mysql_query_rules_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_mysql_query_rules_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_save_to_disk, false); + return true; + } + return rt; + } + if (!strcasecmp(name,"cluster_mysql_servers_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_mysql_servers_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_mysql_servers_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_save_to_disk, false); + return true; } - row_idx++; + return rt; } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement32); - - statsdb->execute("COMMIT"); - delete resultset; -} - -int ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( - const bool reset, const bool copy, const SQLite3_result *resultset, const umap_query_digest *digest_umap, - const umap_query_digest_text *digest_text_umap -) { - statsdb->execute("BEGIN"); - int rc; - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement32=NULL; - char *query1=NULL; - char *query32=NULL; - std::string query32s = ""; - statsdb->execute("DELETE FROM stats_mysql_query_digest_reset"); - statsdb->execute("DELETE FROM stats_mysql_query_digest"); - if (reset) { - query1=(char *)"INSERT INTO stats_mysql_query_digest_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; - query32s = "INSERT INTO stats_mysql_query_digest_reset VALUES " + generate_multi_rows_query(32,14); - query32 = (char *)query32s.c_str(); - } else { - query1=(char *)"INSERT INTO stats_mysql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; - query32s = "INSERT INTO stats_mysql_query_digest VALUES " + generate_multi_rows_query(32,14); - query32 = (char *)query32s.c_str(); + if (!strcasecmp(name,"cluster_mysql_users_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_mysql_users_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_mysql_users_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_save_to_disk, false); + return true; + } + return rt; } - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - rc = statsdb->prepare_v2(query32, &statement32); - ASSERT_SQLITE_OK(rc, statsdb); - int row_idx=0; - int num_rows = resultset ? resultset->rows_count : digest_umap->size(); - int max_bulk_row_idx = num_rows/32; - max_bulk_row_idx=max_bulk_row_idx*32; - auto it = resultset ? digest_umap->cend() : digest_umap->cbegin(); - int i = 0; - - time_t __now; - time(&__now); - unsigned long long curtime=monotonic_time(); - time_t seen_time; - - // If the function do not receives a resultset, it gets the values directly from the digest_umap - while (resultset ? i != resultset->rows_count : it != digest_umap->end()) { - QP_query_digest_stats *qds = (QP_query_digest_stats *)(resultset ? NULL : it->second); - SQLite3_row *row = resultset ? resultset->rows[i] : NULL; - char digest_hex_str[20]; // 2+sizeof(unsigned long long)*2+2 - if (!resultset) { - sprintf(digest_hex_str, "0x%016llX", (long long unsigned int)qds->digest); + if (!strcasecmp(name,"cluster_proxysql_servers_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_proxysql_servers_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_proxysql_servers_save_to_disk, true); + return true; } - int idx=row_idx%32; - if (row_idxfields[11]) : qds->hid); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+2, resultset ? row->fields[0] : qds->schemaname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+3, resultset ? row->fields[1] : qds->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+4, resultset ? row->fields[2] : qds->client_address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+5, resultset ? row->fields[3] : digest_hex_str, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+6, resultset ? row->fields[4] : qds->get_digest_text(digest_text_umap), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+7, resultset ? atoll(row->fields[5]) : qds->count_star); ASSERT_SQLITE_OK(rc, statsdb); - { - seen_time = qds != nullptr ? __now - curtime/1000000 + qds->first_seen/1000000 : 0; - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+8, resultset ? atoll(row->fields[6]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); - } - { - seen_time = qds != nullptr ? __now - curtime/1000000 + qds->last_seen/1000000 : 0; - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+9, resultset ? atoll(row->fields[7]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+10, resultset ? atoll(row->fields[8]) : qds->sum_time); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+11, resultset ? atoll(row->fields[9]) : qds->min_time); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+12, resultset ? atoll(row->fields[10]) : qds->max_time); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+13, resultset ? atoll(row->fields[12]) : qds->rows_affected); ASSERT_SQLITE_OK(rc, statsdb); // rows affected - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+14, resultset ? atoll(row->fields[13]) : qds->rows_sent); ASSERT_SQLITE_OK(rc, statsdb); // rows sent - if (idx==31) { - SAFE_SQLITE3_STEP2(statement32); - rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - } - } else { // single row - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, resultset ? atoll(row->fields[11]) : qds->hid); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 2, resultset ? row->fields[0] : qds->schemaname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 3, resultset ? row->fields[1] : qds->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 4, resultset ? row->fields[2] : qds->client_address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 5, resultset ? row->fields[3] : digest_hex_str, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 6, resultset ? row->fields[4] : qds->get_digest_text(digest_text_umap), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, resultset ? atoll(row->fields[5]) : qds->count_star); ASSERT_SQLITE_OK(rc, statsdb); - { - seen_time = qds != nullptr ? __now - curtime/1000000 + qds->first_seen/1000000 : 0; - rc=(*proxy_sqlite3_bind_int64)(statement1, 8, resultset ? atoll(row->fields[6]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); - } - { - seen_time = qds != nullptr ? __now - curtime/1000000 + qds->last_seen/1000000 : 0; - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, resultset ? atoll(row->fields[7]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); - } - rc=(*proxy_sqlite3_bind_int64)(statement1, 10, resultset ? atoll(row->fields[8]) : qds->sum_time); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 11, resultset ? atoll(row->fields[9]) : qds->min_time); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 12, resultset ? atoll(row->fields[10]) : qds->max_time); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 13, resultset ? atoll(row->fields[12]) : qds->rows_affected); ASSERT_SQLITE_OK(rc, statsdb); // rows affected - rc=(*proxy_sqlite3_bind_int64)(statement1, 14, resultset ? atoll(row->fields[13]) : qds->rows_sent); ASSERT_SQLITE_OK(rc, statsdb); // rows sent - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_proxysql_servers_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_proxysql_servers_save_to_disk, false); + return true; } -#ifdef DEBUG - if (resultset) - assert(row_idx == i); -#endif - row_idx++; - if (resultset) - i++; - else - it++; + return rt; } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement32); - if (reset) { - if (copy) { - statsdb->execute("INSERT INTO stats_mysql_query_digest SELECT * FROM stats_mysql_query_digest_reset"); + if (!strcasecmp(name,"cluster_mysql_variables_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_mysql_variables_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_mysql_variables_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_save_to_disk, false); + return true; } + return rt; } - statsdb->execute("COMMIT"); - - return row_idx; -} - -int ProxySQL_Admin::stats___mysql_query_digests(bool reset, bool copy) { - if (!GloQPro) return 0; - SQLite3_result * resultset=NULL; - if (reset==true) { - resultset=GloQPro->get_query_digests_reset(); - } else { - resultset=GloQPro->get_query_digests(); + if (!strcasecmp(name,"cluster_admin_variables_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_admin_variables_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_admin_variables_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_save_to_disk, false); + return true; + } + return rt; } - if (resultset==NULL) return 0; - statsdb->execute("BEGIN"); - int rc; - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement32=NULL; - - char *query1=NULL; - char *query32=NULL; - std::string query32s = ""; - // ALWAYS delete from both tables - statsdb->execute("DELETE FROM stats_mysql_query_digest_reset"); - statsdb->execute("DELETE FROM stats_mysql_query_digest"); - - if (reset) { - query1=(char *)"INSERT INTO stats_mysql_query_digest_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; - query32s = "INSERT INTO stats_mysql_query_digest_reset VALUES " + generate_multi_rows_query(32,14); - query32 = (char *)query32s.c_str(); - } else { - query1=(char *)"INSERT INTO stats_mysql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; - query32s = "INSERT INTO stats_mysql_query_digest VALUES " + generate_multi_rows_query(32,14); - query32 = (char *)query32s.c_str(); + if (!strcasecmp(name,"cluster_ldap_variables_save_to_disk")) { + bool rt = false; + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.cluster_ldap_variables_save_to_disk=true; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_save_to_disk, true); + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.cluster_ldap_variables_save_to_disk=false; + rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_save_to_disk, false); + return true; + } + return rt; } - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - rc = statsdb->prepare_v2(query32, &statement32); - ASSERT_SQLITE_OK(rc, statsdb); - int row_idx=0; - int max_bulk_row_idx=resultset->rows_count/32; - max_bulk_row_idx=max_bulk_row_idx*32; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r1=*it; - int idx=row_idx%32; - if (row_idxfields[11])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+2, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+3, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+4, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+5, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+6, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+7, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+8, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+9, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+10, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+11, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+12, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // rows affected - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+14, atoll(r1->fields[13])); ASSERT_SQLITE_OK(rc, statsdb); // rows sent - if (idx==31) { - SAFE_SQLITE3_STEP2(statement32); - rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - } - } else { // single row - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[11])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 12, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // rows affected - rc=(*proxy_sqlite3_bind_int64)(statement1, 14, atoll(r1->fields[13])); ASSERT_SQLITE_OK(rc, statsdb); // rows sent - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + if (!strcasecmp(name,"checksum_mysql_query_rules")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + checksum_variables.checksum_mysql_query_rules=true; + return true; } - row_idx++; + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + checksum_variables.checksum_mysql_query_rules=false; + variables.cluster_mysql_query_rules_diffs_before_sync = 0; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_query_rules_diffs_before_sync, 0); + proxy_warning("Disabling deprecated 'admin-checksum_mysql_query_rules', setting 'admin-cluster_mysql_query_rules_diffs_before_sync=0'\n"); + return true; + } + return false; } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement32); + if (!strcasecmp(name,"checksum_mysql_servers")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + checksum_variables.checksum_mysql_servers=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + checksum_variables.checksum_mysql_servers=false; + variables.cluster_mysql_servers_diffs_before_sync = 0; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_servers_diffs_before_sync, 0); + proxy_warning("Disabling deprecated 'admin-checksum_mysql_servers', setting 'admin-cluster_mysql_servers_diffs_before_sync=0'\n"); + return true; + } - if (reset) { - if (copy) { - statsdb->execute("INSERT INTO stats_mysql_query_digest SELECT * FROM stats_mysql_query_digest_reset"); + return false; + } + if (!strcasecmp(name,"checksum_mysql_users")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + checksum_variables.checksum_mysql_users=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + checksum_variables.checksum_mysql_users=false; + variables.cluster_mysql_users_diffs_before_sync = 0; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_users_diffs_before_sync, 0); + proxy_warning("Disabling deprecated 'admin-checksum_mysql_users', setting 'admin-cluster_mysql_users_diffs_before_sync=0'\n"); + return true; } + return false; } - statsdb->execute("COMMIT"); - delete resultset; - - return row_idx; -} - -int ProxySQL_Admin::stats___mysql_query_digests_v2(bool reset, bool copy, bool use_resultset) { - if (!GloQPro) return 0; - std::pair res; - if (reset == true) { - res = GloQPro->get_query_digests_reset_v2(copy, use_resultset); - } else { - res = GloQPro->get_query_digests_v2(use_resultset); + if (!strcasecmp(name,"checksum_mysql_variables")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + checksum_variables.checksum_mysql_variables=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + checksum_variables.checksum_mysql_variables=false; + variables.cluster_mysql_variables_diffs_before_sync = 0; + __sync_lock_test_and_set(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync, 0); + proxy_warning("Disabling deprecated 'admin-checksum_mysql_variables', setting 'admin-cluster_mysql_variables_diffs_before_sync=0'\n"); + return true; + } + return false; } - - if (res.first == NULL) - return res.second; - - int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite(reset, copy, res.first, NULL, NULL); - delete res.first; - - return num_rows; -} - -void ProxySQL_Admin::stats___mysql_client_host_cache(bool reset) { - if (!GloQPro) return; - - SQLite3_result* resultset = GloMTH->get_client_host_cache(reset); - if (resultset==NULL) return; - - statsdb->execute("BEGIN"); - - int rc = 0; - sqlite3_stmt* statement=NULL; - char* query = NULL; - - if (reset) { - query=(char*)"INSERT INTO stats_mysql_client_host_cache_reset VALUES (?1, ?2, ?3)"; - } else { - query=(char*)"INSERT INTO stats_mysql_client_host_cache VALUES (?1, ?2, ?3)"; + if (!strcasecmp(name,"checksum_admin_variables")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + checksum_variables.checksum_admin_variables=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + checksum_variables.checksum_admin_variables=false; + variables.cluster_admin_variables_diffs_before_sync = 0; + __sync_lock_test_and_set(&GloProxyCluster->cluster_admin_variables_diffs_before_sync, 0); + proxy_warning("Disabling deprecated 'admin-checksum_admin_variables', setting 'admin-cluster_admin_variables_diffs_before_sync=0'\n"); + return true; + } + return false; } - - statsdb->execute("DELETE FROM stats_mysql_client_host_cache_reset"); - statsdb->execute("DELETE FROM stats_mysql_client_host_cache"); - - rc = statsdb->prepare_v2(query, &statement); - ASSERT_SQLITE_OK(rc, statsdb); - - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *row = *it; - - rc=(*proxy_sqlite3_bind_text)(statement, 1, row->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoll(row->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoll(row->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); - - SAFE_SQLITE3_STEP2(statement); - rc=(*proxy_sqlite3_clear_bindings)(statement); - rc=(*proxy_sqlite3_reset)(statement); + if (!strcasecmp(name,"checksum_ldap_variables")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + checksum_variables.checksum_ldap_variables=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + checksum_variables.checksum_ldap_variables=false; + variables.cluster_ldap_variables_diffs_before_sync = 0; + __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync, 0); + proxy_warning("Disabling deprecated 'admin-checksum_ldap_variables', setting 'admin-cluster_ldap_variables_diffs_before_sync=0'\n"); + return true; + } + return false; } - - (*proxy_sqlite3_finalize)(statement); - - if (reset) { - statsdb->execute("INSERT INTO stats_mysql_client_host_cache SELECT * FROM stats_mysql_client_host_cache_reset"); + if (!strcasecmp(name,"read_only")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.admin_read_only=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.admin_read_only=false; + return true; + } + return false; } - - statsdb->execute("COMMIT"); - delete resultset; -} - -void ProxySQL_Admin::stats___mysql_errors(bool reset) { - if (!GloQPro) return; - SQLite3_result * resultset=NULL; - if (reset==true) { - resultset=MyHGM->get_mysql_errors(true); - } else { - resultset=MyHGM->get_mysql_errors(false); + if (!strcasecmp(name,"prometheus_memory_metrics_interval")) { + const auto fval = atoi(value); + if (fval > 0 && fval < 7*24*3600) { + variables.p_memory_metrics_interval = fval; + return true; + } else { + return false; + } } - if (resultset==NULL) return; - statsdb->execute("BEGIN"); - int rc; - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement32=NULL; - char *query1=NULL; - char *query32=NULL; - std::string query32s = ""; - if (reset) { - statsdb->execute("DELETE FROM stats_mysql_errors_reset"); - } else { - statsdb->execute("DELETE FROM stats_mysql_errors"); +#ifdef DEBUG + if (!strcasecmp(name,"debug")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.debug=true; + GloVars.global.gdbg=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.debug=false; + GloVars.global.gdbg=false; + return true; + } + return false; } - if (reset) { - query1=(char *)"INSERT INTO stats_mysql_errors_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; - query32s = "INSERT INTO stats_mysql_errors_reset VALUES " + generate_multi_rows_query(32,11); - query32 = (char *)query32s.c_str(); - } else { - query1=(char *)"INSERT INTO stats_mysql_errors VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; - query32s = "INSERT INTO stats_mysql_errors VALUES " + generate_multi_rows_query(32,11); - query32 = (char *)query32s.c_str(); + if (!strcasecmp(name,"debug_output")) { + const auto fval = atoi(value); + if (fval > 0 && fval <= 3) { + debug_output = fval; + proxysql_set_admin_debug_output(debug_output); + return true; + } else { + return false; + } + return false; } - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - rc = statsdb->prepare_v2(query32, &statement32); - ASSERT_SQLITE_OK(rc, statsdb); - int row_idx=0; - int max_bulk_row_idx=resultset->rows_count/32; - max_bulk_row_idx=max_bulk_row_idx*32; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r1=*it; - int idx=row_idx%32; - if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+8, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - if (idx==31) { - SAFE_SQLITE3_STEP2(statement32); - rc=(*proxy_sqlite3_clear_bindings)(statement32); //ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement32); //ASSERT_SQLITE_OK(rc, statsdb); - } - } else { // single row - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); //ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); //ASSERT_SQLITE_OK(rc, statsdb); +#endif /* DEBUG */ + if (!strcasecmp(name,"coredump_generation_interval_ms")) { + int intv=atoi(value); + if (intv >= 0 && intv < INT_MAX) { + variables.coredump_generation_interval_ms=intv; + coredump_generation_interval_ms=intv; + return true; + } else { + return false; } - row_idx++; } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement32); - statsdb->execute("COMMIT"); - delete resultset; -} - -/* -void ProxySQL_Admin::stats___mysql_query_digests_reset() { - if (!GloQPro) return; - SQLite3_result * resultset=GloQPro->get_query_digests_reset(); - if (resultset==NULL) return; - statsdb->execute("BEGIN"); - statsdb->execute("DELETE FROM stats_mysql_query_digest_reset"); - char *a=(char *)"INSERT INTO stats_mysql_query_digest_reset VALUES (%s,\"%s\",\"%s\",\"%s\",\"%s\",%s,%s,%s,%s,%s,%s)"; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - int arg_len=0; - for (int i=0; i<11; i++) { - arg_len+=strlen(r->fields[i]); + if (!strcasecmp(name,"coredump_generation_threshold")) { + int intv=atoi(value); + if (intv > 0 && intv <= 500) { + variables.coredump_generation_threshold=intv; + coredump_generation_threshold=intv; + return true; + } else { + return false; } - char *query=(char *)malloc(strlen(a)+arg_len+32); - sprintf(query,a,r->fields[10],r->fields[0],r->fields[1],r->fields[2],r->fields[3],r->fields[4],r->fields[5],r->fields[6],r->fields[7],r->fields[8],r->fields[9]); - statsdb->execute(query); - free(query); } - statsdb->execute("COMMIT"); - delete resultset; + if (!strcasecmp(name, "ssl_keylog_file")) { + if (strcmp(variables.ssl_keylog_file, value)) { + if (vallen == 0 || strcmp(value, "(null)") == 0) { + proxysql_keylog_close(); + free(variables.ssl_keylog_file); + variables.ssl_keylog_file = strdup(""); + GloVars.global.ssl_keylog_enabled = false; + } else { + char* sslkeylogfile = NULL; + const bool is_absolute_path = (value[0] == '/'); + if (is_absolute_path) { // absolute path + sslkeylogfile = strdup(value); + } else { // relative path + sslkeylogfile = (char*)malloc(strlen(GloVars.datadir) + strlen(value) + 2); + sprintf(sslkeylogfile, "%s/%s", GloVars.datadir, value); + } + if (proxysql_keylog_open(sslkeylogfile) == false) { + free(sslkeylogfile); + proxy_warning("Cannot open SSLKEYLOGFILE '%s' for writing.\n", value); + return false; + } + free(variables.ssl_keylog_file); + if (is_absolute_path) { + variables.ssl_keylog_file = sslkeylogfile; + sslkeylogfile = NULL; + } else { + variables.ssl_keylog_file = strdup(value); + } + if (sslkeylogfile) + free(sslkeylogfile); + GloVars.global.ssl_keylog_enabled = true; + } + } + return true; + } + return false; } -*/ void ProxySQL_Admin::save_mysql_query_rules_fast_routing_from_runtime(bool _runtime) { if (_runtime) { @@ -10724,7 +4198,7 @@ void ProxySQL_Admin::save_mysql_query_rules_fast_routing_from_runtime(bool _runt } else { admindb->execute("DELETE FROM mysql_query_rules_fast_routing"); } - SQLite3_result * resultset=GloQPro->get_current_query_rules_fast_routing(); + SQLite3_result * resultset=GloMyQPro->get_current_query_rules_fast_routing(); if (resultset) { int rc; sqlite3_stmt *statement1=NULL; @@ -10783,13 +4257,82 @@ void ProxySQL_Admin::save_mysql_query_rules_fast_routing_from_runtime(bool _runt resultset = NULL; } +void ProxySQL_Admin::save_pgsql_query_rules_fast_routing_from_runtime(bool _runtime) { + if (_runtime) { + admindb->execute("DELETE FROM runtime_pgsql_query_rules_fast_routing"); + } + else { + admindb->execute("DELETE FROM pgsql_query_rules_fast_routing"); + } + SQLite3_result* resultset = GloPgQPro->get_current_query_rules_fast_routing(); + if (resultset) { + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + //sqlite3 *mydb3=admindb->get_db(); + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + if (_runtime) { + query1 = (char*)"INSERT INTO runtime_pgsql_query_rules_fast_routing VALUES (?1, ?2, ?3, ?4, ?5)"; + query32s = "INSERT INTO runtime_pgsql_query_rules_fast_routing VALUES " + generate_multi_rows_query(32, 5); + query32 = (char*)query32s.c_str(); + } + else { + query1 = (char*)"INSERT INTO pgsql_query_rules_fast_routing VALUES (?1, ?2, ?3, ?4, ?5)"; + query32s = "INSERT INTO pgsql_query_rules_fast_routing VALUES " + generate_multi_rows_query(32, 5); + query32 = (char*)query32s.c_str(); + } + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = admindb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, admindb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = admindb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, admindb); + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 5) + 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 5) + 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 5) + 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 5) + 4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 5) + 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, admindb); + } + } + else { // single row + rc = (*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + } + if (resultset) delete resultset; + resultset = NULL; +} + void ProxySQL_Admin::save_mysql_query_rules_from_runtime(bool _runtime) { if (_runtime) { admindb->execute("DELETE FROM runtime_mysql_query_rules"); } else { admindb->execute("DELETE FROM mysql_query_rules"); } - SQLite3_result * resultset=GloQPro->get_current_query_rules(); + SQLite3_result * resultset=GloMyQPro->get_current_query_rules(); if (resultset==NULL) return; //char *a=(char *)"INSERT INTO mysql_query_rules VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; char *a=NULL; @@ -10868,6 +4411,92 @@ void ProxySQL_Admin::save_mysql_query_rules_from_runtime(bool _runtime) { delete resultset; } +void ProxySQL_Admin::save_pgsql_query_rules_from_runtime(bool _runtime) { + if (_runtime) { + admindb->execute("DELETE FROM runtime_pgsql_query_rules"); + } + else { + admindb->execute("DELETE FROM pgsql_query_rules"); + } + SQLite3_result* resultset = GloPgQPro->get_current_query_rules(); + if (resultset == NULL) return; + //char *a=(char *)"INSERT INTO pgsql_query_rules VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; + char* a = NULL; + if (_runtime) { + a = (char*)"INSERT INTO runtime_pgsql_query_rules (rule_id, active, username, database, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, log, apply, attributes, comment) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; + } else { + a = (char*)"INSERT INTO pgsql_query_rules (rule_id, active, username, database, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, log, apply, attributes, comment) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; + } + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + int arg_len = 0; + char* buffs[34]; // number of fields + for (int i = 0; i < 34; i++) { + if (r->fields[i]) { + char* o = escape_string_single_quotes(r->fields[i], false); + int l = strlen(o) + 4; + arg_len += l; + buffs[i] = (char*)malloc(l); + sprintf(buffs[i], "'%s'", o); + if (o != r->fields[i]) { // there was a copy + free(o); + } + } + else { + int l = 9; + arg_len += l; + buffs[i] = (char*)malloc(l); + sprintf(buffs[i], "NULL"); + } + } + char* query = (char*)malloc(strlen(a) + arg_len + 32); + + sprintf(query, a, + buffs[0], + buffs[1], + buffs[2], + buffs[3], + (strcmp(r->fields[4], "-1") == 0 ? "NULL" : r->fields[4]), // flagIN + buffs[5], // client_addr + buffs[6], // proxy_addr + (strcmp(r->fields[7], "-1") == 0 ? "NULL" : r->fields[7]), // proxy_port + buffs[8], // digest + buffs[9], // match_digest + buffs[10], // match_pattern + r->fields[11], // negate + buffs[12], // re_modifiers + (strcmp(r->fields[13], "-1") == 0 ? "NULL" : r->fields[13]), // flagOUT + buffs[14], // replace_pattern + (strcmp(r->fields[15], "-1") == 0 ? "NULL" : r->fields[15]), // destination_hostgroup + (strcmp(r->fields[16], "-1") == 0 ? "NULL" : r->fields[16]), // cache_ttl + (strcmp(r->fields[17], "-1") == 0 ? "NULL" : r->fields[17]), // cache_empty_result + (strcmp(r->fields[18], "-1") == 0 ? "NULL" : r->fields[18]), // cache_timeout + (strcmp(r->fields[19], "-1") == 0 ? "NULL" : r->fields[19]), // reconnect + (strcmp(r->fields[20], "-1") == 0 ? "NULL" : r->fields[20]), // timeout + (strcmp(r->fields[21], "-1") == 0 ? "NULL" : r->fields[21]), // retries + (strcmp(r->fields[22], "-1") == 0 ? "NULL" : r->fields[22]), // delay + (strcmp(r->fields[23], "-1") == 0 ? "NULL" : r->fields[23]), // next_query_flagIN + (strcmp(r->fields[24], "-1") == 0 ? "NULL" : r->fields[24]), // mirror_flagOUT + (strcmp(r->fields[25], "-1") == 0 ? "NULL" : r->fields[25]), // mirror_hostgroup + buffs[26], // error_msg + buffs[27], // OK_msg + (strcmp(r->fields[28], "-1") == 0 ? "NULL" : r->fields[28]), // sticky_conn + (strcmp(r->fields[29], "-1") == 0 ? "NULL" : r->fields[29]), // multiplex + (strcmp(r->fields[30], "-1") == 0 ? "NULL" : r->fields[30]), // log + (strcmp(r->fields[31], "-1") == 0 ? "NULL" : r->fields[31]), // apply + buffs[32], // attributes + buffs[33] // comment + ); + //fprintf(stderr,"%s\n",query); + admindb->execute(query); + for (int i = 0; i < 34; i++) { + free(buffs[i]); + } + free(query); + } + delete resultset; +} + void ProxySQL_Admin::save_mysql_firewall_whitelist_sqli_fingerprints_from_runtime(bool _runtime, SQLite3_result *resultset) { // NOTE: this function doesn't delete resultset. The caller must do it if (resultset) { @@ -10918,6 +4547,58 @@ void ProxySQL_Admin::save_mysql_firewall_whitelist_sqli_fingerprints_from_runtim } } +void ProxySQL_Admin::save_pgsql_firewall_whitelist_sqli_fingerprints_from_runtime(bool _runtime, SQLite3_result* resultset) { + // NOTE: this function doesn't delete resultset. The caller must do it + if (resultset) { + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + if (_runtime) { + query1 = (char*)"INSERT INTO runtime_pgsql_firewall_whitelist_sqli_fingerprints VALUES (?1, ?2)"; + query32s = "INSERT INTO runtime_pgsql_firewall_whitelist_sqli_fingerprints VALUES " + generate_multi_rows_query(32, 2); + query32 = (char*)query32s.c_str(); + } + else { + query1 = (char*)"INSERT INTO pgsql_firewall_whitelist_sqli_fingerprints VALUES (?1, ?2)"; + query32s = "INSERT INTO pgsql_firewall_whitelist_sqli_fingerprints VALUES " + generate_multi_rows_query(32, 2); + query32 = (char*)query32s.c_str(); + } + rc = admindb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, admindb); + rc = admindb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, admindb); + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 2) + 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 2) + 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, admindb); + } + } + else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + } +} + void ProxySQL_Admin::save_mysql_firewall_whitelist_users_from_runtime(bool _runtime, SQLite3_result *resultset) { // NOTE: this function doesn't delete resultset. The caller must do it if (resultset) { @@ -10974,6 +4655,65 @@ void ProxySQL_Admin::save_mysql_firewall_whitelist_users_from_runtime(bool _runt } } +void ProxySQL_Admin::save_pgsql_firewall_whitelist_users_from_runtime(bool _runtime, SQLite3_result* resultset) { + // NOTE: this function doesn't delete resultset. The caller must do it + if (resultset) { + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + if (_runtime) { + query1 = (char*)"INSERT INTO runtime_pgsql_firewall_whitelist_users VALUES (?1, ?2, ?3, ?4, ?5)"; + query32s = "INSERT INTO runtime_pgsql_firewall_whitelist_users VALUES " + generate_multi_rows_query(32, 5); + query32 = (char*)query32s.c_str(); + } + else { + query1 = (char*)"INSERT INTO pgsql_firewall_whitelist_users VALUES (?1, ?2, ?3, ?4, ?5)"; + query32s = "INSERT INTO pgsql_firewall_whitelist_users VALUES " + generate_multi_rows_query(32, 5); + query32 = (char*)query32s.c_str(); + } + rc = admindb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, admindb); + rc = admindb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, admindb); + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 5) + 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 5) + 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 5) + 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 5) + 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 5) + 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, admindb); + } + } + else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + } +} + + void ProxySQL_Admin::save_mysql_firewall_whitelist_rules_from_runtime(bool _runtime, SQLite3_result *resultset) { // NOTE: this function doesn't delete resultset. The caller must do it if (resultset) { @@ -11015,17 +4755,79 @@ void ProxySQL_Admin::save_mysql_firewall_whitelist_rules_from_runtime(bool _runt rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, admindb); } - } else { // single row - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + } +} + +void ProxySQL_Admin::save_pgsql_firewall_whitelist_rules_from_runtime(bool _runtime, SQLite3_result* resultset) { + // NOTE: this function doesn't delete resultset. The caller must do it + if (resultset) { + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + if (_runtime) { + query1 = (char*)"INSERT INTO runtime_pgsql_firewall_whitelist_rules VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"; + query32s = "INSERT INTO runtime_pgsql_firewall_whitelist_rules VALUES " + generate_multi_rows_query(32, 7); + query32 = (char*)query32s.c_str(); + } + else { + query1 = (char*)"INSERT INTO pgsql_firewall_whitelist_rules VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"; + query32s = "INSERT INTO pgsql_firewall_whitelist_rules VALUES " + generate_multi_rows_query(32, 7); + query32 = (char*)query32s.c_str(); + } + rc = admindb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, admindb); + rc = admindb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, admindb); + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 7) + 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 7) + 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 7) + 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 7) + 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 7) + 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 7) + 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 7) + 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, admindb); + } + } + else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); } row_idx++; } @@ -11049,7 +4851,7 @@ void ProxySQL_Admin::save_mysql_firewall_from_runtime(bool _runtime) { SQLite3_result * resultset_users = NULL; SQLite3_result * resultset_sqli_fingerprints = NULL; - GloQPro->get_current_mysql_firewall_whitelist(&resultset_users, &resultset_rules, &resultset_sqli_fingerprints); + GloMyQPro->get_current_firewall_whitelist(&resultset_users, &resultset_rules, &resultset_sqli_fingerprints); if (resultset_users) { save_mysql_firewall_whitelist_users_from_runtime(_runtime, resultset_users); @@ -11071,68 +4873,42 @@ void ProxySQL_Admin::save_mysql_firewall_from_runtime(bool _runtime) { } } -void ProxySQL_Admin::flush_admin_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Flushing ADMIN variables. Replace:%d, Delete:%d, Only_If_Empty:%d\n", replace, del, onlyifempty); - if (onlyifempty) { - char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; - char *q=(char *)"SELECT COUNT(*) FROM global_variables WHERE variable_name LIKE 'admin-%'"; - db->execute_statement(q, &error , &cols , &affected_rows , &resultset); - int matching_rows=0; - if (error) { - proxy_error("Error on %s : %s\n", q, error); - return; - } else { - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - matching_rows+=atoi(r->fields[0]); - } - } - if (resultset) delete resultset; - if (matching_rows) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Table global_variables has ADMIN variables - skipping\n"); - return; - } - } - if (del) { - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Deleting ADMIN variables from global_variables\n"); - db->execute("DELETE FROM global_variables WHERE variable_name LIKE 'admin-%'"); +void ProxySQL_Admin::save_pgsql_firewall_from_runtime(bool _runtime) { + unsigned long long curtime1 = monotonic_time(); + if (_runtime) { + admindb->execute("DELETE FROM runtime_pgsql_firewall_whitelist_rules"); + admindb->execute("DELETE FROM runtime_pgsql_firewall_whitelist_users"); + admindb->execute("DELETE FROM runtime_pgsql_firewall_whitelist_sqli_fingerprints"); } - if (runtime) { - db->execute("DELETE FROM runtime_global_variables WHERE variable_name LIKE 'admin-%'"); + else { + admindb->execute("DELETE FROM pgsql_firewall_whitelist_rules"); + admindb->execute("DELETE FROM pgsql_firewall_whitelist_users"); + admindb->execute("DELETE FROM pgsql_firewall_whitelist_sqli_fingerprints"); } - char *a; - char *b=(char *)"INSERT INTO runtime_global_variables(variable_name, variable_value) VALUES(\"admin-%s\",\"%s\")"; - if (replace) { - a=(char *)"REPLACE INTO global_variables(variable_name, variable_value) VALUES(\"admin-%s\",\"%s\")"; - } else { - a=(char *)"INSERT OR IGNORE INTO global_variables(variable_name, variable_value) VALUES(\"admin-%s\",\"%s\")"; - } - int l=strlen(a)+200; + SQLite3_result* resultset_rules = NULL; + SQLite3_result* resultset_users = NULL; + SQLite3_result* resultset_sqli_fingerprints = NULL; - char **varnames=get_variables_list(); - for (int i=0; varnames[i]; i++) { - char *val=get_variable(varnames[i]); - l+=( varnames[i] ? strlen(varnames[i]) : 6); - l+=( val ? strlen(val) : 6); - char *query=(char *)malloc(l); - sprintf(query, a, varnames[i], val); - db->execute(query); - if (runtime) { - sprintf(query, b, varnames[i], val); - db->execute(query); - } - if (val) - free(val); - free(query); + GloPgQPro->get_current_firewall_whitelist(&resultset_users, &resultset_rules, &resultset_sqli_fingerprints); + + if (resultset_users) { + save_pgsql_firewall_whitelist_users_from_runtime(_runtime, resultset_users); + delete resultset_users; } - for (int i=0; varnames[i]; i++) { - free(varnames[i]); + if (resultset_rules) { + save_pgsql_firewall_whitelist_rules_from_runtime(_runtime, resultset_rules); + delete resultset_rules; + } + if (resultset_sqli_fingerprints) { + save_pgsql_firewall_whitelist_sqli_fingerprints_from_runtime(_runtime, resultset_sqli_fingerprints); + delete resultset_sqli_fingerprints; + } + unsigned long long curtime2 = monotonic_time(); + curtime1 = curtime1 / 1000; + curtime2 = curtime2 / 1000; + if (curtime2 - curtime1 > 1000) { + proxy_info("locked for %llums\n", curtime2 - curtime1); } - free(varnames); - } #ifdef DEBUG @@ -11278,6 +5054,12 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { BQE1(admindb, mysql_query_rules_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); admindb->execute("INSERT OR REPLACE INTO main.mysql_users SELECT * FROM disk.mysql_users"); BQE1(admindb, mysql_firewall_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); + + BQE1(admindb, pgsql_servers_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); + BQE1(admindb, pgsql_query_rules_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); + admindb->execute("INSERT OR REPLACE INTO main.pgsql_users SELECT * FROM disk.pgsql_users"); + BQE1(admindb, pgsql_firewall_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); + { // online upgrade of mysql-session_idle_ms char *error=NULL; @@ -11302,6 +5084,33 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { } if (resultset) delete resultset; } + + { + // online upgrade of mysql-session_idle_ms + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + std::string q = "SELECT variable_value FROM disk.global_variables WHERE variable_name=\"pgsql-session_idle_ms\""; + admindb->execute_statement(q.c_str(), &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", q.c_str(), error); + } + else { + if (resultset->rows_count == 1) { + SQLite3_row* r = resultset->rows[0]; + if (strcmp(r->fields[0], "1000") == 0) { + proxy_warning("Detected pgsql-session_idle_ms=1000 : automatically setting it to 1 assuming this is an upgrade from an older version.\n"); + proxy_warning("Benchmarks and users show that the old default (1000) of pgsql-session_idle_ms is not optimal.\n"); + proxy_warning("This release prevents the value of pgsql-session_idle_ms to be 1000.\n"); + proxy_warning("If you really want to set pgsql-session_idle_ms close to 1000 , it is recommended to set it to a closer value like 999 or 1001\n"); + admindb->execute("UPDATE disk.global_variables SET variable_value=\"1\" WHERE variable_name=\"pgsql-session_idle_ms\""); + } + } + } + if (resultset) delete resultset; + } + admindb->execute("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables"); BQE1(admindb, scheduler_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); BQE1(admindb, restapi_tablenames, "", "INSERT OR REPLACE INTO main.", " SELECT * FROM disk."); @@ -11333,6 +5142,12 @@ void ProxySQL_Admin::__insert_or_replace_disktable_select_maintable() { BQE1(admindb, scheduler_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); BQE1(admindb, restapi_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); BQE1(admindb, proxysql_servers_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); + + BQE1(admindb, pgsql_servers_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); + BQE1(admindb, pgsql_query_rules_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); + admindb->execute("INSERT OR REPLACE INTO disk.pgsql_users SELECT * FROM main.pgsql_users"); + BQE1(admindb, pgsql_firewall_tablenames, "", "INSERT OR REPLACE INTO disk.", " SELECT * FROM main."); + #ifdef DEBUG admindb->execute("INSERT OR REPLACE INTO disk.debug_levels SELECT * FROM main.debug_levels"); admindb->execute("INSERT OR REPLACE INTO disk.debug_filters SELECT * FROM main.debug_filters"); @@ -11347,7 +5162,6 @@ void ProxySQL_Admin::__insert_or_replace_disktable_select_maintable() { } } - void ProxySQL_Admin::flush_mysql_users__from_disk_to_memory() { admindb->wrlock(); admindb->execute("PRAGMA foreign_keys = OFF"); @@ -11374,6 +5188,33 @@ void ProxySQL_Admin::flush_mysql_users__from_memory_to_disk() { admindb->wrunlock(); } +void ProxySQL_Admin::flush_pgsql_users__from_memory_to_disk() { + admindb->wrlock(); + admindb->execute("PRAGMA foreign_keys = OFF"); + admindb->execute("DELETE FROM disk.pgsql_users"); + admindb->execute("INSERT INTO disk.pgsql_users SELECT * FROM main.pgsql_users"); + if (GloMyLdapAuth) { + admindb->execute("DELETE FROM disk.pgsql_ldap_mapping"); + admindb->execute("INSERT INTO disk.pgsql_ldap_mapping SELECT * FROM main.pgsql_ldap_mapping"); + } + admindb->execute("PRAGMA foreign_keys = ON"); + admindb->wrunlock(); +} + +void ProxySQL_Admin::flush_pgsql_users__from_disk_to_memory() { + admindb->wrlock(); + admindb->execute("PRAGMA foreign_keys = OFF"); + admindb->execute("DELETE FROM main.pgsql_users"); + admindb->execute("INSERT INTO main.pgsql_users SELECT * FROM disk.pgsql_users"); + if (GloMyLdapAuth) { + admindb->execute("DELETE FROM main.pgsql_ldap_mapping"); + admindb->execute("INSERT INTO main.pgsql_ldap_mapping SELECT * FROM disk.pgsql_ldap_mapping"); + } + admindb->execute("PRAGMA foreign_keys = ON"); + admindb->wrunlock(); +} + + #ifdef PROXYSQLCLICKHOUSE void ProxySQL_Admin::flush_clickhouse_users__from_disk_to_memory() { admindb->wrlock(); @@ -11453,6 +5294,14 @@ void ProxySQL_Admin::init_users( pthread_mutex_unlock(&users_mutex); } +void ProxySQL_Admin::init_pgsql_users( + unique_ptr&& pgsql_users_resultset, const std::string& checksum, const time_t epoch +) { + pthread_mutex_lock(&users_mutex); + __refresh_pgsql_users(std::move(pgsql_users_resultset), checksum, epoch); + pthread_mutex_unlock(&users_mutex); +} + #ifdef PROXYSQLCLICKHOUSE void ProxySQL_Admin::init_clickhouse_users() { pthread_mutex_lock(&users_mutex); @@ -11467,6 +5316,12 @@ void ProxySQL_Admin::init_mysql_servers() { mysql_servers_wrunlock(); } +void ProxySQL_Admin::init_pgsql_servers() { + pgsql_servers_wrlock(); + load_pgsql_servers_to_runtime(); + pgsql_servers_wrunlock(); +} + void ProxySQL_Admin::init_proxysql_servers() { load_proxysql_servers_to_runtime(); } @@ -11479,13 +5334,22 @@ void ProxySQL_Admin::init_mysql_firewall() { load_mysql_firewall_to_runtime(); } +void ProxySQL_Admin::init_pgsql_query_rules() { + load_pgsql_query_rules_to_runtime(); +} + +void ProxySQL_Admin::init_pgsql_firewall() { + load_pgsql_firewall_to_runtime(); +} + +template void ProxySQL_Admin::add_admin_users() { #ifdef DEBUG - add_credentials((char *)"admin",variables.admin_credentials, ADMIN_HOSTGROUP); - add_credentials((char *)"stats",variables.stats_credentials, STATS_HOSTGROUP); + add_credentials((char *)"admin",variables.admin_credentials, ADMIN_HOSTGROUP); + add_credentials((char *)"stats",variables.stats_credentials, STATS_HOSTGROUP); #else - add_credentials(variables.admin_credentials, ADMIN_HOSTGROUP); - add_credentials(variables.stats_credentials, STATS_HOSTGROUP); + add_credentials(variables.admin_credentials, ADMIN_HOSTGROUP); + add_credentials(variables.stats_credentials, STATS_HOSTGROUP); #endif /* DEBUG */ } @@ -11496,13 +5360,13 @@ void ProxySQL_Admin::__refresh_users( // Checksums are always generated - 'admin-checksum_*' deprecated pthread_mutex_lock(&GloVars.checksum_mutex); - __delete_inactive_users(USERNAME_BACKEND); - __delete_inactive_users(USERNAME_FRONTEND); + __delete_inactive_users(USERNAME_BACKEND); + __delete_inactive_users(USERNAME_FRONTEND); GloMyAuth->set_all_inactive(USERNAME_BACKEND); GloMyAuth->set_all_inactive(USERNAME_FRONTEND); - add_admin_users(); + add_admin_users(); - SQLite3_result* added_users { __add_active_users(USERNAME_NONE, NULL, mysql_users_resultset.get()) }; + SQLite3_result* added_users { __add_active_users(USERNAME_NONE, NULL, mysql_users_resultset.get()) }; if (mysql_users_resultset == nullptr && added_users != nullptr) { mysql_users_resultset.reset(added_users); } @@ -11576,32 +5440,163 @@ void ProxySQL_Admin::__refresh_clickhouse_users() { } #endif /* PROXYSQLCLICKHOUSE */ -void ProxySQL_Admin::send_MySQL_OK(MySQL_Protocol *myprot, char *msg, int rows) { - assert(myprot); - MySQL_Data_Stream *myds=myprot->get_myds(); - myds->DSS=STATE_QUERY_SENT_DS; - myprot->generate_pkt_OK(true,NULL,NULL,1,rows,0,2,0,msg,false); - myds->DSS=STATE_SLEEP; +// PostgreSQL +void ProxySQL_Admin::__refresh_pgsql_users( + std::unique_ptr&& pgsql_users_resultset, const std::string& checksum, const time_t epoch +) { + bool no_resultset_supplied = pgsql_users_resultset == nullptr; + // Checksums are always generated - 'admin-checksum_*' deprecated + pthread_mutex_lock(&GloVars.checksum_mutex); + + __delete_inactive_users(USERNAME_BACKEND); + __delete_inactive_users(USERNAME_FRONTEND); + GloPgAuth->set_all_inactive(USERNAME_BACKEND); + GloPgAuth->set_all_inactive(USERNAME_FRONTEND); + add_admin_users(); + + SQLite3_result* added_users{ __add_active_users(USERNAME_NONE, NULL, pgsql_users_resultset.get()) }; + if (pgsql_users_resultset == nullptr && added_users != nullptr) { + pgsql_users_resultset.reset(added_users); + } + //if (GloMyLdapAuth) { + // __add_active_users_ldap(); + //} + GloPgAuth->remove_inactives(USERNAME_BACKEND); + GloPgAuth->remove_inactives(USERNAME_FRONTEND); + set_variable((char*)"admin_credentials", (char*)""); + + // Checksums are always generated - 'admin-checksum_*' deprecated + { + char* buff = nullptr; + char buf[20] = { 0 }; + + if (no_resultset_supplied) { + uint64_t hash1 = GloPgAuth->get_runtime_checksum(); + //if (GloMyLdapAuth) { + // hash1 += GloMyLdapAuth->get_ldap_mapping_runtime_checksum(); + //} + uint32_t d32[2]; + memcpy(&d32, &hash1, sizeof(hash1)); + sprintf(buf, "0x%0X%0X", d32[0], d32[1]); + + buff = buf; + } + else { + buff = const_cast(checksum.c_str()); + } + + GloVars.checksums_values.pgsql_users.set_checksum(buff); + GloVars.checksums_values.pgsql_users.version++; + time_t t = time(NULL); + + const bool same_checksum = no_resultset_supplied == false; + const bool matching_checksums = same_checksum || (GloVars.checksums_values.pgsql_users.checksum == checksum); + + if (epoch != 0 && checksum != "" && matching_checksums) { + GloVars.checksums_values.pgsql_users.epoch = epoch; + } + else { + GloVars.checksums_values.pgsql_users.epoch = t; + } + + GloVars.epoch_version = t; + GloVars.generate_global_checksum(); + GloVars.checksums_values.updates_cnt++; + + // store the new 'added_users' resultset after generating the new checksum + GloPgAuth->save_pgsql_users(std::move(pgsql_users_resultset)); + } + pthread_mutex_unlock(&GloVars.checksum_mutex); + + proxy_info( + "Computed checksum for 'LOAD PGSQL USERS TO RUNTIME' was '%s', with epoch '%llu'\n", + GloVars.checksums_values.pgsql_users.checksum, GloVars.checksums_values.pgsql_users.epoch + ); +} + +/* + * @brief Sends an OK message to a client based on the connection type. + * + * This function is used to send an OK message and some additional data + * (number of rows or query) to the client depending on its database + * management system (MySQL or PostgreSQL). + * + * @tparam S The type of session object passed as argument. + * @param[in, out] sess A reference to a valid session object. + * @param msg An OK message string that will be sent to the client. + * @param rows The number of rows affected by the query for MySQL clients. + * @param query The query executed for PostgreSQL clients. + */ +template +void ProxySQL_Admin::send_ok_msg_to_client(S* sess, const char* msg, int rows, const char* query) { + assert(sess->client_myds); + if constexpr (std::is_same_v) { + // Code for MySQL clients + MySQL_Data_Stream* myds = sess->client_myds; + myds->DSS = STATE_QUERY_SENT_DS; + myds->myprot.generate_pkt_OK(true, NULL, NULL, 1, rows, 0, 2, 0, (char*)msg, false); + myds->DSS = STATE_SLEEP; + } else if constexpr (std::is_same_v) { + // Code for PostgreSQL clients + PgSQL_Data_Stream* myds = sess->client_myds; + myds->DSS = STATE_QUERY_SENT_DS; + myds->myprot.generate_ok_packet(true, true, msg, rows, query); + myds->DSS = STATE_SLEEP; + } else { + assert(0); + } } -void ProxySQL_Admin::send_MySQL_ERR(MySQL_Protocol *myprot, char *msg, uint32_t code) { - assert(myprot); - MySQL_Data_Stream *myds=myprot->get_myds(); - myds->DSS=STATE_QUERY_SENT_DS; - char *a = (char *)"ProxySQL Admin Error: "; - char *new_msg = (char *)malloc(strlen(msg)+strlen(a)+1); - sprintf(new_msg, "%s%s", a, msg); - myprot->generate_pkt_ERR(true,NULL,NULL,1,code,(char *)"28000",new_msg); - free(new_msg); - myds->DSS=STATE_SLEEP; +/* + * @brief Sends an error message to a client based on its database management system. + * + * This function is used to send an error message with a given code and message + * (if applicable) to the client depending on its database management system + * (MySQL or PostgreSQL). + * + * @tparam S The type of the session object passed as argument. + * @param[in, out] sess A reference to a valid session object. + * @param msg An error message that will be sent to the client. + * @param mysqlerrcode (For MySQL clients) The error code associated with this + * error message. +*/ +template +void ProxySQL_Admin::send_error_msg_to_client(S* sess, const char *msg, uint16_t mysql_err_code /*, bool fatal*/ ) { + assert(sess->client_myds); + const char prefix_msg[] = "ProxySQL Admin Error: "; + if constexpr (std::is_same_v) { + // Code for MySQL clients + MySQL_Data_Stream* myds = sess->client_myds; + myds->DSS = STATE_QUERY_SENT_DS; + char* new_msg = (char*)malloc(strlen(msg) + sizeof(prefix_msg)); + sprintf(new_msg, "%s%s", prefix_msg, msg); + myds->myprot.generate_pkt_ERR(true, NULL, NULL, 1, mysql_err_code, (char*)"28000", new_msg); + free(new_msg); + myds->DSS = STATE_SLEEP; + } else if constexpr (std::is_same_v) { + // Code for PostgreSQL clients + PgSQL_Data_Stream* myds = sess->client_myds; + char* new_msg = (char*)malloc(strlen(msg) + sizeof(prefix_msg)); + sprintf(new_msg, "%s%s", prefix_msg, msg); + myds->myprot.generate_error_packet(true, true, new_msg, PGSQL_ERROR_CODES::ERRCODE_RAISE_EXCEPTION, false); + free(new_msg); + myds->DSS = STATE_SLEEP; + } else { + assert(0); + } } +template void ProxySQL_Admin::__delete_inactive_users(enum cred_username_type usertype) { char *error=NULL; int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *str=(char *)"SELECT username FROM main.mysql_users WHERE %s=1 AND active=0"; + char* str = nullptr; + if constexpr (pt == SERVER_TYPE_MYSQL) + str=(char *)"SELECT username FROM main.mysql_users WHERE %s=1 AND active=0"; + else if constexpr (pt == SERVER_TYPE_PGSQL) + str = (char*)"SELECT username FROM main.pgsql_users WHERE %s=1 AND active=0"; char *query=(char *)malloc(strlen(str)+15); sprintf(query,str,(usertype==USERNAME_BACKEND ? "backend" : "frontend")); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); @@ -11610,7 +5605,11 @@ void ProxySQL_Admin::__delete_inactive_users(enum cred_username_type usertype) { } else { for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; - GloMyAuth->del(r->fields[0], usertype); + + if constexpr (pt == SERVER_TYPE_MYSQL) + GloMyAuth->del(r->fields[0], usertype); + else if constexpr (pt == SERVER_TYPE_PGSQL) + GloPgAuth->del(r->fields[0], usertype); } } if (resultset) delete resultset; @@ -11660,7 +5659,7 @@ void ProxySQL_Admin::__add_active_users_ldap() { } - +template SQLite3_result* ProxySQL_Admin::__add_active_users( enum cred_username_type usertype, char *__user, SQLite3_result* mysql_users_resultset ) { @@ -11674,13 +5673,21 @@ SQLite3_result* ProxySQL_Admin::__add_active_users( if (__user==NULL) { if (mysql_users_resultset == nullptr) { - str = (char*)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment FROM main.mysql_users WHERE active=1 AND default_hostgroup>=0"; + if constexpr (pt == SERVER_TYPE_MYSQL) { + str = (char*)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment FROM main.mysql_users WHERE active=1 AND default_hostgroup>=0"; + } else if constexpr (pt == SERVER_TYPE_PGSQL) { + str = (char*)"SELECT username,password,use_ssl,default_hostgroup,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment FROM main.pgsql_users WHERE active=1 AND default_hostgroup>=0"; + } admindb->execute_statement(str, &error, &cols, &affected_rows, &resultset); } else { resultset = mysql_users_resultset; } } else { - str=(char *)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.mysql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0 AND username='%s'"; + if constexpr (pt == SERVER_TYPE_MYSQL) { + str = (char*)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.mysql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0 AND username='%s'"; + } else if constexpr (pt == SERVER_TYPE_PGSQL) { + str = (char*)"SELECT username,password,use_ssl,default_hostgroup,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.pgsql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0 AND username='%s'"; + } query=(char *)malloc(strlen(str)+strlen(__user)+15); sprintf(query,str,(usertype==USERNAME_BACKEND ? "backend" : "frontend"),__user); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); @@ -11701,7 +5708,6 @@ SQLite3_result* ProxySQL_Admin::__add_active_users( } } - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; char *password=NULL; @@ -11710,47 +5716,83 @@ SQLite3_result* ProxySQL_Admin::__add_active_users( } else { password=(char *)""; } -// } std::vector usertypes {}; char* max_connections = nullptr; char* attributes = nullptr; char* comment = nullptr; - if (__user != nullptr) { - usertypes.push_back(usertype); + if constexpr (pt == SERVER_TYPE_MYSQL) { + if (__user != nullptr) { + usertypes.push_back(usertype); - max_connections = r->fields[8]; - attributes = r->fields[9]; - comment = r->fields[10]; - } else { - if (strcasecmp(r->fields[8], "1") == 0) { - usertypes.push_back(USERNAME_BACKEND); + max_connections = r->fields[8]; + attributes = r->fields[9]; + comment = r->fields[10]; } - if (strcasecmp(r->fields[9], "1") == 0) { - usertypes.push_back(USERNAME_FRONTEND); + else { + if (strcasecmp(r->fields[8], "1") == 0) { + usertypes.push_back(USERNAME_BACKEND); + } + if (strcasecmp(r->fields[9], "1") == 0) { + usertypes.push_back(USERNAME_FRONTEND); + } + + max_connections = r->fields[10]; + attributes = r->fields[11]; + comment = r->fields[12]; } + } else if constexpr (pt == SERVER_TYPE_PGSQL) { + if (__user != nullptr) { + usertypes.push_back(usertype); - max_connections = r->fields[10]; - attributes = r->fields[11]; - comment = r->fields[12]; - } + max_connections = r->fields[6]; + attributes = r->fields[7]; + comment = r->fields[8]; + } + else { + if (strcasecmp(r->fields[6], "1") == 0) { + usertypes.push_back(USERNAME_BACKEND); + } + if (strcasecmp(r->fields[7], "1") == 0) { + usertypes.push_back(USERNAME_FRONTEND); + } + max_connections = r->fields[8]; + attributes = r->fields[9]; + comment = r->fields[10]; + } + } for (const enum cred_username_type usertype : usertypes) { - GloMyAuth->add( - r->fields[0], // username - password, // before #676, wewere always passing the password. Now it is possible that the password can be hashed - usertype, // backend/frontend - (strcmp(r->fields[2],"1")==0 ? true : false) , // use_ssl - atoi(r->fields[3]), // default_hostgroup - (r->fields[4]==NULL ? (char *)"" : r->fields[4]), //default_schema - (strcmp(r->fields[5],"1")==0 ? true : false) , // schema_locked - (strcmp(r->fields[6],"1")==0 ? true : false) , // transaction_persistent - (strcmp(r->fields[7],"1")==0 ? true : false), // fast_forward - ( atoi(max_connections)>0 ? atoi(max_connections) : 0), // max_connections - (attributes == NULL ? (char *)"" : attributes), // attributes - (comment ==NULL ? (char *)"" : comment) //comment - ); + if constexpr (pt == SERVER_TYPE_MYSQL) { + GloMyAuth->add( + r->fields[0], // username + password, // before #676, wewere always passing the password. Now it is possible that the password can be hashed + usertype, // backend/frontend + (strcmp(r->fields[2], "1") == 0 ? true : false), // use_ssl + atoi(r->fields[3]), // default_hostgroup + (r->fields[4] == NULL ? (char*)"" : r->fields[4]), //default_schema + (strcmp(r->fields[5], "1") == 0 ? true : false), // schema_locked + (strcmp(r->fields[6], "1") == 0 ? true : false), // transaction_persistent + (strcmp(r->fields[7], "1") == 0 ? true : false), // fast_forward + (atoi(max_connections) > 0 ? atoi(max_connections) : 0), // max_connections + (attributes == NULL ? (char*)"" : attributes), // attributes + (comment == NULL ? (char*)"" : comment) //comment + ); + } else if constexpr (pt == SERVER_TYPE_PGSQL) { + GloPgAuth->add( + r->fields[0], // username + password, // before #676, wewere always passing the password. Now it is possible that the password can be hashed + usertype, // backend/frontend + (strcmp(r->fields[2], "1") == 0 ? true : false), // use_ssl + atoi(r->fields[3]), // default_hostgroup + (strcmp(r->fields[4], "1") == 0 ? true : false), // transaction_persistent + (strcmp(r->fields[5], "1") == 0 ? true : false), // fast_forward + (atoi(max_connections) > 0 ? atoi(max_connections) : 0), // max_connections + (attributes == NULL ? (char*)"" : attributes), // attributes + (comment == NULL ? (char*)"" : comment) //comment + ); + } } if (sqlite_result != nullptr) { @@ -11965,6 +6007,41 @@ void ProxySQL_Admin::dump_checksums_values_table() { rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + //PostgreSQL + rc = (*proxy_sqlite3_bind_text)(statement1, 1, "pgsql_query_rules", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.pgsql_query_rules.version); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.pgsql_query_rules.epoch); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.pgsql_query_rules.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + + rc = (*proxy_sqlite3_bind_text)(statement1, 1, "pgsql_servers", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.pgsql_servers.version); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.pgsql_servers.epoch); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.pgsql_servers.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + + rc = (*proxy_sqlite3_bind_text)(statement1, 1, "pgsql_users", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.pgsql_users.version); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.pgsql_users.epoch); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.pgsql_users.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + + rc = (*proxy_sqlite3_bind_text)(statement1, 1, "pgsql_variables", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.pgsql_variables.version); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.pgsql_variables.epoch); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.pgsql_variables.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + // + + if (GloMyLdapAuth) { rc=(*proxy_sqlite3_bind_text)(statement1, 1, "ldap_variables", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); rc=(*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.ldap_variables.version); ASSERT_SQLITE_OK(rc, admindb); @@ -12061,6 +6138,116 @@ void ProxySQL_Admin::save_mysql_users_runtime_to_database(bool _runtime) { free(ads); } +void ProxySQL_Admin::save_pgsql_users_runtime_to_database(bool _runtime) { + char* query = NULL; + if (_runtime) { + query = (char*)"DELETE FROM main.runtime_pgsql_users"; + admindb->execute(query); + } + else { + char* qd = (char*)"UPDATE pgsql_users SET active=0"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", qd); + admindb->execute(qd); + } + pgsql_account_details_t** ads = NULL; + int num_users; + int i; + int rc; + // char *qf=(char *)"REPLACE INTO pgsql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) VALUES('%s','%s',1,%d,%d,'%s',%d,%d,%d,COALESCE((SELECT backend FROM mysql_users WHERE username='%s' AND frontend=1),0),1,%d)"; + // char *qb=(char *)"REPLACE INTO pgsql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) VALUES('%s','%s',1,%d,%d,'%s',%d,%d,%d,1,COALESCE((SELECT frontend FROM mysql_users WHERE username='%s' AND backend=1),0),%d)"; + // char *qfr=(char *)"REPLACE INTO runtime_pgsql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) VALUES('%s','%s',1,%d,%d,'%s',%d,%d,%d,COALESCE((SELECT backend FROM runtime_mysql_users WHERE username='%s' AND frontend=1),0),1,%d)"; + // char *qbr=(char *)"REPLACE INTO runtime_pgsql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) VALUES('%s','%s',1,%d,%d,'%s',%d,%d,%d,1,COALESCE((SELECT frontend FROM runtime_mysql_users WHERE username='%s' AND backend=1),0),%d)"; + + char* qf_stmt1 = (char*)"REPLACE INTO pgsql_users(username,password,active,use_ssl,default_hostgroup,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment) VALUES(?1,?2,1,?3,?4,?5,?6,0,1,?7,?8,?9)"; + char* qb_stmt1 = (char*)"REPLACE INTO pgsql_users(username,password,active,use_ssl,default_hostgroup,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment) VALUES(?1,?2,1,?3,?4,?5,?6,1,0,?7,?8,?9)"; + char* qfr_stmt1 = (char*)"REPLACE INTO runtime_pgsql_users(username,password,active,use_ssl,default_hostgroup,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment) VALUES(?1,?2,1,?3,?4,?5,?6,0,1,?7,?8,?9)"; + char* qbr_stmt1 = (char*)"REPLACE INTO runtime_pgsql_users(username,password,active,use_ssl,default_hostgroup,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment) VALUES(?1,?2,1,?3,?4,?5,?6,1,0,?7,?8,?9)"; + num_users = GloPgAuth->dump_all_users(&ads); + if (num_users == 0) return; + char* q_stmt1_f = NULL; + char* q_stmt1_b = NULL; + sqlite3_stmt* f_statement1 = NULL; + sqlite3_stmt* b_statement1 = NULL; + //sqlite3 *mydb3=admindb->get_db(); + if (_runtime) { + q_stmt1_f = qfr_stmt1; + q_stmt1_b = qbr_stmt1; + } + else { + q_stmt1_f = qf_stmt1; + q_stmt1_b = qb_stmt1; + } + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, q_stmt1_f, -1, &f_statement1, 0); + rc = admindb->prepare_v2(q_stmt1_f, &f_statement1); + ASSERT_SQLITE_OK(rc, admindb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, q_stmt1_b, -1, &b_statement1, 0); + rc = admindb->prepare_v2(q_stmt1_b, &b_statement1); + ASSERT_SQLITE_OK(rc, admindb); + for (i = 0; i < num_users; i++) { + //fprintf(stderr,"%s %d\n", ads[i]->username, ads[i]->default_hostgroup); + pgsql_account_details_t* ad = ads[i]; + sqlite3_stmt* statement1 = NULL; + if (ads[i]->default_hostgroup >= 0) { + /* + char *q=NULL; + if (_runtime==false) { + if (ad->__frontend) { + q=qf; + } else { + q=qb; + } + } else { // _runtime==true + if (ad->__frontend) { + q=qfr; + statement1=f_statement1; + } else { + q=qbr; + statement1=b_statement1; + } + } + */ + if (ad->__frontend) { + statement1 = f_statement1; + } + else { + statement1 = b_statement1; + } + /* + if (_runtime==false) { + query=(char *)malloc(strlen(q)+strlen(ad->username)*2+strlen(ad->password)+strlen(ad->default_schema)+256); + sprintf(query, q, ad->username, ad->password, ad->use_ssl, ad->default_hostgroup, ad->default_schema, ad->schema_locked, ad->transaction_persistent, ad->fast_forward, ad->username, ad->max_connections); + //fprintf(stderr,"%s\n",query); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + free(query); + } else { + */ + rc = (*proxy_sqlite3_bind_text)(statement1, 1, ad->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, ad->password, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, ad->use_ssl); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 4, ad->default_hostgroup); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 5, ad->transaction_persistent); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 6, ad->fast_forward); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 7, ad->max_connections); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 8, ad->attributes, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 9, ad->comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + } + free(ad->username); + free(ad->password); // this is not initialized with dump_all_users( , false) + free(ad->comment); + free(ad->attributes); + free(ad); + } + if (_runtime) { + (*proxy_sqlite3_finalize)(f_statement1); + (*proxy_sqlite3_finalize)(b_statement1); + } + free(ads); +} + void ProxySQL_Admin::save_mysql_ldap_mapping_runtime_to_database(bool _runtime) { if (GloMyLdapAuth==NULL) { return; @@ -12109,27 +6296,102 @@ void ProxySQL_Admin::save_mysql_ldap_mapping_runtime_to_database(bool _runtime) rc=(*proxy_sqlite3_bind_text)(statement8, (idx*7)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); if (idx==7) { SAFE_SQLITE3_STEP2(statement8); - rc=(*proxy_sqlite3_clear_bindings)(statement8); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement8); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_clear_bindings)(statement8); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement8); ASSERT_SQLITE_OK(rc, admindb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement8); + } + if(resultset) delete resultset; + resultset=NULL; +} + +void ProxySQL_Admin::save_pgsql_ldap_mapping_runtime_to_database(bool _runtime) { + if (GloMyLdapAuth == NULL) { + return; + } + char* query = NULL; + SQLite3_result* resultset = NULL; + if (_runtime) { + query = (char*)"DELETE FROM main.runtime_pgsql_ldap_mapping"; + } + else { + query = (char*)"DELETE FROM main.pgsql_ldap_mapping"; + } + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + resultset = GloMyLdapAuth->dump_table_pgsql_ldap_mapping(); + if (resultset) { + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement8 = NULL; + //sqlite3 *mydb3=admindb->get_db(); + char* query1 = NULL; + char* query8 = NULL; + std::string query8s = ""; + if (_runtime) { + query1 = (char*)"INSERT INTO runtime_pgsql_ldap_mapping VALUES (?1, ?2, ?3, ?4)"; + query8s = "INSERT INTO runtime_pgsql_ldap_mapping VALUES " + generate_multi_rows_query(8, 4); + query8 = (char*)query8s.c_str(); + } + else { + query1 = (char*)"INSERT INTO pgsql_ldap_mapping VALUES (?1, ?2, ?3, ?4)"; + query8s = "INSERT INTO pgsql_ldap_mapping VALUES " + generate_multi_rows_query(8, 4); + query8 = (char*)query8s.c_str(); + } + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = admindb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, admindb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query8, -1, &statement8, 0); + rc = admindb->prepare_v2(query8, &statement8); + ASSERT_SQLITE_OK(rc, admindb); + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 8; + max_bulk_row_idx = max_bulk_row_idx * 8; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 8; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement8, (idx * 7) + 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement8, (idx * 7) + 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement8, (idx * 7) + 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement8, (idx * 7) + 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (idx == 7) { + SAFE_SQLITE3_STEP2(statement8); + rc = (*proxy_sqlite3_clear_bindings)(statement8); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement8); ASSERT_SQLITE_OK(rc, admindb); } - } else { // single row - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + } + else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); } row_idx++; } (*proxy_sqlite3_finalize)(statement1); (*proxy_sqlite3_finalize)(statement8); } - if(resultset) delete resultset; - resultset=NULL; + if (resultset) delete resultset; + resultset = NULL; } + #ifdef PROXYSQLCLICKHOUSE void ProxySQL_Admin::save_clickhouse_users_runtime_to_database(bool _runtime) { int rc; @@ -12215,109 +6477,6 @@ void ProxySQL_Admin::save_clickhouse_users_runtime_to_database(bool _runtime) { } #endif /* PROXYSQLCLICKHOUSE */ -void ProxySQL_Admin::stats___mysql_users() { - account_details_t **ads=NULL; - statsdb->execute("DELETE FROM stats_mysql_users"); - - int num_users=GloMyAuth->dump_all_users(&ads, false); - if (num_users==0) return; - - const char q[] { - "INSERT INTO stats_mysql_users(username,frontend_connections,frontend_max_connections) VALUES ('%s',%d,%d)" - }; - char buf[256] = { 0 }; - - for (int i=0; idefault_hostgroup>= 0) { // only not admin/stats - cfmt_t q_fmt = cstr_format(buf, q, ad->username, ad->num_connections_used, ad->max_connections); - - if (q_fmt.str.size()) { - statsdb->execute(q_fmt.str.c_str()); - } else { - statsdb->execute(buf); - } - } - free(ad->username); - free(ad); - } - - if (GloMyLdapAuth) { - std::unique_ptr ldap_users { GloMyLdapAuth->dump_all_users() }; - - for (const SQLite3_row* row : ldap_users->rows) { - const char* username = row->fields[LDAP_USER_FIELD_IDX::USERNAME]; - int f_conns = atoi(row->fields[LDAP_USER_FIELD_IDX::FRONTEND_CONNECTIONS]); - int f_max_conns = atoi(row->fields[LDAP_USER_FIELD_IDX::FRONTED_MAX_CONNECTIONS]); - - cfmt_t q_fmt = cstr_format(buf, q, username, f_conns, f_max_conns); - - if (q_fmt.str.size()) { - statsdb->execute(q_fmt.str.c_str()); - } else { - statsdb->execute(buf); - } - } - } - - free(ads); -} - -void ProxySQL_Admin::stats___mysql_gtid_executed() { - statsdb->execute("DELETE FROM stats_mysql_gtid_executed"); - SQLite3_result *resultset=NULL; - resultset = MyHGM->get_stats_mysql_gtid_executed(); - if (resultset) { - int rc; - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement32=NULL; - - char *query1=NULL; - char *query32=NULL; - std::string query32s = ""; - query1=(char *)"INSERT INTO stats_mysql_gtid_executed VALUES (?1, ?2, ?3, ?4)"; - query32s = "INSERT INTO stats_mysql_gtid_executed VALUES " + generate_multi_rows_query(32,4); - query32 = (char *)query32s.c_str(); - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - rc = statsdb->prepare_v2(query32, &statement32); - ASSERT_SQLITE_OK(rc, statsdb); - - int row_idx=0; - int max_bulk_row_idx=resultset->rows_count/32; - max_bulk_row_idx=max_bulk_row_idx*32; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r1=*it; - int idx=row_idx%32; - if (row_idxfields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*4)+2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement32, (idx*4)+3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*4)+4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); - if (idx==31) { - SAFE_SQLITE3_STEP(statement32); - rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - } - } else { // single row - rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); - SAFE_SQLITE3_STEP(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - } - row_idx++; - } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement32); - delete resultset; - resultset = NULL; - } -} - void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { char *query=NULL; if (_runtime) { @@ -12747,6 +6906,177 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { resultset=NULL; } +void ProxySQL_Admin::save_pgsql_servers_runtime_to_database(bool _runtime) { + // make sure that the caller has called pgsql_servers_wrlock() + char* query = NULL; + string StrQuery; + SQLite3_result* resultset = NULL; + // dump pgsql_servers + if (_runtime) { + query = (char*)"DELETE FROM main.runtime_pgsql_servers"; + } + else { + query = (char*)"DELETE FROM main.pgsql_servers"; + } + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + resultset = PgHGM->dump_table_pgsql("pgsql_servers"); + if (resultset) { + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + //sqlite3 *mydb3=admindb->get_db(); + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + if (_runtime) { + query1 = (char*)"INSERT INTO runtime_pgsql_servers VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + query32s = "INSERT INTO runtime_pgsql_servers VALUES " + generate_multi_rows_query(32, 11); + query32 = (char*)query32s.c_str(); + } + else { + query1 = (char*)"INSERT INTO pgsql_servers VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + query32s = "INSERT INTO pgsql_servers VALUES " + generate_multi_rows_query(32, 11); + query32 = (char*)query32s.c_str(); + } + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = admindb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, admindb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = admindb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, admindb); + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 4, (_runtime ? r1->fields[3] : (strcmp(r1->fields[4], "SHUNNED") == 0 ? "ONLINE" : r1->fields[3])), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 10, atoi(r1->fields[9])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, admindb); + } + } + else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atoi(r1->fields[0])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, (_runtime ? r1->fields[3] : (strcmp(r1->fields[3], "SHUNNED") == 0 ? "ONLINE" : r1->fields[3])), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 10, atoi(r1->fields[9])); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_bind_text)(statement1, 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + } + if (resultset) delete resultset; + resultset = NULL; + + // dump pgsql_replication_hostgroups + if (_runtime) { + query = (char*)"DELETE FROM main.runtime_pgsql_replication_hostgroups"; + } + else { + query = (char*)"DELETE FROM main.pgsql_replication_hostgroups"; + } + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + resultset = PgHGM->dump_table_pgsql("pgsql_replication_hostgroups"); + if (resultset) { + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + int l = 0; + if (r->fields[3]) l = strlen(r->fields[3]); + char* q = NULL; + if (_runtime) { + q = (char*)"INSERT INTO runtime_pgsql_replication_hostgroups VALUES(%s,%s,'%s','%s')"; + } + else { + q = (char*)"INSERT INTO pgsql_replication_hostgroups VALUES(%s,%s,'%s','%s')"; + } + char* query = (char*)malloc(strlen(q) + strlen(r->fields[0]) + strlen(r->fields[1]) + strlen(r->fields[2]) + 16 + l); + if (r->fields[3]) { + char* o = escape_string_single_quotes(r->fields[3], false); + sprintf(query, q, r->fields[0], r->fields[1], r->fields[2], o); + if (o != r->fields[3]) { // there was a copy + free(o); + } + //} else { + //sprintf(query, q, r->fields[0], r->fields[1], r->fields[2], r->fields[3]); + } + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + admindb->execute(query); + free(query); + } + } + if (resultset) delete resultset; + resultset = NULL; + + // dump pgsql_hostgroup_attributes + + StrQuery = "DELETE FROM main."; + if (_runtime) + StrQuery += "runtime_"; + StrQuery += "pgsql_hostgroup_attributes"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", StrQuery.c_str()); + admindb->execute(StrQuery.c_str()); + resultset = PgHGM->dump_table_pgsql("pgsql_hostgroup_attributes"); + if (resultset) { + int rc; + sqlite3_stmt* statement = NULL; + StrQuery = "INSERT INTO "; + if (_runtime) + StrQuery += "runtime_"; + StrQuery += "pgsql_hostgroup_attributes (hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, hostgroup_settings, servers_defaults, comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + rc = admindb->prepare_v2(StrQuery.c_str(), &statement); + ASSERT_SQLITE_OK(rc, admindb); + //proxy_info("New pgsql_aws_aurora_hostgroups table\n"); + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + rc = (*proxy_sqlite3_bind_int64)(statement, 1, atol(r->fields[0])); ASSERT_SQLITE_OK(rc, admindb); // hostgroup_id + rc = (*proxy_sqlite3_bind_int64)(statement, 2, atol(r->fields[1])); ASSERT_SQLITE_OK(rc, admindb); // max_num_online_servers + rc = (*proxy_sqlite3_bind_int64)(statement, 3, atol(r->fields[2])); ASSERT_SQLITE_OK(rc, admindb); // autocommit + rc = (*proxy_sqlite3_bind_int64)(statement, 4, atol(r->fields[3])); ASSERT_SQLITE_OK(rc, admindb); // free_connections_pct + rc = (*proxy_sqlite3_bind_text)(statement, 5, r->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // variable_name + rc = (*proxy_sqlite3_bind_int64)(statement, 6, atol(r->fields[5])); ASSERT_SQLITE_OK(rc, admindb); // multiplex + rc = (*proxy_sqlite3_bind_int64)(statement, 7, atol(r->fields[6])); ASSERT_SQLITE_OK(rc, admindb); // connection_warming + rc = (*proxy_sqlite3_bind_int64)(statement, 8, atol(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); // throttle_connections_per_sec + rc = (*proxy_sqlite3_bind_text)(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // ignore_session_variables + rc = (*proxy_sqlite3_bind_text)(statement, 10, r->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // hostgroup_settings + rc = (*proxy_sqlite3_bind_text)(statement, 11, r->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // servers_defaults + rc = (*proxy_sqlite3_bind_text)(statement, 12, r->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // comment + + SAFE_SQLITE3_STEP2(statement); + rc = (*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb); + rc = (*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, admindb); + } + (*proxy_sqlite3_finalize)(statement); + } + if (resultset) delete resultset; + resultset = NULL; +} + void ProxySQL_Admin::load_scheduler_to_runtime() { char *error=NULL; @@ -13006,6 +7336,119 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const incoming_servers_t& inc } } +void ProxySQL_Admin::load_pgsql_servers_to_runtime(const incoming_pgsql_servers_t& incoming_pgsql_servers, + const runtime_pgsql_servers_checksum_t& peer_runtime_pgsql_server, const pgsql_servers_v2_checksum_t& peer_pgsql_server_v2) { + // make sure that the caller has called pgsql_servers_wrlock() + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* resultset = NULL; + SQLite3_result* resultset_servers = NULL; + SQLite3_result* resultset_replication = NULL; + SQLite3_result* resultset_hostgroup_attributes = NULL; + + SQLite3_result* runtime_pgsql_servers = incoming_pgsql_servers.runtime_pgsql_servers; + SQLite3_result* incoming_replication_hostgroups = incoming_pgsql_servers.incoming_replication_hostgroups; + SQLite3_result* incoming_hostgroup_attributes = incoming_pgsql_servers.incoming_hostgroup_attributes; + SQLite3_result* incoming_pgsql_servers_v2 = incoming_pgsql_servers.incoming_pgsql_servers_v2; + + const char* query = (char*)"SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.pgsql_servers ORDER BY hostgroup_id, hostname, port"; + if (runtime_pgsql_servers == nullptr) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset_servers); + } + else { + resultset_servers = runtime_pgsql_servers; + } + //MyHGH->wrlock(); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } + else { + PgHGM->servers_add(resultset_servers); + } + // memory leak was detected here. The following few lines fix that + if (runtime_pgsql_servers == nullptr) { + if (resultset_servers != nullptr) { + delete resultset_servers; + resultset_servers = nullptr; + } + } + resultset = NULL; + + query = (char*)"SELECT a.* FROM pgsql_replication_hostgroups a JOIN pgsql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } + else { + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + proxy_error("Incompatible entry in pgsql_replication_hostgroups will be ignored : ( %s , %s )\n", r->fields[0], r->fields[1]); + } + } + if (resultset) delete resultset; + resultset = NULL; + + query = (char*)"SELECT a.* FROM pgsql_replication_hostgroups a LEFT JOIN pgsql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + if (incoming_replication_hostgroups == nullptr) { + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset_replication); + } + else { + resultset_replication = incoming_replication_hostgroups; + } + + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } + else { + // Pass the resultset to PgHGM + PgHGM->save_incoming_pgsql_table(resultset_replication, "pgsql_replication_hostgroups"); + } + //if (resultset) delete resultset; + //resultset=NULL; + + + // support for hostgroup attributes, table pgsql_hostgroup_attributes + query = (char*)"SELECT * FROM pgsql_hostgroup_attributes ORDER BY hostgroup_id"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + if (incoming_hostgroup_attributes == nullptr) { + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset_hostgroup_attributes); + } + else { + resultset_hostgroup_attributes = incoming_hostgroup_attributes; + } + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } + else { + // Pass the resultset to PgHGM + PgHGM->save_incoming_pgsql_table(resultset_hostgroup_attributes, "pgsql_hostgroup_attributes"); + } + + // commit all the changes + PgHGM->commit( + { runtime_pgsql_servers, peer_runtime_pgsql_server }, + { incoming_pgsql_servers_v2, peer_pgsql_server_v2 }, + false, true + ); + + // quering runtime table will update and return latest records, so this is not needed. + // GloAdmin->save_pgsql_servers_runtime_to_database(true); + + // clean up + if (resultset) delete resultset; + resultset = NULL; + if (resultset_replication) { + delete resultset_replication; + resultset_replication = NULL; + } + if (resultset_hostgroup_attributes) { + resultset_hostgroup_attributes = NULL; + } +} char * ProxySQL_Admin::load_mysql_firewall_to_runtime() { // NOTE: firewall is currently NOT part of Cluster @@ -13020,7 +7463,7 @@ char * ProxySQL_Admin::load_mysql_firewall_to_runtime() { int cols_sqli_fingerprints=0; int affected_rows_sqli_fingerprints=0; bool success = false; - if (GloQPro==NULL) return (char *)"Global Query Processor not started: command impossible to run"; + if (GloMyQPro==NULL) return (char *)"Global Query Processor not started: command impossible to run"; char *query_users = (char *)"SELECT * FROM mysql_firewall_whitelist_users"; char *query_rules = (char *)"SELECT * FROM mysql_firewall_whitelist_rules"; char *query_sqli_fingerprints = (char *)"SELECT * FROM mysql_firewall_whitelist_sqli_fingerprints"; @@ -13032,13 +7475,71 @@ char * ProxySQL_Admin::load_mysql_firewall_to_runtime() { admindb->execute_statement(query_sqli_fingerprints, &error_sqli_fingerprints , &cols_sqli_fingerprints , &affected_rows_sqli_fingerprints , &resultset_sqli_fingerprints); if (error_users) { proxy_error("Error on %s : %s\n", query_users, error_users); - } else if (error_rules) { + } else if (error_rules) { + proxy_error("Error on %s : %s\n", query_rules, error_rules); + } else if (error_sqli_fingerprints) { + proxy_error("Error on %s : %s\n", query_sqli_fingerprints, error_sqli_fingerprints); + } else { + success = true; + GloMyQPro->load_firewall(resultset_users, resultset_rules, resultset_sqli_fingerprints); + } + if (success == false) { + // clean up + if (resultset_users) { + free(resultset_users); + } + if (resultset_rules) { + free(resultset_rules); + } + if (resultset_sqli_fingerprints) { + free(resultset_sqli_fingerprints); + } + } + unsigned long long curtime2=monotonic_time(); + curtime1 = curtime1/1000; + curtime2 = curtime2/1000; + if (curtime2-curtime1 > 1000) { + proxy_info("locked for %llums\n", curtime2-curtime1); + } + + return NULL; +} + +char* ProxySQL_Admin::load_pgsql_firewall_to_runtime() { + // NOTE: firewall is currently NOT part of Cluster + unsigned long long curtime1 = monotonic_time(); + char* error_users = NULL; + int cols_users = 0; + int affected_rows_users = 0; + char* error_rules = NULL; + int cols_rules = 0; + int affected_rows_rules = 0; + char* error_sqli_fingerprints = NULL; + int cols_sqli_fingerprints = 0; + int affected_rows_sqli_fingerprints = 0; + bool success = false; + if (GloPgQPro == NULL) return (char*)"Global Query Processor not started: command impossible to run"; + char* query_users = (char*)"SELECT * FROM pgsql_firewall_whitelist_users"; + char* query_rules = (char*)"SELECT * FROM pgsql_firewall_whitelist_rules"; + char* query_sqli_fingerprints = (char*)"SELECT * FROM pgsql_firewall_whitelist_sqli_fingerprints"; + SQLite3_result* resultset_users = NULL; + SQLite3_result* resultset_rules = NULL; + SQLite3_result* resultset_sqli_fingerprints = NULL; + admindb->execute_statement(query_users, &error_users, &cols_users, &affected_rows_users, &resultset_users); + admindb->execute_statement(query_rules, &error_rules, &cols_rules, &affected_rows_rules, &resultset_rules); + admindb->execute_statement(query_sqli_fingerprints, &error_sqli_fingerprints, &cols_sqli_fingerprints, &affected_rows_sqli_fingerprints, &resultset_sqli_fingerprints); + if (error_users) { + proxy_error("Error on %s : %s\n", query_users, error_users); + } + else if (error_rules) { proxy_error("Error on %s : %s\n", query_rules, error_rules); - } else if (error_sqli_fingerprints) { + } + else if (error_sqli_fingerprints) { proxy_error("Error on %s : %s\n", query_sqli_fingerprints, error_sqli_fingerprints); - } else { + } + else { success = true; - GloQPro->load_mysql_firewall(resultset_users, resultset_rules, resultset_sqli_fingerprints); + GloPgQPro->load_firewall(resultset_users, resultset_rules, resultset_sqli_fingerprints); } if (success == false) { // clean up @@ -13052,11 +7553,11 @@ char * ProxySQL_Admin::load_mysql_firewall_to_runtime() { free(resultset_sqli_fingerprints); } } - unsigned long long curtime2=monotonic_time(); - curtime1 = curtime1/1000; - curtime2 = curtime2/1000; - if (curtime2-curtime1 > 1000) { - proxy_info("locked for %llums\n", curtime2-curtime1); + unsigned long long curtime2 = monotonic_time(); + curtime1 = curtime1 / 1000; + curtime2 = curtime2 / 1000; + if (curtime2 - curtime1 > 1000) { + proxy_info("locked for %llums\n", curtime2 - curtime1); } return NULL; @@ -13068,7 +7569,7 @@ char* ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_ char *error=NULL; int cols=0; int affected_rows=0; - if (GloQPro==NULL) return (char *)"Global Query Processor not started: command impossible to run"; + if (GloMyQPro==NULL) return (char *)"Global Query Processor not started: command impossible to run"; SQLite3_result *resultset=NULL; char *query=(char *)"SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, attributes, comment FROM main.mysql_query_rules WHERE active=1 ORDER BY rule_id"; if (SQLite3_query_rules_resultset==NULL) { @@ -13095,7 +7596,7 @@ char* ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_ } else if (error2) { proxy_error("Error on %s : %s\n", query2, error2); } else { - fast_routing_hashmap_t fast_routing_hashmap( GloQPro->create_fast_routing_hashmap(resultset2) ); + fast_routing_hashmap_t fast_routing_hashmap(GloMyQPro->create_fast_routing_hashmap(resultset2) ); #ifdef BENCHMARK_FASTROUTING_LOAD for (int i=0; i<10; i++) { #endif // BENCHMARK_FASTROUTING_LOAD @@ -13112,7 +7613,7 @@ char* ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_ } unsigned long long curtime1 = monotonic_time(); - GloQPro->wrlock(); + GloMyQPro->wrlock(); // Checksums are always generated - 'admin-checksum_*' deprecated { pthread_mutex_lock(&GloVars.checksum_mutex); @@ -13161,7 +7662,7 @@ char* ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_ GloVars.checksums_values.mysql_query_rules.checksum, GloVars.checksums_values.mysql_query_rules.epoch ); } - rules_mem_sts_t prev_rules_data( GloQPro->reset_all(false) ); + rules_mem_sts_t prev_rules_data(GloMyQPro->reset_all(false) ); QP_rule_t * nqpr; for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; @@ -13181,7 +7682,7 @@ char* ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_ } } } - nqpr=GloQPro->new_query_rule( + nqpr=GloMyQPro->new_query_rule( atoi(r->fields[0]), // rule_id true, r->fields[1], // username @@ -13218,25 +7719,25 @@ char* ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_ r->fields[32], // attributes r->fields[33] // comment ); - GloQPro->insert(nqpr, false); + GloMyQPro->insert(nqpr, false); } - GloQPro->sort(false); + GloMyQPro->sort(false); #ifdef BENCHMARK_FASTROUTING_LOAD // load a copy of resultset and resultset2 SQLite3_result *resultset3 = new SQLite3_result(resultset); - GloQPro->save_query_rules(resultset3); + GloMyQPro->save_query_rules(resultset3); SQLite3_result *resultset4 = new SQLite3_result(resultset2); - GloQPro->load_fast_routing(resultset4); + GloMyQPro->load_fast_routing(resultset4); #else // load the original resultset and resultset2 - GloQPro->save_query_rules(resultset); - SQLite3_result* prev_fast_routing_resultset = GloQPro->load_fast_routing(fast_routing_hashmap); + GloMyQPro->save_query_rules(resultset); + SQLite3_result* prev_fast_routing_resultset=GloMyQPro->load_fast_routing(fast_routing_hashmap); #endif // BENCHMARK_FASTROUTING_LOAD - GloQPro->commit(); + GloMyQPro->commit(); #ifdef BENCHMARK_FASTROUTING_LOAD } #endif // BENCHMARK_FASTROUTING_LOAD - GloQPro->wrunlock(); + GloMyQPro->wrunlock(); unsigned long long curtime2 = monotonic_time(); unsigned long long elapsed_ms = (curtime2/1000) - (curtime1/1000); if (elapsed_ms > 5) { @@ -13255,11 +7756,216 @@ char* ProxySQL_Admin::load_mysql_query_rules_to_runtime(SQLite3_result* SQLite3_ __reset_rules(&prev_rules_data.query_rules); } } - // if (resultset) delete resultset; // never delete it. GloQPro saves it - // if (resultset2) delete resultset2; // never delete it. GloQPro saves it + // if (resultset) delete resultset; // never delete it. GloMyQPro saves it + // if (resultset2) delete resultset2; // never delete it. GloMyQPro saves it + return NULL; +} + +char* ProxySQL_Admin::load_pgsql_query_rules_to_runtime(SQLite3_result* SQLite3_query_rules_resultset, SQLite3_result* SQLite3_query_rules_fast_routing_resultset, const std::string& checksum, const time_t epoch) { + // About the queries used here, see notes about CLUSTER_QUERY_PGSQL_QUERY_RULES and + // CLUSTER_QUERY_PGSQL_QUERY_RULES_FAST_ROUTING in ProxySQL_Cluster.hpp + char* error = NULL; + int cols = 0; + int affected_rows = 0; + if (GloPgQPro == NULL) return (char*)"Global Query Processor not started: command impossible to run"; + SQLite3_result* resultset = NULL; + char* query = (char*)"SELECT rule_id, username, database, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, log, apply, attributes, comment FROM main.pgsql_query_rules WHERE active=1 ORDER BY rule_id"; + if (SQLite3_query_rules_resultset == NULL) { + admindb->execute_statement(query, &error, &cols, &affected_rows, &resultset); + } + else { + // Cluster can pass SQLite3_query_rules_resultset , to absolutely speed up + // the process and therefore there is no need to run any query + resultset = SQLite3_query_rules_resultset; + } + char* error2 = NULL; + int cols2 = 0; + int affected_rows2 = 0; + SQLite3_result* resultset2 = NULL; + char* query2 = (char*)"SELECT username, database, flagIN, destination_hostgroup, comment FROM main.pgsql_query_rules_fast_routing ORDER BY username, database, flagIN"; + if (SQLite3_query_rules_fast_routing_resultset == NULL) { + admindb->execute_statement(query2, &error2, &cols2, &affected_rows2, &resultset2); + } + else { + // Cluster can pass SQLite3_query_rules_fast_routing_resultset , to absolutely speed up + // the process and therefore there is no need to run any query + resultset2 = SQLite3_query_rules_fast_routing_resultset; + } + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } + else if (error2) { + proxy_error("Error on %s : %s\n", query2, error2); + } + else { + fast_routing_hashmap_t fast_routing_hashmap(GloPgQPro->create_fast_routing_hashmap(resultset2)); +#ifdef BENCHMARK_FASTROUTING_LOAD + for (int i = 0; i < 10; i++) { +#endif // BENCHMARK_FASTROUTING_LOAD + // Computed resultsets checksums outside of critical sections + uint64_t hash1 = 0; + uint64_t hash2 = 0; + + if ( + SQLite3_query_rules_resultset == nullptr || + SQLite3_query_rules_fast_routing_resultset == nullptr + ) { + hash1 = resultset->raw_checksum(); + hash2 = resultset2->raw_checksum(); + } + + unsigned long long curtime1 = monotonic_time(); + GloPgQPro->wrlock(); + // Checksums are always generated - 'admin-checksum_*' deprecated + { + pthread_mutex_lock(&GloVars.checksum_mutex); + char* buff = nullptr; + char buf[20]; + + // If both the resultsets are supplied, then the supplied checksum is the already computed one. + if ( + SQLite3_query_rules_resultset == nullptr || + SQLite3_query_rules_fast_routing_resultset == nullptr + ) { + hash1 += hash2; + uint32_t d32[2]; + memcpy(&d32, &hash1, sizeof(hash1)); + sprintf(buf, "0x%0X%0X", d32[0], d32[1]); + + buff = buf; + } + else { + buff = const_cast(checksum.c_str()); + } + + GloVars.checksums_values.pgsql_query_rules.set_checksum(buff); + GloVars.checksums_values.pgsql_query_rules.version++; + time_t t = time(NULL); + + // Since the supplied checksum is the already computed one and both resultset are + // supplied there is no need for comparsion, because we will be comparing it with itself. + bool same_checksum = + SQLite3_query_rules_resultset != nullptr && + SQLite3_query_rules_fast_routing_resultset != nullptr; + bool matching_checksums = + same_checksum || (GloVars.checksums_values.pgsql_query_rules.checksum == checksum); + + if (epoch != 0 && checksum != "" && matching_checksums) { + GloVars.checksums_values.pgsql_query_rules.epoch = epoch; + } + else { + GloVars.checksums_values.pgsql_query_rules.epoch = t; + } + + GloVars.epoch_version = t; + GloVars.generate_global_checksum(); + GloVars.checksums_values.updates_cnt++; + pthread_mutex_unlock(&GloVars.checksum_mutex); + proxy_info( + "Computed checksum for 'LOAD PGSQL QUERY RULES TO RUNTIME' was '%s', with epoch '%llu'\n", + GloVars.checksums_values.pgsql_query_rules.checksum, GloVars.checksums_values.pgsql_query_rules.epoch + ); + } + rules_mem_sts_t prev_rules_data(GloPgQPro->reset_all(false)); + QP_rule_t* nqpr; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + if (r->fields[4]) { + char* pct = NULL; + if (strlen(r->fields[4]) >= INET6_ADDRSTRLEN) { + proxy_error("Query rule with rule_id=%s has an invalid client_addr: %s\n", r->fields[0], r->fields[4]); + continue; + } + pct = strchr(r->fields[4], '%'); + if (pct) { // there is a wildcard + if (strlen(pct) == 1) { + // % is at the end of the string, good + } + else { + proxy_error("Query rule with rule_id=%s has a wildcard that is not at the end of client_addr: %s\n", r->fields[0], r->fields[4]); + continue; + } + } + } + nqpr = GloPgQPro->new_query_rule( + atoi(r->fields[0]), // rule_id + true, + r->fields[1], // username + r->fields[2], // schemaname + atoi(r->fields[3]), // flagIN + r->fields[4], // client_addr + r->fields[5], // proxy_addr + (r->fields[6] == NULL ? -1 : atol(r->fields[6])), // proxy_port + r->fields[7], // digest + r->fields[8], // match_digest + r->fields[9], // match_pattern + (atoi(r->fields[10]) == 1 ? true : false), // negate_match_pattern + r->fields[11], // re_modifiers + (r->fields[12] == NULL ? -1 : atol(r->fields[12])), // flagOUT + r->fields[13], // replae_pattern + (r->fields[14] == NULL ? -1 : atoi(r->fields[14])), // destination_hostgroup + (r->fields[15] == NULL ? -1 : atol(r->fields[15])), // cache_ttl + (r->fields[16] == NULL ? -1 : atol(r->fields[16])), // cache_empty_result + (r->fields[17] == NULL ? -1 : atol(r->fields[17])), // cache_timeout + (r->fields[18] == NULL ? -1 : atol(r->fields[18])), // reconnect + (r->fields[19] == NULL ? -1 : atol(r->fields[19])), // timeout + (r->fields[20] == NULL ? -1 : atol(r->fields[20])), // retries + (r->fields[21] == NULL ? -1 : atol(r->fields[21])), // delay + (r->fields[22] == NULL ? -1 : atol(r->fields[22])), // next_query_flagIN + (r->fields[23] == NULL ? -1 : atol(r->fields[23])), // mirror_flagOUT + (r->fields[24] == NULL ? -1 : atol(r->fields[24])), // mirror_hostgroup + r->fields[25], // error_msg + r->fields[26], // OK_msg + (r->fields[27] == NULL ? -1 : atol(r->fields[27])), // sticky_conn + (r->fields[28] == NULL ? -1 : atol(r->fields[28])), // multiplex + (r->fields[29] == NULL ? -1 : atol(r->fields[29])), // log + (atoi(r->fields[30]) == 1 ? true : false), + r->fields[31], // attributes + r->fields[32] // comment + ); + GloPgQPro->insert(nqpr, false); + } + GloPgQPro->sort(false); +#ifdef BENCHMARK_FASTROUTING_LOAD + // load a copy of resultset and resultset2 + SQLite3_result* resultset3 = new SQLite3_result(resultset); + GloPgQPro->save_query_rules(resultset3); + SQLite3_result* resultset4 = new SQLite3_result(resultset2); + GloPgQPro->load_fast_routing(resultset4); +#else + // load the original resultset and resultset2 + GloPgQPro->save_query_rules(resultset); + SQLite3_result* prev_fast_routing_resultset = GloPgQPro->load_fast_routing(fast_routing_hashmap); +#endif // BENCHMARK_FASTROUTING_LOAD + GloPgQPro->commit(); +#ifdef BENCHMARK_FASTROUTING_LOAD + } +#endif // BENCHMARK_FASTROUTING_LOAD + GloPgQPro->wrunlock(); + unsigned long long curtime2 = monotonic_time(); + unsigned long long elapsed_ms = (curtime2 / 1000) - (curtime1 / 1000); + if (elapsed_ms > 5) { + proxy_info("Query processor locked for %llums\n", curtime2 - curtime1); + } + + // Free previous 'fast_routing' structures outside of critical section + { + delete prev_fast_routing_resultset; + if (prev_rules_data.rules_fast_routing) { + kh_destroy(khStrInt, prev_rules_data.rules_fast_routing); + } + if (prev_rules_data.rules_fast_routing___keys_values) { + free(prev_rules_data.rules_fast_routing___keys_values); + } + __reset_rules(&prev_rules_data.query_rules); + } + } + // if (resultset) delete resultset; // never delete it. GloPgQPro saves it + // if (resultset2) delete resultset2; // never delete it. GloPgQPro saves it return NULL; } + extern "C" ProxySQL_Admin * create_ProxySQL_Admin_func() { return new ProxySQL_Admin(); } @@ -13301,820 +8007,6 @@ void ProxySQL_Admin::flush_error_log() { } } -void ProxySQL_Admin::disk_upgrade_mysql_query_rules() { - // this function is called only for configdb table - // it is responsible to upgrade table mysql_query_rules if its structure is from a previous version - int rci; - configdb->execute("PRAGMA foreign_keys = OFF"); - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_1_0); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.1.0 of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v110 - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v110"); - // rename current table to add suffix _v110 - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v110"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,apply) SELECT rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,apply FROM mysql_query_rules_v110"); - } - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0a); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.0a of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v120a - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v120a"); - // rename current table to add suffix _v120a - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v120a"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,mirror_flagOUT,mirror_hostgroup,apply) SELECT rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,mirror_flagOUT,mirror_hostgroup,apply FROM mysql_query_rules_v120a"); - } - // upgrade related to issue #643 , adding comment in mysql_query_rules table - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0g); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.0g of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v120g - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v120g"); - // rename current table to add suffix _v120g - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v120g"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply FROM mysql_query_rules_v120g"); - } - // upgrade related to issue #643 , adding comment in mysql_query_rules table - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_2); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.2 of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v122 - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v122"); - // rename current table to add suffix _v122 - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v122"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply,comment FROM mysql_query_rules_v122"); - } - // upgrade related to issue #643 , adding comment in mysql_query_rules table - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_3_1); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.3.1 of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v131 - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v131"); - // rename current table to add suffix _v131 - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v131"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment FROM mysql_query_rules_v131"); - } - - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0a); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.4.0a of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v140a - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v140a"); - // rename current table to add suffix _v140a - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v40a"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment FROM mysql_query_rules_v140a"); - } - - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0b); - if (rci) { // note: upgrade from V1_4_0a or V1_4_0b is the same - // upgrade is required - proxy_warning("Detected version v1.4.0b of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v140b - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v140b"); - // rename current table to add suffix _v140b - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v140b"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment FROM mysql_query_rules_v140b"); - } - - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_1); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.4.1 of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v141 - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v141"); - // rename current table to add suffix _v141 - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v141"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment FROM mysql_query_rules_v141"); - } - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0a); - if (rci) { - // upgrade is required - proxy_warning("Detected version v2.0.0a of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v200a - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200a"); - // rename current table to add suffix _v200a - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200a"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200a"); - } - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0b); - if (rci) { - // upgrade is required - proxy_warning("Detected version v2.0.0b of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v200b - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200b"); - // rename current table to add suffix _v200b - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200b"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200b"); - } - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0c); - if (rci) { - // upgrade is required - proxy_warning("Detected version v2.0.0c of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v200c - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200c"); - // rename current table to add suffix _v200c - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200c"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200c"); - } - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0d); - if (rci) { - // upgrade is required - proxy_warning("Detected version v2.0.0d of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v200d - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200d"); - // rename current table to add suffix _v200d - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200d"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200d"); - } - rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0e); - if (rci) { - // upgrade is required - proxy_warning("Detected version v2.1.0e of table mysql_query_rules\n"); - proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); - // drop any existing table with suffix _v200e - configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200e"); - // rename current table to add suffix _v200e - configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200e"); - // create new table - configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200e"); - } - configdb->execute("PRAGMA foreign_keys = ON"); -} - -void ProxySQL_Admin::disk_upgrade_scheduler() { - // this function is called only for configdb table - // it is responsible to upgrade table scheduler if its structure is from a previous version - int rci; - configdb->execute("PRAGMA foreign_keys = OFF"); - rci=configdb->check_table_structure((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_0); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.0 of table scheduler\n"); - proxy_warning("ONLINE UPGRADE of table scheduler in progress\n"); - // drop any existing table with suffix _v120 - configdb->execute("DROP TABLE IF EXISTS scheduler_v120"); - // rename current table to add suffix _v120 - configdb->execute("ALTER TABLE scheduler RENAME TO scheduler_v120"); - // create new table - configdb->build_table((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER,false); - // copy fields from old table - configdb->execute("INSERT INTO scheduler (id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5) SELECT id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5 FROM scheduler_v120"); - } - rci=configdb->check_table_structure((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2a); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.2a of table scheduler\n"); - proxy_warning("ONLINE UPGRADE of table scheduler in progress\n"); - // drop any existing table with suffix _v122a - configdb->execute("DROP TABLE IF EXISTS scheduler_v122a"); - // rename current table to add suffix _v122a - configdb->execute("ALTER TABLE scheduler RENAME TO scheduler_v122a"); - // create new table - configdb->build_table((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER,false); - // copy fields from old table - configdb->execute("INSERT INTO scheduler (id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5,comment) SELECT id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5,comment FROM scheduler_v122a"); - } - rci=configdb->check_table_structure((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2b); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.2b of table scheduler\n"); - proxy_warning("ONLINE UPGRADE of table scheduler in progress\n"); - // drop any existing table with suffix _v122b - configdb->execute("DROP TABLE IF EXISTS scheduler_v122b"); - // rename current table to add suffix _v122b - configdb->execute("ALTER TABLE scheduler RENAME TO scheduler_v122b"); - // create new table - configdb->build_table((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER,false); - // copy fields from old table - configdb->execute("INSERT INTO scheduler (id,active,interval_ms,filename,arg1,arg2,arg3,arg4,arg5,comment) SELECT id,active,interval_ms,filename,arg1,arg2,arg3,arg4,arg5,comment FROM scheduler_v122b"); - } - - configdb->execute("PRAGMA foreign_keys = ON"); -} - -void ProxySQL_Admin::disk_upgrade_mysql_servers() { - // this function is called only for configdb table - // it is responsible to upgrade table mysql_servers if its structure is from a previous version - int rci; - configdb->execute("PRAGMA foreign_keys = OFF"); - rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_1_0); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.1.0 of table mysql_servers\n"); - proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); - // drop any existing table with suffix _v110 - configdb->execute("DROP TABLE IF EXISTS mysql_servers_v110"); - // rename current table to add suffix _v110 - configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v110"); - // create new table - configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); - // fix bug #1224 - configdb->execute("UPDATE mysql_servers_v110 SET weight = 10000000 WHERE weight > 10000000"); - // fix bug #962 - configdb->execute("UPDATE mysql_servers_v110 SET compression = 1 WHERE compression > 0"); - // copy fields from old table - configdb->execute("INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag FROM mysql_servers_v110"); - } - rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_0e); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.0 of table mysql_servers\n"); - proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); - // drop any existing table with suffix _v120 - configdb->execute("DROP TABLE IF EXISTS mysql_servers_v120"); - // rename current table to add suffix _v120 - configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v120"); - // create new table - configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); - // fix bug #1224 - configdb->execute("UPDATE mysql_servers_v120 SET weight = 10000000 WHERE weight > 10000000"); - // fix bug #962 - configdb->execute("UPDATE mysql_servers_v120 SET compression = 1 WHERE compression > 0"); - // copy fields from old table - configdb->execute("INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms FROM mysql_servers_v120"); - } - rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_2); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.2 of table mysql_servers\n"); - proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); - //drop any existing table with suffix _v122 - configdb->execute("DROP TABLE IF EXISTS mysql_servers_v122"); - // rename current table to add suffix _v122 - configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v122"); - // create new table - configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); - // fix bug #1224 - configdb->execute("UPDATE mysql_servers_v122 SET weight = 10000000 WHERE weight > 10000000"); - // fix bug #962 - configdb->execute("UPDATE mysql_servers_v122 SET compression = 1 WHERE compression > 0"); - // copy fields from old table - configdb->execute("INSERT OR IGNORE INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_v122"); - } - rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_4_4); // 1.4.4 has the same column of 1.2.2 - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.4.4 (pre-2.0.0) of table mysql_servers\n"); - proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); - //drop any existing table with suffix _v144 - configdb->execute("DROP TABLE IF EXISTS mysql_servers_v144"); - // rename current table to add suffix _v144 - configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v144"); - // create new table - configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); - // fix bug #1224 - configdb->execute("UPDATE mysql_servers_v144 SET weight = 10000000 WHERE weight > 10000000"); - // fix bug #962 - configdb->execute("UPDATE mysql_servers_v144 SET compression = 1 WHERE compression > 0"); - // copy fields from old table - configdb->execute("INSERT OR IGNORE INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_v144"); - } - rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0a); - if (rci) { - // upgrade is required - proxy_warning("Detected version 2.0.0a of table mysql_servers\n"); - proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); - //drop any existing table with suffix _v200a - configdb->execute("DROP TABLE IF EXISTS mysql_servers_v200a"); - // rename current table to add suffix _v200a - configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v200a"); - // create new table - configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); - // fix bug #1224 - configdb->execute("UPDATE mysql_servers_v200a SET weight = 10000000 WHERE weight > 10000000"); - // fix bug #962 - configdb->execute("UPDATE mysql_servers_v200a SET compression = 1 WHERE compression > 0"); - // copy fields from old table - configdb->execute("INSERT OR IGNORE INTO mysql_servers SELECT * FROM mysql_servers_v200a"); - } - rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0b); - if (rci) { - // upgrade is required - proxy_warning("Detected version 2.0.0b of table mysql_servers\n"); - proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); - //drop any existing table with suffix _v200b - configdb->execute("DROP TABLE IF EXISTS mysql_servers_v200b"); - // rename current table to add suffix _v200b - configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v200b"); - // create new table - configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); - // fix bug #1224 - configdb->execute("UPDATE mysql_servers_v200b SET weight = 10000000 WHERE weight > 10000000"); - // fix bug #962 - configdb->execute("UPDATE mysql_servers_v200b SET compression = 1 WHERE compression > 0"); - // copy fields from old table - configdb->execute("INSERT OR IGNORE INTO mysql_servers SELECT * FROM mysql_servers_v200b"); - } - rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0c); - if (rci) { - // upgrade is required to fix issue #1923 - proxy_warning("Detected version 2.0.0c (pre-2.0.11) of table mysql_servers\n"); - proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); - //drop any existing table with suffix _v200c - configdb->execute("DROP TABLE IF EXISTS mysql_servers_v200c"); - // rename current table to add suffix _v200c - configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v200c"); - // create new table - configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); - configdb->execute("INSERT OR IGNORE INTO mysql_servers SELECT * FROM mysql_servers_v200c"); - } - rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_0); // issue #643 - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.0 of table mysql_replication_hostgroups\n"); - proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); - // drop any existing table with suffix _v100 - configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v100"); - // rename current table to add suffix _v100 - configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v100"); - // create new table - configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_replication_hostgroups (writer_hostgroup,reader_hostgroup) SELECT writer_hostgroup , reader_hostgroup FROM mysql_replication_hostgroups_v100"); - } - rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_2_2); // issue #1304 - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.2.2 (pre-1.4.5) of table mysql_replication_hostgroups\n"); - proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); - // drop any existing table with suffix _v122 - configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v122"); - // rename current table to add suffix _v122 - configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v122"); - // create new table - configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_replication_hostgroups (writer_hostgroup,reader_hostgroup,comment) SELECT writer_hostgroup , reader_hostgroup , COALESCE(comment,'') FROM mysql_replication_hostgroups_v122"); - } - rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_4_5); // issue #1304 - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.4.5 (pre-2.0.0) of table mysql_replication_hostgroups\n"); - proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); - // drop any existing table with suffix _v145 - configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v145"); - // rename current table to add suffix _v145 - configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v145"); - // create new table - configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_replication_hostgroups (writer_hostgroup,reader_hostgroup,comment) SELECT writer_hostgroup , reader_hostgroup , comment FROM mysql_replication_hostgroups_v145"); - } - rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V2_0_0); // issue #2186 - if (rci) { - // upgrade is required - proxy_warning("Detected version v2.0.0 (pre-2.0.8) of table mysql_replication_hostgroups\n"); - proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); - // drop any existing table with suffix _v200 - configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v200"); - // rename current table to add suffix _v200 - configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v200"); - // create new table - configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_replication_hostgroups SELECT * FROM mysql_replication_hostgroups_v200"); - } - - // upgrade mysql_group_replication_hostgroups - rci=configdb->check_table_structure((char *)"mysql_group_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS_V1_4); - if (rci) { - // upgrade is required - proxy_warning("Detected version v1.4 (pre-2.0.0) of mysql_group_replication_hostgroups\n"); - proxy_warning("ONLINE UPGRADE of table mysql_group_replication_hostgroups in progress\n"); - // drop any existing table with suffix _v14 - configdb->execute("DROP TABLE IF EXISTS mysql_group_replication_hostgroups_v14"); - // rename current table to add suffix _v14 - configdb->execute("ALTER TABLE mysql_group_replication_hostgroups RENAME TO mysql_group_replication_hostgroups_v14"); - // create new table - configdb->build_table((char *)"mysql_group_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_group_replication_hostgroups SELECT * FROM mysql_group_replication_hostgroups_v14"); - } - - - // upgrade mysql_galera_hostgroups - rci=configdb->check_table_structure((char *)"mysql_galera_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS_V2_0_0a); - if (rci) { - // upgrade is required - proxy_warning("Detected version v2.0.0a (pre-2.0.0b) of mysql_galera_hostgroups\n"); - proxy_warning("ONLINE UPGRADE of table mysql_galera_hostgroups in progress\n"); - // drop any existing table with suffix _v200a - configdb->execute("DROP TABLE IF EXISTS mysql_galera_hostgroups_v200a"); - // rename current table to add suffix _v200a - configdb->execute("ALTER TABLE mysql_galera_hostgroups RENAME TO mysql_galera_hostgroups_v200a"); - // create new table - configdb->build_table((char *)"mysql_galera_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_galera_hostgroups SELECT * FROM mysql_galera_hostgroups_v200a"); - } - - // upgrade mysql_aws_aurora_hostgroups - rci=configdb->check_table_structure((char *)"mysql_aws_aurora_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_8); - if (rci) { - // upgrade is required - proxy_warning("Detected version pre-v2.0.9 of mysql_aws_aurora_hostgroups\n"); - proxy_warning("ONLINE UPGRADE of table mysql_aws_aurora_hostgroups in progress\n"); - // drop mysql_aws_aurora_hostgroups table with suffix _v208 - configdb->execute("DROP TABLE IF EXISTS mysql_aws_aurora_hostgroups_v208"); - // rename current table to add suffix _v208 - configdb->execute("ALTER TABLE mysql_aws_aurora_hostgroups RENAME TO mysql_aws_aurora_hostgroups_v208"); - // create new table - configdb->build_table((char *)"mysql_aws_aurora_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, " - "max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, comment) " - "SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " - "check_timeout_ms, writer_is_also_reader, new_reader_weight, comment FROM mysql_aws_aurora_hostgroups_v208"); - } - - // upgrade mysql_hostgroup_attributes - rci=configdb->check_table_structure((char *)"mysql_hostgroup_attributes",(char *)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0); - if (rci) { - // upgrade is required - proxy_warning("Detected version pre-v2.5.2 of mysql_hostgroup_attributes\n"); - proxy_warning("ONLINE UPGRADE of table mysql_hostgroup_attributes in progress\n"); - // drop mysql_hostgroup_attributes table with suffix _v250 - configdb->execute("DROP TABLE IF EXISTS mysql_hostgroup_attributes_v250"); - // rename current table to add suffix _v250 - configdb->execute("ALTER TABLE mysql_hostgroup_attributes RENAME TO mysql_hostgroup_attributes_v250"); - // create new table - configdb->build_table((char *)"mysql_hostgroup_attributes",(char *)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES,false); - // copy fields from old table - configdb->execute( - "INSERT INTO mysql_hostgroup_attributes (" - " hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex," - " connection_warming, throttle_connections_per_sec, ignore_session_variables, comment" - ") SELECT" - " hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex," - " connection_warming, throttle_connections_per_sec, ignore_session_variables, comment" - " FROM mysql_hostgroup_attributes_v250" - ); - } - rci = configdb->check_table_structure((char*)"mysql_hostgroup_attributes", (char*)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2); - if (rci) { - // upgrade is required - proxy_warning("Detected version pre-v2.6.0 of mysql_hostgroup_attributes\n"); - proxy_warning("ONLINE UPGRADE of table mysql_hostgroup_attributes in progress\n"); - // drop mysql_hostgroup_attributes table with suffix _v252 - configdb->execute("DROP TABLE IF EXISTS mysql_hostgroup_attributes_v252"); - // rename current table to add suffix _v252 - configdb->execute("ALTER TABLE mysql_hostgroup_attributes RENAME TO mysql_hostgroup_attributes_v252"); - // create new table - configdb->build_table((char*)"mysql_hostgroup_attributes", (char*)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES, false); - // copy fields from old table - configdb->execute( - "INSERT INTO mysql_hostgroup_attributes (" - " hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex," - " connection_warming, throttle_connections_per_sec, ignore_session_variables, servers_defaults, comment" - ") SELECT" - " hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex," - " connection_warming, throttle_connections_per_sec, ignore_session_variables, servers_defaults, comment" - " FROM mysql_hostgroup_attributes_v252" - ); - } - configdb->execute("PRAGMA foreign_keys = ON"); - -} - - -void ProxySQL_Admin::disk_upgrade_mysql_users() { - // this function is called only for configdb table - // it is responsible to upgrade table mysql_users if its structure is from a previous version - int rci; - configdb->execute("PRAGMA foreign_keys = OFF"); - // change transaction_persistent=1 by default . See #793 - rci=configdb->check_table_structure((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS_V1_3_0); - if (rci) { - // upgrade is required - proxy_warning("Detected version pre-1.4 of table mysql_users\n"); - proxy_warning("ONLINE UPGRADE of table mysql_users in progress\n"); - // drop any existing table with suffix _v130 - configdb->execute("DROP TABLE IF EXISTS mysql_users_v130"); - // rename current table to add suffix _v130 - configdb->execute("ALTER TABLE mysql_users RENAME TO mysql_users_v130"); - // create new table - configdb->build_table((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) SELECT * FROM mysql_users_v130"); - } - // adding mysql_users.commment . See #1633 - rci=configdb->check_table_structure((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS_V1_4_0); - if (rci) { - // upgrade is required - proxy_warning("Detected version pre-2.0 of table mysql_users\n"); - proxy_warning("ONLINE UPGRADE of table mysql_users in progress\n"); - // drop any existing table with suffix _v140 - configdb->execute("DROP TABLE IF EXISTS mysql_users_v140"); - // rename current table to add suffix _v140 - configdb->execute("ALTER TABLE mysql_users RENAME TO mysql_users_v140"); - // create new table - configdb->build_table((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) SELECT * FROM mysql_users_v140"); - } - // adding mysql_users.attributes. See #3083 - rci=configdb->check_table_structure((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS_V2_0_0); - if (rci) { - // upgrade is required - proxy_warning("Detected version pre-2.1.0 of table mysql_users\n"); - proxy_warning("ONLINE UPGRADE of table mysql_users in progress\n"); - // drop any existing table with suffix _v210 - configdb->execute("DROP TABLE IF EXISTS mysql_users_v200"); - // rename current table to add suffix _v210 - configdb->execute("ALTER TABLE mysql_users RENAME TO mysql_users_v200"); - // create new table - configdb->build_table((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS,false); - // copy fields from old table - configdb->execute("INSERT INTO mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections,comment) SELECT * FROM mysql_users_v200"); - } - configdb->execute("PRAGMA foreign_keys = ON"); -} - - -void ProxySQL_Admin::disk_upgrade_rest_api_routes() { - int rci; - configdb->execute("PRAGMA foreign_keys = OFF"); - - rci=configdb->check_table_structure((char *)"restapi_routes",(char *)ADMIN_SQLITE_TABLE_RESTAPI_ROUTES_V2_0_15); - if (rci) { - // upgrade is required - proxy_warning("Detected version pre-2.1.0 of table restapi_routes\n"); - proxy_warning("ONLINE UPGRADE of table restapi_routes in progress\n"); - // drop any existing table with suffix _v2015 - configdb->execute("DROP TABLE IF EXISTS restapi_routes_v2015"); - // rename current table to add suffix _v2015 - configdb->execute("ALTER TABLE restapi_routes RENAME TO restapi_routes_v2015"); - // create new table - configdb->build_table((char *)"restapi_routes",(char *)ADMIN_SQLITE_TABLE_RESTAPI_ROUTES,false); - // copy fields from old table - configdb->execute("INSERT INTO restapi_routes(id,active,timeout_ms,method,uri,script,comment) SELECT id,active,interval_ms,method,uri,script,comment FROM restapi_routes_v2015"); - } - - configdb->execute("PRAGMA foreign_keys = ON"); -} - -Scheduler_Row::Scheduler_Row(unsigned int _id, bool _is_active, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5, char *_comment) { - int i; - id=_id; - is_active=_is_active; - interval_ms=_in; - filename=strdup(_f); - args=(char **)malloc(6*sizeof(char *)); - for (i=0;i<6;i++) { - args[i]=NULL; - } - // only copy fields if the previous one is not null - if (a1) { - args[0]=strdup(a1); - if (a2) { - args[1]=strdup(a2); - if (a3) { - args[2]=strdup(a3); - if (a4) { - args[3]=strdup(a4); - if (a5) { - args[4]=strdup(a5); - } - } - } - } - } - comment=strdup(_comment); -} - -Scheduler_Row::~Scheduler_Row() { - int i; - for (i=0;i<6;i++) { - if (args[i]) { - free(args[i]); - } - args[i]=NULL; - } - if (filename) { - free(filename); - } - free(args); - free(comment); - args=NULL; -} - -ProxySQL_External_Scheduler::ProxySQL_External_Scheduler() { -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_init(&rwlock,NULL); -#else - spinlock_rwlock_init(&rwlock); -#endif - last_version=0; - version=0; - next_run=0; -} - -ProxySQL_External_Scheduler::~ProxySQL_External_Scheduler() { -} - -void ProxySQL_External_Scheduler::update_table(SQLite3_result *resultset) { -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_wrlock(&rwlock); -#else - spin_wrlock(&rwlock); -#endif - // delete all current rows - Scheduler_Row *sr; - for (std::vector::iterator it=Scheduler_Rows.begin(); it!=Scheduler_Rows.end(); ++it) { - sr=*it; - delete sr; - } - Scheduler_Rows.clear(); - - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - unsigned int id=strtoul(r->fields[0], NULL, 10); - bool is_active=false; - if (atoi(r->fields[1])) { - is_active=true; - } - unsigned int interval_ms=strtoul(r->fields[2], NULL, 10); - Scheduler_Row *sr=new Scheduler_Row(id, is_active, interval_ms, - r->fields[3], - r->fields[4], r->fields[5], - r->fields[6], r->fields[7], - r->fields[8], - r->fields[9] // comment, issue #643 - ); - Scheduler_Rows.push_back(sr); - } - // increase version - __sync_fetch_and_add(&version,1); - // unlock -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_unlock(&rwlock); -#else - spin_wrunlock(&rwlock); -#endif -} - -// this fuction will be called as a deatached thread -static void * waitpid_thread(void *arg) { - pid_t *cpid_ptr=(pid_t *)arg; - int status; - waitpid(*cpid_ptr, &status, 0); - free(cpid_ptr); - return NULL; -} - -unsigned long long ProxySQL_External_Scheduler::run_once() { - Scheduler_Row *sr=NULL; - unsigned long long curtime=monotonic_time(); - curtime=curtime/1000; -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_rdlock(&rwlock); -#else - spin_rdlock(&rwlock); -#endif - if (__sync_add_and_fetch(&version,0) > last_version) { // version was changed - next_run=0; - last_version=version; - for (std::vector::iterator it=Scheduler_Rows.begin(); it!=Scheduler_Rows.end(); ++it) { - sr=*it; - if (sr->is_active==false) { - continue; - } - sr->next=curtime+sr->interval_ms; - if (next_run==0) { - next_run=sr->next; - } else { - if (sr->next < next_run) { // we try to find the first event that needs to be executed - next_run=sr->next; - } - } - } - } - if (curtime >= next_run) { - next_run=0; - for (std::vector::iterator it=Scheduler_Rows.begin(); it!=Scheduler_Rows.end(); ++it) { - sr=*it; - if (sr->is_active==false) { - continue; - } - if (curtime >= sr->next) { - // the event is scheduled for execution - sr->next=curtime+sr->interval_ms; - char **newargs=(char **)malloc(7*sizeof(char *)); - for (int i=1;i<7;i++) { - newargs[i]=sr->args[i-1]; - } - newargs[0]=sr->filename; - proxy_info("Scheduler starting id: %u , filename: %s\n", sr->id, sr->filename); - pid_t cpid; - cpid = fork(); - if (cpid == -1) { - perror("fork"); - exit(EXIT_FAILURE); - } - if (cpid == 0) { - close_all_non_term_fd({}); - char *newenviron[] = { NULL }; - int rc; - rc=execve(sr->filename, newargs, newenviron); - if (rc) { - proxy_error("Scheduler: Failed to run %s\n", sr->filename); - perror("execve()"); - exit(EXIT_FAILURE); - } - } else { - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pthread_attr_setstacksize (&attr, 64*1024); - pid_t *cpid_ptr=(pid_t *)malloc(sizeof(pid_t)); - *cpid_ptr=cpid; - pthread_t thr; - if (pthread_create(&thr, &attr, waitpid_thread, (void *)cpid_ptr) !=0 ) { - perror("Thread creation"); - exit(EXIT_FAILURE); - } - } - free(newargs); - } - if (next_run==0) { - next_run=sr->next; - } else { - if (sr->next < next_run) { // we try to find the first event that needs to be executed - next_run=sr->next; - } - } - } - } - // find the smaller next_run - for (std::vector::iterator it=Scheduler_Rows.begin(); it!=Scheduler_Rows.end(); ++it) { - sr=*it; - if (next_run==0) { - } - } -#ifdef PA_PTHREAD_MUTEX - pthread_rwlock_unlock(&rwlock); -#else - spin_rdunlock(&rwlock); -#endif - return next_run; -} - void ProxySQL_Admin::load_proxysql_servers_to_runtime(bool _lock, const std::string& checksum, const time_t epoch) { // make sure that the caller has called mysql_servers_wrlock() char *error=NULL; @@ -14292,71 +8184,6 @@ void ProxySQL_Admin::dump_coredump_filter_values_table() { (*proxy_sqlite3_finalize)(stmt); } -void ProxySQL_Admin::stats___mysql_prepared_statements_info() { - if (!GloMyStmt) return; - SQLite3_result * resultset=NULL; - resultset=GloMyStmt->get_prepared_statements_global_infos(); - if (resultset==NULL) return; - statsdb->execute("BEGIN"); - int rc; - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement32=NULL; - - char *query1=NULL; - char *query32=NULL; - std::string query32s = ""; - statsdb->execute("DELETE FROM stats_mysql_prepared_statements_info"); - query1=(char *)"INSERT INTO stats_mysql_prepared_statements_info VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; - query32s = "INSERT INTO stats_mysql_prepared_statements_info VALUES " + generate_multi_rows_query(32,9); - query32 = (char *)query32s.c_str(); - - rc = statsdb->prepare_v2(query1, &statement1); - ASSERT_SQLITE_OK(rc, statsdb); - rc = statsdb->prepare_v2(query32, &statement32); - ASSERT_SQLITE_OK(rc, statsdb); - int row_idx=0; - int max_bulk_row_idx=resultset->rows_count/32; - max_bulk_row_idx=max_bulk_row_idx*32; - for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r1=*it; - int idx=row_idx%32; - if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_text(statement32, (idx*9)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_text(statement32, (idx*9)+3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_text(statement32, (idx*9)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_int64(statement32, (idx*9)+5, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_int64(statement32, (idx*9)+6, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_int64(statement32, (idx*9)+7, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_int64(statement32, (idx*9)+8, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_text(statement32, (idx*9)+9, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - if (idx==31) { - SAFE_SQLITE3_STEP2(statement32); - rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); - } - } else { // single row - rc=sqlite3_bind_int64(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_text(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_text(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_text(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_int64(statement1, 5, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_int64(statement1, 6, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_int64(statement1, 7, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_int64(statement1, 8, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); - rc=sqlite3_bind_text(statement1, 9, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); - } - row_idx++; - } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement32); - statsdb->execute("COMMIT"); - delete resultset; -} - #ifdef TEST_GALERA void ProxySQL_Admin::enable_galera_testing() { proxy_info("Admin is enabling Galera Testing using SQLite3 Server and HGs from 2271 and 2290\n"); @@ -14550,170 +8377,3 @@ void ProxySQL_Admin::enable_replicationlag_testing() { } #endif // TEST_REPLICATIONLAG -void ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters() { - mysql_servers_wrlock(); - admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id BETWEEN 10001 AND 20000"); - admindb->execute("DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup BETWEEN 10001 AND 20000"); - char *q1 = (char *)"INSERT INTO mysql_servers (hostgroup_id, hostname, port) VALUES (?1, ?2, ?3), (?4, ?5, ?6), (?7, ?8, ?9)"; - char *q2 = (char *)"INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup) VALUES (?1, ?2)"; - int rc; - sqlite3_stmt *statement1=NULL; - sqlite3_stmt *statement2=NULL; - rc=admindb->prepare_v2(q1, &statement1); - ASSERT_SQLITE_OK(rc, admindb); - rc=admindb->prepare_v2(q2, &statement2); - ASSERT_SQLITE_OK(rc, admindb); - char hostnamebuf1[32]; - char hostnamebuf2[32]; - char hostnamebuf3[32]; - for (int i=1000; i<2000; i++) { - sprintf(hostnamebuf1,"hostname%d", i*10+1); - sprintf(hostnamebuf2,"hostname%d", i*10+2); - sprintf(hostnamebuf3,"hostname%d", i*10+3); - rc=(*proxy_sqlite3_bind_int64)(statement1, 1, i*10+1); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 2, hostnamebuf1, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 3, 3306); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 4, i*10+2); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 5, hostnamebuf2, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 6, 3306); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 7, i*10+2); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_text)(statement1, 8, hostnamebuf3, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement1, 9, 3306); ASSERT_SQLITE_OK(rc, admindb); - SAFE_SQLITE3_STEP2(statement1); - rc=(*proxy_sqlite3_bind_int64)(statement2, 1, i*10+1); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_bind_int64)(statement2, 2, i*10+2); ASSERT_SQLITE_OK(rc, admindb); - SAFE_SQLITE3_STEP2(statement2); - rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, admindb); - rc=(*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, admindb); - } - (*proxy_sqlite3_finalize)(statement1); - (*proxy_sqlite3_finalize)(statement2); - load_mysql_servers_to_runtime(); - mysql_servers_wrunlock(); -} -unsigned long long ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_read_only_action() { - // we immediately exit. This is just for developer - return 0; - ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters(); - char hostnamebuf1[32]; - char hostnamebuf2[32]; - char hostnamebuf3[32]; - unsigned long long t1 = monotonic_time(); - //for (int j=0 ; j<500; j++) { - for (int j=0 ; j<1000; j++) { - for (int i=1000; i<2000; i++) { - sprintf(hostnamebuf1,"hostname%d", i*10+1); - sprintf(hostnamebuf2,"hostname%d", i*10+2); - sprintf(hostnamebuf3,"hostname%d", i*10+3); - MyHGM->read_only_action_v2( std::list { - read_only_server_t { std::string(hostnamebuf1), 3306, 0 }, - read_only_server_t { std::string(hostnamebuf2), 3306, 1 }, - read_only_server_t { std::string(hostnamebuf3), 3306, 1 } - } ); - } - } - unsigned long long t2 = monotonic_time(); - t1 /= 1000; - t2 /= 1000; - unsigned long long d = t2-t1; - return d; -} - -#ifdef DEBUG -// NEVER USED THIS FUNCTION IN PRODUCTION. -// THIS IS FOR TESTING PURPOSE ONLY -// IT ACCESSES MyHGM without lock -unsigned long long ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_HG_lookup() { - // we immediately exit. This is just for developer - return 0; - ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters(); - unsigned long long t1 = monotonic_time(); - unsigned int hid = 0; - MyHGC * myhgc = NULL; - for (int j=0 ; j<100000; j++) { - for (unsigned int i=1000; i<2000; i++) { - // NEVER USED THIS FUNCTION IN PRODUCTION. - // THIS IS FOR TESTING PURPOSE ONLY - // IT ACCESSES MyHGM without lock - hid = i*10+1; // writer hostgroup - myhgc = MyHGM->MyHGC_lookup(hid); - assert(myhgc); - hid++; // reader hostgroup - myhgc = MyHGM->MyHGC_lookup(hid); - assert(myhgc); - } - } - unsigned long long t2 = monotonic_time(); - t1 /= 1000; - t2 /= 1000; - unsigned long long d = t2-t1; - return d; -} - -// NEVER USED THIS FUNCTION IN PRODUCTION. -// THIS IS FOR TESTING PURPOSE ONLY -// IT ACCESSES MyHGM without lock -unsigned long long ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_Balancing_HG5211() { - unsigned long long t1 = monotonic_time(); - const unsigned int NS = 4; - unsigned int cu[NS] = { 50, 10, 10, 0 }; - MyHGC * myhgc = NULL; - myhgc = MyHGM->MyHGC_lookup(5211); - assert(myhgc); - assert(myhgc->mysrvs->servers->len == NS); - unsigned int cnt[NS]; - for (unsigned int i=0; imysrvs->servers->index(i); - m->ConnectionsUsed->conns->len=cu[i]; - } - unsigned int NL = 1000; - for (unsigned int i=0; iget_random_MySrvC(NULL, 0, -1, NULL); - assert(mysrvc); - for (unsigned int k=0; kmysrvs->servers->index(k); - if (m == mysrvc) - cnt[k]++; - } - } - { - unsigned int tc = 0; - for (unsigned int k=0; kexecute("PRAGMA foreign_keys = OFF"); + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_1_0); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.1.0 of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v110 + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v110"); + // rename current table to add suffix _v110 + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v110"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,apply) SELECT rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,apply FROM mysql_query_rules_v110"); + } + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0a); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.0a of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v120a + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v120a"); + // rename current table to add suffix _v120a + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v120a"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,mirror_flagOUT,mirror_hostgroup,apply) SELECT rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,mirror_flagOUT,mirror_hostgroup,apply FROM mysql_query_rules_v120a"); + } + // upgrade related to issue #643 , adding comment in mysql_query_rules table + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0g); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.0g of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v120g + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v120g"); + // rename current table to add suffix _v120g + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v120g"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply FROM mysql_query_rules_v120g"); + } + // upgrade related to issue #643 , adding comment in mysql_query_rules table + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_2); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.2 of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v122 + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v122"); + // rename current table to add suffix _v122 + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v122"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply,comment FROM mysql_query_rules_v122"); + } + // upgrade related to issue #643 , adding comment in mysql_query_rules table + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_3_1); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.3.1 of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v131 + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v131"); + // rename current table to add suffix _v131 + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v131"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment FROM mysql_query_rules_v131"); + } + + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0a); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.4.0a of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v140a + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v140a"); + // rename current table to add suffix _v140a + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v40a"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment FROM mysql_query_rules_v140a"); + } + + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0b); + if (rci) { // note: upgrade from V1_4_0a or V1_4_0b is the same + // upgrade is required + proxy_warning("Detected version v1.4.0b of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v140b + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v140b"); + // rename current table to add suffix _v140b + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v140b"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment FROM mysql_query_rules_v140b"); + } + + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_1); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.4.1 of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v141 + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v141"); + // rename current table to add suffix _v141 + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v141"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,re_modifiers,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,sticky_conn,multiplex,log,apply,comment FROM mysql_query_rules_v141"); + } + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0a); + if (rci) { + // upgrade is required + proxy_warning("Detected version v2.0.0a of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v200a + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200a"); + // rename current table to add suffix _v200a + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200a"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200a"); + } + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0b); + if (rci) { + // upgrade is required + proxy_warning("Detected version v2.0.0b of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v200b + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200b"); + // rename current table to add suffix _v200b + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200b"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200b"); + } + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0c); + if (rci) { + // upgrade is required + proxy_warning("Detected version v2.0.0c of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v200c + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200c"); + // rename current table to add suffix _v200c + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200c"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200c"); + } + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0d); + if (rci) { + // upgrade is required + proxy_warning("Detected version v2.0.0d of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v200d + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200d"); + // rename current table to add suffix _v200d + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200d"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200d"); + } + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0e); + if (rci) { + // upgrade is required + proxy_warning("Detected version v2.1.0e of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v200e + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200e"); + // rename current table to add suffix _v200e + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200e"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200e"); + } + configdb->execute("PRAGMA foreign_keys = ON"); +} + +void ProxySQL_Admin::disk_upgrade_scheduler() { + // this function is called only for configdb table + // it is responsible to upgrade table scheduler if its structure is from a previous version + int rci; + configdb->execute("PRAGMA foreign_keys = OFF"); + rci=configdb->check_table_structure((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_0); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.0 of table scheduler\n"); + proxy_warning("ONLINE UPGRADE of table scheduler in progress\n"); + // drop any existing table with suffix _v120 + configdb->execute("DROP TABLE IF EXISTS scheduler_v120"); + // rename current table to add suffix _v120 + configdb->execute("ALTER TABLE scheduler RENAME TO scheduler_v120"); + // create new table + configdb->build_table((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER,false); + // copy fields from old table + configdb->execute("INSERT INTO scheduler (id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5) SELECT id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5 FROM scheduler_v120"); + } + rci=configdb->check_table_structure((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2a); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.2a of table scheduler\n"); + proxy_warning("ONLINE UPGRADE of table scheduler in progress\n"); + // drop any existing table with suffix _v122a + configdb->execute("DROP TABLE IF EXISTS scheduler_v122a"); + // rename current table to add suffix _v122a + configdb->execute("ALTER TABLE scheduler RENAME TO scheduler_v122a"); + // create new table + configdb->build_table((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER,false); + // copy fields from old table + configdb->execute("INSERT INTO scheduler (id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5,comment) SELECT id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5,comment FROM scheduler_v122a"); + } + rci=configdb->check_table_structure((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2b); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.2b of table scheduler\n"); + proxy_warning("ONLINE UPGRADE of table scheduler in progress\n"); + // drop any existing table with suffix _v122b + configdb->execute("DROP TABLE IF EXISTS scheduler_v122b"); + // rename current table to add suffix _v122b + configdb->execute("ALTER TABLE scheduler RENAME TO scheduler_v122b"); + // create new table + configdb->build_table((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER,false); + // copy fields from old table + configdb->execute("INSERT INTO scheduler (id,active,interval_ms,filename,arg1,arg2,arg3,arg4,arg5,comment) SELECT id,active,interval_ms,filename,arg1,arg2,arg3,arg4,arg5,comment FROM scheduler_v122b"); + } + + configdb->execute("PRAGMA foreign_keys = ON"); +} + +void ProxySQL_Admin::disk_upgrade_mysql_servers() { + // this function is called only for configdb table + // it is responsible to upgrade table mysql_servers if its structure is from a previous version + int rci; + configdb->execute("PRAGMA foreign_keys = OFF"); + rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_1_0); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.1.0 of table mysql_servers\n"); + proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); + // drop any existing table with suffix _v110 + configdb->execute("DROP TABLE IF EXISTS mysql_servers_v110"); + // rename current table to add suffix _v110 + configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v110"); + // create new table + configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); + // fix bug #1224 + configdb->execute("UPDATE mysql_servers_v110 SET weight = 10000000 WHERE weight > 10000000"); + // fix bug #962 + configdb->execute("UPDATE mysql_servers_v110 SET compression = 1 WHERE compression > 0"); + // copy fields from old table + configdb->execute("INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag FROM mysql_servers_v110"); + } + rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_0e); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.0 of table mysql_servers\n"); + proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); + // drop any existing table with suffix _v120 + configdb->execute("DROP TABLE IF EXISTS mysql_servers_v120"); + // rename current table to add suffix _v120 + configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v120"); + // create new table + configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); + // fix bug #1224 + configdb->execute("UPDATE mysql_servers_v120 SET weight = 10000000 WHERE weight > 10000000"); + // fix bug #962 + configdb->execute("UPDATE mysql_servers_v120 SET compression = 1 WHERE compression > 0"); + // copy fields from old table + configdb->execute("INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms FROM mysql_servers_v120"); + } + rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_2); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.2 of table mysql_servers\n"); + proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); + //drop any existing table with suffix _v122 + configdb->execute("DROP TABLE IF EXISTS mysql_servers_v122"); + // rename current table to add suffix _v122 + configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v122"); + // create new table + configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); + // fix bug #1224 + configdb->execute("UPDATE mysql_servers_v122 SET weight = 10000000 WHERE weight > 10000000"); + // fix bug #962 + configdb->execute("UPDATE mysql_servers_v122 SET compression = 1 WHERE compression > 0"); + // copy fields from old table + configdb->execute("INSERT OR IGNORE INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_v122"); + } + rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_4_4); // 1.4.4 has the same column of 1.2.2 + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.4.4 (pre-2.0.0) of table mysql_servers\n"); + proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); + //drop any existing table with suffix _v144 + configdb->execute("DROP TABLE IF EXISTS mysql_servers_v144"); + // rename current table to add suffix _v144 + configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v144"); + // create new table + configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); + // fix bug #1224 + configdb->execute("UPDATE mysql_servers_v144 SET weight = 10000000 WHERE weight > 10000000"); + // fix bug #962 + configdb->execute("UPDATE mysql_servers_v144 SET compression = 1 WHERE compression > 0"); + // copy fields from old table + configdb->execute("INSERT OR IGNORE INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_v144"); + } + rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0a); + if (rci) { + // upgrade is required + proxy_warning("Detected version 2.0.0a of table mysql_servers\n"); + proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); + //drop any existing table with suffix _v200a + configdb->execute("DROP TABLE IF EXISTS mysql_servers_v200a"); + // rename current table to add suffix _v200a + configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v200a"); + // create new table + configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); + // fix bug #1224 + configdb->execute("UPDATE mysql_servers_v200a SET weight = 10000000 WHERE weight > 10000000"); + // fix bug #962 + configdb->execute("UPDATE mysql_servers_v200a SET compression = 1 WHERE compression > 0"); + // copy fields from old table + configdb->execute("INSERT OR IGNORE INTO mysql_servers SELECT * FROM mysql_servers_v200a"); + } + rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0b); + if (rci) { + // upgrade is required + proxy_warning("Detected version 2.0.0b of table mysql_servers\n"); + proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); + //drop any existing table with suffix _v200b + configdb->execute("DROP TABLE IF EXISTS mysql_servers_v200b"); + // rename current table to add suffix _v200b + configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v200b"); + // create new table + configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); + // fix bug #1224 + configdb->execute("UPDATE mysql_servers_v200b SET weight = 10000000 WHERE weight > 10000000"); + // fix bug #962 + configdb->execute("UPDATE mysql_servers_v200b SET compression = 1 WHERE compression > 0"); + // copy fields from old table + configdb->execute("INSERT OR IGNORE INTO mysql_servers SELECT * FROM mysql_servers_v200b"); + } + rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V2_0_0c); + if (rci) { + // upgrade is required to fix issue #1923 + proxy_warning("Detected version 2.0.0c (pre-2.0.11) of table mysql_servers\n"); + proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); + //drop any existing table with suffix _v200c + configdb->execute("DROP TABLE IF EXISTS mysql_servers_v200c"); + // rename current table to add suffix _v200c + configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v200c"); + // create new table + configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); + configdb->execute("INSERT OR IGNORE INTO mysql_servers SELECT * FROM mysql_servers_v200c"); + } + rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_0); // issue #643 + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.0 of table mysql_replication_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); + // drop any existing table with suffix _v100 + configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v100"); + // rename current table to add suffix _v100 + configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v100"); + // create new table + configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_replication_hostgroups (writer_hostgroup,reader_hostgroup) SELECT writer_hostgroup , reader_hostgroup FROM mysql_replication_hostgroups_v100"); + } + rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_2_2); // issue #1304 + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.2 (pre-1.4.5) of table mysql_replication_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); + // drop any existing table with suffix _v122 + configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v122"); + // rename current table to add suffix _v122 + configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v122"); + // create new table + configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_replication_hostgroups (writer_hostgroup,reader_hostgroup,comment) SELECT writer_hostgroup , reader_hostgroup , COALESCE(comment,'') FROM mysql_replication_hostgroups_v122"); + } + rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_4_5); // issue #1304 + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.4.5 (pre-2.0.0) of table mysql_replication_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); + // drop any existing table with suffix _v145 + configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v145"); + // rename current table to add suffix _v145 + configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v145"); + // create new table + configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_replication_hostgroups (writer_hostgroup,reader_hostgroup,comment) SELECT writer_hostgroup , reader_hostgroup , comment FROM mysql_replication_hostgroups_v145"); + } + rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V2_0_0); // issue #2186 + if (rci) { + // upgrade is required + proxy_warning("Detected version v2.0.0 (pre-2.0.8) of table mysql_replication_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); + // drop any existing table with suffix _v200 + configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v200"); + // rename current table to add suffix _v200 + configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v200"); + // create new table + configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_replication_hostgroups SELECT * FROM mysql_replication_hostgroups_v200"); + } + + // upgrade mysql_group_replication_hostgroups + rci=configdb->check_table_structure((char *)"mysql_group_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS_V1_4); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.4 (pre-2.0.0) of mysql_group_replication_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_group_replication_hostgroups in progress\n"); + // drop any existing table with suffix _v14 + configdb->execute("DROP TABLE IF EXISTS mysql_group_replication_hostgroups_v14"); + // rename current table to add suffix _v14 + configdb->execute("ALTER TABLE mysql_group_replication_hostgroups RENAME TO mysql_group_replication_hostgroups_v14"); + // create new table + configdb->build_table((char *)"mysql_group_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_group_replication_hostgroups SELECT * FROM mysql_group_replication_hostgroups_v14"); + } + + + // upgrade mysql_galera_hostgroups + rci=configdb->check_table_structure((char *)"mysql_galera_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS_V2_0_0a); + if (rci) { + // upgrade is required + proxy_warning("Detected version v2.0.0a (pre-2.0.0b) of mysql_galera_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_galera_hostgroups in progress\n"); + // drop any existing table with suffix _v200a + configdb->execute("DROP TABLE IF EXISTS mysql_galera_hostgroups_v200a"); + // rename current table to add suffix _v200a + configdb->execute("ALTER TABLE mysql_galera_hostgroups RENAME TO mysql_galera_hostgroups_v200a"); + // create new table + configdb->build_table((char *)"mysql_galera_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_galera_hostgroups SELECT * FROM mysql_galera_hostgroups_v200a"); + } + + // upgrade mysql_aws_aurora_hostgroups + rci=configdb->check_table_structure((char *)"mysql_aws_aurora_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS_V2_0_8); + if (rci) { + // upgrade is required + proxy_warning("Detected version pre-v2.0.9 of mysql_aws_aurora_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_aws_aurora_hostgroups in progress\n"); + // drop mysql_aws_aurora_hostgroups table with suffix _v208 + configdb->execute("DROP TABLE IF EXISTS mysql_aws_aurora_hostgroups_v208"); + // rename current table to add suffix _v208 + configdb->execute("ALTER TABLE mysql_aws_aurora_hostgroups RENAME TO mysql_aws_aurora_hostgroups_v208"); + // create new table + configdb->build_table((char *)"mysql_aws_aurora_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_AWS_AURORA_HOSTGROUPS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, " + "max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, comment) " + "SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, " + "check_timeout_ms, writer_is_also_reader, new_reader_weight, comment FROM mysql_aws_aurora_hostgroups_v208"); + } + + // upgrade mysql_hostgroup_attributes + rci=configdb->check_table_structure((char *)"mysql_hostgroup_attributes",(char *)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0); + if (rci) { + // upgrade is required + proxy_warning("Detected version pre-v2.5.2 of mysql_hostgroup_attributes\n"); + proxy_warning("ONLINE UPGRADE of table mysql_hostgroup_attributes in progress\n"); + // drop mysql_hostgroup_attributes table with suffix _v250 + configdb->execute("DROP TABLE IF EXISTS mysql_hostgroup_attributes_v250"); + // rename current table to add suffix _v250 + configdb->execute("ALTER TABLE mysql_hostgroup_attributes RENAME TO mysql_hostgroup_attributes_v250"); + // create new table + configdb->build_table((char *)"mysql_hostgroup_attributes",(char *)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES,false); + // copy fields from old table + configdb->execute( + "INSERT INTO mysql_hostgroup_attributes (" + " hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex," + " connection_warming, throttle_connections_per_sec, ignore_session_variables, comment" + ") SELECT" + " hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex," + " connection_warming, throttle_connections_per_sec, ignore_session_variables, comment" + " FROM mysql_hostgroup_attributes_v250" + ); + } + rci = configdb->check_table_structure((char*)"mysql_hostgroup_attributes", (char*)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2); + if (rci) { + // upgrade is required + proxy_warning("Detected version pre-v2.6.0 of mysql_hostgroup_attributes\n"); + proxy_warning("ONLINE UPGRADE of table mysql_hostgroup_attributes in progress\n"); + // drop mysql_hostgroup_attributes table with suffix _v252 + configdb->execute("DROP TABLE IF EXISTS mysql_hostgroup_attributes_v252"); + // rename current table to add suffix _v252 + configdb->execute("ALTER TABLE mysql_hostgroup_attributes RENAME TO mysql_hostgroup_attributes_v252"); + // create new table + configdb->build_table((char*)"mysql_hostgroup_attributes", (char*)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES, false); + // copy fields from old table + configdb->execute( + "INSERT INTO mysql_hostgroup_attributes (" + " hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex," + " connection_warming, throttle_connections_per_sec, ignore_session_variables, servers_defaults, comment" + ") SELECT" + " hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex," + " connection_warming, throttle_connections_per_sec, ignore_session_variables, servers_defaults, comment" + " FROM mysql_hostgroup_attributes_v252" + ); + } + configdb->execute("PRAGMA foreign_keys = ON"); + +} + + +void ProxySQL_Admin::disk_upgrade_mysql_users() { + // this function is called only for configdb table + // it is responsible to upgrade table mysql_users if its structure is from a previous version + int rci; + configdb->execute("PRAGMA foreign_keys = OFF"); + // change transaction_persistent=1 by default . See #793 + rci=configdb->check_table_structure((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS_V1_3_0); + if (rci) { + // upgrade is required + proxy_warning("Detected version pre-1.4 of table mysql_users\n"); + proxy_warning("ONLINE UPGRADE of table mysql_users in progress\n"); + // drop any existing table with suffix _v130 + configdb->execute("DROP TABLE IF EXISTS mysql_users_v130"); + // rename current table to add suffix _v130 + configdb->execute("ALTER TABLE mysql_users RENAME TO mysql_users_v130"); + // create new table + configdb->build_table((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) SELECT * FROM mysql_users_v130"); + } + // adding mysql_users.commment . See #1633 + rci=configdb->check_table_structure((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS_V1_4_0); + if (rci) { + // upgrade is required + proxy_warning("Detected version pre-2.0 of table mysql_users\n"); + proxy_warning("ONLINE UPGRADE of table mysql_users in progress\n"); + // drop any existing table with suffix _v140 + configdb->execute("DROP TABLE IF EXISTS mysql_users_v140"); + // rename current table to add suffix _v140 + configdb->execute("ALTER TABLE mysql_users RENAME TO mysql_users_v140"); + // create new table + configdb->build_table((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) SELECT * FROM mysql_users_v140"); + } + // adding mysql_users.attributes. See #3083 + rci=configdb->check_table_structure((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS_V2_0_0); + if (rci) { + // upgrade is required + proxy_warning("Detected version pre-2.1.0 of table mysql_users\n"); + proxy_warning("ONLINE UPGRADE of table mysql_users in progress\n"); + // drop any existing table with suffix _v210 + configdb->execute("DROP TABLE IF EXISTS mysql_users_v200"); + // rename current table to add suffix _v210 + configdb->execute("ALTER TABLE mysql_users RENAME TO mysql_users_v200"); + // create new table + configdb->build_table((char *)"mysql_users",(char *)ADMIN_SQLITE_TABLE_MYSQL_USERS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections,comment) SELECT * FROM mysql_users_v200"); + } + configdb->execute("PRAGMA foreign_keys = ON"); +} + + +void ProxySQL_Admin::disk_upgrade_rest_api_routes() { + int rci; + configdb->execute("PRAGMA foreign_keys = OFF"); + + rci=configdb->check_table_structure((char *)"restapi_routes",(char *)ADMIN_SQLITE_TABLE_RESTAPI_ROUTES_V2_0_15); + if (rci) { + // upgrade is required + proxy_warning("Detected version pre-2.1.0 of table restapi_routes\n"); + proxy_warning("ONLINE UPGRADE of table restapi_routes in progress\n"); + // drop any existing table with suffix _v2015 + configdb->execute("DROP TABLE IF EXISTS restapi_routes_v2015"); + // rename current table to add suffix _v2015 + configdb->execute("ALTER TABLE restapi_routes RENAME TO restapi_routes_v2015"); + // create new table + configdb->build_table((char *)"restapi_routes",(char *)ADMIN_SQLITE_TABLE_RESTAPI_ROUTES,false); + // copy fields from old table + configdb->execute("INSERT INTO restapi_routes(id,active,timeout_ms,method,uri,script,comment) SELECT id,active,interval_ms,method,uri,script,comment FROM restapi_routes_v2015"); + } + + configdb->execute("PRAGMA foreign_keys = ON"); +} diff --git a/lib/ProxySQL_Admin_Scheduler.cpp b/lib/ProxySQL_Admin_Scheduler.cpp new file mode 100644 index 0000000000..c03ce74088 --- /dev/null +++ b/lib/ProxySQL_Admin_Scheduler.cpp @@ -0,0 +1,215 @@ +#include // std::cout +#include // std::stringstream +#include +#include // std::sort +#include +#include // std::vector +#include +#include "MySQL_HostGroups_Manager.h" +#include "proxysql_admin.h" + +Scheduler_Row::Scheduler_Row(unsigned int _id, bool _is_active, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5, char *_comment) { + int i; + id=_id; + is_active=_is_active; + interval_ms=_in; + filename=strdup(_f); + args=(char **)malloc(6*sizeof(char *)); + for (i=0;i<6;i++) { + args[i]=NULL; + } + // only copy fields if the previous one is not null + if (a1) { + args[0]=strdup(a1); + if (a2) { + args[1]=strdup(a2); + if (a3) { + args[2]=strdup(a3); + if (a4) { + args[3]=strdup(a4); + if (a5) { + args[4]=strdup(a5); + } + } + } + } + } + comment=strdup(_comment); +} + +Scheduler_Row::~Scheduler_Row() { + int i; + for (i=0;i<6;i++) { + if (args[i]) { + free(args[i]); + } + args[i]=NULL; + } + if (filename) { + free(filename); + } + free(args); + free(comment); + args=NULL; +} + +ProxySQL_External_Scheduler::ProxySQL_External_Scheduler() { +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_init(&rwlock,NULL); +#else + spinlock_rwlock_init(&rwlock); +#endif + last_version=0; + version=0; + next_run=0; +} + +ProxySQL_External_Scheduler::~ProxySQL_External_Scheduler() { +} + +void ProxySQL_External_Scheduler::update_table(SQLite3_result *resultset) { +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_wrlock(&rwlock); +#else + spin_wrlock(&rwlock); +#endif + // delete all current rows + Scheduler_Row *sr; + for (std::vector::iterator it=Scheduler_Rows.begin(); it!=Scheduler_Rows.end(); ++it) { + sr=*it; + delete sr; + } + Scheduler_Rows.clear(); + + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + unsigned int id=strtoul(r->fields[0], NULL, 10); + bool is_active=false; + if (atoi(r->fields[1])) { + is_active=true; + } + unsigned int interval_ms=strtoul(r->fields[2], NULL, 10); + Scheduler_Row *sr=new Scheduler_Row(id, is_active, interval_ms, + r->fields[3], + r->fields[4], r->fields[5], + r->fields[6], r->fields[7], + r->fields[8], + r->fields[9] // comment, issue #643 + ); + Scheduler_Rows.push_back(sr); + } + // increase version + __sync_fetch_and_add(&version,1); + // unlock +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_unlock(&rwlock); +#else + spin_wrunlock(&rwlock); +#endif +} + +// this fuction will be called as a deatached thread +static void * waitpid_thread(void *arg) { + pid_t *cpid_ptr=(pid_t *)arg; + int status; + waitpid(*cpid_ptr, &status, 0); + free(cpid_ptr); + return NULL; +} + +unsigned long long ProxySQL_External_Scheduler::run_once() { + Scheduler_Row *sr=NULL; + unsigned long long curtime=monotonic_time(); + curtime=curtime/1000; +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_rdlock(&rwlock); +#else + spin_rdlock(&rwlock); +#endif + if (__sync_add_and_fetch(&version,0) > last_version) { // version was changed + next_run=0; + last_version=version; + for (std::vector::iterator it=Scheduler_Rows.begin(); it!=Scheduler_Rows.end(); ++it) { + sr=*it; + if (sr->is_active==false) { + continue; + } + sr->next=curtime+sr->interval_ms; + if (next_run==0) { + next_run=sr->next; + } else { + if (sr->next < next_run) { // we try to find the first event that needs to be executed + next_run=sr->next; + } + } + } + } + if (curtime >= next_run) { + next_run=0; + for (std::vector::iterator it=Scheduler_Rows.begin(); it!=Scheduler_Rows.end(); ++it) { + sr=*it; + if (sr->is_active==false) { + continue; + } + if (curtime >= sr->next) { + // the event is scheduled for execution + sr->next=curtime+sr->interval_ms; + char **newargs=(char **)malloc(7*sizeof(char *)); + for (int i=1;i<7;i++) { + newargs[i]=sr->args[i-1]; + } + newargs[0]=sr->filename; + proxy_info("Scheduler starting id: %u , filename: %s\n", sr->id, sr->filename); + pid_t cpid; + cpid = fork(); + if (cpid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + if (cpid == 0) { + close_all_non_term_fd({}); + char *newenviron[] = { NULL }; + int rc; + rc=execve(sr->filename, newargs, newenviron); + if (rc) { + proxy_error("Scheduler: Failed to run %s\n", sr->filename); + perror("execve()"); + exit(EXIT_FAILURE); + } + } else { + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + pthread_attr_setstacksize (&attr, 64*1024); + pid_t *cpid_ptr=(pid_t *)malloc(sizeof(pid_t)); + *cpid_ptr=cpid; + pthread_t thr; + if (pthread_create(&thr, &attr, waitpid_thread, (void *)cpid_ptr) !=0 ) { + perror("Thread creation"); + exit(EXIT_FAILURE); + } + } + free(newargs); + } + if (next_run==0) { + next_run=sr->next; + } else { + if (sr->next < next_run) { // we try to find the first event that needs to be executed + next_run=sr->next; + } + } + } + } + // find the smaller next_run + for (std::vector::iterator it=Scheduler_Rows.begin(); it!=Scheduler_Rows.end(); ++it) { + sr=*it; + if (next_run==0) { + } + } +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_unlock(&rwlock); +#else + spin_rdunlock(&rwlock); +#endif + return next_run; +} diff --git a/lib/ProxySQL_Admin_Stats.cpp b/lib/ProxySQL_Admin_Stats.cpp new file mode 100644 index 0000000000..4e9ceb3301 --- /dev/null +++ b/lib/ProxySQL_Admin_Stats.cpp @@ -0,0 +1,2342 @@ +#include // std::cout +#include // std::stringstream +#include +#include // std::sort +#include +#include // std::vector +#include +#include "cpp.h" + +#include "MySQL_Authentication.hpp" +#include "PgSQL_Authentication.h" +#include "MySQL_LDAP_Authentication.hpp" +#include "MySQL_PreparedStatement.h" +#include "ProxySQL_Cluster.hpp" + +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" + +#define SAFE_SQLITE3_STEP(_stmt) do {\ + do {\ + rc=(*proxy_sqlite3_step)(_stmt);\ + if (rc!=SQLITE_DONE) {\ + assert(rc==SQLITE_LOCKED);\ + usleep(100);\ + }\ + } while (rc!=SQLITE_DONE);\ +} while (0) + +extern bool admin_proxysql_mysql_paused; +extern bool admin_proxysql_pgsql_paused; +extern MySQL_Authentication *GloMyAuth; +extern PgSQL_Authentication* GloPgAuth; +extern MySQL_LDAP_Authentication *GloMyLdapAuth; +extern Query_Cache *GloQC; +extern ProxySQL_Admin *GloAdmin; +extern MySQL_Threads_Handler *GloMTH; +extern PgSQL_Threads_Handler* GloPTH; +extern MySQL_STMT_Manager_v14 *GloMyStmt; +extern MySQL_Query_Processor* GloMyQPro; +extern PgSQL_Query_Processor* GloPgQPro; +extern ProxySQL_Cluster *GloProxyCluster; + +void ProxySQL_Admin::p_update_metrics() { + // Update proxysql_uptime + auto t1 = monotonic_time(); + auto new_uptime = (t1 - GloVars.global.start_time)/1000/1000; + auto cur_uptime = this->metrics.p_counter_array[p_admin_counter::uptime]->Value(); + this->metrics.p_counter_array[p_admin_counter::uptime]->Increment(new_uptime - cur_uptime); + + // Update memory metrics + this->p_stats___memory_metrics(); + // Update stmt metrics + this->p_update_stmt_metrics(); + + // updated mysql_listener_paused + int st = ( admin_proxysql_mysql_paused == true ? 1 : 0); + this->metrics.p_gauge_array[p_admin_gauge::mysql_listener_paused]->Set(st); + + // updated pgsql_listener_paused + st = ( admin_proxysql_pgsql_paused == true ? 1 : 0); + this->metrics.p_gauge_array[p_admin_gauge::pgsql_listener_paused]->Set(st); +} + + + +/** + * @brief Gets the number of currently opened file descriptors. In case of error '-1' is + * returned and error is logged. + * @return On success, the number of currently opened file descriptors, '-1' otherwise. + */ +int32_t get_open_fds() { + DIR* dir = opendir("/proc/self/fd"); + if (dir == NULL) { + proxy_error("'opendir()' failed with error: '%d'\n", errno); + return -1; + } + + struct dirent* dp = nullptr; + int32_t count = -3; + + while ((dp = readdir(dir)) != NULL) { + count++; + } + + closedir(dir); + + return count; +} + + +void ProxySQL_Admin::p_stats___memory_metrics() { + if (!GloMTH) return; + + // Check that last execution is older than the specified interval + unsigned long long new_ts = monotonic_time() / 1000 / 1000; + if (new_ts < last_p_memory_metrics_ts + variables.p_memory_metrics_interval) { + return; + } + // Update the 'memory_metrics' last exec timestamp + last_p_memory_metrics_ts = new_ts; + + // proxysql_connpool_memory_bytes metric + const auto connpool_mem = MyHGM->Get_Memory_Stats(); + this->metrics.p_gauge_array[p_admin_gauge::connpool_memory_bytes]->Set(connpool_mem); + + // proxysql_sqlite3_memory_bytes metric + int highwater = 0; + int current = 0; + (*proxy_sqlite3_status)(SQLITE_STATUS_MEMORY_USED, ¤t, &highwater, 0); + this->metrics.p_gauge_array[p_admin_gauge::sqlite3_memory_bytes]->Set(current); + + // proxysql_jemalloc_* memory metrics + // =============================================================== + size_t + allocated = 0, + resident = 0, + active = 0, + mapped = 0, + metadata = 0, + retained = 0, + sz = sizeof(size_t); + +#ifndef NOJEM + mallctl("stats.resident", &resident, &sz, NULL, 0); + mallctl("stats.active", &active, &sz, NULL, 0); + mallctl("stats.allocated", &allocated, &sz, NULL, 0); + mallctl("stats.mapped", &mapped, &sz, NULL, 0); + mallctl("stats.metadata", &metadata, &sz, NULL, 0); + mallctl("stats.retained", &retained, &sz, NULL, 0); +#endif // NOJEM + + this->metrics.p_gauge_array[p_admin_gauge::jemalloc_resident]->Set(resident); + this->metrics.p_gauge_array[p_admin_gauge::jemalloc_active]->Set(active); + const auto cur_allocated = this->metrics.p_counter_array[p_admin_counter::jemalloc_allocated]->Value(); + this->metrics.p_counter_array[p_admin_counter::jemalloc_allocated]->Increment(allocated - cur_allocated); + this->metrics.p_gauge_array[p_admin_gauge::jemalloc_mapped]->Set(mapped); + this->metrics.p_gauge_array[p_admin_gauge::jemalloc_metadata]->Set(metadata); + this->metrics.p_gauge_array[p_admin_gauge::jemalloc_retained]->Set(retained); + + // =============================================================== + + // proxysql_auth_memory metric + unsigned long mu = GloMyAuth->memory_usage(); + this->metrics.p_gauge_array[p_admin_gauge::auth_memory_bytes]->Set(mu); + + // proxysql_query_digest_memory metric + const auto& query_digest_t_size = GloMyQPro->get_query_digests_total_size(); + this->metrics.p_gauge_array[p_admin_gauge::query_digest_memory_bytes]->Set(query_digest_t_size); + + // mysql_query_rules_memory metric + const auto& rules_mem_used = GloMyQPro->get_rules_mem_used(); + this->metrics.p_gauge_array[p_admin_gauge::mysql_query_rules_memory_bytes]->Set(rules_mem_used); + + // mysql_firewall_users_table metric + const auto& firewall_users_table = GloMyQPro->get_firewall_memory_users_table(); + this->metrics.p_gauge_array[p_admin_gauge::mysql_firewall_users_table]->Set(firewall_users_table); + + // mysql_firewall_users_config metric + const auto& firewall_users_config = GloMyQPro->get_firewall_memory_users_config(); + this->metrics.p_gauge_array[p_admin_gauge::mysql_firewall_users_config]->Set(firewall_users_config); + + // mysql_firewall_rules_table metric + const auto& firewall_rules_table = GloMyQPro->get_firewall_memory_rules_table(); + this->metrics.p_gauge_array[p_admin_gauge::mysql_firewall_rules_table]->Set(firewall_rules_table); + + // mysql_firewall_rules_table metric + const auto& firewall_rules_config = GloMyQPro->get_firewall_memory_rules_config(); + this->metrics.p_gauge_array[p_admin_gauge::mysql_firewall_rules_config]->Set(firewall_rules_config); + + // proxysql_stack_memory_mysql_threads + const auto& stack_memory_mysql_threads = + __sync_fetch_and_add(&GloVars.statuses.stack_memory_mysql_threads, 0); + this->metrics.p_gauge_array[p_admin_gauge::stack_memory_mysql_threads]->Set(stack_memory_mysql_threads); + + // proxysql_stack_memory_admin_threads + const auto& stack_memory_admin_threads = + __sync_fetch_and_add(&GloVars.statuses.stack_memory_admin_threads, 0); + this->metrics.p_gauge_array[p_admin_gauge::stack_memory_admin_threads]->Set(stack_memory_admin_threads); + + // proxysql_stack_memory_cluster_threads + const auto& stack_memory_cluster_threads = + __sync_fetch_and_add(&GloVars.statuses.stack_memory_cluster_threads, 0); + this->metrics.p_gauge_array[p_admin_gauge::stack_memory_cluster_threads]->Set(stack_memory_cluster_threads); + + // proxysql_prepare_statement_memory metric + uint64_t prepare_stmt_metadata_mem_used; + uint64_t prepare_stmt_backend_mem_used; + GloMyStmt->get_memory_usage(prepare_stmt_metadata_mem_used, prepare_stmt_backend_mem_used); + this->metrics.p_gauge_array[p_admin_gauge::prepare_stmt_metadata_memory_bytes]->Set(prepare_stmt_metadata_mem_used); + this->metrics.p_gauge_array[p_admin_gauge::prepare_stmt_backend_memory_bytes]->Set(prepare_stmt_backend_mem_used); + + // Update opened file descriptors + int32_t cur_fds = get_open_fds(); + if (cur_fds != -1) { + this->metrics.p_gauge_array[p_admin_gauge::fds_in_use]->Set(cur_fds); + } +} + + +void ProxySQL_Admin::stats___memory_metrics() { + if (!GloMTH) return; + SQLite3_result * resultset = NULL; + + int highwater; + int current; + char bu[32]; + char *vn=NULL; + char *query=NULL; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_memory_metrics"); + char *a=(char *)"INSERT INTO stats_memory_metrics VALUES (\"%s\",\"%s\")"; + if (resultset) { + delete resultset; + resultset=NULL; + } + (*proxy_sqlite3_status)(SQLITE_STATUS_MEMORY_USED, ¤t, &highwater, 0); + vn=(char *)"SQLite3_memory_bytes"; + sprintf(bu,"%d",current); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); +#ifndef NOJEM + { + uint64_t epoch = 1; + size_t allocated = 0, resident = 0, active = 0, mapped = 0 , metadata = 0, retained = 0 , sz = sizeof(size_t); + mallctl("epoch", &epoch, &sz, &epoch, sz); + mallctl("stats.resident", &resident, &sz, NULL, 0); + mallctl("stats.active", &active, &sz, NULL, 0); + mallctl("stats.allocated", &allocated, &sz, NULL, 0); + mallctl("stats.mapped", &mapped, &sz, NULL, 0); + mallctl("stats.metadata", &metadata, &sz, NULL, 0); + mallctl("stats.retained", &retained, &sz, NULL, 0); +// float frag_pct = ((float)active / allocated)*100 - 100; +// size_t frag_bytes = active - allocated; +// float rss_pct = ((float)resident / allocated)*100 - 100; +// size_t rss_bytes = resident - allocated; +// float metadata_pct = ((float)metadata / resident)*100; + vn=(char *)"jemalloc_resident"; + sprintf(bu,"%lu",resident); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"jemalloc_active"; + sprintf(bu,"%lu",active); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"jemalloc_allocated"; + sprintf(bu,"%lu",allocated); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"jemalloc_mapped"; + sprintf(bu,"%lu",mapped); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"jemalloc_metadata"; + sprintf(bu,"%lu",metadata); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"jemalloc_retained"; + sprintf(bu,"%lu",retained); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } +#endif + { + if (GloMyAuth) { + unsigned long mu = GloMyAuth->memory_usage(); + vn=(char *)"Auth_memory"; + sprintf(bu,"%lu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } + } + { + if (GloMyQPro) { + unsigned long long mu = GloMyQPro->get_query_digests_total_size(); + vn=(char *)"mysql_query_digest_memory"; + sprintf(bu,"%llu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } + if (GloMyQPro) { + unsigned long long mu = GloMyQPro->get_rules_mem_used(); + vn=(char *)"mysql_query_rules_memory"; + sprintf(bu,"%llu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } + if (GloPgQPro) { + unsigned long long mu = GloPgQPro->get_query_digests_total_size(); + vn = (char*)"pgsql_query_digest_memory"; + sprintf(bu, "%llu", mu); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + } + if (GloPgQPro) { + unsigned long long mu = GloPgQPro->get_rules_mem_used(); + vn = (char*)"pgsql_query_rules_memory"; + sprintf(bu, "%llu", mu); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + } + if (GloMyStmt) { + uint64_t prep_stmt_metadata_mem_usage; + uint64_t prep_stmt_backend_mem_usage; + GloMyStmt->get_memory_usage(prep_stmt_metadata_mem_usage, prep_stmt_backend_mem_usage); + vn = (char*)"prepare_statement_metadata_memory"; + sprintf(bu, "%lu", prep_stmt_metadata_mem_usage); + query=(char*)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + vn = (char*)"prepare_statement_backend_memory"; + sprintf(bu, "%lu", prep_stmt_backend_mem_usage); + query=(char*)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + } + if (GloMyQPro) { + unsigned long long mu = 0; + mu = GloMyQPro->get_firewall_memory_users_table(); + vn=(char *)"mysql_firewall_users_table"; + sprintf(bu,"%llu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + mu = GloMyQPro->get_firewall_memory_users_config(); + vn=(char *)"mysql_firewall_users_config"; + sprintf(bu,"%llu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + mu = GloMyQPro->get_firewall_memory_rules_table(); + vn=(char *)"mysql_firewall_rules_table"; + sprintf(bu,"%llu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + mu = GloMyQPro->get_firewall_memory_rules_config(); + vn=(char *)"mysql_firewall_rules_config"; + sprintf(bu,"%llu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } + } + { + unsigned long mu; + mu = __sync_fetch_and_add(&GloVars.statuses.stack_memory_mysql_threads,0); + vn=(char *)"stack_memory_mysql_threads"; + sprintf(bu,"%lu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + mu = __sync_fetch_and_add(&GloVars.statuses.stack_memory_admin_threads,0); + vn=(char *)"stack_memory_admin_threads"; + sprintf(bu,"%lu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + mu = __sync_fetch_and_add(&GloVars.statuses.stack_memory_cluster_threads,0); + vn=(char *)"stack_memory_cluster_threads"; + sprintf(bu,"%lu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } + statsdb->execute("COMMIT"); +} + +void ProxySQL_Admin::p_update_stmt_metrics() { + if (GloMyStmt) { + uint64_t stmt_client_active_unique { 0 }; + uint64_t stmt_client_active_total { 0 }; + uint64_t stmt_max_stmt_id { 0 }; + uint64_t stmt_cached { 0 }; + uint64_t stmt_server_active_unique { 0 }; + uint64_t stmt_server_active_total { 0 }; + GloMyStmt->get_metrics( + &stmt_client_active_unique, + &stmt_client_active_total, + &stmt_max_stmt_id, + &stmt_cached, + &stmt_server_active_unique, + &stmt_server_active_total + ); + + this->metrics.p_gauge_array[p_admin_gauge::stmt_client_active_total]->Set(stmt_client_active_total); + this->metrics.p_gauge_array[p_admin_gauge::stmt_client_active_unique]->Set(stmt_client_active_unique); + + this->metrics.p_gauge_array[p_admin_gauge::stmt_server_active_total]->Set(stmt_server_active_total); + this->metrics.p_gauge_array[p_admin_gauge::stmt_server_active_unique]->Set(stmt_server_active_unique); + + this->metrics.p_gauge_array[p_admin_gauge::stmt_max_stmt_id]->Set(stmt_max_stmt_id); + this->metrics.p_gauge_array[p_admin_gauge::stmt_cached]->Set(stmt_cached); + } +} + +void ProxySQL_Admin::stats___mysql_global() { + if (!GloMTH) return; + SQLite3_result * resultset=GloMTH->SQL3_GlobalStatus(true); + if (resultset==NULL) return; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_mysql_global"); + char *a=(char *)"INSERT INTO stats_mysql_global VALUES (\"%s\",\"%s\")"; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<2; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset=NULL; + + resultset=MyHGM->SQL3_Get_ConnPool_Stats(); + if (resultset) { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<2; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset=NULL; + } + + int highwater; + int current; + (*proxy_sqlite3_status)(SQLITE_STATUS_MEMORY_USED, ¤t, &highwater, 0); + char bu[32]; + char *vn=NULL; + char *query=NULL; + vn=(char *)"SQLite3_memory_bytes"; + sprintf(bu,"%d",current); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + + unsigned long long connpool_mem=MyHGM->Get_Memory_Stats(); + vn=(char *)"ConnPool_memory_bytes"; + sprintf(bu,"%llu",connpool_mem); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + + if (GloMyStmt) { + uint64_t stmt_client_active_unique = 0; + uint64_t stmt_client_active_total = 0; + uint64_t stmt_max_stmt_id = 0; + uint64_t stmt_cached = 0; + uint64_t stmt_server_active_unique = 0; + uint64_t stmt_server_active_total = 0; + GloMyStmt->get_metrics(&stmt_client_active_unique,&stmt_client_active_total,&stmt_max_stmt_id,&stmt_cached,&stmt_server_active_unique,&stmt_server_active_total); + vn=(char *)"Stmt_Client_Active_Total"; + sprintf(bu,"%lu",stmt_client_active_total); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"Stmt_Client_Active_Unique"; + sprintf(bu,"%lu",stmt_client_active_unique); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"Stmt_Server_Active_Total"; + sprintf(bu,"%lu",stmt_server_active_total); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"Stmt_Server_Active_Unique"; + sprintf(bu,"%lu",stmt_server_active_unique); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"Stmt_Max_Stmt_id"; + sprintf(bu,"%lu",stmt_max_stmt_id); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + vn=(char *)"Stmt_Cached"; + sprintf(bu,"%lu",stmt_cached); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } + + if (GloQC && (resultset=GloQC->SQL3_getStats())) { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<2; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset=NULL; + } + + if (GloMyLdapAuth) { + resultset=GloMyLdapAuth->SQL3_getStats(); + if (resultset) { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<2; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset=NULL; + } + } + + if (GloMyQPro) { + unsigned long long mu = GloMyQPro->get_new_req_conns_count(); + vn=(char *)"new_req_conns_count"; + sprintf(bu,"%llu",mu); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } + { + vn=(char *)"mysql_listener_paused"; + sprintf(bu, "%s", ( admin_proxysql_mysql_paused==true ? "true" : "false") ); + query=(char *)malloc(strlen(a)+strlen(vn)+strlen(bu)+16); + sprintf(query,a,vn,bu); + statsdb->execute(query); + free(query); + } + statsdb->execute("COMMIT"); +} + +void ProxySQL_Admin::stats___pgsql_global() { + if (!GloPTH) return; + SQLite3_result* resultset = GloPTH->SQL3_GlobalStatus(true); + if (resultset == NULL) return; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_pgsql_global"); + char* a = (char*)"INSERT INTO stats_pgsql_global VALUES (\"%s\",\"%s\")"; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + int arg_len = 0; + for (int i = 0; i < 2; i++) { + arg_len += strlen(r->fields[i]); + } + char* query = (char*)malloc(strlen(a) + arg_len + 32); + sprintf(query, a, r->fields[0], r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset = NULL; + + resultset = PgHGM->SQL3_Get_ConnPool_Stats(); + if (resultset) { + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + int arg_len = 0; + for (int i = 0; i < 2; i++) { + arg_len += strlen(r->fields[i]); + } + char* query = (char*)malloc(strlen(a) + arg_len + 32); + sprintf(query, a, r->fields[0], r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset = NULL; + } + + int highwater; + int current; + (*proxy_sqlite3_status)(SQLITE_STATUS_MEMORY_USED, ¤t, &highwater, 0); + char bu[32]; + char* vn = NULL; + char* query = NULL; + vn = (char*)"SQLite3_memory_bytes"; + sprintf(bu, "%d", current); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + + unsigned long long connpool_mem = PgHGM->Get_Memory_Stats(); + vn = (char*)"ConnPool_memory_bytes"; + sprintf(bu, "%llu", connpool_mem); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + + /*if (GloMyStmt) { + uint64_t stmt_client_active_unique = 0; + uint64_t stmt_client_active_total = 0; + uint64_t stmt_max_stmt_id = 0; + uint64_t stmt_cached = 0; + uint64_t stmt_server_active_unique = 0; + uint64_t stmt_server_active_total = 0; + GloMyStmt->get_metrics(&stmt_client_active_unique, &stmt_client_active_total, &stmt_max_stmt_id, &stmt_cached, &stmt_server_active_unique, &stmt_server_active_total); + vn = (char*)"Stmt_Client_Active_Total"; + sprintf(bu, "%lu", stmt_client_active_total); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + vn = (char*)"Stmt_Client_Active_Unique"; + sprintf(bu, "%lu", stmt_client_active_unique); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + vn = (char*)"Stmt_Server_Active_Total"; + sprintf(bu, "%lu", stmt_server_active_total); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + vn = (char*)"Stmt_Server_Active_Unique"; + sprintf(bu, "%lu", stmt_server_active_unique); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + vn = (char*)"Stmt_Max_Stmt_id"; + sprintf(bu, "%lu", stmt_max_stmt_id); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + vn = (char*)"Stmt_Cached"; + sprintf(bu, "%lu", stmt_cached); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + }*/ + + if (GloQC && (resultset = GloQC->SQL3_getStats())) { + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + int arg_len = 0; + for (int i = 0; i < 2; i++) { + arg_len += strlen(r->fields[i]); + } + char* query = (char*)malloc(strlen(a) + arg_len + 32); + sprintf(query, a, r->fields[0], r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset = NULL; + } + + /*if (GloMyLdapAuth) { + resultset = GloMyLdapAuth->SQL3_getStats(); + if (resultset) { + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + int arg_len = 0; + for (int i = 0; i < 2; i++) { + arg_len += strlen(r->fields[i]); + } + char* query = (char*)malloc(strlen(a) + arg_len + 32); + sprintf(query, a, r->fields[0], r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset = NULL; + } + }*/ + + if (GloPgQPro) { + unsigned long long mu = GloPgQPro->get_new_req_conns_count(); + vn = (char*)"new_req_conns_count"; + sprintf(bu, "%llu", mu); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + } + { + vn = (char*)"pgsql_listener_paused"; + sprintf(bu, "%s", (admin_proxysql_pgsql_paused == true ? "true" : "false")); + query = (char*)malloc(strlen(a) + strlen(vn) + strlen(bu) + 16); + sprintf(query, a, vn, bu); + statsdb->execute(query); + free(query); + } + statsdb->execute("COMMIT"); +} + + +void ProxySQL_Admin::stats___mysql_processlist() { + int rc; + if (!GloMTH) return; + mysql_thread___show_processlist_extended = variables.mysql_show_processlist_extended; + SQLite3_result * resultset=GloMTH->SQL3_Processlist(); + if (resultset==NULL) return; + + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char *query1=NULL; + char *query32=NULL; + std::string query32s = ""; + + query1 = (char *)"INSERT OR IGNORE INTO stats_mysql_processlist VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)"; + query32s = "INSERT OR IGNORE INTO stats_mysql_processlist VALUES " + generate_multi_rows_query(32,16); + query32 = (char *)query32s.c_str(); + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + +/* for reference +CREATE TABLE stats_mysql_processlist ( + ThreadID INT NOT NULL, + SessionID INTEGER PRIMARY KEY, + user VARCHAR, + db VARCHAR, + cli_host VARCHAR, + cli_port INT, + hostgroup INT, + l_srv_host VARCHAR, + l_srv_port INT, + srv_host VARCHAR, + srv_port INT, + command VARCHAR, + time_ms INT NOT NULL, + info VARCHAR, + status_flags INT, + extended_info VARCHAR) +*/ + + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_mysql_processlist"); + + int row_idx=0; + int max_bulk_row_idx=resultset->rows_count/32; + max_bulk_row_idx=max_bulk_row_idx*32; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + int idx=row_idx%32; + if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, statsdb); // ThreadID + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // SessionID + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // db + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // cli_host + if (r1->fields[5]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // cli_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+6); ASSERT_SQLITE_OK(rc, statsdb); + } + if (r1->fields[6]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+8); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_host + if (r1->fields[8]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+9); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+10, r1->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host + if (r1->fields[10]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+11); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // command + if (r1->fields[12]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // time_ms + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+13); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+14, r1->fields[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // info + if (r1->fields[14]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*16)+15, atoll(r1->fields[14])); ASSERT_SQLITE_OK(rc, statsdb); // status_flags + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*16)+15); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*16)+16, r1->fields[15], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // extended_info + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // ThreadID + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // SessionID + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user + rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // db + rc=(*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // cli_host + if (r1->fields[5]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // cli_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 6); ASSERT_SQLITE_OK(rc, statsdb); + } + if (r1->fields[6]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 8); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement1, 8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_host + if (r1->fields[8]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 9); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement1, 10, r1->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host + if (r1->fields[10]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 11); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement1, 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // command + if (r1->fields[12]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // time_ms + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 13); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement1, 14, r1->fields[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // info + if (r1->fields[14]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 15, atoll(r1->fields[14])); ASSERT_SQLITE_OK(rc, statsdb); // status_flags + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 15); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement1, 16, r1->fields[15], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // extended_info + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___pgsql_processlist() { + int rc; + if (!GloPTH) return; + pgsql_thread___show_processlist_extended = variables.pgsql_show_processlist_extended; + SQLite3_result* resultset = GloPTH->SQL3_Processlist(); + if (resultset == NULL) return; + + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + + query1 = (char*)"INSERT OR IGNORE INTO stats_pgsql_processlist VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)"; + query32s = "INSERT OR IGNORE INTO stats_pgsql_processlist VALUES " + generate_multi_rows_query(32, 16); + query32 = (char*)query32s.c_str(); + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_pgsql_processlist"); + + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 16) + 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // ThreadID + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 16) + 2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // SessionID + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 16) + 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 16) + 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // database + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 16) + 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // cli_host + if (r1->fields[5]) { + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 16) + 6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // cli_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx * 16) + 6); ASSERT_SQLITE_OK(rc, statsdb); + } + if (r1->fields[6]) { + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 16) + 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx * 16) + 8); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 16) + 8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_host + if (r1->fields[8]) { + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 16) + 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx * 16) + 9); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 16) + 10, r1->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host + if (r1->fields[10]) { + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 16) + 11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx * 16) + 11); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 16) + 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // command + if (r1->fields[12]) { + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 16) + 13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // time_ms + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx * 16) + 13); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 16) + 14, r1->fields[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // info + if (r1->fields[14]) { + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 16) + 15, atoll(r1->fields[14])); ASSERT_SQLITE_OK(rc, statsdb); // status_flags + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx * 16) + 15); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 16) + 16, r1->fields[15], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // extended_info + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // ThreadID + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // SessionID + rc = (*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user + rc = (*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // database + rc = (*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // cli_host + if (r1->fields[5]) { + rc = (*proxy_sqlite3_bind_int64)(statement1, 6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // cli_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 6); ASSERT_SQLITE_OK(rc, statsdb); + } + if (r1->fields[6]) { + rc = (*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 8); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement1, 8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_host + if (r1->fields[8]) { + rc = (*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); // l_srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 9); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement1, 10, r1->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host + if (r1->fields[10]) { + rc = (*proxy_sqlite3_bind_int64)(statement1, 11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 11); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement1, 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // command + if (r1->fields[12]) { + rc = (*proxy_sqlite3_bind_int64)(statement1, 13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // time_ms + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 13); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement1, 14, r1->fields[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // info + if (r1->fields[14]) { + rc = (*proxy_sqlite3_bind_int64)(statement1, 15, atoll(r1->fields[14])); ASSERT_SQLITE_OK(rc, statsdb); // status_flags + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 15); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement1, 16, r1->fields[15], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // extended_info + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___mysql_connection_pool(bool _reset) { + + if (!MyHGM) return; + SQLite3_result * resultset=MyHGM->SQL3_Connection_Pool(_reset); + if (resultset==NULL) return; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_mysql_connection_pool"); + char *a=(char *)"INSERT INTO stats_mysql_connection_pool VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<14; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1],r->fields[2],r->fields[3],r->fields[4],r->fields[5],r->fields[6],r->fields[7],r->fields[8],r->fields[9],r->fields[10],r->fields[11],r->fields[12],r->fields[13]); + statsdb->execute(query); + free(query); + } + if (_reset) { + statsdb->execute("DELETE FROM stats_mysql_connection_pool_reset"); + statsdb->execute("INSERT INTO stats_mysql_connection_pool_reset SELECT * FROM stats_mysql_connection_pool"); + } + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___pgsql_connection_pool(bool _reset) { + if (!PgHGM) return; + SQLite3_result* resultset = PgHGM->SQL3_Connection_Pool(_reset); + if (resultset == NULL) return; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_pgsql_connection_pool"); + char* a = (char*)"INSERT INTO stats_pgsql_connection_pool VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r = *it; + int arg_len = 0; + for (int i = 0; i < 13; i++) { + arg_len += strlen(r->fields[i]); + } + char* query = (char*)malloc(strlen(a) + arg_len + 32); + sprintf(query, a, r->fields[0], r->fields[1], r->fields[2], r->fields[3], r->fields[4], r->fields[5], r->fields[6], r->fields[7], r->fields[8], r->fields[9], r->fields[10], r->fields[11], r->fields[12]); + statsdb->execute(query); + free(query); + } + if (_reset) { + statsdb->execute("DELETE FROM stats_pgsql_connection_pool_reset"); + statsdb->execute("INSERT INTO stats_pgsql_connection_pool_reset SELECT * FROM stats_pgsql_connection_pool"); + } + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___mysql_free_connections() { + int rc; + if (!MyHGM) return; + SQLite3_result * resultset=MyHGM->SQL3_Free_Connections(); + if (resultset==NULL) return; + + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char *query1=NULL; + char *query32=NULL; + std::string query32s = ""; + + query1 = (char *)"INSERT INTO stats_mysql_free_connections VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)"; + query32s = "INSERT INTO stats_mysql_free_connections VALUES " + generate_multi_rows_query(32,13); + query32 = (char *)query32s.c_str(); + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_mysql_free_connections"); + + int row_idx=0; + int max_bulk_row_idx=resultset->rows_count/32; + max_bulk_row_idx=max_bulk_row_idx*32; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + int idx=row_idx%32; + if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, statsdb); // FD + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*13)+2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host + if (r1->fields[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*13)+4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*13)+4); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // db + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // init_connect + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // time_zone + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+9, r1->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // sql_mode + if (r1->fields[9]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*13)+10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); // autocommit + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*13)+10); ASSERT_SQLITE_OK(rc, statsdb); + } + if (r1->fields[10]) { + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*13)+11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // idle_ms + } else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx*13)+11); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // statistics + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*13)+13, r1->fields[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // mysql_info + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // FD + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host + if (r1->fields[3]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 4); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user + rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // db + rc=(*proxy_sqlite3_bind_text)(statement1, 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // init_connect + rc=(*proxy_sqlite3_bind_text)(statement1, 8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // time_zone + rc=(*proxy_sqlite3_bind_text)(statement1, 9, r1->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // sql_mode + if (r1->fields[9]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); // autocommit + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 10); ASSERT_SQLITE_OK(rc, statsdb); + } + if (r1->fields[10]) { + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); // idle_ms + } else { + rc = (*proxy_sqlite3_bind_null)(statement1, 11); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_text)(statement1, 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // statistics + rc=(*proxy_sqlite3_bind_text)(statement1, 13, r1->fields[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // mysql_info + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + statsdb->execute("COMMIT"); + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + delete resultset; +} + +void ProxySQL_Admin::stats___pgsql_free_connections() { + int rc; + if (!PgHGM) return; + SQLite3_result* resultset = PgHGM->SQL3_Free_Connections(); + if (resultset == NULL) return; + + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + + query1 = (char*)"INSERT INTO stats_pgsql_free_connections VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)"; + query32s = "INSERT INTO stats_pgsql_free_connections VALUES " + generate_multi_rows_query(32, 12); + query32 = (char*)query32s.c_str(); + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_pgsql_free_connections"); + + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 12) + 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // FD + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 12) + 2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 12) + 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host + if (r1->fields[3]) { + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 12) + 4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port + } + else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx * 12) + 4); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 12) + 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 12) + 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // database + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 12) + 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // init_connect + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 12) + 8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // time_zone + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 12) + 9, r1->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // sql_mode + if (r1->fields[9]) { + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 12) + 10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); // idle_ms + } + else { + rc = (*proxy_sqlite3_bind_null)(statement32, (idx * 12) + 10); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 12) + 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // statistics + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 12) + 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // pgsql_info + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } + else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // FD + rc = (*proxy_sqlite3_bind_int64)(statement1, 2, atoll(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); // hostgroup + rc = (*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // srv_host + if (r1->fields[3]) { + rc = (*proxy_sqlite3_bind_int64)(statement1, 4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); // srv_port + } + else { + rc = (*proxy_sqlite3_bind_null)(statement1, 4); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // user + rc = (*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // database + rc = (*proxy_sqlite3_bind_text)(statement1, 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // init_connect + rc = (*proxy_sqlite3_bind_text)(statement1, 8, r1->fields[7], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // time_zone + rc = (*proxy_sqlite3_bind_text)(statement1, 9, r1->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // sql_mode + if (r1->fields[9]) { + rc = (*proxy_sqlite3_bind_int64)(statement1, 10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); // idle_ms + } + else { + rc = (*proxy_sqlite3_bind_null)(statement1, 10); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_text)(statement1, 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // statistics + rc = (*proxy_sqlite3_bind_text)(statement1, 12, r1->fields[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // pgsql_info + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + statsdb->execute("COMMIT"); + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + delete resultset; +} + + +void ProxySQL_Admin::stats___mysql_commands_counters() { + if (!GloMyQPro) return; + SQLite3_result * resultset=GloMyQPro->get_stats_commands_counters(); + if (resultset==NULL) return; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_mysql_commands_counters"); + char *a=(char *)"INSERT INTO stats_mysql_commands_counters VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<15; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1],r->fields[2],r->fields[3],r->fields[4],r->fields[5],r->fields[6],r->fields[7],r->fields[8],r->fields[9],r->fields[10],r->fields[11],r->fields[12],r->fields[13],r->fields[14]); + statsdb->execute(query); + free(query); + } + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___mysql_query_rules() { + if (!GloMyQPro) return; + SQLite3_result * resultset=GloMyQPro->get_stats_query_rules(); + if (resultset==NULL) return; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_mysql_query_rules"); + char *a=(char *)"INSERT INTO stats_mysql_query_rules VALUES (\"%s\",\"%s\")"; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<2; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1]); + statsdb->execute(query); + free(query); + } + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___proxysql_servers_checksums() { + // NOTE: This mutex unlock is required due to a race condition created when: + // - One Admin session has the following callstack: + // + admin_session_handler -> locks on 'sql_query_global_mutex' + // | GenericRefreshStatistics + // | stats___proxysql_servers_checksums + // | get_stats_proxysql_servers_checksums + // + stats_proxysql_servers_checksums -> tries to lock on 'ProxySQL_Cluster_Nodes::mutex' + // - One ProxySQL_Cluster thread has the following callstack: + // + ProxySQL_Cluster::Update_Node_Checksums + // + ProxySQL_Cluster_Nodes::Update_Node_Checksums -> locks on 'ProxySQL_Cluster_Nodes::mutex' + // | ProxySQL_Node_Entry::set_checksums + // + ProxySQL_Cluster::pull_mysql_query_rules_from_peer -> tries to lock on 'sql_query_global_mutex' + // Producing a deadlock scenario between the two threads. + pthread_mutex_unlock(&this->sql_query_global_mutex); + SQLite3_result* resultset = GloProxyCluster->get_stats_proxysql_servers_checksums(); + pthread_mutex_lock(&this->sql_query_global_mutex); + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_proxysql_servers_checksums"); + if (resultset) { + int rc; + sqlite3_stmt *statement1=NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char *query1=NULL; + query1=(char *)"INSERT INTO stats_proxysql_servers_checksums VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoi(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + (*proxy_sqlite3_finalize)(statement1); + } + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___proxysql_servers_metrics() { + //SQLite3_result * resultset=GloProxyCluster->get_stats_proxysql_servers_metrics(); + //if (resultset==NULL) return; + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_proxysql_servers_metrics"); + SQLite3_result *resultset=NULL; + resultset=GloProxyCluster->get_stats_proxysql_servers_metrics(); + if (resultset) { + int rc; + sqlite3_stmt *statement1=NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char *query1=NULL; + query1=(char *)"INSERT INTO stats_proxysql_servers_metrics VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"; + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoi(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoi(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoi(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoi(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoi(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoi(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoi(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + (*proxy_sqlite3_finalize)(statement1); + } + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___proxysql_message_metrics(bool reset) { + SQLite3_result* resultset = proxysql_get_message_stats(reset); + if (resultset == NULL) return; + + statsdb->execute("BEGIN"); + if (reset) { + statsdb->execute("DELETE FROM stats_proxysql_message_metrics_reset"); + } else { + statsdb->execute("DELETE FROM stats_proxysql_message_metrics"); + } + + char* query1 = nullptr; + char* query32 = nullptr; + std::string query32s = ""; + + if (reset) { + query1=(char*)"INSERT INTO stats_proxysql_message_metrics_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"; + query32s = "INSERT INTO stats_proxysql_message_metrics_reset VALUES " + generate_multi_rows_query(32,7); + query32 = (char *)query32s.c_str(); + } else { + query1=(char*)"INSERT INTO stats_proxysql_message_metrics VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)"; + query32s = "INSERT INTO stats_proxysql_message_metrics VALUES " + generate_multi_rows_query(32,7); + query32 = (char *)query32s.c_str(); + } + + sqlite3_stmt* statement1 = nullptr; + sqlite3_stmt* statement32 = nullptr; + int rc = 0; + + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count/32; + max_bulk_row_idx = max_bulk_row_idx*32; + + for (SQLite3_row* r1 : resultset->rows) { + int idx=row_idx%32; + + if (row_idxfields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // message_id + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*7)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // filename + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*7)+3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); // line + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*7)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // func + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*7)+5, atoll(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); // count_star + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*7)+6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // first_seen + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*7)+7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // last_seen + + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // message_id + rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // filename + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); // line + rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // func + rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(r1->fields[4])); ASSERT_SQLITE_OK(rc, statsdb); // count_star + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); // first_seen + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); // last_seen + + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + + statsdb->execute("COMMIT"); + delete resultset; +} + +int ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( + const bool reset, const bool copy, const SQLite3_result *resultset, const umap_query_digest *digest_umap, + const umap_query_digest_text *digest_text_umap +) { + statsdb->execute("BEGIN"); + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + char *query1=NULL; + char *query32=NULL; + std::string query32s = ""; + statsdb->execute("DELETE FROM stats_mysql_query_digest_reset"); + statsdb->execute("DELETE FROM stats_mysql_query_digest"); + if (reset) { + query1=(char *)"INSERT INTO stats_mysql_query_digest_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + query32s = "INSERT INTO stats_mysql_query_digest_reset VALUES " + generate_multi_rows_query(32,14); + query32 = (char *)query32s.c_str(); + } else { + query1=(char *)"INSERT INTO stats_mysql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + query32s = "INSERT INTO stats_mysql_query_digest VALUES " + generate_multi_rows_query(32,14); + query32 = (char *)query32s.c_str(); + } + + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + int row_idx=0; + int num_rows = resultset ? resultset->rows_count : digest_umap->size(); + int max_bulk_row_idx = num_rows/32; + max_bulk_row_idx=max_bulk_row_idx*32; + auto it = resultset ? digest_umap->cend() : digest_umap->cbegin(); + int i = 0; + + time_t __now; + time(&__now); + unsigned long long curtime=monotonic_time(); + time_t seen_time; + + // If the function do not receives a resultset, it gets the values directly from the digest_umap + while (resultset ? i != resultset->rows_count : it != digest_umap->end()) { + QP_query_digest_stats *qds = (QP_query_digest_stats *)(resultset ? NULL : it->second); + SQLite3_row *row = resultset ? resultset->rows[i] : NULL; + char digest_hex_str[20]; // 2+sizeof(unsigned long long)*2+2 + if (!resultset) { + sprintf(digest_hex_str, "0x%016llX", (long long unsigned int)qds->digest); + } + int idx=row_idx%32; + if (row_idxfields[11]) : qds->hid); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+2, resultset ? row->fields[0] : qds->schemaname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+3, resultset ? row->fields[1] : qds->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+4, resultset ? row->fields[2] : qds->client_address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+5, resultset ? row->fields[3] : digest_hex_str, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+6, resultset ? row->fields[4] : qds->get_digest_text(digest_text_umap), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+7, resultset ? atoll(row->fields[5]) : qds->count_star); ASSERT_SQLITE_OK(rc, statsdb); + { + seen_time = qds != nullptr ? __now - curtime/1000000 + qds->first_seen/1000000 : 0; + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+8, resultset ? atoll(row->fields[6]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); + } + { + seen_time = qds != nullptr ? __now - curtime/1000000 + qds->last_seen/1000000 : 0; + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+9, resultset ? atoll(row->fields[7]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+10, resultset ? atoll(row->fields[8]) : qds->sum_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+11, resultset ? atoll(row->fields[9]) : qds->min_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+12, resultset ? atoll(row->fields[10]) : qds->max_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+13, resultset ? atoll(row->fields[12]) : qds->rows_affected); ASSERT_SQLITE_OK(rc, statsdb); // rows affected + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+14, resultset ? atoll(row->fields[13]) : qds->rows_sent); ASSERT_SQLITE_OK(rc, statsdb); // rows sent + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, resultset ? atoll(row->fields[11]) : qds->hid); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, resultset ? row->fields[0] : qds->schemaname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, resultset ? row->fields[1] : qds->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 4, resultset ? row->fields[2] : qds->client_address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 5, resultset ? row->fields[3] : digest_hex_str, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 6, resultset ? row->fields[4] : qds->get_digest_text(digest_text_umap), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, resultset ? atoll(row->fields[5]) : qds->count_star); ASSERT_SQLITE_OK(rc, statsdb); + { + seen_time = qds != nullptr ? __now - curtime/1000000 + qds->first_seen/1000000 : 0; + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, resultset ? atoll(row->fields[6]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); + } + { + seen_time = qds != nullptr ? __now - curtime/1000000 + qds->last_seen/1000000 : 0; + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, resultset ? atoll(row->fields[7]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); + } + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, resultset ? atoll(row->fields[8]) : qds->sum_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, resultset ? atoll(row->fields[9]) : qds->min_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 12, resultset ? atoll(row->fields[10]) : qds->max_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 13, resultset ? atoll(row->fields[12]) : qds->rows_affected); ASSERT_SQLITE_OK(rc, statsdb); // rows affected + rc=(*proxy_sqlite3_bind_int64)(statement1, 14, resultset ? atoll(row->fields[13]) : qds->rows_sent); ASSERT_SQLITE_OK(rc, statsdb); // rows sent + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } +#ifdef DEBUG + if (resultset) + assert(row_idx == i); +#endif + row_idx++; + if (resultset) + i++; + else + it++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + if (reset) { + if (copy) { + statsdb->execute("INSERT INTO stats_mysql_query_digest SELECT * FROM stats_mysql_query_digest_reset"); + } + } + statsdb->execute("COMMIT"); + + return row_idx; +} + +int ProxySQL_Admin::stats___mysql_query_digests(bool reset, bool copy) { + if (!GloMyQPro) return 0; + SQLite3_result * resultset=NULL; + if (reset==true) { + resultset=GloMyQPro->get_query_digests_reset(); + } else { + resultset=GloMyQPro->get_query_digests(); + } + if (resultset==NULL) return 0; + statsdb->execute("BEGIN"); + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char *query1=NULL; + char *query32=NULL; + std::string query32s = ""; + // ALWAYS delete from both tables + //if (reset) { + statsdb->execute("DELETE FROM stats_mysql_query_digest_reset"); + //} else { + statsdb->execute("DELETE FROM stats_mysql_query_digest"); + //} +// char *a=(char *)"INSERT INTO stats_mysql_query_digest VALUES (%s,\"%s\",\"%s\",\"%s\",\"%s\",%s,%s,%s,%s,%s,%s)"; + if (reset) { + query1=(char *)"INSERT INTO stats_mysql_query_digest_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + query32s = "INSERT INTO stats_mysql_query_digest_reset VALUES " + generate_multi_rows_query(32,14); + query32 = (char *)query32s.c_str(); + } else { + query1=(char *)"INSERT INTO stats_mysql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + query32s = "INSERT INTO stats_mysql_query_digest VALUES " + generate_multi_rows_query(32,14); + query32 = (char *)query32s.c_str(); + } + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + int row_idx=0; + int max_bulk_row_idx=resultset->rows_count/32; + max_bulk_row_idx=max_bulk_row_idx*32; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + int idx=row_idx%32; + if (row_idxfields[11])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+2, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+3, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+4, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+5, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+6, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+7, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+8, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+9, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+10, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+11, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+12, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // rows affected + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+14, atoll(r1->fields[13])); ASSERT_SQLITE_OK(rc, statsdb); // rows sent + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[11])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 12, atoll(r1->fields[10])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 13, atoll(r1->fields[12])); ASSERT_SQLITE_OK(rc, statsdb); // rows affected + rc=(*proxy_sqlite3_bind_int64)(statement1, 14, atoll(r1->fields[13])); ASSERT_SQLITE_OK(rc, statsdb); // rows sent + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); +/* + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[10],r->fields[0],r->fields[1],r->fields[2],r->fields[3],r->fields[4],r->fields[5],r->fields[6],r->fields[7],r->fields[8],r->fields[9]); + statsdb->execute(query); + free(query); + } +*/ + if (reset) { + if (copy) { + statsdb->execute("INSERT INTO stats_mysql_query_digest SELECT * FROM stats_mysql_query_digest_reset"); + } + } + statsdb->execute("COMMIT"); + delete resultset; + + return row_idx; +} + +int ProxySQL_Admin::stats___mysql_query_digests_v2(bool reset, bool copy, bool use_resultset) { + if (!GloMyQPro) return 0; + std::pair res; + if (reset == true) { + res=GloMyQPro->get_query_digests_reset_v2(copy, use_resultset); + } else { + res=GloMyQPro->get_query_digests_v2(use_resultset); + } + + if (res.first == NULL) + return res.second; + + int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite(reset, copy, res.first, NULL, NULL); + delete res.first; + + return num_rows; +} + +void ProxySQL_Admin::stats___mysql_client_host_cache(bool reset) { + if (!GloMyQPro) return; + + SQLite3_result* resultset = GloMTH->get_client_host_cache(reset); + if (resultset==NULL) return; + + statsdb->execute("BEGIN"); + + int rc = 0; + sqlite3_stmt* statement=NULL; + char* query = NULL; + + if (reset) { + query=(char*)"INSERT INTO stats_mysql_client_host_cache_reset VALUES (?1, ?2, ?3)"; + } else { + query=(char*)"INSERT INTO stats_mysql_client_host_cache VALUES (?1, ?2, ?3)"; + } + + statsdb->execute("DELETE FROM stats_mysql_client_host_cache_reset"); + statsdb->execute("DELETE FROM stats_mysql_client_host_cache"); + + rc = statsdb->prepare_v2(query, &statement); + ASSERT_SQLITE_OK(rc, statsdb); + + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *row = *it; + + rc=(*proxy_sqlite3_bind_text)(statement, 1, row->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoll(row->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoll(row->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); + + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); + rc=(*proxy_sqlite3_reset)(statement); + } + + (*proxy_sqlite3_finalize)(statement); + + if (reset) { + statsdb->execute("INSERT INTO stats_mysql_client_host_cache SELECT * FROM stats_mysql_client_host_cache_reset"); + } + + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___pgsql_client_host_cache(bool reset) { + if (!GloPTH) return; + + SQLite3_result* resultset = GloPTH->get_client_host_cache(reset); + if (resultset == NULL) return; + + statsdb->execute("BEGIN"); + + int rc = 0; + sqlite3_stmt* statement = NULL; + char* query = NULL; + + if (reset) { + query = (char*)"INSERT INTO stats_pgsql_client_host_cache_reset VALUES (?1, ?2, ?3)"; + } else { + query = (char*)"INSERT INTO stats_pgsql_client_host_cache VALUES (?1, ?2, ?3)"; + } + + statsdb->execute("DELETE FROM stats_pgsql_client_host_cache_reset"); + statsdb->execute("DELETE FROM stats_pgsql_client_host_cache"); + + rc = statsdb->prepare_v2(query, &statement); + ASSERT_SQLITE_OK(rc, statsdb); + + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* row = *it; + + rc = (*proxy_sqlite3_bind_text)(statement, 1, row->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 2, atoll(row->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement, 3, atoll(row->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); + + SAFE_SQLITE3_STEP2(statement); + rc = (*proxy_sqlite3_clear_bindings)(statement); + rc = (*proxy_sqlite3_reset)(statement); + } + + (*proxy_sqlite3_finalize)(statement); + + if (reset) { + statsdb->execute("INSERT INTO stats_pgsql_client_host_cache SELECT * FROM stats_pgsql_client_host_cache_reset"); + } + + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___mysql_errors(bool reset) { + if (!GloMyQPro) return; + SQLite3_result * resultset=NULL; + if (reset==true) { + resultset=MyHGM->get_mysql_errors(true); + } else { + resultset=MyHGM->get_mysql_errors(false); + } + if (resultset==NULL) return; + statsdb->execute("BEGIN"); + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char *query1=NULL; + char *query32=NULL; + std::string query32s = ""; + if (reset) { + statsdb->execute("DELETE FROM stats_mysql_errors_reset"); + } else { + statsdb->execute("DELETE FROM stats_mysql_errors"); + } + if (reset) { + query1=(char *)"INSERT INTO stats_mysql_errors_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + query32s = "INSERT INTO stats_mysql_errors_reset VALUES " + generate_multi_rows_query(32,11); + query32 = (char *)query32s.c_str(); + } else { + query1=(char *)"INSERT INTO stats_mysql_errors VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + query32s = "INSERT INTO stats_mysql_errors VALUES " + generate_multi_rows_query(32,11); + query32 = (char *)query32s.c_str(); + } + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + int row_idx=0; + int max_bulk_row_idx=resultset->rows_count/32; + max_bulk_row_idx=max_bulk_row_idx*32; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + int idx=row_idx%32; + if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+8, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*11)+10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*11)+11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); //ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); //ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); //ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); //ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___pgsql_errors(bool reset) { + if (!PgHGM) return; + SQLite3_result* resultset = PgHGM->get_pgsql_errors(reset); + if (resultset == NULL) return; + statsdb->execute("BEGIN"); + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + if (reset) { + statsdb->execute("DELETE FROM stats_pgsql_errors_reset"); + } + else { + statsdb->execute("DELETE FROM stats_pgsql_errors"); + } + if (reset) { + query1 = (char*)"INSERT INTO stats_pgsql_errors_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + query32s = "INSERT INTO stats_pgsql_errors_reset VALUES " + generate_multi_rows_query(32, 11); + query32 = (char*)query32s.c_str(); + } + else { + query1 = (char*)"INSERT INTO stats_pgsql_errors VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)"; + query32s = "INSERT INTO stats_pgsql_errors VALUES " + generate_multi_rows_query(32, 11); + query32 = (char*)query32s.c_str(); + } + + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + int row_idx = 0; + int max_bulk_row_idx = resultset->rows_count / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + for (std::vector::iterator it = resultset->rows.begin(); it != resultset->rows.end(); ++it) { + SQLite3_row* r1 = *it; + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // hid + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // hostname + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); // port + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // username + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // client_address + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // database + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // sqlstate + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 8, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); // count_star + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); // first_seen + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 11) + 10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); // last_seen + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 11) + 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // last_error + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); //ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement32); //ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); // hid + rc = (*proxy_sqlite3_bind_text)(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // hostname + rc = (*proxy_sqlite3_bind_int64)(statement1, 3, atoll(r1->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); // port + rc = (*proxy_sqlite3_bind_text)(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // username + rc = (*proxy_sqlite3_bind_text)(statement1, 5, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // client_address + rc = (*proxy_sqlite3_bind_text)(statement1, 6, r1->fields[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // database + rc = (*proxy_sqlite3_bind_text)(statement1, 7, r1->fields[6], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // sqlstate + rc = (*proxy_sqlite3_bind_int64)(statement1, 8, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); // count_star + rc = (*proxy_sqlite3_bind_int64)(statement1, 9, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); // first_seen + rc = (*proxy_sqlite3_bind_int64)(statement1, 10, atoll(r1->fields[9])); ASSERT_SQLITE_OK(rc, statsdb); // last_seen + rc = (*proxy_sqlite3_bind_text)(statement1, 11, r1->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); // last_error + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); //ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement1); //ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + statsdb->execute("COMMIT"); + delete resultset; +} + +void ProxySQL_Admin::stats___mysql_users() { + account_details_t **ads=NULL; + statsdb->execute("DELETE FROM stats_mysql_users"); + + int num_users=GloMyAuth->dump_all_users(&ads, false); + if (num_users==0) return; + + const char q[] { + "INSERT INTO stats_mysql_users(username,frontend_connections,frontend_max_connections) VALUES ('%s',%d,%d)" + }; + char buf[256] = { 0 }; + + for (int i=0; idefault_hostgroup>= 0) { // only not admin/stats + cfmt_t q_fmt = cstr_format(buf, q, ad->username, ad->num_connections_used, ad->max_connections); + + if (q_fmt.str.size()) { + statsdb->execute(q_fmt.str.c_str()); + } else { + statsdb->execute(buf); + } + } + free(ad->username); + free(ad); + } + + if (GloMyLdapAuth) { + std::unique_ptr ldap_users { GloMyLdapAuth->dump_all_users() }; + + for (const SQLite3_row* row : ldap_users->rows) { + const char* username = row->fields[LDAP_USER_FIELD_IDX::USERNAME]; + int f_conns = atoi(row->fields[LDAP_USER_FIELD_IDX::FRONTEND_CONNECTIONS]); + int f_max_conns = atoi(row->fields[LDAP_USER_FIELD_IDX::FRONTED_MAX_CONNECTIONS]); + + cfmt_t q_fmt = cstr_format(buf, q, username, f_conns, f_max_conns); + + if (q_fmt.str.size()) { + statsdb->execute(q_fmt.str.c_str()); + } else { + statsdb->execute(buf); + } + } + } + + free(ads); +} + +void ProxySQL_Admin::stats___pgsql_users() { + pgsql_account_details_t** ads = NULL; + statsdb->execute("DELETE FROM stats_pgsql_users"); + + int num_users = GloPgAuth->dump_all_users(&ads, false); + if (num_users == 0) return; + + const char q[] = "INSERT INTO stats_pgsql_users(username,frontend_connections,frontend_max_connections) VALUES ('%s',%d,%d)"; + + char buf[256] = { 0 }; + + for (int i = 0; i < num_users; i++) { + pgsql_account_details_t* ad = ads[i]; + if (ad->default_hostgroup >= 0) { // only not admin/stats + cfmt_t q_fmt = cstr_format(buf, q, ad->username, ad->num_connections_used, ad->max_connections); + + if (q_fmt.str.size()) { + statsdb->execute(q_fmt.str.c_str()); + } + else { + statsdb->execute(buf); + } + } + free(ad->username); + free(ad); + } + + /*if (GloMyLdapAuth) { + std::unique_ptr ldap_users{ GloMyLdapAuth->dump_all_users() }; + + for (const SQLite3_row* row : ldap_users->rows) { + const char* username = row->fields[LDAP_USER_FIELD_IDX::USERNAME]; + int f_conns = atoi(row->fields[LDAP_USER_FIELD_IDX::FRONTEND_CONNECTIONS]); + int f_max_conns = atoi(row->fields[LDAP_USER_FIELD_IDX::FRONTED_MAX_CONNECTIONS]); + + cfmt_t q_fmt = cstr_format(buf, q, username, f_conns, f_max_conns); + + if (q_fmt.str.size()) { + statsdb->execute(q_fmt.str.c_str()); + } + else { + statsdb->execute(buf); + } + } + }*/ + + free(ads); +} + +void ProxySQL_Admin::stats___mysql_gtid_executed() { + statsdb->execute("DELETE FROM stats_mysql_gtid_executed"); + SQLite3_result *resultset=NULL; + resultset = MyHGM->get_stats_mysql_gtid_executed(); + if (resultset) { + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char *query1=NULL; + char *query32=NULL; + std::string query32s = ""; + query1=(char *)"INSERT INTO stats_mysql_gtid_executed VALUES (?1, ?2, ?3, ?4)"; + query32s = "INSERT INTO stats_mysql_gtid_executed VALUES " + generate_multi_rows_query(32,4); + query32 = (char *)query32s.c_str(); + + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + int row_idx=0; + int max_bulk_row_idx=resultset->rows_count/32; + max_bulk_row_idx=max_bulk_row_idx*32; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + int idx=row_idx%32; + if (row_idxfields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*4)+2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*4)+3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*4)+4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); + if (idx==31) { + SAFE_SQLITE3_STEP(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_text)(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 2, atoi(r1->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(r1->fields[3])); ASSERT_SQLITE_OK(rc, statsdb); + SAFE_SQLITE3_STEP(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + delete resultset; + resultset = NULL; + } +} + +void ProxySQL_Admin::stats___mysql_prepared_statements_info() { + if (!GloMyStmt) return; + SQLite3_result * resultset=NULL; + resultset=GloMyStmt->get_prepared_statements_global_infos(); + if (resultset==NULL) return; + statsdb->execute("BEGIN"); + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + //sqlite3 *mydb3=statsdb->get_db(); + char *query1=NULL; + char *query32=NULL; + std::string query32s = ""; + statsdb->execute("DELETE FROM stats_mysql_prepared_statements_info"); + query1=(char *)"INSERT INTO stats_mysql_prepared_statements_info VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; + query32s = "INSERT INTO stats_mysql_prepared_statements_info VALUES " + generate_multi_rows_query(32,9); + query32 = (char *)query32s.c_str(); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query1, -1, &statement1, 0); + //rc=sqlite3_prepare_v2(mydb3, query1, -1, &statement1, 0); + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + //rc=(*proxy_sqlite3_prepare_v2)(mydb3, query32, -1, &statement32, 0); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + int row_idx=0; + int max_bulk_row_idx=resultset->rows_count/32; + max_bulk_row_idx=max_bulk_row_idx*32; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r1=*it; + int idx=row_idx%32; + if (row_idxfields[0])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_text(statement32, (idx*9)+2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_text(statement32, (idx*9)+3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_text(statement32, (idx*9)+4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_int64(statement32, (idx*9)+5, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_int64(statement32, (idx*9)+6, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_int64(statement32, (idx*9)+7, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_int64(statement32, (idx*9)+8, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_text(statement32, (idx*9)+9, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=sqlite3_bind_int64(statement1, 1, atoll(r1->fields[0])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_text(statement1, 2, r1->fields[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_text(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_text(statement1, 4, r1->fields[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_int64(statement1, 5, atoll(r1->fields[5])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_int64(statement1, 6, atoll(r1->fields[6])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_int64(statement1, 7, atoll(r1->fields[7])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_int64(statement1, 8, atoll(r1->fields[8])); ASSERT_SQLITE_OK(rc, statsdb); + rc=sqlite3_bind_text(statement1, 9, r1->fields[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + statsdb->execute("COMMIT"); + delete resultset; +} + +int ProxySQL_Admin::stats___save_pgsql_query_digest_to_sqlite( + const bool reset, const bool copy, const SQLite3_result* resultset, const umap_query_digest* digest_umap, + const umap_query_digest_text* digest_text_umap +) { + statsdb->execute("BEGIN"); + int rc; + sqlite3_stmt* statement1 = NULL; + sqlite3_stmt* statement32 = NULL; + char* query1 = NULL; + char* query32 = NULL; + std::string query32s = ""; + statsdb->execute("DELETE FROM stats_pgsql_query_digest_reset"); + statsdb->execute("DELETE FROM stats_pgsql_query_digest"); + if (reset) { + query1 = (char*)"INSERT INTO stats_pgsql_query_digest_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + query32s = "INSERT INTO stats_pgsql_query_digest_reset VALUES " + generate_multi_rows_query(32, 14); + query32 = (char*)query32s.c_str(); + } + else { + query1 = (char*)"INSERT INTO stats_pgsql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + query32s = "INSERT INTO stats_pgsql_query_digest VALUES " + generate_multi_rows_query(32, 14); + query32 = (char*)query32s.c_str(); + } + + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + int row_idx = 0; + int num_rows = resultset ? resultset->rows_count : digest_umap->size(); + int max_bulk_row_idx = num_rows / 32; + max_bulk_row_idx = max_bulk_row_idx * 32; + auto it = resultset ? digest_umap->cend() : digest_umap->cbegin(); + int i = 0; + + time_t __now; + time(&__now); + unsigned long long curtime = monotonic_time(); + time_t seen_time; + + // If the function do not receives a resultset, it gets the values directly from the digest_umap + while (resultset ? i != resultset->rows_count : it != digest_umap->end()) { + QP_query_digest_stats* qds = (QP_query_digest_stats*)(resultset ? NULL : it->second); + SQLite3_row* row = resultset ? resultset->rows[i] : NULL; + string digest_hex_str; + if (!resultset) { + std::ostringstream digest_stream; + digest_stream << "0x" << std::hex << qds->digest; + digest_hex_str = digest_stream.str(); + } + int idx = row_idx % 32; + if (row_idx < max_bulk_row_idx) { // bulk + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 1, resultset ? atoll(row->fields[11]) : qds->hid); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 14) + 2, resultset ? row->fields[0] : qds->schemaname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 14) + 3, resultset ? row->fields[1] : qds->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 14) + 4, resultset ? row->fields[2] : qds->client_address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 14) + 5, resultset ? row->fields[3] : digest_hex_str.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement32, (idx * 14) + 6, resultset ? row->fields[4] : qds->get_digest_text(digest_text_umap), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 7, resultset ? atoll(row->fields[5]) : qds->count_star); ASSERT_SQLITE_OK(rc, statsdb); + { + seen_time = qds != nullptr ? __now - curtime / 1000000 + qds->first_seen / 1000000 : 0; + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 8, resultset ? atoll(row->fields[6]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); + } + { + seen_time = qds != nullptr ? __now - curtime / 1000000 + qds->last_seen / 1000000 : 0; + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 9, resultset ? atoll(row->fields[7]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 10, resultset ? atoll(row->fields[8]) : qds->sum_time); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 11, resultset ? atoll(row->fields[9]) : qds->min_time); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 12, resultset ? atoll(row->fields[10]) : qds->max_time); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 13, resultset ? atoll(row->fields[12]) : qds->rows_affected); ASSERT_SQLITE_OK(rc, statsdb); // rows affected + rc = (*proxy_sqlite3_bind_int64)(statement32, (idx * 14) + 14, resultset ? atoll(row->fields[13]) : qds->rows_sent); ASSERT_SQLITE_OK(rc, statsdb); // rows sent + if (idx == 31) { + SAFE_SQLITE3_STEP2(statement32); + rc = (*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } + else { // single row + rc = (*proxy_sqlite3_bind_int64)(statement1, 1, resultset ? atoll(row->fields[11]) : qds->hid); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement1, 2, resultset ? row->fields[0] : qds->schemaname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement1, 3, resultset ? row->fields[1] : qds->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement1, 4, resultset ? row->fields[2] : qds->client_address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement1, 5, resultset ? row->fields[3] : digest_hex_str.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_text)(statement1, 6, resultset ? row->fields[4] : qds->get_digest_text(digest_text_umap), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 7, resultset ? atoll(row->fields[5]) : qds->count_star); ASSERT_SQLITE_OK(rc, statsdb); + { + seen_time = qds != nullptr ? __now - curtime / 1000000 + qds->first_seen / 1000000 : 0; + rc = (*proxy_sqlite3_bind_int64)(statement1, 8, resultset ? atoll(row->fields[6]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); + } + { + seen_time = qds != nullptr ? __now - curtime / 1000000 + qds->last_seen / 1000000 : 0; + rc = (*proxy_sqlite3_bind_int64)(statement1, 9, resultset ? atoll(row->fields[7]) : seen_time); ASSERT_SQLITE_OK(rc, statsdb); + } + rc = (*proxy_sqlite3_bind_int64)(statement1, 10, resultset ? atoll(row->fields[8]) : qds->sum_time); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 11, resultset ? atoll(row->fields[9]) : qds->min_time); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 12, resultset ? atoll(row->fields[10]) : qds->max_time); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_bind_int64)(statement1, 13, resultset ? atoll(row->fields[12]) : qds->rows_affected); ASSERT_SQLITE_OK(rc, statsdb); // rows affected + rc = (*proxy_sqlite3_bind_int64)(statement1, 14, resultset ? atoll(row->fields[13]) : qds->rows_sent); ASSERT_SQLITE_OK(rc, statsdb); // rows sent + SAFE_SQLITE3_STEP2(statement1); + rc = (*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc = (*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } +#ifdef DEBUG + if (resultset) + assert(row_idx == i); +#endif + row_idx++; + if (resultset) + i++; + else + it++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + if (reset) { + if (copy) { + statsdb->execute("INSERT INTO stats_pgsql_query_digest SELECT * FROM stats_pgsql_query_digest_reset"); + } + } + statsdb->execute("COMMIT"); + + return row_idx; +} diff --git a/lib/ProxySQL_Admin_Tests.cpp b/lib/ProxySQL_Admin_Tests.cpp new file mode 100644 index 0000000000..1f0fe752e9 --- /dev/null +++ b/lib/ProxySQL_Admin_Tests.cpp @@ -0,0 +1,143 @@ +#include // std::cout +#include // std::stringstream +#include +#include // std::sort +#include +#include // std::vector +#include + +#include "MySQL_Data_Stream.h" + +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" + +extern MySQL_Query_Processor* GloMyQPro; +extern PgSQL_Query_Processor* GloPgQPro; + +int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { + int r = 0; + if (!GloMyQPro) return 0; + if (use_swap == false) { + SQLite3_result * resultset=NULL; + if (reset==true) { + resultset=GloMyQPro->get_query_digests_reset(); + } else { + resultset=GloMyQPro->get_query_digests(); + } + if (resultset==NULL) return 0; + r = resultset->rows_count; + delete resultset; + } else { + umap_query_digest uqd; + umap_query_digest_text uqdt; + GloMyQPro->get_query_digests_reset(&uqd, &uqdt); + r = uqd.size(); + for (std::unordered_map::iterator it=uqd.begin(); it!=uqd.end(); ++it) { + QP_query_digest_stats * qds = (QP_query_digest_stats *)it->second; + delete qds; + } + uqd.erase(uqd.begin(),uqd.end()); + for (std::unordered_map::iterator it=uqdt.begin(); it!=uqdt.end(); ++it) { + free(it->second); + } + uqdt.erase(uqdt.begin(),uqdt.end()); + } + return r; +} + +bool ProxySQL_Test___Refresh_MySQL_Variables(unsigned int cnt) { + MySQL_Thread *mysql_thr=new MySQL_Thread(); + mysql_thr->curtime=monotonic_time(); + for (unsigned int i = 0; i < cnt ; i++) { + mysql_thr->refresh_variables(); + } + delete mysql_thr; + return true; +} + +int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg) { + int r = 0; + r = GloMyQPro->purge_query_digests(async_purge, parallel, msg); + return r; +} + +int ProxySQL_Test___GenerateRandomQueryInDigestTable(int n) { + //unsigned long long queries=n; + //queries *= 1000; + MySQL_Session *sess = new MySQL_Session(); + // When the session is destroyed, client_connections is automatically decreased. + // Because this is not a real connection, we artificially increase + // client_connections + __sync_fetch_and_add(&MyHGM->status.client_connections,1); + sess->client_myds = new MySQL_Data_Stream(); + sess->client_myds->fd=0; + sess->client_myds->init(MYDS_FRONTEND, sess, sess->client_myds->fd); + MySQL_Connection *myconn=new MySQL_Connection(); + sess->client_myds->attach_connection(myconn); + myconn->set_is_client(); // this is used for prepared statements + //unsigned long long cur = monotonic_time(); + SQP_par_t qp; + qp.first_comment=NULL; + qp.query_prefix=NULL; + qp.digest_text = (char *)malloc(1024); + MySQL_Connection_userinfo ui; + char * username_buf = (char *)malloc(32); + char * schemaname_buf = (char *)malloc(64); + //ui.username = username_buf; + //ui.schemaname = schemaname_buf; + strcpy(username_buf,"user_name_"); + strcpy(schemaname_buf,"shard_name_"); + bool orig_norm = mysql_thread___query_digests_normalize_digest_text; + for (int i=0; i ? AND a.c IN (?,?,?) ORDER BY k,l DESC LIMIT ?",i, j); + int digest_text_length = strlen(qp.digest_text); + qp.digest=SpookyHash::Hash64(qp.digest_text, digest_text_length, 0); + for (int k=0; k<10; k++) { + //sprintf(username_buf,"user_%d",k%10); + int _a = fastrand(); + int _k = _a%20; + int _j = _a%7; + for (int _i=0 ; _i<_k ; _i++) { + username_buf[10+_i]='0' + (_j+_i)%10; + } + username_buf[10+_k]='\0'; + for (int l=0; l<10; l++) { + //if (fastrand()%100==0) { + // sprintf(schemaname_buf,"long_shard_name_shard_whatever_%d",l%10); + //} else { + // sprintf(schemaname_buf,"shard_%d",l%10); + //} + int _a = fastrand(); + int _k = _a%30; + int _j = _a%11; + for (int _i=0 ; _i<_k ; _i++) { + schemaname_buf[11+_i]='0' + (_j+_i)%10; + } + schemaname_buf[11+_k]='\0'; + ui.set(username_buf, NULL, schemaname_buf, NULL); + int hg = 0; + uint64_t hash2; + SpookyHash myhash; + myhash.Init(19,3); + myhash.Update(ui.username,strlen(ui.username)); + myhash.Update(&qp.digest,sizeof(qp.digest)); + myhash.Update(ui.schemaname,strlen(ui.schemaname)); + myhash.Update(&hg,sizeof(hg)); + myhash.Final(&qp.digest_total,&hash2); + //update_query_digest(qp, sess->current_hostgroup, ui, t, sess->thread->curtime, NULL, sess); + GloMyQPro->update_query_digest(qp.digest_total,qp.digest,qp.digest_text,hg,&ui,fastrand(),0,"", + sess->CurrentQuery.affected_rows, sess->CurrentQuery.rows_sent); + } + } + } + } + delete sess; + mysql_thread___query_digests_normalize_digest_text = orig_norm; + return n*1000; +} diff --git a/lib/ProxySQL_Admin_Tests2.cpp b/lib/ProxySQL_Admin_Tests2.cpp new file mode 100644 index 0000000000..0edab1449e --- /dev/null +++ b/lib/ProxySQL_Admin_Tests2.cpp @@ -0,0 +1,883 @@ +#include // std::cout +#include // std::stringstream +#include +#include // std::sort +#include +#include // std::vector +#include + +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" + +#include "MySQL_Data_Stream.h" + +static int int_cmp(const void *a, const void *b) { + const unsigned long long *ia = (const unsigned long long *)a; + const unsigned long long *ib = (const unsigned long long *)b; + if (*ia < *ib) return -1; + if (*ia > *ib) return 1; + return 0; +} + +extern MySQL_Query_Processor* GloMyQPro; +extern PgSQL_Query_Processor* GloPgQPro; +extern MySQL_Monitor *GloMyMon; +extern MySQL_Threads_Handler *GloMTH; + +static pthread_mutex_t test_mysql_firewall_whitelist_mutex = PTHREAD_MUTEX_INITIALIZER; +static std::unordered_map map_test_mysql_firewall_whitelist_rules; +static char rand_del[6] = {0}; + +static void init_rand_del() { + if (rand_del[0] == 0) { + static const char alphanum[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + rand_del[0] = '-'; + for (int i = 1; i < 4; i++) { + rand_del[i] = alphanum[rand() % (sizeof(alphanum) - 1)]; + } + rand_del[4] = '-'; + rand_del[5] = 0; + } +} + +int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap); +bool ProxySQL_Test___Refresh_MySQL_Variables(unsigned int cnt); +int ProxySQL_Test___PurgeDigestTable(bool async_purge, bool parallel, char **msg); +int ProxySQL_Test___GenerateRandomQueryInDigestTable(int n); + +void ProxySQL_Admin::map_test_mysql_firewall_whitelist_rules_cleanup() { + for (std::unordered_map::iterator it = map_test_mysql_firewall_whitelist_rules.begin(); it != map_test_mysql_firewall_whitelist_rules.end(); ++it) { + PtrArray* myptrarray = (PtrArray*)it->second; + delete myptrarray; + } + map_test_mysql_firewall_whitelist_rules.clear(); +} + +bool ProxySQL_Admin::ProxySQL_Test___Load_MySQL_Whitelist(int *ret1, int *ret2, int cmd, int loops) { + // cmd == 1 : populate the structure with a global mutex + // cmd == 2 : perform lookup with a global mutex + // cmd == 3 : perform lookup with a mutex for each call + // cmd == 4 : populate the structure with a global mutex , but without cleaning up + // all accept an extra argument that is the number of loops + char *q = (char *)"SELECT * FROM mysql_firewall_whitelist_rules ORDER BY RANDOM()"; + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + bool ret = true; + int _ret1 = 0; + // cleanup + if (cmd == 1 || cmd == 2 || cmd == 4) { + pthread_mutex_lock(&test_mysql_firewall_whitelist_mutex); + } + admindb->execute_statement(q, &error , &cols , &affected_rows , &resultset); + init_rand_del(); + if (error) { + proxy_error("Error on %s : %s\n", q, error); + return false; + } else { + *ret1 = resultset->rows_count; + int loop = 0; + //if (cmd == 1) { + // loop = loops -1; + //} + for ( ; loop < loops ; loop++) { + _ret1 = 0; + if (cmd == 1) { + for (std::unordered_map::iterator it = map_test_mysql_firewall_whitelist_rules.begin() ; it != map_test_mysql_firewall_whitelist_rules.end(); ++it) { + PtrArray * myptrarray = (PtrArray *)it->second; + delete myptrarray; + } + map_test_mysql_firewall_whitelist_rules.clear(); + } + if (cmd == 4) { + for (std::unordered_map::iterator it = map_test_mysql_firewall_whitelist_rules.begin() ; it != map_test_mysql_firewall_whitelist_rules.end(); ++it) { + PtrArray * myptrarray = (PtrArray *)it->second; + myptrarray->reset(); + } + } + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int active = atoi(r->fields[0]); + if (active == 0) { + continue; + } + char * username = r->fields[1]; + char * client_address = r->fields[2]; + char * schemaname = r->fields[3]; + char * flagIN = r->fields[4]; + char * digest_hex = r->fields[5]; + unsigned long long digest_num = strtoull(digest_hex,NULL,0); + string s = username; + s += rand_del; + s += client_address; + s += rand_del; + s += schemaname; + s += rand_del; + s += flagIN; + std::unordered_map:: iterator it2; + if (cmd == 1 || cmd == 4) { + it2 = map_test_mysql_firewall_whitelist_rules.find(s); + if (it2 != map_test_mysql_firewall_whitelist_rules.end()) { + PtrArray * myptrarray = (PtrArray *)it2->second; + myptrarray->add((void *)digest_num); + } else { + PtrArray * myptrarray = new PtrArray(); + myptrarray->add((void *)digest_num); + map_test_mysql_firewall_whitelist_rules[s] = (void *)myptrarray; + //proxy_info("Inserted key: %s\n" , s.c_str()); + } + } else if (cmd == 2 || cmd == 3) { + if (cmd == 3) { + pthread_mutex_lock(&test_mysql_firewall_whitelist_mutex); + } + it2 = map_test_mysql_firewall_whitelist_rules.find(s); + if (it2 != map_test_mysql_firewall_whitelist_rules.end()) { + PtrArray * myptrarray = (PtrArray *)it2->second; + void * r = bsearch(&digest_num, myptrarray->pdata, myptrarray->len, sizeof(unsigned long long), int_cmp); + if (r) _ret1++; + } else { + //proxy_error("Not found: %s %s %s %s\n", username, client_address, schemaname, flagIN); + proxy_error("Not found: %s\n", s.c_str()); + } + if (cmd == 3) { + pthread_mutex_unlock(&test_mysql_firewall_whitelist_mutex); + } + } + } + if (cmd == 1 || cmd == 4) { + std::unordered_map::iterator it = map_test_mysql_firewall_whitelist_rules.begin(); + while (it != map_test_mysql_firewall_whitelist_rules.end()) { + PtrArray * myptrarray = (PtrArray *)it->second; + switch (cmd) { + case 1: + qsort(myptrarray->pdata, myptrarray->len, sizeof(unsigned long long), int_cmp); + it++; + break; + case 4: + if (myptrarray->len) { + qsort(myptrarray->pdata, myptrarray->len, sizeof(unsigned long long), int_cmp); + it++; + } else { + delete myptrarray; + it = map_test_mysql_firewall_whitelist_rules.erase(it); + } + break; + default: + break; + } + } + } + } + } + if (cmd == 2 || cmd == 3) { + *ret2 = _ret1; + } + if (resultset) delete resultset; + if (cmd == 1 || cmd == 2 || cmd == 4) { + pthread_mutex_unlock(&test_mysql_firewall_whitelist_mutex); + } + return ret; +} + +// if dual is not 0 , we call the new search algorithm +bool ProxySQL_Admin::ProxySQL_Test___Verify_mysql_query_rules_fast_routing( + int *ret1, int *ret2, int cnt, int dual, int ths, bool lock, bool maps_per_thread +) { + // A thread param of '0' is equivalent to not testing + if (ths == 0) { ths = 1; } + char *q = (char *)"SELECT username, schemaname, flagIN, destination_hostgroup FROM mysql_query_rules_fast_routing ORDER BY RANDOM()"; + + bool ret = true; + int matching_rows = 0; + + SQLite3_result *resultset=NULL; + { + char *error=NULL; + int cols=0; + int affected_rows=0; + admindb->execute_statement(q, &error , &cols , &affected_rows , &resultset); + + if (error) { + proxy_error("Error on %s : %s\n", q, error); + *ret1 = -1; + return false; + } + } + *ret2 = resultset->rows_count; + + char *query2=(char *)"SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM main.mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN"; + SQLite3_result* resultset2 = nullptr; + + if (maps_per_thread) { + char* error2 = nullptr; + int cols2 = 0; + int affected_rows2 = 0; + admindb->execute_statement(query2, &error2 , &cols2 , &affected_rows2 , &resultset2); + + if (error2) { + proxy_error("Error on %s : %s\n", query2, error2); + return false; + } + } + + vector results(ths, 0); + vector th_hashmaps {}; + + if (maps_per_thread) { + for (uint32_t i = 0; i < static_cast(ths); i++) { + th_hashmaps.push_back(GloMyQPro->create_fast_routing_hashmap(resultset2)); + } + } + + const auto perform_searches = + [&results,&dual](khash_t(khStrInt)* hashmap, SQLite3_result* resultset, uint32_t pos, bool lock) -> void + { + uint32_t matching_rows = 0; + + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int dest_HG = atoi(r->fields[3]); + int ret_HG = -1; + if (dual) { + ret_HG = GloMyQPro->testing___find_HG_in_mysql_query_rules_fast_routing_dual( + hashmap, r->fields[0], r->fields[1], atoi(r->fields[2]), lock + ); + } else { + ret_HG = GloMyQPro->testing___find_HG_in_mysql_query_rules_fast_routing( + r->fields[0], r->fields[1], atoi(r->fields[2]) + ); + } + + if (dest_HG == ret_HG) { + matching_rows++; + } + } + + results[pos] = matching_rows; + }; + + proxy_info("Test with params - cnt: %d, threads: %d, lock: %d, maps_per_thread: %d\n", cnt, ths, lock, maps_per_thread); + + unsigned long long curtime1 = monotonic_time() / 1000; + std::vector workers {}; + + for (int i = 0; i < ths; i++) { + khash_t(khStrInt)* hashmap = maps_per_thread ? th_hashmaps[i].rules_fast_routing : nullptr; + workers.push_back(std::thread(perform_searches, hashmap, resultset, i, lock)); + } + + for (std::thread& w : workers) { + w.join(); + } + + matching_rows = results[0]; + if (matching_rows != resultset->rows_count) { + ret = false; + } + *ret1 = matching_rows; + + if (ret == true) { + if (cnt > 1) { + for (int i=1 ; i < cnt; i++) { + std::vector workers {}; + + for (int i = 0; i < ths; i++) { + khash_t(khStrInt)* hashmap = maps_per_thread ? th_hashmaps[i].rules_fast_routing : nullptr; + workers.push_back(std::thread(perform_searches, hashmap, resultset, i, lock)); + } + + for (std::thread& w : workers) { + w.join(); + } + } + } + } + + unsigned long long curtime2 = monotonic_time() / 1000; + uint32_t total_maps_size = 0; + + for (const fast_routing_hashmap_t& hashmap : th_hashmaps) { + total_maps_size += hashmap.rules_fast_routing___keys_values___size; + total_maps_size += kh_size(hashmap.rules_fast_routing) * ((sizeof(int) + sizeof(char *) + 4)); + + kh_destroy(khStrInt, hashmap.rules_fast_routing); + free(hashmap.rules_fast_routing___keys_values); + } + + proxy_info("Test took %llums\n", curtime2 - curtime1); + proxy_info("Verified rows %d\n", results[0]); + proxy_info("Total maps size %dkb\n", total_maps_size / 1024); + + if (resultset) delete resultset; + if (resultset2) delete resultset2; + + return ret; +} + +unsigned int ProxySQL_Admin::ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(unsigned int cnt, bool empty) { + char *a = (char *)"INSERT OR IGNORE INTO mysql_query_rules_fast_routing VALUES (?1, ?2, ?3, ?4, '')"; + int rc; + sqlite3_stmt *statement1=NULL; + rc=admindb->prepare_v2(a, &statement1); + ASSERT_SQLITE_OK(rc, admindb); + admindb->execute("DELETE FROM mysql_query_rules_fast_routing"); + char * username_buf = (char *)malloc(128); + char * schemaname_buf = (char *)malloc(256); + //ui.username = username_buf; + //ui.schemaname = schemaname_buf; + if (empty==false) { + strcpy(username_buf,"user_name_"); + } else { + strcpy(username_buf,""); + } + strcpy(schemaname_buf,"shard_name_"); + int _k; + for (unsigned int i=0; iget_db())==0) { + i--; + } + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + } + (*proxy_sqlite3_finalize)(statement1); + free(username_buf); + free(schemaname_buf); + return cnt; +} + +void ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters() { + mysql_servers_wrlock(); + admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id BETWEEN 10001 AND 20000"); + admindb->execute("DELETE FROM mysql_replication_hostgroups WHERE writer_hostgroup BETWEEN 10001 AND 20000"); + char *q1 = (char *)"INSERT INTO mysql_servers (hostgroup_id, hostname, port) VALUES (?1, ?2, ?3), (?4, ?5, ?6), (?7, ?8, ?9)"; + char *q2 = (char *)"INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup) VALUES (?1, ?2)"; + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement2=NULL; + rc=admindb->prepare_v2(q1, &statement1); + ASSERT_SQLITE_OK(rc, admindb); + rc=admindb->prepare_v2(q2, &statement2); + ASSERT_SQLITE_OK(rc, admindb); + char hostnamebuf1[32]; + char hostnamebuf2[32]; + char hostnamebuf3[32]; + for (int i=1000; i<2000; i++) { + sprintf(hostnamebuf1,"hostname%d", i*10+1); + sprintf(hostnamebuf2,"hostname%d", i*10+2); + sprintf(hostnamebuf3,"hostname%d", i*10+3); + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, i*10+1); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, hostnamebuf1, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 3, 3306); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 4, i*10+2); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 5, hostnamebuf2, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 6, 3306); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, i*10+2); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_text)(statement1, 8, hostnamebuf3, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, 3306); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_bind_int64)(statement2, 1, i*10+1); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_bind_int64)(statement2, 2, i*10+2); ASSERT_SQLITE_OK(rc, admindb); + SAFE_SQLITE3_STEP2(statement2); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_clear_bindings)(statement2); ASSERT_SQLITE_OK(rc, admindb); + rc=(*proxy_sqlite3_reset)(statement2); ASSERT_SQLITE_OK(rc, admindb); + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement2); + load_mysql_servers_to_runtime(); + mysql_servers_wrunlock(); +} +unsigned long long ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_read_only_action() { + // we immediately exit. This is just for developer + return 0; + ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters(); + char hostnamebuf1[32]; + char hostnamebuf2[32]; + char hostnamebuf3[32]; + unsigned long long t1 = monotonic_time(); + //for (int j=0 ; j<500; j++) { + for (int j=0 ; j<1000; j++) { + for (int i=1000; i<2000; i++) { + sprintf(hostnamebuf1,"hostname%d", i*10+1); + sprintf(hostnamebuf2,"hostname%d", i*10+2); + sprintf(hostnamebuf3,"hostname%d", i*10+3); + MyHGM->read_only_action_v2( std::list { + read_only_server_t { std::string(hostnamebuf1), 3306, 0 }, + read_only_server_t { std::string(hostnamebuf2), 3306, 1 }, + read_only_server_t { std::string(hostnamebuf3), 3306, 1 } + } ); + } + } + unsigned long long t2 = monotonic_time(); + t1 /= 1000; + t2 /= 1000; + unsigned long long d = t2-t1; + return d; +} + +#ifdef DEBUG +// NEVER USED THIS FUNCTION IN PRODUCTION. +// THIS IS FOR TESTING PURPOSE ONLY +// IT ACCESSES MyHGM without lock +unsigned long long ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_HG_lookup() { + // we immediately exit. This is just for developer + return 0; + ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters(); + unsigned long long t1 = monotonic_time(); + unsigned int hid = 0; + MyHGC * myhgc = NULL; + for (int j=0 ; j<100000; j++) { + for (unsigned int i=1000; i<2000; i++) { + // NEVER USED THIS FUNCTION IN PRODUCTION. + // THIS IS FOR TESTING PURPOSE ONLY + // IT ACCESSES MyHGM without lock + hid = i*10+1; // writer hostgroup + myhgc = MyHGM->MyHGC_lookup(hid); + assert(myhgc); + hid++; // reader hostgroup + myhgc = MyHGM->MyHGC_lookup(hid); + assert(myhgc); + } + } + unsigned long long t2 = monotonic_time(); + t1 /= 1000; + t2 /= 1000; + unsigned long long d = t2-t1; + return d; +} + +// NEVER USED THIS FUNCTION IN PRODUCTION. +// THIS IS FOR TESTING PURPOSE ONLY +// IT ACCESSES MyHGM without lock +unsigned long long ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_Balancing_HG5211() { + unsigned long long t1 = monotonic_time(); + const unsigned int NS = 4; + unsigned int cu[NS] = { 50, 10, 10, 0 }; + MyHGC * myhgc = NULL; + myhgc = MyHGM->MyHGC_lookup(5211); + assert(myhgc); + assert(myhgc->mysrvs->servers->len == NS); + unsigned int cnt[NS]; + for (unsigned int i=0; imysrvs->servers->index(i); + m->ConnectionsUsed->conns->len=cu[i]; + } + unsigned int NL = 1000; + for (unsigned int i=0; iget_random_MySrvC(NULL, 0, -1, NULL); + assert(mysrvc); + for (unsigned int k=0; kmysrvs->servers->index(k); + if (m == mysrvc) + cnt[k]++; + } + } + { + unsigned int tc = 0; + for (unsigned int k=0; k(ProxySQL_Admin*, MySQL_Session*, char*, bool&); +template void ProxySQL_Admin::ProxySQL_Test_Handler(ProxySQL_Admin*, PgSQL_Session*, char*, bool&); + +template +void ProxySQL_Admin::ProxySQL_Test_Handler(ProxySQL_Admin *SPA, S* sess, char *query_no_space, bool& run_query) { + if constexpr (std::is_same_v) { + } else if constexpr (std::is_same_v) { + } else { + assert(0); + } + int test_n = 0; + int test_arg1 = 0; + int test_arg2 = 0; + int test_arg3 = -1; + int test_arg4 = -1; + int r1 = 0; + proxy_warning("Received PROXYSQLTEST command: %s\n", query_no_space); + char *msg = NULL; + sscanf(query_no_space+strlen("PROXYSQLTEST "),"%d %d %d %d %d", &test_n, &test_arg1, &test_arg2, &test_arg3, &test_arg4); + if (test_n) { + switch (test_n) { + case 1: + // generate test_arg1*1000 entries in digest map + if (test_arg1==0) { + test_arg1=1; + } + r1 = ProxySQL_Test___GenerateRandomQueryInDigestTable(test_arg1); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 2: + // get all the entries from the digest map, but without writing to DB + // it uses multiple threads + r1 = ProxySQL_Test___GetDigestTable(false, false); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 3: + // get all the entries from the digest map and reset, but without writing to DB + // it uses multiple threads + r1 = ProxySQL_Test___GetDigestTable(true, false); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 4: + // purge the digest map, synchronously, in single thread + r1 = ProxySQL_Test___PurgeDigestTable(false, false, NULL); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 5: + // purge the digest map, synchronously, in multiple threads + r1 = ProxySQL_Test___PurgeDigestTable(false, true, NULL); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 6: + // purge the digest map, asynchronously, in single thread + r1 = ProxySQL_Test___PurgeDigestTable(true, false, &msg); + SPA->send_ok_msg_to_client(sess, msg, r1, query_no_space); + free(msg); + run_query=false; + break; + case 7: + // get all the entries from the digest map and reset, but without writing to DB + // it uses multiple threads + // it locks for a very short time and doesn't use SQLite3_result, but swap + r1 = ProxySQL_Test___GetDigestTable(true, true); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 8: + // get all the entries from the digest map and reset, AND write to DB + r1 = SPA->FlushDigestTableToDisk(SPA->statsdb_disk); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 11: // generate username + case 15: // no username, empty string + // generate random mysql_query_rules_fast_routing + if (test_arg1==0) { + test_arg1=10000; + } + if (test_n==15) { + r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, true); + } else { + r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, false); + } + SPA->send_ok_msg_to_client(sess, (char *)"Generated new mysql_query_rules_fast_routing table", r1, query_no_space); + run_query=false; + break; + case 12: // generate username + case 16: // no username, empty string + // generate random mysql_query_rules_fast_routing and LOAD TO RUNTIME + if (test_arg1==0) { + test_arg1=10000; + } + if (test_n==16) { + r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, true); + } else { + r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, false); + } + msg = SPA->load_mysql_query_rules_to_runtime(); + if (msg==NULL) { + SPA->send_ok_msg_to_client(sess, (char *)"Generated new mysql_query_rules_fast_routing table and loaded to runtime", r1, query_no_space); + } else { + SPA->send_error_msg_to_client(sess, msg); + } + run_query=false; + break; + case 13: + // LOAD MYSQL QUERY RULES TO RUNTIME for N times + if (test_arg1==0) { + test_arg1=1; + } + for (int i=0; iload_mysql_query_rules_to_runtime(); + } + msg = (char *)malloc(128); + sprintf(msg,"Loaded mysql_query_rules_fast_routing to runtime %d times",test_arg1); + SPA->send_ok_msg_to_client(sess, msg, 0, query_no_space); + run_query=false; + free(msg); + break; + case 14: // old algorithm + case 17: // perform dual lookup, with and without username + // Allows to verify and benchmark 'mysql_query_rules_fast_routing'. Every options + // verifies all 'mysql_query_rules_fast_routing' rules: + // - Test num: 14 old algorithm, 17 perform a dual lookup. + // - arg1: 1-N Number of times the computation should be repeated. + // - arg2: 1-N Number of parallel threads for the test. + // - arg3: 1-0 Wether or not to acquire a read_lock before searching in the hashmap. + // - arg4: 1-0 Wether or not to create thread specific hashmaps for the search. + if (test_arg1==0) { + test_arg1=1; + } + // To preserve classic mode + if (test_arg3 == -1) { + test_arg3 = 1; + } + if (test_arg4 == -1) { + test_arg4 = 0; + } + { + int ret1, ret2; + bool bret = SPA->ProxySQL_Test___Verify_mysql_query_rules_fast_routing( + &ret1, &ret2, test_arg1, (test_n==14 ? 0 : 1), test_arg2, test_arg3, test_arg4 + ); + if (bret) { + SPA->send_ok_msg_to_client(sess, (char *)"Verified all rules in mysql_query_rules_fast_routing", ret1, query_no_space); + } else { + if (ret1==-1) { + SPA->send_error_msg_to_client(sess, (char *)"Severe error in verifying rules in mysql_query_rules_fast_routing"); + } else { + msg = (char *)malloc(256); + sprintf(msg,"Error verifying mysql_query_rules_fast_routing. Found %d rows out of %d", ret1, ret2); + SPA->send_error_msg_to_client(sess, msg); + free(msg); + } + } + } + run_query=false; + break; + case 21: + // refresh mysql variables N*1000 times + if (test_arg1==0) { + test_arg1=1; + } + test_arg1 *= 1000; + ProxySQL_Test___Refresh_MySQL_Variables(test_arg1); + msg = (char *)malloc(128); + sprintf(msg,"Refreshed MySQL Variables %d times",test_arg1); + SPA->send_ok_msg_to_client(sess, msg, 0, query_no_space); + run_query=false; + free(msg); + break; + case 22: + // get all the entries from the digest map, but WRITING to DB + // it uses multiple threads + // It locks the maps while generating the resultset + r1 = SPA->stats___mysql_query_digests(false, true); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 23: + // get all the entries from the digest map, but WRITING to DB + // it uses multiple threads for creating the resultset + r1 = SPA->stats___mysql_query_digests_v2(false, false, true); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 24: + // get all the entries from the digest map, but WRITING to DB + // Do not create a resultset, uses the digest_umap + r1 = SPA->stats___mysql_query_digests_v2(false, false, false); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 25: + // get all the entries from the digest map AND RESET, but WRITING to DB + // it uses multiple threads + // It locks the maps while generating the resultset + r1 = SPA->stats___mysql_query_digests(true, true); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 26: + // get all the entries from the digest map AND RESET, but WRITING to DB + // it uses multiple threads for creating the resultset + r1 = SPA->stats___mysql_query_digests_v2(true, true, true); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 27: + // get all the entries from the digest map AND RESET, but WRITING to DB + // Do not create a resultset, uses the digest_umap + r1 = SPA->stats___mysql_query_digests_v2(true, true, false); + SPA->send_ok_msg_to_client(sess, NULL, r1, query_no_space); + run_query=false; + break; + case 31: + { + if (test_arg1==0) { + test_arg1=1; + } + if (test_arg1 > 4) { + test_arg1=1; + } +/* + if (test_arg1 == 2 || test_arg1 == 3) { + if (test_arg2 == 0) { + test_arg2 = 1; + } + } +*/ + int ret1; + int ret2; + SPA->ProxySQL_Test___Load_MySQL_Whitelist(&ret1, &ret2, test_arg1, test_arg2); + if (test_arg1==1 || test_arg1==4) { + SPA->send_ok_msg_to_client(sess, (char *)"Processed all rows from firewall whitelist", ret1, query_no_space); + } else if (test_arg1==2 || test_arg1==3) { + if (ret1 == ret2) { + SPA->send_ok_msg_to_client(sess, (char *)"Verified all rows from firewall whitelist", ret1, query_no_space); + } else { + msg = (char *)malloc(256); + sprintf(msg,"Error verifying firewall whitelist. Found %d entries out of %d", ret2, ret1); + SPA->send_error_msg_to_client(sess, msg); + free(msg); + } + } + run_query=false; + } + break; + case 41: + { + char msg[256]; + unsigned long long d = SPA->ProxySQL_Test___MySQL_HostGroups_Manager_read_only_action(); + sprintf(msg, "Tested in %llums\n", d); + SPA->send_ok_msg_to_client(sess, msg, 0, query_no_space); + run_query=false; + } + break; +#ifdef DEBUG + case 51: + { + char msg[256]; + unsigned long long d = SPA->ProxySQL_Test___MySQL_HostGroups_Manager_HG_lookup(); + sprintf(msg, "Tested in %llums\n", d); + SPA->send_ok_msg_to_client(sess, msg, 0, query_no_space); + run_query=false; + } + break; + case 52: + { + char msg[256]; + SPA->mysql_servers_wrlock(); + SPA->admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id=5211"); + SPA->admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, port, weight) VALUES (5211,'127.0.0.2',3306,10000)"); + SPA->admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, port, weight) VALUES (5211,'127.0.0.3',3306,8000)"); + SPA->admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, port, weight) VALUES (5211,'127.0.0.4',3306,8000)"); + SPA->admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, port, weight) VALUES (5211,'127.0.0.5',3306,7000)"); + SPA->load_mysql_servers_to_runtime(); + SPA->mysql_servers_wrunlock(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded mysql servers to RUNTIME\n"); + unsigned long long d = SPA->ProxySQL_Test___MySQL_HostGroups_Manager_Balancing_HG5211(); + sprintf(msg, "Tested in %llums\n", d); + SPA->mysql_servers_wrlock(); + SPA->admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id=5211"); + SPA->load_mysql_servers_to_runtime(); + SPA->mysql_servers_wrunlock(); + SPA->send_ok_msg_to_client(sess, msg, 0, query_no_space); + run_query=false; + } + break; + case 53: + { + // Test monitor tasks timeout + // test_arg1: 1 = ON, 0 = OFF + char msg[256]; + GloMyMon->proxytest_forced_timeout = (test_arg1) ? true : false; + sprintf(msg, "Monitor task timeout flag is:%s\n", GloMyMon->proxytest_forced_timeout ? "ON" : "OFF"); + SPA->send_ok_msg_to_client(sess, msg, 0, query_no_space); + run_query = false; + } + break; + case 54: + { + run_query = false; + if (test_arg1 == 0) { + test_arg1 = 1000; + } + if (GloMTH->variables.ssl_p2s_ca == NULL && + GloMTH->variables.ssl_p2s_capath == NULL) { + SPA->send_error_msg_to_client(sess, (char *)"'mysql-ssl_p2s_ca' and 'mysql-ssl_p2s_capath' have not been configured"); + break; + } + char msg[256]; + uint64_t duration = 0ULL; + if (SPA->ProxySQL_Test___CA_Certificate_Load_And_Verify(&duration, test_arg1, GloMTH->variables.ssl_p2s_ca, + GloMTH->variables.ssl_p2s_capath)) { + sprintf(msg, "Took %lums in loading and verifying CA Certificate for %d times\n", duration, test_arg1); + SPA->send_ok_msg_to_client(sess, msg, 0, query_no_space); + } + else { + SPA->send_error_msg_to_client(sess, (char *)"Unable to verify CA Certificate"); + } + } + break; +#endif // DEBUG + default: + SPA->send_error_msg_to_client(sess, (char *)"Invalid test"); + run_query=false; + break; + } + } else { + SPA->send_error_msg_to_client(sess, (char *)"Invalid test"); + } +} diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 5146ad9a85..f4139446ab 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -242,7 +242,6 @@ void * ProxySQL_Cluster_Monitor_thread(void *args) { } } else { proxy_warning("Cluster: unable to connect to peer %s:%d . Error: %s\n", node->hostname, node->port, mysql_error(conn)); - node->remove_dns_record(); node->resolve_hostname(); mysql_close(conn); conn = mysql_init(NULL); @@ -4483,11 +4482,3 @@ void ProxySQL_Node_Address::resolve_hostname() { } } } - -void ProxySQL_Node_Address::remove_dns_record() { - // make sure hostname is not NULL and port is not 0 (UNIX socket) - if (hostname && port) { - MySQL_Monitor::remove_dns_record_from_dns_cache(hostname); - } -} - diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index dc7360471a..ae7a8a0c03 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -1336,12 +1336,12 @@ int ProxySQL_Config::Read_ProxySQL_Servers_from_configfile() { int rows=0; admindb->execute("PRAGMA foreign_keys = OFF"); if (root.exists("proxysql_servers")==true) { - const Setting &mysql_servers = root["proxysql_servers"]; - int count = mysql_servers.getLength(); + const Setting & proxysql_servers = root["proxysql_servers"]; + int count = proxysql_servers.getLength(); //fprintf(stderr, "Found %d servers\n",count); char *q=(char *)"INSERT OR REPLACE INTO proxysql_servers (hostname, port, weight, comment) VALUES (\"%s\", %d, %d, '%s')"; for (i=0; i< count; i++) { - const Setting &server = mysql_servers[i]; + const Setting &server = proxysql_servers[i]; std::string address; int port; int weight=0; @@ -1419,3 +1419,623 @@ int ProxySQL_Config::Write_Global_Variables_to_configfile(std::string& data) { return 0; } + +int ProxySQL_Config::Write_PgSQL_Servers_to_configfile(std::string& data) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* sqlite_resultset = NULL; + + char* query = (char*)"SELECT * FROM pgsql_servers"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from pgsql_servers : %s\n", error); + return -1; + } + else { + if (sqlite_resultset) { + data += "pgsql_servers:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "hostgroup_id", r->fields[0], ""); + addField(data, "hostname", r->fields[1]); + addField(data, "port", r->fields[2], ""); + addField(data, "status", r->fields[3]); + addField(data, "weight", r->fields[4], ""); + addField(data, "compression", r->fields[5], ""); + addField(data, "max_connections", r->fields[6], ""); + addField(data, "max_replication_lag", r->fields[7], ""); + addField(data, "use_ssl", r->fields[8], ""); + addField(data, "max_latency_ms", r->fields[9], ""); + addField(data, "comment", r->fields[10]); + + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + + if (sqlite_resultset) + delete sqlite_resultset; + + query = (char*)"SELECT * FROM pgsql_replication_hostgroups"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from pgsql_replication_hostgroups : %s\n", error); + return -1; + } + else { + if (sqlite_resultset) { + data += "pgsql_replication_hostgroups:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "writer_hostgroup", r->fields[0], ""); + addField(data, "reader_hostgroup", r->fields[1], ""); + addField(data, "check_type", r->fields[2]); + addField(data, "comment", r->fields[3]); + + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + + if (sqlite_resultset) + delete sqlite_resultset; + + return 0; +} + +int ProxySQL_Config::Read_PgSQL_Servers_from_configfile() { + const Setting& root = GloVars.confFile->cfg.getRoot(); + int i; + int rows = 0; + admindb->execute("PRAGMA foreign_keys = OFF"); + if (root.exists("pgsql_servers") == true) { + const Setting& pgsql_servers = root["pgsql_servers"]; + int count = pgsql_servers.getLength(); + //fprintf(stderr, "Found %d servers\n",count); + char* q = (char*)"INSERT OR REPLACE INTO pgsql_servers (hostname, port, hostgroup_id, compression, weight, status, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) VALUES (\"%s\", %d, %d, %d, %d, \"%s\", %d, %d, %d, %d, '%s')"; + for (i = 0; i < count; i++) { + const Setting& server = pgsql_servers[i]; + std::string address; + std::string status = "ONLINE"; + int port = 5432; + int hostgroup; + int weight = 1; + int compression = 0; + int max_connections = 1000; // default + int max_replication_lag = 0; // default + int use_ssl = 0; + int max_latency_ms = 0; + std::string comment = ""; + if (server.lookupValue("address", address) == false) { + if (server.lookupValue("hostname", address) == false) { + proxy_error("Admin: detected a pgsql_servers in config file without a mandatory hostname\n"); + continue; + } + } + server.lookupValue("port", port); + if (server.lookupValue("hostgroup", hostgroup) == false) { + if (server.lookupValue("hostgroup_id", hostgroup) == false) { + proxy_error("Admin: detected a pgsql_servers in config file without a mandatory hostgroup_id\n"); + continue; + } + } + server.lookupValue("status", status); + if ( + (strcasecmp(status.c_str(), (char*)"ONLINE")) + && (strcasecmp(status.c_str(), (char*)"SHUNNED")) + && (strcasecmp(status.c_str(), (char*)"OFFLINE_SOFT")) + && (strcasecmp(status.c_str(), (char*)"OFFLINE_HARD")) + ) { + status = "ONLINE"; + } + server.lookupValue("compression", compression); + server.lookupValue("weight", weight); + server.lookupValue("max_connections", max_connections); + server.lookupValue("max_replication_lag", max_replication_lag); + server.lookupValue("use_ssl", use_ssl); + server.lookupValue("max_latency_ms", max_latency_ms); + server.lookupValue("comment", comment); + char* o1 = strdup(comment.c_str()); + char* o = escape_string_single_quotes(o1, false); + char* query = (char*)malloc(strlen(q) + strlen(status.c_str()) + strlen(address.c_str()) + strlen(o) + 128); + sprintf(query, q, address.c_str(), port, hostgroup, compression, weight, status.c_str(), max_connections, max_replication_lag, use_ssl, max_latency_ms, o); + //fprintf(stderr, "%s\n", query); + admindb->execute(query); + if (o != o1) free(o); + free(o1); + free(query); + rows++; + } + } + if (root.exists("pgsql_replication_hostgroups") == true) { + const Setting& pgsql_replication_hostgroups = root["pgsql_replication_hostgroups"]; + int count = pgsql_replication_hostgroups.getLength(); + char* q = (char*)"INSERT OR REPLACE INTO pgsql_replication_hostgroups (writer_hostgroup, reader_hostgroup, comment, check_type) VALUES (%d, %d, '%s', '%s')"; + for (i = 0; i < count; i++) { + const Setting& line = pgsql_replication_hostgroups[i]; + int writer_hostgroup; + int reader_hostgroup; + std::string comment = ""; + std::string check_type = ""; + if (line.lookupValue("writer_hostgroup", writer_hostgroup) == false) { + proxy_error("Admin: detected a pgsql_replication_hostgroups in config file without a mandatory writer_hostgroup\n"); + continue; + } + if (line.lookupValue("reader_hostgroup", reader_hostgroup) == false) { + proxy_error("Admin: detected a pgsql_replication_hostgroups in config file without a mandatory reader_hostgroup\n"); + continue; + } + line.lookupValue("comment", comment); + char* o1 = strdup(comment.c_str()); + char* o = escape_string_single_quotes(o1, false); + line.lookupValue("check_type", check_type); + if ( + (strcasecmp(check_type.c_str(), (char*)"read_only")) + && (strcasecmp(check_type.c_str(), (char*)"innodb_read_only")) + && (strcasecmp(check_type.c_str(), (char*)"super_read_only")) + ) { + check_type = "read_only"; + } + char* t1 = strdup(check_type.c_str()); + char* t = escape_string_single_quotes(t1, false); + char* query = (char*)malloc(strlen(q) + strlen(o) + strlen(t) + 32); + sprintf(query, q, writer_hostgroup, reader_hostgroup, o, t); + //fprintf(stderr, "%s\n", query); + admindb->execute(query); + if (o != o1) free(o); + free(o1); + if (t != t1) free(t); + free(t1); + free(query); + rows++; + } + } + admindb->execute("PRAGMA foreign_keys = ON"); + return rows; +} + +int ProxySQL_Config::Write_PgSQL_Users_to_configfile(std::string& data) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* sqlite_resultset = NULL; + + char* query = (char*)"SELECT * FROM pgsql_users"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from pgsql_users: %s\n", error); + return -1; + } + else { + if (sqlite_resultset) { + data += "pgsql_users:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "username", r->fields[0]); + addField(data, "password", r->fields[1]); + addField(data, "active", r->fields[2], ""); + addField(data, "use_ssl", r->fields[3], ""); + addField(data, "default_hostgroup", r->fields[4], ""); + addField(data, "transaction_persistent", r->fields[5], ""); + addField(data, "fast_forward", r->fields[6], ""); + addField(data, "backend", r->fields[7], ""); + addField(data, "frontend", r->fields[8], ""); + addField(data, "max_connections", r->fields[9], ""); + addField(data, "attributes", r->fields[10]); + addField(data, "comment", r->fields[11]); + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + + if (sqlite_resultset) + delete sqlite_resultset; + + return 0; +} + +int ProxySQL_Config::Read_PgSQL_Users_from_configfile() { + const Setting& root = GloVars.confFile->cfg.getRoot(); + if (root.exists("pgsql_users") == false) return 0; + const Setting& pgsql_users = root["pgsql_users"]; + int count = pgsql_users.getLength(); + //fprintf(stderr, "Found %d users\n",count); + int i; + int rows = 0; + admindb->execute("PRAGMA foreign_keys = OFF"); + char* q = (char*)"INSERT OR REPLACE INTO pgsql_users (username, password, active, use_ssl, default_hostgroup, transaction_persistent, fast_forward, max_connections, attributes, comment) VALUES ('%s', '%s', %d, %d, %d, %d, %d, %d, '%s','%s')"; + for (i = 0; i < count; i++) { + const Setting& user = pgsql_users[i]; + std::string username; + std::string password = ""; + int active = 1; + int use_ssl = 0; + int default_hostgroup = 0; + int transaction_persistent = 1; + int fast_forward = 0; + int max_connections = 10000; + std::string comment = ""; + std::string attributes = ""; + if (user.lookupValue("username", username) == false) { + proxy_error("Admin: detected a pgsql_users in config file without a mandatory username\n"); + continue; + } + user.lookupValue("password", password); + user.lookupValue("default_hostgroup", default_hostgroup); + user.lookupValue("active", active); + user.lookupValue("use_ssl", use_ssl); + //if (user.lookupValue("default_schema", default_schema)==false) default_schema=""; + user.lookupValue("transaction_persistent", transaction_persistent); + user.lookupValue("fast_forward", fast_forward); + user.lookupValue("max_connections", max_connections); + user.lookupValue("attributes", attributes); + user.lookupValue("comment", comment); + char* o1 = strdup(comment.c_str()); + char* o = escape_string_single_quotes(o1, false); + char* query = (char*)malloc(strlen(q) + strlen(username.c_str()) + strlen(password.c_str()) + strlen(o) + strlen(attributes.c_str()) + 128); + sprintf(query, q, username.c_str(), password.c_str(), active, use_ssl, default_hostgroup, transaction_persistent, fast_forward, max_connections, attributes.c_str(), o); + admindb->execute(query); + if (o != o1) free(o); + free(o1); + free(query); + rows++; + } + admindb->execute("PRAGMA foreign_keys = ON"); + return rows; +} + +int ProxySQL_Config::Write_PgSQL_Query_Rules_to_configfile(std::string& data) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* sqlite_resultset = NULL; + + char* query = (char*)"SELECT * FROM pgsql_query_rules"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from pgsql_query_rules : %s\n", error); + return -1; + } + else { + if (sqlite_resultset) { + std::string prefix; + data += "pgsql_query_rules:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "rule_id", r->fields[0], ""); + addField(data, "active", r->fields[1], ""); + addField(data, "username", r->fields[2]); + addField(data, "database", r->fields[3]); + addField(data, "flagIN", r->fields[4], ""); + addField(data, "client_addr", r->fields[5]); + addField(data, "proxy_addr", r->fields[6]); + addField(data, "proxy_port", r->fields[7], ""); + addField(data, "digest", r->fields[8]); + addField(data, "match_digest", r->fields[9]); + addField(data, "match_pattern", r->fields[10]); + addField(data, "negate_match_pattern", r->fields[11], ""); + addField(data, "re_modifiers", r->fields[12]); + addField(data, "flagOUT", r->fields[13], ""); + addField(data, "replace_pattern", r->fields[14]); + addField(data, "destination_hostgroup", r->fields[15], ""); + addField(data, "cache_ttl", r->fields[16], ""); + addField(data, "cache_empty_result", r->fields[17], ""); + addField(data, "cache_timeout", r->fields[18], ""); + addField(data, "reconnect", r->fields[19], ""); + addField(data, "timeout", r->fields[20], ""); + addField(data, "retries", r->fields[21], ""); + addField(data, "delay", r->fields[22], ""); + addField(data, "next_query_flagIN", r->fields[23], ""); + addField(data, "mirror_flagOUT", r->fields[24], ""); + addField(data, "mirror_hostgroup", r->fields[25], ""); + addField(data, "error_msg", r->fields[26]); + addField(data, "OK_msg", r->fields[27]); + addField(data, "sticky_conn", r->fields[28], ""); + addField(data, "multiplex", r->fields[29], ""); + addField(data, "log", r->fields[30], ""); + addField(data, "apply", r->fields[31], ""); + addField(data, "attributes", r->fields[32]); + addField(data, "comment", r->fields[33]); + + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + + if (sqlite_resultset) + delete sqlite_resultset; + + return 0; +} + + +int ProxySQL_Config::Read_PgSQL_Query_Rules_from_configfile() { + const Setting& root = GloVars.confFile->cfg.getRoot(); + if (root.exists("pgsql_query_rules") == false) return 0; + const Setting& pgsql_query_rules = root["pgsql_query_rules"]; + int count = pgsql_query_rules.getLength(); + //fprintf(stderr, "Found %d users\n",count); + int i; + int rows = 0; + admindb->execute("PRAGMA foreign_keys = OFF"); + char* q = (char*)"INSERT OR REPLACE INTO pgsql_query_rules (rule_id, active, username, database, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, log, apply, attributes, comment) VALUES (%d, %d, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s, %s)"; + for (i = 0; i < count; i++) { + const Setting& rule = pgsql_query_rules[i]; + int rule_id; + int active = 1; + bool username_exists = false; + std::string username; + bool database_exists = false; + std::string database; + int flagIN = 0; + + // variables for parsing client_addr + bool client_addr_exists = false; + std::string client_addr; + + // variables for parsing proxy_addr + bool proxy_addr_exists = false; + std::string proxy_addr; + + // variable for parsing proxy_port + int proxy_port = -1; + + // variables for parsing digest + bool digest_exists = false; + std::string digest; + + + bool match_digest_exists = false; + std::string match_digest; + bool match_pattern_exists = false; + std::string match_pattern; + int negate_match_pattern = 0; + + bool re_modifiers_exists = false; + std::string re_modifiers; + + int flagOUT = -1; + bool replace_pattern_exists = false; + std::string replace_pattern; + int destination_hostgroup = -1; + int next_query_flagIN = -1; + int mirror_flagOUT = -1; + int mirror_hostgroup = -1; + int cache_ttl = -1; + int cache_empty_result = -1; + int cache_timeout = -1; + int reconnect = -1; + int timeout = -1; + int retries = -1; + int delay = -1; + bool error_msg_exists = false; + std::string error_msg; + bool OK_msg_exists = false; + std::string OK_msg; + + int sticky_conn = -1; + int multiplex = -1; + + // variable for parsing log + int log = -1; + + int apply = 0; + + // attributes + bool attributes_exists = false; + std::string attributes{}; + + bool comment_exists = false; + std::string comment; + + // validate arguments + if (rule.lookupValue("rule_id", rule_id) == false) { + proxy_error("Admin: detected a pgsql_query_rules in config file without a mandatory rule_id\n"); + continue; + } + rule.lookupValue("active", active); + if (rule.lookupValue("username", username)) username_exists = true; + if (rule.lookupValue("database", database)) database_exists = true; + rule.lookupValue("flagIN", flagIN); + + if (rule.lookupValue("client_addr", client_addr)) client_addr_exists = true; + if (rule.lookupValue("proxy_addr", proxy_addr)) proxy_addr_exists = true; + rule.lookupValue("proxy_port", proxy_port); + if (rule.lookupValue("digest", digest)) digest_exists = true; + + if (rule.lookupValue("match_digest", match_digest)) match_digest_exists = true; + if (rule.lookupValue("match_pattern", match_pattern)) match_pattern_exists = true; + rule.lookupValue("negate_match_pattern", negate_match_pattern); + if (rule.lookupValue("re_modifiers", re_modifiers)) { + } + else { + re_modifiers = "CASELESS"; + } + re_modifiers_exists = true; + rule.lookupValue("flagOUT", flagOUT); + if (rule.lookupValue("replace_pattern", replace_pattern)) replace_pattern_exists = true; + rule.lookupValue("destination_hostgroup", destination_hostgroup); + rule.lookupValue("next_query_flagIN", next_query_flagIN); + rule.lookupValue("mirror_flagOUT", mirror_flagOUT); + rule.lookupValue("mirror_hostgroup", mirror_hostgroup); + rule.lookupValue("cache_ttl", cache_ttl); + rule.lookupValue("cache_empty_result", cache_empty_result); + rule.lookupValue("cache_timeout", cache_timeout); + rule.lookupValue("reconnect", reconnect); + rule.lookupValue("timeout", timeout); + rule.lookupValue("retries", retries); + rule.lookupValue("delay", delay); + + if (rule.lookupValue("error_msg", error_msg)) error_msg_exists = true; + if (rule.lookupValue("OK_msg", OK_msg)) OK_msg_exists = true; + + rule.lookupValue("sticky_conn", sticky_conn); + rule.lookupValue("multiplex", multiplex); + + rule.lookupValue("log", log); + + rule.lookupValue("apply", apply); + if (rule.lookupValue("comment", comment)) comment_exists = true; + if (rule.lookupValue("attributes", attributes)) attributes_exists = true; + + + //if (user.lookupValue("default_schema", default_schema)==false) default_schema=""; + int query_len = 0; + query_len += strlen(q) + + strlen(std::to_string(rule_id).c_str()) + + strlen(std::to_string(active).c_str()) + + (username_exists ? strlen(username.c_str()) : 0) + 4 + + (database_exists ? strlen(database.c_str()) : 0) + 4 + + strlen(std::to_string(flagIN).c_str()) + 4 + + + (client_addr_exists ? strlen(client_addr.c_str()) : 0) + 4 + + (proxy_addr_exists ? strlen(proxy_addr.c_str()) : 0) + 4 + + strlen(std::to_string(proxy_port).c_str()) + 4 + + + (match_digest_exists ? strlen(match_digest.c_str()) : 0) + 4 + + (match_pattern_exists ? strlen(match_pattern.c_str()) : 0) + 4 + + strlen(std::to_string(negate_match_pattern).c_str()) + 4 + + (re_modifiers_exists ? strlen(re_modifiers.c_str()) : 0) + 4 + + strlen(std::to_string(flagOUT).c_str()) + 4 + + (replace_pattern_exists ? strlen(replace_pattern.c_str()) : 0) + 4 + + strlen(std::to_string(destination_hostgroup).c_str()) + 4 + + strlen(std::to_string(cache_ttl).c_str()) + 4 + + strlen(std::to_string(cache_empty_result).c_str()) + 4 + + strlen(std::to_string(cache_timeout).c_str()) + 4 + + strlen(std::to_string(reconnect).c_str()) + 4 + + strlen(std::to_string(timeout).c_str()) + 4 + + strlen(std::to_string(next_query_flagIN).c_str()) + 4 + + strlen(std::to_string(mirror_flagOUT).c_str()) + 4 + + strlen(std::to_string(mirror_hostgroup).c_str()) + 4 + + strlen(std::to_string(retries).c_str()) + 4 + + strlen(std::to_string(delay).c_str()) + 4 + + (error_msg_exists ? strlen(error_msg.c_str()) : 0) + 4 + + (OK_msg_exists ? strlen(OK_msg.c_str()) : 0) + 4 + + strlen(std::to_string(sticky_conn).c_str()) + 4 + + strlen(std::to_string(multiplex).c_str()) + 4 + + strlen(std::to_string(log).c_str()) + 4 + + strlen(std::to_string(apply).c_str()) + 4 + + (attributes_exists ? strlen(attributes.c_str()) : 0) + 4 + + (comment_exists ? strlen(comment.c_str()) : 0) + 4 + + 64; + char* query = (char*)malloc(query_len); + if (username_exists) + username = "\"" + username + "\""; + else + username = "NULL"; + if (database_exists) + database = "\"" + database + "\""; + else + database = "NULL"; + + if (client_addr_exists) + client_addr = "\"" + client_addr + "\""; + else + client_addr = "NULL"; + if (proxy_addr_exists) + proxy_addr = "\"" + proxy_addr + "\""; + else + proxy_addr = "NULL"; + if (digest_exists) + digest = "\"" + digest + "\""; + else + digest = "NULL"; + + if (match_digest_exists) + match_digest = "\"" + match_digest + "\""; + else + match_digest = "NULL"; + if (match_pattern_exists) + match_pattern = "\"" + match_pattern + "\""; + else + match_pattern = "NULL"; + if (replace_pattern_exists) + replace_pattern = "\"" + replace_pattern + "\""; + else + replace_pattern = "NULL"; + if (error_msg_exists) + error_msg = "\"" + error_msg + "\""; + else + error_msg = "NULL"; + if (OK_msg_exists) + OK_msg = "\"" + OK_msg + "\""; + else + OK_msg = "NULL"; + if (re_modifiers_exists) + re_modifiers = "\"" + re_modifiers + "\""; + else + re_modifiers = "NULL"; + if (attributes_exists) + attributes = "'" + attributes + "'"; + else + attributes = "NULL"; + if (comment_exists) + comment = "'" + comment + "'"; + else + comment = "NULL"; + + + sprintf(query, q, + rule_id, active, + username.c_str(), + database.c_str(), + (flagIN >= 0 ? std::to_string(flagIN).c_str() : "NULL"), + client_addr.c_str(), + proxy_addr.c_str(), + (proxy_port >= 0 ? std::to_string(proxy_port).c_str() : "NULL"), + digest.c_str(), + match_digest.c_str(), + match_pattern.c_str(), + (negate_match_pattern == 0 ? 0 : 1), + re_modifiers.c_str(), + (flagOUT >= 0 ? std::to_string(flagOUT).c_str() : "NULL"), + replace_pattern.c_str(), + (destination_hostgroup >= 0 ? std::to_string(destination_hostgroup).c_str() : "NULL"), + (cache_ttl >= 0 ? std::to_string(cache_ttl).c_str() : "NULL"), + (cache_empty_result >= 0 ? std::to_string(cache_empty_result).c_str() : "NULL"), + (cache_timeout >= 0 ? std::to_string(cache_timeout).c_str() : "NULL"), + (reconnect >= 0 ? std::to_string(reconnect).c_str() : "NULL"), + (timeout >= 0 ? std::to_string(timeout).c_str() : "NULL"), + (retries >= 0 ? std::to_string(retries).c_str() : "NULL"), + (delay >= 0 ? std::to_string(delay).c_str() : "NULL"), + (next_query_flagIN >= 0 ? std::to_string(next_query_flagIN).c_str() : "NULL"), + (mirror_flagOUT >= 0 ? std::to_string(mirror_flagOUT).c_str() : "NULL"), + (mirror_hostgroup >= 0 ? std::to_string(mirror_hostgroup).c_str() : "NULL"), + error_msg.c_str(), + OK_msg.c_str(), + (sticky_conn >= 0 ? std::to_string(sticky_conn).c_str() : "NULL"), + (multiplex >= 0 ? std::to_string(multiplex).c_str() : "NULL"), + (log >= 0 ? std::to_string(log).c_str() : "NULL"), + (apply == 0 ? 0 : 1), + attributes.c_str(), + comment.c_str() + ); + //fprintf(stderr, "%s\n", query); + admindb->execute(query); + free(query); + rows++; + } + admindb->execute("PRAGMA foreign_keys = ON"); + return rows; +} diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index 7cca7d8475..b14aedbbfe 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -114,6 +114,10 @@ ProxySQL_GlobalVariables::~ProxySQL_GlobalVariables() { checksums_values.mysql_variables.in_shutdown = true; checksums_values.ldap_variables.in_shutdown = true; checksums_values.proxysql_servers.in_shutdown = true; + checksums_values.pgsql_query_rules.in_shutdown = true; + checksums_values.pgsql_servers.in_shutdown = true; + checksums_values.pgsql_users.in_shutdown = true; + checksums_values.pgsql_variables.in_shutdown = true; if (global.gr_bootstrap_uri) { free(global.gr_bootstrap_uri); global.gr_bootstrap_uri = nullptr; @@ -578,6 +582,31 @@ uint64_t ProxySQL_GlobalVariables::generate_global_checksum() { myhash.Update(v->checksum,strlen(v->checksum)); myhash.Update(&v->version,sizeof(v->version)); } + v = &checksums_values.pgsql_query_rules; + if (v->version) { + myhash.Update(v->checksum, strlen(v->checksum)); + myhash.Update(&v->version, sizeof(v->version)); + } + v = &checksums_values.pgsql_servers; + if (v->version) { + myhash.Update(v->checksum, strlen(v->checksum)); + myhash.Update(&v->version, sizeof(v->version)); + } + v = &checksums_values.pgsql_servers_v2; + if (v->version) { + myhash.Update(v->checksum, strlen(v->checksum)); + myhash.Update(&v->version, sizeof(v->version)); + } + v = &checksums_values.pgsql_users; + if (v->version) { + myhash.Update(v->checksum, strlen(v->checksum)); + myhash.Update(&v->version, sizeof(v->version)); + } + v = &checksums_values.pgsql_variables; + if (v->version) { + myhash.Update(v->checksum, strlen(v->checksum)); + myhash.Update(&v->version, sizeof(v->version)); + } v = &checksums_values.proxysql_servers; if (v->version) { myhash.Update(v->checksum,strlen(v->checksum)); diff --git a/lib/ProxySQL_HTTP_Server.cpp b/lib/ProxySQL_HTTP_Server.cpp index d8affaf396..35c8f946db 100644 --- a/lib/ProxySQL_HTTP_Server.cpp +++ b/lib/ProxySQL_HTTP_Server.cpp @@ -377,7 +377,6 @@ int ProxySQL_HTTP_Server::handler(void *cls, struct MHD_Connection *connection, char *username; - char *password = NULL; const char *realm = "Access to ProxySQL status page"; username = MHD_digest_auth_get_username(connection); @@ -387,30 +386,14 @@ int ProxySQL_HTTP_Server::handler(void *cls, struct MHD_Connection *connection, MHD_destroy_response(response); return ret; } + account_details_t ad { GloMyAuth->lookup(username, USERNAME_FRONTEND, { false, false, false }) }; { - int default_hostgroup = -1; - char *default_schema = NULL; - bool schema_locked; - bool transaction_persistent; - bool fast_forward; - bool _ret_use_ssl = false; - int max_connections; - void *sha1_pass = NULL; - password=GloMyAuth->lookup(username, USERNAME_FRONTEND, &_ret_use_ssl, &default_hostgroup, &default_schema, &schema_locked, &transaction_persistent, &fast_forward, &max_connections, &sha1_pass, NULL); - if (default_schema) { // unused - free(default_schema); - } - if (sha1_pass) { // unused - free(sha1_pass); - } if ( - (default_hostgroup != STATS_HOSTGROUP) + (ad.default_hostgroup != STATS_HOSTGROUP) || - (password == NULL) + (ad.password == NULL) ) { - if (password) { - free(password); // cleanup - } + free_account_details(ad); free(username); // cleanup response = MHD_create_response_from_buffer(strlen(DENIED), (void *)DENIED, MHD_RESPMEM_PERSISTENT); ret = MHD_queue_auth_fail_response(connection, realm, OPAQUE, response, MHD_NO); @@ -418,9 +401,9 @@ int ProxySQL_HTTP_Server::handler(void *cls, struct MHD_Connection *connection, return ret; } } - ret = MHD_digest_auth_check(connection, realm, username, password, 300); + ret = MHD_digest_auth_check(connection, realm, username, ad.password, 300); free(username); - free(password); + free_account_details(ad); if ( (ret == MHD_INVALID_NONCE) || (ret == MHD_NO) ) { response = MHD_create_response_from_buffer(strlen(DENIED), (void *)DENIED, MHD_RESPMEM_PERSISTENT); if (NULL == response) diff --git a/lib/ProxySQL_Poll.cpp b/lib/ProxySQL_Poll.cpp index 8876e5f373..78cb47c463 100644 --- a/lib/ProxySQL_Poll.cpp +++ b/lib/ProxySQL_Poll.cpp @@ -3,12 +3,12 @@ #include #include "StatCounters.h" #include "MySQL_Data_Stream.h" +#include "PgSQL_Data_Stream.h" #include "ProxySQL_Poll.h" #include "proxysql_structs.h" #include #include "cpp.h" - /** * @file ProxySQL_Poll.cpp * @@ -24,10 +24,11 @@ * This function reduces the size of the ProxySQL_Poll object by reallocating memory to fit the current number of elements. * It adjusts the size of internal arrays to a size that is a power of two near the current number of elements. */ -void ProxySQL_Poll::shrink() { + template +void ProxySQL_Poll::shrink() { unsigned int new_size=l_near_pow_2(len+1); fds=(struct pollfd *)realloc(fds,new_size*sizeof(struct pollfd)); - myds=(MySQL_Data_Stream **)realloc(myds,new_size*sizeof(MySQL_Data_Stream *)); + myds=(T **)realloc(myds,new_size*sizeof(T *)); last_recv=(unsigned long long *)realloc(last_recv,new_size*sizeof(unsigned long long)); last_sent=(unsigned long long *)realloc(last_sent,new_size*sizeof(unsigned long long)); size=new_size; @@ -41,23 +42,26 @@ void ProxySQL_Poll::shrink() { * * @param more The number of additional elements to accommodate. */ -void ProxySQL_Poll::expand(unsigned int more) { + template +void ProxySQL_Poll::expand(unsigned int more) { if ( (len+more) > size ) { unsigned int new_size=l_near_pow_2(len+more); fds=(struct pollfd *)realloc(fds,new_size*sizeof(struct pollfd)); - myds=(MySQL_Data_Stream **)realloc(myds,new_size*sizeof(MySQL_Data_Stream *)); + myds=(T **)realloc(myds,new_size*sizeof(T *)); last_recv=(unsigned long long *)realloc(last_recv,new_size*sizeof(unsigned long long)); last_sent=(unsigned long long *)realloc(last_sent,new_size*sizeof(unsigned long long)); size=new_size; } } + /** * @brief Constructs a new ProxySQL_Poll object. * * This constructor initializes a new ProxySQL_Poll object with default values and allocates memory for internal arrays. */ -ProxySQL_Poll::ProxySQL_Poll() { +template +ProxySQL_Poll::ProxySQL_Poll() { loop_counters=new StatCounters(15,10); poll_timeout=0; loops=0; @@ -67,7 +71,7 @@ ProxySQL_Poll::ProxySQL_Poll() { bootstrapping_listeners = true; size=MIN_POLL_LEN; fds=(struct pollfd *)malloc(size*sizeof(struct pollfd)); - myds=(MySQL_Data_Stream **)malloc(size*sizeof(MySQL_Data_Stream *)); + myds=(T**)malloc(size*sizeof(T *)); last_recv=(unsigned long long *)malloc(size*sizeof(unsigned long long)); last_sent=(unsigned long long *)malloc(size*sizeof(unsigned long long)); } @@ -77,7 +81,8 @@ ProxySQL_Poll::ProxySQL_Poll() { * * This destructor deallocates memory for internal arrays and releases resources associated with the ProxySQL_Poll object. */ -ProxySQL_Poll::~ProxySQL_Poll() { +template +ProxySQL_Poll::~ProxySQL_Poll() { unsigned int i; for (i=0;i +void ProxySQL_Poll::add(uint32_t _events, int _fd, T *_myds, unsigned long long sent_time) { if (len==size) { expand(1); } @@ -129,7 +135,8 @@ void ProxySQL_Poll::add(uint32_t _events, int _fd, MySQL_Data_Stream *_myds, uns * * @param i The index of the file descriptor (FD) to remove. */ -void ProxySQL_Poll::remove_index_fast(unsigned int i) { +template +void ProxySQL_Poll::remove_index_fast(unsigned int i) { if ((int)i==-1) return; myds[i]->poll_fds_idx=-1; // this prevents further delete if (i != (len-1)) { @@ -156,7 +163,8 @@ void ProxySQL_Poll::remove_index_fast(unsigned int i) { * @param fd The file descriptor (FD) to search for. * @return The index of the file descriptor (FD) if found, otherwise -1. */ -int ProxySQL_Poll::find_index(int fd) { +template +int ProxySQL_Poll::find_index(int fd) { unsigned int i; for (i=0; i; +template class ProxySQL_Poll; \ No newline at end of file diff --git a/lib/ProxySQL_RESTAPI_Server.cpp b/lib/ProxySQL_RESTAPI_Server.cpp index 24c3f2e667..6d7a2978eb 100644 --- a/lib/ProxySQL_RESTAPI_Server.cpp +++ b/lib/ProxySQL_RESTAPI_Server.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include "proxysql.h" #include "cpp.h" #include "httpserver.hpp" @@ -17,7 +21,7 @@ extern ProxySQL_Admin *GloAdmin; using namespace httpserver; -using nlohmann::json; +//using nlohmann::json; class sync_resource : public http_resource { private: diff --git a/lib/ProxySQL_Statistics.cpp b/lib/ProxySQL_Statistics.cpp index 523623109f..f1a83fbb34 100644 --- a/lib/ProxySQL_Statistics.cpp +++ b/lib/ProxySQL_Statistics.cpp @@ -96,6 +96,9 @@ void ProxySQL_Statistics::init() { insert_into_tables_defs(tables_defs_statsdb_disk,"myhgm_connections_day", STATSDB_SQLITE_TABLE_MYHGM_CONNECTIONS_DAY); insert_into_tables_defs(tables_defs_statsdb_disk,"history_mysql_query_digest", STATSDB_SQLITE_TABLE_HISTORY_MYSQL_QUERY_DIGEST); + insert_into_tables_defs(tables_defs_statsdb_disk,"history_pgsql_query_digest", STATSDB_SQLITE_TABLE_HISTORY_PGSQL_QUERY_DIGEST); + insert_into_tables_defs(tables_defs_statsdb_disk,"history_pgsql_status_variables", STATSDB_SQLITE_TABLE_HISTORY_PGSQL_STATUS_VARIABLES); + insert_into_tables_defs(tables_defs_statsdb_disk,"history_pgsql_status_variables_lookup", STATSDB_SQLITE_TABLE_HISTORY_PGSQL_STATUS_VARIABLES_LOOKUP); disk_upgrade_mysql_connections(); @@ -110,6 +113,10 @@ void ProxySQL_Statistics::init() { // statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_mysql_query_digest_last_seen ON history_mysql_query_digest (last_seen)"); statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_mysql_query_digest_dump_time ON history_mysql_query_digest (dump_time)"); statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_mysql_status_variable_id_timestamp ON history_mysql_status_variables(variable_id,timestamp)"); + + statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_pgsql_query_digest_first_seen ON history_pgsql_query_digest (first_seen)"); + statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_pgsql_query_digest_dump_time ON history_pgsql_query_digest (dump_time)"); + statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_pgsql_status_variable_id_timestamp ON history_pgsql_status_variables(variable_id,timestamp)"); } void ProxySQL_Statistics::disk_upgrade_mysql_connections() { diff --git a/lib/QP_query_digest_stats.cpp b/lib/QP_query_digest_stats.cpp index 509951b506..ff0750a79b 100644 --- a/lib/QP_query_digest_stats.cpp +++ b/lib/QP_query_digest_stats.cpp @@ -25,39 +25,40 @@ static void my_itoa(char s[], unsigned long long n) } -QP_query_digest_stats::QP_query_digest_stats(char *u, char *s, uint64_t d, char *dt, int h, char *ca) { - digest=d; +QP_query_digest_stats::QP_query_digest_stats(const char* _user, const char* _schema, uint64_t _digest, const char* _digest_text, + int _hid, const char* _client_addr, int query_digests_max_digest_length) { + digest=_digest; digest_text=NULL; - if (dt) { - digest_text=strndup(dt, mysql_thread___query_digests_max_digest_length); + if (_digest_text) { + digest_text=strndup(_digest_text, query_digests_max_digest_length); } - if (strlen(u) < sizeof(username_buf)) { - strcpy(username_buf,u); + if (strlen(_user) < sizeof(username_buf)) { + strcpy(username_buf, _user); username = username_buf; } else { - username=strdup(u); + username = strdup(_user); } - if (strlen(s) < sizeof(schemaname_buf)) { - strcpy(schemaname_buf,s); + if (strlen(_schema) < sizeof(schemaname_buf)) { + strcpy(schemaname_buf, _schema); schemaname = schemaname_buf; } else { - schemaname=strdup(s); + schemaname = strdup(_schema); } - if (strlen(ca) < sizeof(client_address_buf)) { - strcpy(client_address_buf,ca); + if (strlen(_client_addr) < sizeof(client_address_buf)) { + strcpy(client_address_buf, _client_addr); client_address = client_address_buf; } else { - client_address=strdup(ca); + client_address = strdup(_client_addr); } - count_star=0; - first_seen=0; - last_seen=0; - sum_time=0; - min_time=0; - max_time=0; - rows_affected=0; - rows_sent=0; - hid=h; + count_star = 0; + first_seen = 0; + last_seen = 0; + sum_time = 0; + min_time = 0; + max_time = 0; + rows_affected = 0; + rows_sent = 0; + hid = _hid; } void QP_query_digest_stats::add_time( unsigned long long t, unsigned long long n, unsigned long long ra, unsigned long long rs, diff --git a/lib/QP_rule_text.cpp b/lib/QP_rule_text.cpp index cd1a340425..01dc59da18 100644 --- a/lib/QP_rule_text.cpp +++ b/lib/QP_rule_text.cpp @@ -1,22 +1,10 @@ -/* -#include // std::cout -#include // std::sort -#include // std::vector -#include "re2/re2.h" -#include "re2/regexp.h" -#include "proxysql.h" -#include "cpp.h" -#include "MySQL_PreparedStatement.h" -#include "MySQL_Data_Stream.h" -*/ -#include "query_processor.h" #include #include "proxysql_macros.h" - +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" #include "QP_rule_text.h" - QP_rule_text_hitsonly::QP_rule_text_hitsonly(QP_rule_t *QPr) { pta=NULL; pta=(char **)malloc(sizeof(char *)*2); @@ -31,69 +19,12 @@ QP_rule_text_hitsonly::~QP_rule_text_hitsonly() { free(pta); } -QP_rule_text::QP_rule_text(QP_rule_t *QPr) { - num_fields=36; // this count the number of fields - pta=NULL; - pta=(char **)malloc(sizeof(char *)*num_fields); - itostr(pta[0], (long long)QPr->rule_id); - itostr(pta[1], (long long)QPr->active); - pta[2]=strdup_null(QPr->username); - pta[3]=strdup_null(QPr->schemaname); - itostr(pta[4], (long long)QPr->flagIN); - - pta[5]=strdup_null(QPr->client_addr); - pta[6]=strdup_null(QPr->proxy_addr); - itostr(pta[7], (long long)QPr->proxy_port); +QP_rule_text::QP_rule_text() : pta(NULL), num_fields(0) { - char buf[20]; - if (QPr->digest) { - sprintf(buf,"0x%016llX", (long long unsigned int)QPr->digest); - pta[8]=strdup(buf); - } else { - pta[8]=NULL; - } - - pta[9]=strdup_null(QPr->match_digest); - pta[10]=strdup_null(QPr->match_pattern); - itostr(pta[11], (long long)QPr->negate_match_pattern); - std::string re_mod; - re_mod=""; - if ((QPr->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) re_mod = "CASELESS"; - if ((QPr->re_modifiers & QP_RE_MOD_GLOBAL) == QP_RE_MOD_GLOBAL) { - if (re_mod.length()) { - re_mod = re_mod + ","; - } - re_mod = re_mod + "GLOBAL"; - } - pta[12]=strdup_null((char *)re_mod.c_str()); // re_modifiers - itostr(pta[13], (long long)QPr->flagOUT); - pta[14]=strdup_null(QPr->replace_pattern); - itostr(pta[15], (long long)QPr->destination_hostgroup); - itostr(pta[16], (long long)QPr->cache_ttl); - itostr(pta[17], (long long)QPr->cache_empty_result); - itostr(pta[18], (long long)QPr->cache_timeout); - itostr(pta[19], (long long)QPr->reconnect); - itostr(pta[20], (long long)QPr->timeout); - itostr(pta[21], (long long)QPr->retries); - itostr(pta[22], (long long)QPr->delay); - itostr(pta[23], (long long)QPr->next_query_flagIN); - itostr(pta[24], (long long)QPr->mirror_flagOUT); - itostr(pta[25], (long long)QPr->mirror_hostgroup); - pta[26]=strdup_null(QPr->error_msg); - pta[27]=strdup_null(QPr->OK_msg); - itostr(pta[28], (long long)QPr->sticky_conn); - itostr(pta[29], (long long)QPr->multiplex); - itostr(pta[30], (long long)QPr->gtid_from_hostgroup); - itostr(pta[31], (long long)QPr->log); - itostr(pta[32], (long long)QPr->apply); - pta[33]=strdup_null(QPr->attributes); - pta[34]=strdup_null(QPr->comment); // issue #643 - itostr(pta[35], (long long)QPr->hits); } - QP_rule_text::~QP_rule_text() { - for(int i=0; i // std::cout #include // std::sort #include // std::vector +#include +#include #include "re2/re2.h" #include "re2/regexp.h" +#include "pcrecpp.h" #include "proxysql.h" #include "cpp.h" -#include "MySQL_PreparedStatement.h" +#include "PgSQL_Data_Stream.h" #include "MySQL_Data_Stream.h" #include "query_processor.h" - -#ifndef SPOOKYV2 -#include "SpookyV2.h" -#define SPOOKYV2 -#endif - -#include "pcrecpp.h" +#include "QP_rule_text.h" +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" #ifdef DEBUG #define DEB "_DEBUG" #else #define DEB "" #endif /* DEBUG */ -#define QUERY_PROCESSOR_VERSION "2.0.6.0805" DEB +#define QUERY_PROCESSOR_VERSION "3.0.0.0004" DEB +#define GET_THREAD_VARIABLE(VARIABLE_NAME) \ +({((std::is_same_v) ? mysql_thread___##VARIABLE_NAME : pgsql_thread___##VARIABLE_NAME) ;}) +template +class Query_Processor; -#include -#include - -#include "QP_rule_text.h" +template +class Query_Processor; extern MySQL_Threads_Handler *GloMTH; +extern PgSQL_Threads_Handler* GloPTH; extern ProxySQL_Admin *GloAdmin; +// per thread variables +__thread unsigned int _thr_SQP_version; +__thread std::vector* _thr_SQP_rules; +__thread khash_t(khStrInt)* _thr_SQP_rules_fast_routing; +__thread char* _thr___rules_fast_routing___keys_values; + +struct __RE2_objects_t { + pcrecpp::RE_Options* opt1; + pcrecpp::RE* re1; + re2::RE2::Options* opt2; + RE2* re2; +}; + +typedef struct __RE2_objects_t re2_t; + static int int_cmp(const void *a, const void *b) { const unsigned long long *ia = (const unsigned long long *)a; const unsigned long long *ib = (const unsigned long long *)b; @@ -42,18 +63,9 @@ static int int_cmp(const void *a, const void *b) { return 0; } - -struct __RE2_objects_t { - pcrecpp::RE_Options *opt1; - pcrecpp::RE *re1; - re2::RE2::Options *opt2; - RE2 *re2; -}; - -typedef struct __RE2_objects_t re2_t; - -static bool rules_sort_comp_function (QP_rule_t * a, QP_rule_t * b) { return (a->rule_id < b->rule_id); } - +static bool rules_sort_comp_function (QP_rule_t * a, QP_rule_t * b) { + return (a->rule_id < b->rule_id); +} static unsigned long long mem_used_rule(QP_rule_t *qr) { unsigned long long s = sizeof(QP_rule_t); @@ -87,13 +99,13 @@ static unsigned long long mem_used_rule(QP_rule_t *qr) { return s; } -static re2_t * compile_query_rule(QP_rule_t *qr, int i) { +static re2_t * compile_query_rule(QP_rule_t *qr, int i, int query_processor_regex) { re2_t *r=(re2_t *)malloc(sizeof(re2_t)); r->opt1=NULL; r->re1=NULL; r->opt2=NULL; r->re2=NULL; - if (mysql_thread___query_processor_regex==2) { + if (query_processor_regex==2) { r->opt2=new re2::RE2::Options(RE2::Quiet); if ((qr->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) { r->opt2->set_case_sensitive(false); @@ -185,14 +197,8 @@ void __reset_rules(std::vector * qrs) { qrs->clear(); } -// per thread variables -__thread unsigned int _thr_SQP_version; -__thread std::vector * _thr_SQP_rules; -__thread khash_t(khStrInt) * _thr_SQP_rules_fast_routing; -__thread char * _thr___rules_fast_routing___keys_values; -__thread Command_Counter * _thr_commands_counters[MYSQL_COM_QUERY___NONE]; - -Query_Processor::Query_Processor() { +template +Query_Processor::Query_Processor(int _query_rules_fast_routing_algorithm) { #ifdef DEBUG if (glovars.has_debug==false) { #else @@ -204,76 +210,20 @@ Query_Processor::Query_Processor() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Initializing Query Processor with version=0\n"); // firewall - pthread_mutex_init(&global_mysql_firewall_whitelist_mutex, NULL); - global_mysql_firewall_whitelist_users_runtime = NULL; - global_mysql_firewall_whitelist_rules_runtime = NULL; - global_mysql_firewall_whitelist_sqli_fingerprints_runtime = NULL; - global_mysql_firewall_whitelist_users_map___size = 0; - global_mysql_firewall_whitelist_users_result___size = 0; - global_mysql_firewall_whitelist_rules_map___size = 0; - global_mysql_firewall_whitelist_rules_result___size = 0; + pthread_mutex_init(&global_firewall_whitelist_mutex, NULL); + global_firewall_whitelist_users_runtime = NULL; + global_firewall_whitelist_rules_runtime = NULL; + global_firewall_whitelist_sqli_fingerprints_runtime = NULL; + global_firewall_whitelist_users_map___size = 0; + global_firewall_whitelist_users_result___size = 0; + global_firewall_whitelist_rules_map___size = 0; + global_firewall_whitelist_rules_result___size = 0; pthread_rwlock_init(&rwlock, NULL); pthread_rwlock_init(&digest_rwlock, NULL); version=0; rules_mem_used=0; - for (int i=0; iget_variable_int("query_rules_fast_routing_algorithm"); - } -}; +} -Query_Processor::~Query_Processor() { - for (int i=0; i +Query_Processor::~Query_Processor() { __reset_rules(&rules); if (rules_fast_routing) { kh_destroy(khStrInt, rules_fast_routing); @@ -317,8 +264,8 @@ Query_Processor::~Query_Processor() { for (std::unordered_map::iterator it=digest_text_umap.begin(); it!=digest_text_umap.end(); ++it) { free(it->second); } - digest_umap.erase(digest_umap.begin(),digest_umap.end()); - digest_text_umap.erase(digest_text_umap.begin(),digest_text_umap.end()); + digest_umap.clear(); + digest_text_umap.clear(); if (query_rules_resultset) { delete query_rules_resultset; query_rules_resultset = NULL; @@ -327,33 +274,33 @@ Query_Processor::~Query_Processor() { delete fast_routing_resultset; fast_routing_resultset = NULL; } - if (global_mysql_firewall_whitelist_users_runtime) { - delete global_mysql_firewall_whitelist_users_runtime; - global_mysql_firewall_whitelist_users_runtime = NULL; + if (global_firewall_whitelist_users_runtime) { + delete global_firewall_whitelist_users_runtime; + global_firewall_whitelist_users_runtime = NULL; } - if (global_mysql_firewall_whitelist_rules_runtime) { - delete global_mysql_firewall_whitelist_rules_runtime; - global_mysql_firewall_whitelist_rules_runtime = NULL; + if (global_firewall_whitelist_rules_runtime) { + delete global_firewall_whitelist_rules_runtime; + global_firewall_whitelist_rules_runtime = NULL; } - if (global_mysql_firewall_whitelist_sqli_fingerprints_runtime) { - delete global_mysql_firewall_whitelist_sqli_fingerprints_runtime; - global_mysql_firewall_whitelist_sqli_fingerprints_runtime = NULL; + if (global_firewall_whitelist_sqli_fingerprints_runtime) { + delete global_firewall_whitelist_sqli_fingerprints_runtime; + global_firewall_whitelist_sqli_fingerprints_runtime = NULL; } -}; +} // This function is called by each thread when it starts. It create a Query Processor Table for each thread -void Query_Processor::init_thread() { +template +void Query_Processor::init_thread() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Initializing Per-Thread Query Processor Table with version=0\n"); _thr_SQP_version=0; _thr_SQP_rules=new std::vector; // per-thread 'rules_fast_routing' structures are created on demand _thr_SQP_rules_fast_routing = nullptr; _thr___rules_fast_routing___keys_values = NULL; - for (int i=0; i +void Query_Processor::end_thread() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Destroying Per-Thread Query Processor Table with version=%d\n", _thr_SQP_version); __reset_rules(_thr_SQP_rules); delete _thr_SQP_rules; @@ -364,22 +311,30 @@ void Query_Processor::end_thread() { free(_thr___rules_fast_routing___keys_values); _thr___rules_fast_routing___keys_values = NULL; } - for (int i=0; i +void Query_Processor::print_version() { fprintf(stderr,"Standard Query Processor rev. %s -- %s -- %s\n", QUERY_PROCESSOR_VERSION, __FILE__, __TIMESTAMP__); -}; +} -void Query_Processor::wrlock() { +template +void Query_Processor::wrlock() { pthread_rwlock_wrlock(&rwlock); -}; +} + +template +void Query_Processor::rdlock() { + pthread_rwlock_rdlock(&rwlock); +} -void Query_Processor::wrunlock() { +template +void Query_Processor::wrunlock() { pthread_rwlock_unlock(&rwlock); -}; +} -unsigned long long Query_Processor::get_rules_mem_used() { +template +unsigned long long Query_Processor::get_rules_mem_used() { unsigned long long s = 0; wrlock(); s = rules_mem_used; @@ -387,135 +342,20 @@ unsigned long long Query_Processor::get_rules_mem_used() { return s; } -unsigned long long Query_Processor::get_new_req_conns_count() { +template +unsigned long long Query_Processor::get_new_req_conns_count() { return __sync_fetch_and_add(&new_req_conns_count, 0); } -QP_rule_t * Query_Processor::new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *client_addr, char *proxy_addr, int proxy_port, char *digest, char *match_digest, char *match_pattern, bool negate_match_pattern, char *re_modifiers, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int cache_empty_result, int cache_timeout , int reconnect, int timeout, int retries, int delay, int next_query_flagIN, int mirror_flagOUT, int mirror_hostgroup, char *error_msg, char *OK_msg, int sticky_conn, int multiplex, int gtid_from_hostgroup, int log, bool apply, char *attributes, char *comment) { - QP_rule_t * newQR=(QP_rule_t *)malloc(sizeof(QP_rule_t)); - newQR->rule_id=rule_id; - newQR->active=active; - newQR->username=(username ? strdup(username) : NULL); - newQR->schemaname=(schemaname ? strdup(schemaname) : NULL); - newQR->flagIN=flagIN; - newQR->match_digest=(match_digest ? strdup(match_digest) : NULL); - newQR->match_pattern=(match_pattern ? strdup(match_pattern) : NULL); - newQR->negate_match_pattern=negate_match_pattern; - newQR->re_modifiers=0; - { - tokenizer_t tok; - tokenizer( &tok, re_modifiers, ",", TOKENIZER_NO_EMPTIES ); - const char* token; - for (token = tokenize( &tok ); token; token = tokenize( &tok )) { - if (strncasecmp(token,(char *)"CASELESS",strlen((char *)"CASELESS"))==0) { - newQR->re_modifiers|=QP_RE_MOD_CASELESS; - } - if (strncasecmp(token,(char *)"GLOBAL",strlen((char *)"GLOBAL"))==0) { - newQR->re_modifiers|=QP_RE_MOD_GLOBAL; - } - } - free_tokenizer( &tok ); - } - newQR->flagOUT=flagOUT; - newQR->replace_pattern=(replace_pattern ? strdup(replace_pattern) : NULL); - newQR->destination_hostgroup=destination_hostgroup; - newQR->cache_ttl=cache_ttl; - newQR->cache_empty_result=cache_empty_result; - newQR->cache_timeout=cache_timeout; - newQR->reconnect=reconnect; - newQR->timeout=timeout; - newQR->retries=retries; - newQR->delay=delay; - newQR->next_query_flagIN=next_query_flagIN; - newQR->mirror_flagOUT=mirror_flagOUT; - newQR->mirror_hostgroup=mirror_hostgroup; - newQR->error_msg=(error_msg ? strdup(error_msg) : NULL); - newQR->OK_msg=(OK_msg ? strdup(OK_msg) : NULL); - newQR->sticky_conn=sticky_conn; - newQR->multiplex=multiplex; - newQR->gtid_from_hostgroup = gtid_from_hostgroup; - newQR->apply=apply; - newQR->attributes=(attributes ? strdup(attributes) : NULL); - newQR->comment=(comment ? strdup(comment) : NULL); // see issue #643 - newQR->regex_engine1=NULL; - newQR->regex_engine2=NULL; - newQR->hits=0; - - newQR->client_addr_wildcard_position = -1; // not existing by default - newQR->client_addr=(client_addr ? strdup(client_addr) : NULL); - if (newQR->client_addr) { - char *pct = strchr(newQR->client_addr,'%'); - if (pct) { // there is a wildcard . We assume Admin did already all the input validation - if (pct == newQR->client_addr) { - // client_addr == '%' - // % is at the end of the string, but also at the beginning - // becoming a catch all - newQR->client_addr_wildcard_position = 0; - } else { - // this math is valid also if (pct == newQR->client_addr) - // but we separate it to clarify that client_addr_wildcard_position is a match all - newQR->client_addr_wildcard_position = strlen(newQR->client_addr) - strlen(pct); - } - } - } - newQR->proxy_addr=(proxy_addr ? strdup(proxy_addr) : NULL); - newQR->proxy_port=proxy_port; - newQR->log=log; - newQR->digest=0; - if (digest) { - unsigned long long num=strtoull(digest,NULL,0); - if (num!=ULLONG_MAX && num!=0) { - newQR->digest=num; - } else { - proxy_error("Incorrect digest for rule_id %d : %s\n" , rule_id, digest); - } - } - newQR->flagOUT_weights_total = 0; - newQR->flagOUT_ids = NULL; - newQR->flagOUT_weights = NULL; - if (newQR->attributes != NULL) { - if (strlen(newQR->attributes)) { - nlohmann::json j_attributes = nlohmann::json::parse(newQR->attributes); - if ( j_attributes.find("flagOUTs") != j_attributes.end() ) { - newQR->flagOUT_ids = new vector; - newQR->flagOUT_weights = new vector; - const nlohmann::json& flagOUTs = j_attributes["flagOUTs"]; - if (flagOUTs.type() == nlohmann::json::value_t::array) { - for (auto it = flagOUTs.begin(); it != flagOUTs.end(); it++) { - bool parsed = false; - const nlohmann::json& j = *it; - if (j.find("id") != j.end() && j.find("weight") != j.end()) { - if (j["id"].type() == nlohmann::json::value_t::number_unsigned && j["weight"].type() == nlohmann::json::value_t::number_unsigned) { - int id = j["id"]; - int weight = j["weight"]; - newQR->flagOUT_ids->push_back(id); - newQR->flagOUT_weights->push_back(weight); - newQR->flagOUT_weights_total += weight; - parsed = true; - } - } - if (parsed == false) { - proxy_error("Failed to parse flagOUTs in JSON on attributes for rule_id %d : %s\n" , newQR->rule_id, j.dump().c_str()); - } - } - } else { - proxy_error("Failed to parse flagOUTs attributes for rule_id %d : %s\n" , newQR->rule_id, flagOUTs.dump().c_str()); - } - } - } - } - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Creating new rule in %p : rule_id:%d, active:%d, username=%s, schemaname=%s, flagIN:%d, %smatch_digest=\"%s\", %smatch_pattern=\"%s\", flagOUT:%d replace_pattern=\"%s\", destination_hostgroup:%d, apply:%d\n", newQR, newQR->rule_id, newQR->active, newQR->username, newQR->schemaname, newQR->flagIN, (newQR->negate_match_pattern ? "(!)" : "") , newQR->match_digest, (newQR->negate_match_pattern ? "(!)" : "") , newQR->match_pattern, newQR->flagOUT, newQR->replace_pattern, newQR->destination_hostgroup, newQR->apply); - return newQR; -}; - - -void Query_Processor::delete_query_rule(QP_rule_t *qr) { +template +void Query_Processor::delete_query_rule(QP_rule_t *qr) { __delete_query_rule(qr); -}; +} -rules_mem_sts_t Query_Processor::reset_all(bool lock) { +template +rules_mem_sts_t Query_Processor::reset_all(bool lock) { if (lock) - pthread_rwlock_wrlock(&rwlock); + wrlock(); rules_mem_sts_t hashmaps_data {}; this->rules.swap(hashmaps_data.query_rules); @@ -532,68 +372,47 @@ rules_mem_sts_t Query_Processor::reset_all(bool lock) { } if (lock) - pthread_rwlock_unlock(&rwlock); + wrunlock(); rules_mem_used=0; return hashmaps_data; -}; +} -bool Query_Processor::insert(QP_rule_t *qr, bool lock) { +template +bool Query_Processor::insert(QP_rule_t *qr, bool lock) { bool ret=true; if (lock) - pthread_rwlock_wrlock(&rwlock); + wrlock(); rules.push_back(qr); rules_mem_used += mem_used_rule(qr); if (lock) - pthread_rwlock_unlock(&rwlock); + wrunlock(); return ret; -}; +} -void Query_Processor::sort(bool lock) { +template +void Query_Processor::sort(bool lock) { if (lock) - pthread_rwlock_wrlock(&rwlock); + wrlock(); proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Sorting rules\n"); std::sort (rules.begin(), rules.end(), rules_sort_comp_function); if (lock) - pthread_rwlock_unlock(&rwlock); -}; + wrunlock(); +} // when commit is called, the version number is increased and the this will trigger the mysql threads to get a new Query Processor Table // The operation is asynchronous -void Query_Processor::commit() { +template +void Query_Processor::commit() { __sync_add_and_fetch(&version,1); proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Increasing version number to %d - all threads will notice this and refresh their rules\n", version); -}; - -SQLite3_result * Query_Processor::get_stats_commands_counters() { - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping commands counters\n"); - SQLite3_result *result=new SQLite3_result(15); - result->add_column_definition(SQLITE_TEXT,"Command"); - result->add_column_definition(SQLITE_TEXT,"Total_Cnt"); - result->add_column_definition(SQLITE_TEXT,"Total_Time_us"); - result->add_column_definition(SQLITE_TEXT,"cnt_100us"); - result->add_column_definition(SQLITE_TEXT,"cnt_500us"); - result->add_column_definition(SQLITE_TEXT,"cnt_1ms"); - result->add_column_definition(SQLITE_TEXT,"cnt_5ms"); - result->add_column_definition(SQLITE_TEXT,"cnt_10ms"); - result->add_column_definition(SQLITE_TEXT,"cnt_50ms"); - result->add_column_definition(SQLITE_TEXT,"cnt_100ms"); - result->add_column_definition(SQLITE_TEXT,"cnt_500ms"); - result->add_column_definition(SQLITE_TEXT,"cnt_1s"); - result->add_column_definition(SQLITE_TEXT,"cnt_5s"); - result->add_column_definition(SQLITE_TEXT,"cnt_10s"); - result->add_column_definition(SQLITE_TEXT,"cnt_INFs"); - for (int i=0 ; i < MYSQL_COM_QUERY__UNINITIALIZED ; i++) { - char **pta=commands_counters[i]->get_row(); - result->add_row(pta); - commands_counters[i]->free_row(pta); - } - return result; } -SQLite3_result * Query_Processor::get_stats_query_rules() { + +template +SQLite3_result * Query_Processor::get_stats_query_rules() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping query rules statistics, using Global version %d\n", version); SQLite3_result *result=new SQLite3_result(2); - pthread_rwlock_rdlock(&rwlock); + rdlock(); QP_rule_t *qr1; result->add_column_definition(SQLITE_TEXT,"rule_id"); result->add_column_definition(SQLITE_TEXT,"hits"); @@ -606,88 +425,44 @@ SQLite3_result * Query_Processor::get_stats_query_rules() { delete qt; } } - pthread_rwlock_unlock(&rwlock); - return result; -} - -SQLite3_result * Query_Processor::get_current_query_rules() { - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query rules, using Global version %d\n", version); - SQLite3_result *result=new SQLite3_result(35); - pthread_rwlock_rdlock(&rwlock); - QP_rule_t *qr1; - result->add_column_definition(SQLITE_TEXT,"rule_id"); - result->add_column_definition(SQLITE_TEXT,"active"); - result->add_column_definition(SQLITE_TEXT,"username"); - result->add_column_definition(SQLITE_TEXT,"schemaname"); - result->add_column_definition(SQLITE_TEXT,"flagIN"); - result->add_column_definition(SQLITE_TEXT,"client_addr"); - result->add_column_definition(SQLITE_TEXT,"proxy_addr"); - result->add_column_definition(SQLITE_TEXT,"proxy_port"); - result->add_column_definition(SQLITE_TEXT,"digest"); - result->add_column_definition(SQLITE_TEXT,"match_digest"); - result->add_column_definition(SQLITE_TEXT,"match_pattern"); - result->add_column_definition(SQLITE_TEXT,"negate_match_pattern"); - result->add_column_definition(SQLITE_TEXT,"re_modifiers"); - result->add_column_definition(SQLITE_TEXT,"flagOUT"); - result->add_column_definition(SQLITE_TEXT,"replace_pattern"); - result->add_column_definition(SQLITE_TEXT,"destination_hostgroup"); - result->add_column_definition(SQLITE_TEXT,"cache_ttl"); - result->add_column_definition(SQLITE_TEXT,"cache_empty_result"); - result->add_column_definition(SQLITE_TEXT,"cache_timeout"); - result->add_column_definition(SQLITE_TEXT,"reconnect"); - result->add_column_definition(SQLITE_TEXT,"timeout"); - result->add_column_definition(SQLITE_TEXT,"retries"); - result->add_column_definition(SQLITE_TEXT,"delay"); - result->add_column_definition(SQLITE_TEXT,"next_query_flagIN"); - result->add_column_definition(SQLITE_TEXT,"mirror_flagOUT"); - result->add_column_definition(SQLITE_TEXT,"mirror_hostgroup"); - result->add_column_definition(SQLITE_TEXT,"error_msg"); - result->add_column_definition(SQLITE_TEXT,"OK_msg"); - result->add_column_definition(SQLITE_TEXT,"sticky_conn"); - result->add_column_definition(SQLITE_TEXT,"multiplex"); - result->add_column_definition(SQLITE_TEXT,"gtid_from_hostgroup"); - result->add_column_definition(SQLITE_TEXT,"log"); - result->add_column_definition(SQLITE_TEXT,"apply"); - result->add_column_definition(SQLITE_TEXT,"attributes"); - result->add_column_definition(SQLITE_TEXT,"comment"); // issue #643 - result->add_column_definition(SQLITE_TEXT,"hits"); - for (std::vector::iterator it=rules.begin(); it!=rules.end(); ++it) { - qr1=*it; - QP_rule_text *qt=new QP_rule_text(qr1); - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping Query Rule id: %d\n", qr1->rule_id); - result->add_row(qt->pta); - delete qt; - } - pthread_rwlock_unlock(&rwlock); + wrunlock(); return result; } -int Query_Processor::get_current_query_rules_fast_routing_count() { +template +int Query_Processor::get_current_query_rules_fast_routing_count() { int result = 0; - pthread_rwlock_rdlock(&rwlock); + rdlock(); result = fast_routing_resultset->rows_count; - pthread_rwlock_unlock(&rwlock); + wrunlock(); return result; } // we return the resultset fast_routing_resultset // the caller of this function must lock Query Processor -SQLite3_result * Query_Processor::get_current_query_rules_fast_routing_inner() { +template +SQLite3_result * Query_Processor::get_current_query_rules_fast_routing_inner() { return fast_routing_resultset; } // we return the resultset query_rules_resultset // the caller of this function must lock Query Processor -SQLite3_result * Query_Processor::get_current_query_rules_inner() { +template +SQLite3_result * Query_Processor::get_current_query_rules_inner() { return query_rules_resultset; } -SQLite3_result * Query_Processor::get_current_query_rules_fast_routing() { +template +SQLite3_result * Query_Processor::get_current_query_rules_fast_routing() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query rules fast_routing, using Global version %d\n", version); SQLite3_result *result=new SQLite3_result(5); - pthread_rwlock_rdlock(&rwlock); + rdlock(); //QP_rule_t *qr1; result->add_column_definition(SQLITE_TEXT,"username"); - result->add_column_definition(SQLITE_TEXT,"schemaname"); + if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "schemaname"); + } else if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "database"); + } result->add_column_definition(SQLITE_TEXT,"flagIN"); result->add_column_definition(SQLITE_TEXT,"destination_hostgroup"); result->add_column_definition(SQLITE_TEXT,"comment"); @@ -704,11 +479,12 @@ SQLite3_result * Query_Processor::get_current_query_rules_fast_routing() { SQLite3_row *r=*it; result->add_row(r); } - pthread_rwlock_unlock(&rwlock); + wrunlock(); return result; } -int Query_Processor::search_rules_fast_routing_dest_hg( +template +int Query_Processor::search_rules_fast_routing_dest_hg( khash_t(khStrInt)** __rules_fast_routing, const char* u, const char* s, int flagIN, bool lock ) { int dest_hg = -1; @@ -724,7 +500,7 @@ int Query_Processor::search_rules_fast_routing_dest_hg( sprintf(keybuf_ptr,"%s%s%s---%d", u, rand_del, s, flagIN); if (lock) { - pthread_rwlock_rdlock(&this->rwlock); + rdlock(); } khash_t(khStrInt)* _rules_fast_routing = *__rules_fast_routing; khiter_t k = kh_get(khStrInt, _rules_fast_routing, keybuf_ptr); @@ -738,7 +514,7 @@ int Query_Processor::search_rules_fast_routing_dest_hg( dest_hg = kh_val(_rules_fast_routing,k); } if (lock) { - pthread_rwlock_unlock(&this->rwlock); + wrunlock(); } if (keylen >= sizeof(keybuf)) { @@ -873,7 +649,8 @@ void * purge_query_digests_parallel(void *_arg) { return NULL; } -unsigned long long Query_Processor::purge_query_digests(bool async_purge, bool parallel, char **msg) { +template +unsigned long long Query_Processor::purge_query_digests(bool async_purge, bool parallel, char **msg) { unsigned long long ret = 0; if (async_purge) { ret = purge_query_digests_async(msg); @@ -883,7 +660,8 @@ unsigned long long Query_Processor::purge_query_digests(bool async_purge, bool p return ret; } -unsigned long long Query_Processor::purge_query_digests_async(char **msg) { +template +unsigned long long Query_Processor::purge_query_digests_async(char **msg) { unsigned long long ret = 0; umap_query_digest digest_umap_aux; umap_query_digest_text digest_text_umap_aux; @@ -921,8 +699,8 @@ unsigned long long Query_Processor::purge_query_digests_async(char **msg) { return ret; } - -unsigned long long Query_Processor::purge_query_digests_sync(bool parallel) { +template +unsigned long long Query_Processor::purge_query_digests_sync(bool parallel) { unsigned long long ret = 0; pthread_rwlock_wrlock(&digest_rwlock); size_t map_size = digest_umap.size(); @@ -962,7 +740,8 @@ unsigned long long Query_Processor::purge_query_digests_sync(bool parallel) { return ret; } -unsigned long long Query_Processor::get_query_digests_total_size() { +template +unsigned long long Query_Processor::get_query_digests_total_size() { unsigned long long ret=0; pthread_rwlock_rdlock(&digest_rwlock); size_t map_size = digest_umap.size(); @@ -1017,7 +796,8 @@ unsigned long long Query_Processor::get_query_digests_total_size() { return ret; } -std::pair Query_Processor::get_query_digests_v2(const bool use_resultset) { +template +std::pair Query_Processor::get_query_digests_v2(const bool use_resultset) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query digest\n"); SQLite3_result *result = NULL; // Create two auxiliary maps and swap its content with the main maps. This @@ -1041,7 +821,11 @@ std::pair Query_Processor::get_query_digests_v2(const boo result = new SQLite3_result(14); } result->add_column_definition(SQLITE_TEXT,"hid"); - result->add_column_definition(SQLITE_TEXT,"schemaname"); + if constexpr (std::is_same_v){ + result->add_column_definition(SQLITE_TEXT, "schemaname"); + } else if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "database"); + } result->add_column_definition(SQLITE_TEXT,"username"); result->add_column_definition(SQLITE_TEXT,"client_address"); result->add_column_definition(SQLITE_TEXT,"digest"); @@ -1088,9 +872,15 @@ std::pair Query_Processor::get_query_digests_v2(const boo } } } else { - num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( - false, false, NULL, &digest_umap_aux, &digest_text_umap_aux - ); + if constexpr (std::is_same_v) { + num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( + false, false, NULL, &digest_umap_aux, &digest_text_umap_aux + ); + } else if constexpr (std::is_same_v) { + num_rows = GloAdmin->stats___save_pgsql_query_digest_to_sqlite( + false, false, NULL, &digest_umap_aux, &digest_text_umap_aux + ); + } } if (map_size >= DIGEST_STATS_FAST_MINSIZE) { curtime2=monotonic_time(); @@ -1161,7 +951,8 @@ std::pair Query_Processor::get_query_digests_v2(const boo return res; } -SQLite3_result * Query_Processor::get_query_digests() { +template +SQLite3_result * Query_Processor::get_query_digests() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query digest\n"); SQLite3_result *result = NULL; pthread_rwlock_rdlock(&digest_rwlock); @@ -1175,7 +966,11 @@ SQLite3_result * Query_Processor::get_query_digests() { result = new SQLite3_result(14); } result->add_column_definition(SQLITE_TEXT,"hid"); - result->add_column_definition(SQLITE_TEXT,"schemaname"); + if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "schemaname"); + } else if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "database"); + } result->add_column_definition(SQLITE_TEXT,"username"); result->add_column_definition(SQLITE_TEXT,"client_address"); result->add_column_definition(SQLITE_TEXT,"digest"); @@ -1228,7 +1023,8 @@ SQLite3_result * Query_Processor::get_query_digests() { return result; } -std::pair Query_Processor::get_query_digests_reset_v2( +template +std::pair Query_Processor::get_query_digests_reset_v2( const bool copy, const bool use_resultset ) { SQLite3_result *result = NULL; @@ -1256,7 +1052,11 @@ std::pair Query_Processor::get_query_digests_reset_v2( result = new SQLite3_result(14); } result->add_column_definition(SQLITE_TEXT,"hid"); - result->add_column_definition(SQLITE_TEXT,"schemaname"); + if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "schemaname"); + } else if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "database"); + } result->add_column_definition(SQLITE_TEXT,"username"); result->add_column_definition(SQLITE_TEXT,"client_address"); result->add_column_definition(SQLITE_TEXT,"digest"); @@ -1305,9 +1105,15 @@ std::pair Query_Processor::get_query_digests_reset_v2( } } } else { - num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( - true, copy, result, &digest_umap_aux, &digest_text_umap_aux - ); + if constexpr (std::is_same_v) { + num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( + true, copy, result, &digest_umap_aux, &digest_text_umap_aux + ); + } else if constexpr (std::is_same_v) { + num_rows = GloAdmin->stats___save_pgsql_query_digest_to_sqlite( + true, copy, result, &digest_umap_aux, &digest_text_umap_aux + ); + } for ( std::unordered_map::iterator it = digest_umap_aux.begin(); it != digest_umap_aux.end(); @@ -1345,14 +1151,16 @@ std::pair Query_Processor::get_query_digests_reset_v2( return res; } -void Query_Processor::get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt) { +template +void Query_Processor::get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt) { pthread_rwlock_wrlock(&digest_rwlock); digest_umap.swap(*uqd); digest_text_umap.swap(*uqdt); pthread_rwlock_unlock(&digest_rwlock); } -SQLite3_result * Query_Processor::get_query_digests_reset() { +template +SQLite3_result * Query_Processor::get_query_digests_reset() { SQLite3_result *result = NULL; pthread_rwlock_wrlock(&digest_rwlock); unsigned long long curtime1; @@ -1369,7 +1177,11 @@ SQLite3_result * Query_Processor::get_query_digests_reset() { result = new SQLite3_result(14); } result->add_column_definition(SQLITE_TEXT,"hid"); - result->add_column_definition(SQLITE_TEXT,"schemaname"); + if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "schemaname"); + } else if constexpr (std::is_same_v) { + result->add_column_definition(SQLITE_TEXT, "database"); + } result->add_column_definition(SQLITE_TEXT,"username"); result->add_column_definition(SQLITE_TEXT,"client_address"); result->add_column_definition(SQLITE_TEXT,"digest"); @@ -1445,50 +1257,16 @@ SQLite3_result * Query_Processor::get_query_digests_reset() { } return result; } - - -Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *sess, void *ptr, unsigned int size, Query_Info *qi) { +template +Query_Processor_Output* Query_Processor::process_query(TypeSession* sess, bool stmt_exec, const char *query, + unsigned int len, Query_Processor_Output* ret, SQP_par_t* qp) { // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE // to avoid unnecssary deallocation/allocation, we initialize qpo witout new allocation - Query_Processor_Output *ret=sess->qpo; - ret->init(); - - SQP_par_t stmt_exec_qp; - SQP_par_t *qp=NULL; - if (qi) { - // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE - if (ptr) { - qp=(SQP_par_t *)&qi->QueryParserArgs; - } else { - qp=&stmt_exec_qp; - qp->digest = qi->stmt_info->digest; - qp->digest_text = qi->stmt_info->digest_text; - qp->first_comment = qi->stmt_info->first_comment; - } - } -#define stackbuffer_size 128 - char stackbuffer[stackbuffer_size]; - unsigned int len=0; - char *query=NULL; - // NOTE: if ptr == NULL , we are calling process_mysql_query() on an STMT_EXECUTE - if (ptr) { - len = size-sizeof(mysql_hdr)-1; - if (len < stackbuffer_size) { - query=stackbuffer; - } else { - query=(char *)l_alloc(len+1); - } - memcpy(query,(char *)ptr+sizeof(mysql_hdr)+1,len); - query[len]=0; - } else { - query = qi->stmt_info->query; - len = qi->stmt_info->query_length; - } if (__sync_add_and_fetch(&version,0) > _thr_SQP_version) { // update local rules; proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Detected a changed in version. Global:%d , local:%d . Refreshing...\n", version, _thr_SQP_version); - pthread_rwlock_rdlock(&rwlock); + rdlock(); _thr_SQP_version=__sync_add_and_fetch(&version,0); __reset_rules(_thr_SQP_rules); QP_rule_t *qr1; @@ -1497,40 +1275,15 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses qr1=*it; if (qr1->active) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Copying Query Rule id: %d\n", qr1->rule_id); - char buf[20]; - if (qr1->digest) { // not 0 - sprintf(buf,"0x%016llX", (long long unsigned int)qr1->digest); - } - std::string re_mod; - re_mod=""; - if ((qr1->re_modifiers & QP_RE_MOD_CASELESS) == QP_RE_MOD_CASELESS) re_mod = "CASELESS"; - if ((qr1->re_modifiers & QP_RE_MOD_GLOBAL) == QP_RE_MOD_GLOBAL) { - if (re_mod.length()) { - re_mod = re_mod + ","; - } - re_mod = re_mod + "GLOBAL"; - } - qr2=new_query_rule(qr1->rule_id, qr1->active, qr1->username, qr1->schemaname, qr1->flagIN, - qr1->client_addr, qr1->proxy_addr, qr1->proxy_port, - ( qr1->digest ? buf : NULL ) , - qr1->match_digest, qr1->match_pattern, qr1->negate_match_pattern, (char *)re_mod.c_str(), - qr1->flagOUT, qr1->replace_pattern, qr1->destination_hostgroup, - qr1->cache_ttl, qr1->cache_empty_result, qr1->cache_timeout, - qr1->reconnect, qr1->timeout, qr1->retries, qr1->delay, - qr1->next_query_flagIN, qr1->mirror_flagOUT, qr1->mirror_hostgroup, - qr1->error_msg, qr1->OK_msg, qr1->sticky_conn, qr1->multiplex, - qr1->gtid_from_hostgroup, - qr1->log, qr1->apply, - qr1->attributes, - qr1->comment); + qr2=(static_cast(this))->new_query_rule(static_cast(qr1)); qr2->parent=qr1; // pointer to parent to speed up parent update (hits) if (qr2->match_digest) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Compiling regex for rule_id: %d, match_digest: %s\n", qr2->rule_id, qr2->match_digest); - qr2->regex_engine1=(void *)compile_query_rule(qr2,1); + qr2->regex_engine1=(void *)compile_query_rule(qr2,1, GET_THREAD_VARIABLE(query_processor_regex)); } if (qr2->match_pattern) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Compiling regex for rule_id: %d, match_pattern: %s\n", qr2->rule_id, qr2->match_pattern); - qr2->regex_engine2=(void *)compile_query_rule(qr2,2); + qr2->regex_engine2=(void *)compile_query_rule(qr2,2, GET_THREAD_VARIABLE(query_processor_regex)); } _thr_SQP_rules->push_back(qr2); } @@ -1571,7 +1324,7 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses //for (std::unordered_map::iterator it = rules_fast_routing.begin(); it != rules_fast_routing.end(); ++it) { // _thr_SQP_rules_fast_routing->insert( //} - pthread_rwlock_unlock(&rwlock); + wrunlock(); } QP_rule_t *qr = NULL; re2_t *re2p; @@ -1580,7 +1333,7 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses if (sess->next_query_flagIN >= 0) { flagIN=sess->next_query_flagIN; } - int reiterate=mysql_thread___query_processor_iterations; + int reiterate=GET_THREAD_VARIABLE(query_processor_iterations); if (sess->mirror==true) { // we are into a mirror session // we immediately set a destination_hostgroup @@ -1746,7 +1499,7 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses } if (qr->delay >= 0) { // Note: negative delay means this rule doesn't change - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set delay: %d. Session will%s be paused for %dms\n", qr->rule_id, qr->delay, (qr->delay == 0 ? " NOT" : "" ) , qr->delay); + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set delay: %d. Session will%s be paused for %dms\n", qr->rule_id, qr->delay, (qr->delay == 0 ? " NOT" : "" ) , qr->delay); ret->delay=qr->delay; } if (qr->next_query_flagIN >= 0) { @@ -1756,8 +1509,8 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses } if (qr->mirror_flagOUT >= 0) { // Note: negative mirror_flagOUT means this rule doesn't change the mirror flagOUT - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set mirror flagOUT: %d\n", qr->rule_id, qr->mirror_flagOUT); - ret->mirror_flagOUT=qr->mirror_flagOUT; + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set mirror flagOUT: %d\n", qr->rule_id, qr->mirror_flagOUT); + ret->mirror_flagOUT=qr->mirror_flagOUT; } if (qr->mirror_hostgroup >= 0) { // Note: negative mirror_hostgroup means this rule doesn't change the mirror @@ -1799,11 +1552,6 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set multiplex: %d. Connection will%s multiplex\n", qr->rule_id, qr->multiplex, (qr->multiplex == 0 ? " NOT" : "" )); ret->multiplex=qr->multiplex; } - if (qr->gtid_from_hostgroup >= 0) { - // Note: negative gtid_from_hostgroup means this rule doesn't change the gtid_from_hostgroup - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set gtid from hostgroup: %d. A new session will be created\n", qr->rule_id, qr->gtid_from_hostgroup); - ret->gtid_from_hostgroup = qr->gtid_from_hostgroup; - } if (qr->log >= 0) { // Note: negative log means this rule doesn't change proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set log: %d. Query will%s logged\n", qr->rule_id, qr->log, (qr->log == 0 ? " NOT" : "" )); @@ -1813,8 +1561,11 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses // Note: negative hostgroup means this rule doesn't change proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set destination hostgroup: %d\n", qr->rule_id, qr->destination_hostgroup); ret->destination_hostgroup=qr->destination_hostgroup; + } + if constexpr (has_process_query_extended::value) { + (static_cast(this))->process_query_extended(static_cast(ret), static_cast(qr)); } - if (ptr) { // we aren't processing a STMT_EXECUTE + if (stmt_exec == false) { // we aren't processing a STMT_EXECUTE if (qr->replace_pattern) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d on match_pattern \"%s\" has a replace_pattern \"%s\" to apply\n", qr->rule_id, qr->match_pattern, qr->replace_pattern); if (ret->new_query==NULL) ret->new_query=new std::string(query); @@ -1872,21 +1623,14 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses ret->destination_hostgroup = dst_hg; } } - // FIXME : there is too much data being copied around - if (len < stackbuffer_size) { - // query is in the stack - } else { - if (ptr) { - l_free(len+1,query); - } - } + if (sess->mirror==false) { // we process comments only on original queries, not on mirrors if (qp && qp->first_comment) { // we have a comment to parse query_parser_first_comment(ret, qp->first_comment); } } - if (mysql_thread___firewall_whitelist_enabled) { + if (GET_THREAD_VARIABLE(firewall_whitelist_enabled)) { char *username = NULL; char *client_address = NULL; bool check_run = true; @@ -1897,7 +1641,7 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses check_run = true; username = sess->client_myds->myconn->userinfo->username; client_address = sess->client_myds->addr.addr; - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_lock(&global_firewall_whitelist_mutex); // FIXME // for now this function search for either username@ip or username@'' int wus_status = find_firewall_whitelist_user(username, client_address); @@ -1919,7 +1663,7 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses if (wus_status == WUS_PROTECTING) { if (ret->error_msg == NULL) { // change error message only if not already set - ret->error_msg = strdup(mysql_thread___firewall_whitelist_errormsg); + ret->error_msg = strdup(GET_THREAD_VARIABLE(firewall_whitelist_errormsg)); } } } @@ -1927,7 +1671,7 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses ret->firewall_whitelist_mode = WUS_OFF; } } - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_unlock(&global_firewall_whitelist_mutex); if (ret->firewall_whitelist_mode == WUS_DETECTING || ret->firewall_whitelist_mode == WUS_PROTECTING) { char buf[32]; if (qp && qp->digest) { @@ -1956,14 +1700,15 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses return ret; }; -int Query_Processor::find_firewall_whitelist_user(char *username, char *client) { +template +int Query_Processor::find_firewall_whitelist_user(char *username, char *client) { int ret = WUS_NOT_FOUND; string s = username; s += rand_del; s += client; std::unordered_map:: iterator it2; - it2 = global_mysql_firewall_whitelist_users.find(s); - if (it2 != global_mysql_firewall_whitelist_users.end()) { + it2 = global_firewall_whitelist_users.find(s); + if (it2 != global_firewall_whitelist_users.end()) { ret = it2->second; return ret; } @@ -1971,7 +1716,8 @@ int Query_Processor::find_firewall_whitelist_user(char *username, char *client) return ret; } -bool Query_Processor::find_firewall_whitelist_rule(char *username, char *client_address, char *schemaname, int flagIN, uint64_t digest) { +template +bool Query_Processor::find_firewall_whitelist_rule(char *username, char *client_address, char *schemaname, int flagIN, uint64_t digest) { bool ret = false; string s = username; s += rand_del; @@ -1981,8 +1727,8 @@ bool Query_Processor::find_firewall_whitelist_rule(char *username, char *client_ s += rand_del; s += to_string(flagIN); std::unordered_map:: iterator it; - it = global_mysql_firewall_whitelist_rules.find(s); - if (it != global_mysql_firewall_whitelist_rules.end()) { + it = global_firewall_whitelist_rules.find(s); + if (it != global_firewall_whitelist_rules.end()) { PtrArray *myptrarray = (PtrArray *)it->second; void * found = bsearch(&digest, myptrarray->pdata, myptrarray->len, sizeof(unsigned long long), int_cmp); if (found) { @@ -1993,7 +1739,8 @@ bool Query_Processor::find_firewall_whitelist_rule(char *username, char *client_ } // this function is called by mysql_session to free the result generated by process_mysql_query() -void Query_Processor::delete_QP_out(Query_Processor_Output *o) { +template +void Query_Processor::delete_QP_out(Query_Processor_Output *o) { //l_free(sizeof(QP_out_t),o); if (o) { //delete o; // do not deallocate, but "destroy" it @@ -2001,7 +1748,8 @@ void Query_Processor::delete_QP_out(Query_Processor_Output *o) { } }; -void Query_Processor::update_query_processor_stats() { +template +void Query_Processor::update_query_processor_stats() { // Note: // this function is called by each thread to update global query statistics // @@ -2011,7 +1759,7 @@ void Query_Processor::update_query_processor_stats() { // It acquires a read lock to ensure that the rules table doesn't change // Yet, because it has to update vales, it uses atomic operations proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 8, "Updating query rules statistics\n"); - pthread_rwlock_rdlock(&rwlock); + rdlock(); if (__sync_add_and_fetch(&version,0) == _thr_SQP_version) { QP_rule_t *qr; for (std::vector::iterator it=_thr_SQP_rules->begin(); it!=_thr_SQP_rules->end(); ++it) { @@ -2022,31 +1770,31 @@ void Query_Processor::update_query_processor_stats() { } } } - pthread_rwlock_unlock(&rwlock); - for (int i=0; icounters[j]) { - __sync_fetch_and_add(&commands_counters[i]->counters[j],_thr_commands_counters[i]->counters[j]); - _thr_commands_counters[i]->counters[j]=0; - } - } - if (_thr_commands_counters[i]->total_time) - __sync_fetch_and_add(&commands_counters[i]->total_time,_thr_commands_counters[i]->total_time); - _thr_commands_counters[i]->total_time=0; - } + wrunlock(); + }; - -void Query_Processor::query_parser_init(SQP_par_t *qp, char *query, int query_length, int flags) { +template +void Query_Processor::query_parser_init(SQP_par_t *qp, const char *query, int query_length, int flags) { // trying to get rid of libinjection // instead of initializing qp->sf , we copy query info later in this function qp->digest_text=NULL; qp->first_comment=NULL; qp->query_prefix=NULL; - if (mysql_thread___query_digests) { - qp->digest_text=mysql_query_digest_and_first_comment_2(query, query_length, &qp->first_comment, ((query_length < QUERY_DIGEST_BUF) ? qp->buf : NULL)); + if (GET_THREAD_VARIABLE(query_digests)) { + options opts; + opts.lowercase = GET_THREAD_VARIABLE(query_digests_lowercase); + opts.replace_null = GET_THREAD_VARIABLE(query_digests_replace_null); + opts.replace_number = GET_THREAD_VARIABLE(query_digests_no_digits); + opts.grouping_limit = GET_THREAD_VARIABLE(query_digests_grouping_limit); + opts.groups_grouping_limit = GET_THREAD_VARIABLE(query_digests_groups_grouping_limit); + opts.keep_comment = GET_THREAD_VARIABLE(query_digests_keep_comment); + opts.max_query_length = GET_THREAD_VARIABLE(query_digests_max_query_length); + + qp->digest_text=query_digest_and_first_comment_2(query, query_length, &qp->first_comment, + ((query_length < QUERY_DIGEST_BUF) ? qp->buf : NULL), &opts); // the hash is computed only up to query_digests_max_digest_length bytes - int digest_text_length=strnlen(qp->digest_text, mysql_thread___query_digests_max_digest_length); + const int digest_text_length=strnlen(qp->digest_text, GET_THREAD_VARIABLE(query_digests_max_digest_length)); qp->digest=SpookyHash::Hash64(qp->digest_text, digest_text_length, 0); #ifdef DEBUG if (qp->first_comment && strlen(qp->first_comment)) { @@ -2054,7 +1802,7 @@ void Query_Processor::query_parser_init(SQP_par_t *qp, char *query, int query_le } #endif /* DEBUG */ } else { - if (mysql_thread___commands_stats) { + if (GET_THREAD_VARIABLE(commands_stats)) { size_t sl=32; if ((unsigned int)query_length < sl) { sl=query_length; @@ -2064,25 +1812,20 @@ void Query_Processor::query_parser_init(SQP_par_t *qp, char *query, int query_le } }; -enum MYSQL_COM_QUERY_command Query_Processor::query_parser_command_type(SQP_par_t *qp) { - enum MYSQL_COM_QUERY_command ret=__query_parser_command_type(qp); - return ret; -} -unsigned long long Query_Processor::query_parser_update_counters(MySQL_Session *sess, enum MYSQL_COM_QUERY_command c, SQP_par_t *qp, unsigned long long t) { - if (c>=MYSQL_COM_QUERY___NONE) return 0; - unsigned long long ret=_thr_commands_counters[c]->add_time(t); +template +void Query_Processor::query_parser_update_counters(TypeSession* sess, uint64_t digest_total, uint64_t digest, + char* digest_text, unsigned long long t) { - char *ca = (char *)""; - if (mysql_thread___query_digests_track_hostname) { - if (sess->client_myds) { - if (sess->client_myds->addr.addr) { - ca = sess->client_myds->addr.addr; + if (digest_text) { + char* ca = (char*)""; + if (GET_THREAD_VARIABLE(query_digests_track_hostname)) { + if (sess->client_myds) { + if (sess->client_myds->addr.addr) { + ca = sess->client_myds->addr.addr; + } } } - } - - if (sess->CurrentQuery.stmt_info==NULL && qp->digest_text) { // this code is executed only if digest_text is not NULL , that means mysql_thread___query_digests was true when the query started uint64_t hash2; SpookyHash myhash; @@ -2091,100 +1834,49 @@ unsigned long long Query_Processor::query_parser_update_counters(MySQL_Session * assert(sess->client_myds); assert(sess->client_myds->myconn); assert(sess->client_myds->myconn->userinfo); - MySQL_Connection_userinfo *ui=sess->client_myds->myconn->userinfo; + auto *ui=sess->client_myds->myconn->userinfo; assert(ui->username); assert(ui->schemaname); myhash.Update(ui->username,strlen(ui->username)); - myhash.Update(&qp->digest,sizeof(qp->digest)); + myhash.Update(&digest,sizeof(digest)); myhash.Update(ui->schemaname,strlen(ui->schemaname)); - myhash.Update(&sess->current_hostgroup,sizeof(sess->default_hostgroup)); + myhash.Update(&sess->current_hostgroup,sizeof(sess->current_hostgroup)); myhash.Update(ca,strlen(ca)); - myhash.Final(&qp->digest_total,&hash2); - update_query_digest(qp, sess->current_hostgroup, ui, t, sess->thread->curtime, NULL, sess); + myhash.Final(&digest_total,&hash2); + update_query_digest(digest_total, digest, digest_text, sess->current_hostgroup, ui, t, sess->thread->curtime, ca, + sess->CurrentQuery.affected_rows, sess->CurrentQuery.rows_sent); } - if (sess->CurrentQuery.stmt_info && sess->CurrentQuery.stmt_info->digest_text) { - uint64_t hash2; - SpookyHash myhash; - myhash.Init(19,3); - assert(sess); - assert(sess->client_myds); - assert(sess->client_myds->myconn); - assert(sess->client_myds->myconn->userinfo); - MySQL_Connection_userinfo *ui=sess->client_myds->myconn->userinfo; - assert(ui->username); - assert(ui->schemaname); - MySQL_STMT_Global_info *stmt_info=sess->CurrentQuery.stmt_info; - myhash.Update(ui->username,strlen(ui->username)); - myhash.Update(&stmt_info->digest,sizeof(qp->digest)); - myhash.Update(ui->schemaname,strlen(ui->schemaname)); - myhash.Update(&sess->current_hostgroup,sizeof(sess->default_hostgroup)); - myhash.Update(ca,strlen(ca)); - myhash.Final(&qp->digest_total,&hash2); - //delete myhash; - update_query_digest(qp, sess->current_hostgroup, ui, t, sess->thread->curtime, stmt_info, sess); - } - return ret; } -void Query_Processor::update_query_digest(SQP_par_t *qp, int hid, MySQL_Connection_userinfo *ui, unsigned long long t, unsigned long long n, MySQL_STMT_Global_info *_stmt_info, MySQL_Session *sess) { +template +void Query_Processor::update_query_digest(uint64_t digest_total, uint64_t digest, char* digest_text, int hid, + TypeConnInfo* ui, unsigned long long t, unsigned long long n, const char* client_addr, unsigned long long rows_affected, + unsigned long long rows_sent) { pthread_rwlock_wrlock(&digest_rwlock); QP_query_digest_stats *qds; - unsigned long long rows_affected = 0; - unsigned long long rows_sent = 0; - - if (sess) { - rows_affected = sess->CurrentQuery.affected_rows; - rows_sent = sess->CurrentQuery.rows_sent; - } - std::unordered_map::iterator it; - it=digest_umap.find(qp->digest_total); + it=digest_umap.find(digest_total); if (it != digest_umap.end()) { // found qds=(QP_query_digest_stats *)it->second; - qds->add_time(t,n, rows_affected,rows_sent); + qds->add_time(t,n,rows_affected,rows_sent); } else { char *dt = NULL; - if (mysql_thread___query_digests_normalize_digest_text==false) { - if (_stmt_info==NULL) { - dt = qp->digest_text; - } else { - dt = _stmt_info->digest_text; - } - } - char *ca = (char *)""; - if (mysql_thread___query_digests_track_hostname) { - if (sess->client_myds) { - if (sess->client_myds->addr.addr) { - ca = sess->client_myds->addr.addr; - } - } - } - if (_stmt_info==NULL) { - qds=new QP_query_digest_stats(ui->username, ui->schemaname, qp->digest, dt, hid, ca); - } else { - qds=new QP_query_digest_stats(ui->username, ui->schemaname, _stmt_info->digest, dt, hid, ca); + if (GET_THREAD_VARIABLE(query_digests_normalize_digest_text)==false) { + dt = digest_text; } + qds=new QP_query_digest_stats(ui->username, ui->schemaname, digest, dt, hid, client_addr, GET_THREAD_VARIABLE(query_digests_max_digest_length)); qds->add_time(t,n, rows_affected,rows_sent); - digest_umap.insert(std::make_pair(qp->digest_total,(void *)qds)); - if (mysql_thread___query_digests_normalize_digest_text==true) { - uint64_t dig = 0; - if (_stmt_info==NULL) { - dig = qp->digest; - } else { - dig = _stmt_info->digest; - } + digest_umap.insert(std::make_pair(digest_total,(void *)qds)); + if (GET_THREAD_VARIABLE(query_digests_normalize_digest_text)==true) { + const uint64_t dig = digest; std::unordered_map::iterator it2; it2=digest_text_umap.find(dig); if (it2 != digest_text_umap.end()) { // found } else { - if (_stmt_info==NULL) { - dt = strdup(qp->digest_text); - } else { - dt = strdup(_stmt_info->digest_text); - } + dt = strdup(digest_text); digest_text_umap.insert(std::make_pair(dig,dt)); } } @@ -2193,404 +1885,20 @@ void Query_Processor::update_query_digest(SQP_par_t *qp, int hid, MySQL_Connecti pthread_rwlock_unlock(&digest_rwlock); } -char * Query_Processor::get_digest_text(SQP_par_t *qp) { +template +char * Query_Processor::get_digest_text(SQP_par_t *qp) { if (qp==NULL) return NULL; return qp->digest_text; } -uint64_t Query_Processor::get_digest(SQP_par_t *qp) { +template +uint64_t Query_Processor::get_digest(SQP_par_t *qp) { if (qp==NULL) return 0; return qp->digest; } -enum MYSQL_COM_QUERY_command Query_Processor::__query_parser_command_type(SQP_par_t *qp) { - char *text=NULL; // this new variable is a pointer to either qp->digest_text , or to the query - if (qp->digest_text) { - text=qp->digest_text; - } else { - text=qp->query_prefix; - } - - enum MYSQL_COM_QUERY_command ret=MYSQL_COM_QUERY_UNKNOWN; - char c1; - - tokenizer_t tok; - tokenizer( &tok, text, " ", TOKENIZER_NO_EMPTIES ); - char* token=NULL; -__get_token: - token=(char *)tokenize(&tok); - if (token==NULL) { - goto __exit__query_parser_command_type; - } -__remove_paranthesis: - if (token[0] == '(') { - if (strlen(token) > 1) { - token++; - goto __remove_paranthesis; - } else { - goto __get_token; - } - } - c1=token[0]; - proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Command:%s Prefix:%c\n", token, c1); - switch (c1) { - case 'a': - case 'A': - if (!mystrcasecmp("ALTER",token)) { // ALTER [ONLINE | OFFLINE] [IGNORE] TABLE - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!mystrcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_ALTER_TABLE; - break; - } else { - if (!mystrcasecmp("OFFLINE",token) || !mystrcasecmp("ONLINE",token)) { - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!mystrcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_ALTER_TABLE; - break; - } else { - if (!mystrcasecmp("IGNORE",token)) { - if (token==NULL) break; - token=(char *)tokenize(&tok); - if (!mystrcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_ALTER_TABLE; - break; - } - } - } - } else { - if (!mystrcasecmp("IGNORE",token)) { - if (token==NULL) break; - token=(char *)tokenize(&tok); - if (!mystrcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_ALTER_TABLE; - break; - } - } - } - } - if (!mystrcasecmp("VIEW",token)) { - ret=MYSQL_COM_QUERY_ALTER_VIEW; - break; - } - break; - } - if (!mystrcasecmp("ANALYZE",token)) { // ANALYZE [NO_WRITE_TO_BINLOG | LOCAL] TABLE - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_ANALYZE_TABLE; - } else { - if (!strcasecmp("NO_WRITE_TO_BINLOG",token) || !strcasecmp("LOCAL",token)) { - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_ANALYZE_TABLE; - } - } - } - break; - } - break; - case 'b': - case 'B': - if (!strcasecmp("BEGIN",token)) { // BEGIN - ret=MYSQL_COM_QUERY_BEGIN; - } - break; - case 'c': - case 'C': - if (!strcasecmp("CALL",token)) { // CALL - ret=MYSQL_COM_QUERY_CALL; - break; - } - if (!strcasecmp("CHANGE",token)) { // CHANGE - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("MASTER",token)) { - ret=MYSQL_COM_QUERY_CHANGE_MASTER; - break; - } - break; - } - if (!strcasecmp("COMMIT",token)) { // COMMIT - ret=MYSQL_COM_QUERY_COMMIT; - break; - } - if (!strcasecmp("CREATE",token)) { // CREATE - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("DATABASE",token)) { - ret=MYSQL_COM_QUERY_CREATE_DATABASE; - break; - } - if (!strcasecmp("INDEX",token)) { - ret=MYSQL_COM_QUERY_CREATE_INDEX; - break; - } - if (!strcasecmp("SCHEMA",token)) { - ret=MYSQL_COM_QUERY_CREATE_DATABASE; - break; - } - if (!strcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_CREATE_TABLE; - break; - } - if (!strcasecmp("TEMPORARY",token)) { - ret=MYSQL_COM_QUERY_CREATE_TEMPORARY; - break; - } - if (!strcasecmp("TRIGGER",token)) { - ret=MYSQL_COM_QUERY_CREATE_TRIGGER; - break; - } - if (!strcasecmp("USER",token)) { - ret=MYSQL_COM_QUERY_CREATE_USER; - break; - } - if (!strcasecmp("VIEW",token)) { - ret=MYSQL_COM_QUERY_CREATE_VIEW; - break; - } - break; - } - break; - case 'd': - case 'D': - if (!strcasecmp("DEALLOCATE",token)) { // DEALLOCATE PREPARE - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("PREPARE",token)) { - ret=MYSQL_COM_QUERY_DEALLOCATE; - break; - } - } - if (!strcasecmp("DELETE",token)) { // DELETE - ret=MYSQL_COM_QUERY_DELETE; - break; - } - if (!strcasecmp("DESCRIBE",token)) { // DESCRIBE - ret=MYSQL_COM_QUERY_DESCRIBE; - break; - } - if (!strcasecmp("DROP",token)) { // DROP - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_DROP_TABLE; - break; - } - if (!strcasecmp("TRIGGER",token)) { - ret=MYSQL_COM_QUERY_DROP_TRIGGER; - break; - } - if (!strcasecmp("USER",token)) { - ret=MYSQL_COM_QUERY_DROP_USER; - break; - } - if (!strcasecmp("VIEW",token)) { - ret=MYSQL_COM_QUERY_DROP_VIEW; - break; - } - } - break; - case 'e': - case 'E': - if (!strcasecmp("EXECUTE",token)) { // EXECUTE - ret=MYSQL_COM_QUERY_EXECUTE; - } - break; - case 'f': - case 'F': - if (!strcasecmp("FLUSH",token)) { // FLUSH - ret=MYSQL_COM_QUERY_FLUSH; - break; - } - break; - case 'g': - case 'G': - if (!strcasecmp("GRANT",token)) { // GRANT - ret=MYSQL_COM_QUERY_GRANT; - break; - } - break; - case 'i': - case 'I': - if (!strcasecmp("INSERT",token)) { // INSERT - ret=MYSQL_COM_QUERY_INSERT; - break; - } - break; - case 'k': - case 'K': - if (!strcasecmp("KILL",token)) { // KILL - ret=MYSQL_COM_QUERY_KILL; - break; - } - break; - case 'l': - case 'L': - if (!strcasecmp("LOCK",token)) { // LOCK - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_LOCK_TABLE; - break; - } - } - if (!strcasecmp("LOAD",token)) { // LOAD - ret=MYSQL_COM_QUERY_LOAD; - break; - } - break; - case 'o': - case 'O': - if (!strcasecmp("OPTIMIZE",token)) { // OPTIMIZE - ret=MYSQL_COM_QUERY_OPTIMIZE; - break; - } - break; - case 'p': - case 'P': - if (!strcasecmp("PREPARE",token)) { // PREPARE - ret=MYSQL_COM_QUERY_PREPARE; - break; - } - if (!strcasecmp("PURGE",token)) { // PURGE - ret=MYSQL_COM_QUERY_PURGE; - break; - } - break; - case 'r': - case 'R': - if (!strcasecmp("RELEASE",token)) { // RELEASE - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("SAVEPOINT",token)) { - ret=MYSQL_COM_QUERY_RELEASE_SAVEPOINT; - break; - } - } - if (!strcasecmp("RENAME",token)) { // RENAME - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_RENAME_TABLE; - break; - } - } - if (!strcasecmp("REPLACE",token)) { // REPLACE - ret=MYSQL_COM_QUERY_REPLACE; - break; - } - if (!strcasecmp("RESET",token)) { // RESET - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("MASTER",token)) { - ret=MYSQL_COM_QUERY_RESET_MASTER; - break; - } - if (!strcasecmp("SLAVE",token)) { - ret=MYSQL_COM_QUERY_RESET_SLAVE; - break; - } - break; - } - if (!strcasecmp("REVOKE",token)) { // REVOKE - ret=MYSQL_COM_QUERY_REVOKE; - break; - } - if (!strcasecmp("ROLLBACK",token)) { // ROLLBACK - token=(char *)tokenize(&tok); - if (token==NULL) { - ret=MYSQL_COM_QUERY_ROLLBACK; - break; - } else { - if (!strcasecmp("TO",token)) { - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("SAVEPOINT",token)) { - ret=MYSQL_COM_QUERY_ROLLBACK_SAVEPOINT; - break; - } - } - } - break; - } - break; - case 's': - case 'S': - if (!mystrcasecmp("SAVEPOINT",token)) { // SAVEPOINT - ret=MYSQL_COM_QUERY_SAVEPOINT; - break; - } - if (!mystrcasecmp("SELECT",token)) { // SELECT - ret=MYSQL_COM_QUERY_SELECT; - break; - // FIXME: SELECT FOR UPDATE is not implemented - } - if (!mystrcasecmp("SET",token)) { // SET - ret=MYSQL_COM_QUERY_SET; - break; - } - if (!mystrcasecmp("SHOW",token)) { // SHOW - ret=MYSQL_COM_QUERY_SHOW; - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("TABLE",token)) { - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("STATUS",token)) { - ret=MYSQL_COM_QUERY_SHOW_TABLE_STATUS; - } - } - break; - } - if (!mystrcasecmp("START",token)) { // START - token=(char *)tokenize(&tok); - if (token==NULL) break; - if (!strcasecmp("TRANSACTION",token)) { - ret=MYSQL_COM_QUERY_START_TRANSACTION; - } - break; - } - break; - case 't': - case 'T': - if (!strcasecmp("TRUNCATE",token)) { // TRUNCATE - if (token==NULL) break; - if (!strcasecmp("TABLE",token)) { - ret=MYSQL_COM_QUERY_TRUNCATE_TABLE; - break; - } - } - break; - case 'u': - case 'U': - if (!strcasecmp("UNLOCK",token)) { // UNLOCK - ret=MYSQL_COM_QUERY_UNLOCK_TABLES; - break; - } - if (!strcasecmp("UPDATE",token)) { // UPDATE - ret=MYSQL_COM_QUERY_UPDATE; - break; - } - break; - default: - break; - } - -__exit__query_parser_command_type: - free_tokenizer( &tok ); - if (qp->query_prefix) { - free(qp->query_prefix); - qp->query_prefix=NULL; - } - return ret; -} - -bool Query_Processor::query_parser_first_comment(Query_Processor_Output *qpo, char *fc) { +template +bool Query_Processor::query_parser_first_comment(Query_Processor_Output *qpo, char *fc) { bool ret=false; tokenizer_t tok; tokenizer( &tok, fc, ";", TOKENIZER_NO_EMPTIES ); @@ -2598,7 +1906,7 @@ bool Query_Processor::query_parser_first_comment(Query_Processor_Output *qpo, ch for ( token = tokenize( &tok ) ; token ; token = tokenize( &tok ) ) { char *key=NULL; char *value=NULL; - c_split_2(token, "=", &key, &value); + c_split_2(token, "=", &key, &value); remove_spaces(key); remove_spaces(value); if (strlen(key)) { @@ -2659,23 +1967,15 @@ bool Query_Processor::query_parser_first_comment(Query_Processor_Output *qpo, ch } } } - if (!strcasecmp(key,"min_gtid")) { - size_t l = strlen(value); - if (is_valid_gtid(value, l)) { - char *buf=(char*)malloc(l+1); - strncpy(buf, value, l); - buf[l+1] = '\0'; - qpo->min_gtid = buf; - } else { - proxy_warning("Invalid gtid value=%s\n", value); - } - } if (!strcasecmp(key, "create_new_connection")) { int32_t val = atoi(value); if (val == 1) { qpo->create_new_conn = true; } } + if constexpr (has_query_parser_first_comment_extended::value) { + (static_cast(this))->query_parser_first_comment_extended(key, value, static_cast(qpo)); + } } proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Variables in comment %s , key=%s , value=%s\n", token, key, value); @@ -2686,25 +1986,8 @@ bool Query_Processor::query_parser_first_comment(Query_Processor_Output *qpo, ch return ret; } -bool Query_Processor::is_valid_gtid(char *gtid, size_t gtid_len) { - if (gtid_len < 3) { - return false; - } - char *sep_pos = index(gtid, ':'); - if (sep_pos == NULL) { - return false; - } - size_t uuid_len = sep_pos - gtid; - if (uuid_len < 1) { - return false; - } - if (gtid_len < uuid_len + 2) { - return false; - } - return true; -} - -void Query_Processor::query_parser_free(SQP_par_t *qp) { +template +void Query_Processor::query_parser_free(SQP_par_t *qp) { if (qp->digest_text) { if (qp->digest_text != qp->buf) { free(qp->digest_text); @@ -2717,21 +2000,23 @@ void Query_Processor::query_parser_free(SQP_par_t *qp) { } }; -bool Query_Processor::whitelisted_sqli_fingerprint(char *_s) { +template +bool Query_Processor::whitelisted_sqli_fingerprint(char *_s) { bool ret = false; string s = _s; - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - for (std::vector::iterator it = global_mysql_firewall_whitelist_sqli_fingerprints.begin() ; ret == false && it != global_mysql_firewall_whitelist_sqli_fingerprints.end(); ++it) { + pthread_mutex_lock(&global_firewall_whitelist_mutex); + for (std::vector::iterator it = global_firewall_whitelist_sqli_fingerprints.begin() ; ret == false && it != global_firewall_whitelist_sqli_fingerprints.end(); ++it) { if (s == *it) { ret = true; } } - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_unlock(&global_firewall_whitelist_mutex); return ret; } -void Query_Processor::load_mysql_firewall_sqli_fingerprints(SQLite3_result *resultset) { - global_mysql_firewall_whitelist_sqli_fingerprints.erase(global_mysql_firewall_whitelist_sqli_fingerprints.begin(), global_mysql_firewall_whitelist_sqli_fingerprints.end()); +template +void Query_Processor::load_firewall_sqli_fingerprints(SQLite3_result *resultset) { + global_firewall_whitelist_sqli_fingerprints.erase(global_firewall_whitelist_sqli_fingerprints.begin(), global_firewall_whitelist_sqli_fingerprints.end()); // perform the inserts for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; @@ -2741,14 +2026,15 @@ void Query_Processor::load_mysql_firewall_sqli_fingerprints(SQLite3_result *resu } char * fingerprint = r->fields[1]; string s = fingerprint; - global_mysql_firewall_whitelist_sqli_fingerprints.push_back(s); + global_firewall_whitelist_sqli_fingerprints.push_back(s); } } -void Query_Processor::load_mysql_firewall_users(SQLite3_result *resultset) { +template +void Query_Processor::load_firewall_users(SQLite3_result *resultset) { unsigned long long tot_size = 0; std::unordered_map::iterator it; - for (it = global_mysql_firewall_whitelist_users.begin() ; it != global_mysql_firewall_whitelist_users.end(); ++it) { + for (it = global_firewall_whitelist_users.begin() ; it != global_firewall_whitelist_users.end(); ++it) { it->second = WUS_NOT_FOUND; } // perform the inserts/updates @@ -2765,8 +2051,8 @@ void Query_Processor::load_mysql_firewall_users(SQLite3_result *resultset) { s += rand_del; s += client_address; std::unordered_map:: iterator it2; - it2 = global_mysql_firewall_whitelist_users.find(s); - if (it2 != global_mysql_firewall_whitelist_users.end()) { + it2 = global_firewall_whitelist_users.find(s); + if (it2 != global_firewall_whitelist_users.end()) { if (strcmp(mode,(char *)"DETECTING")==0) { it2->second = WUS_DETECTING; } else if (strcmp(mode,(char *)"PROTECTING")==0) { @@ -2783,12 +2069,12 @@ void Query_Processor::load_mysql_firewall_users(SQLite3_result *resultset) { m = WUS_PROTECTING; } //wus->myptrarray = new PtrArray(); - global_mysql_firewall_whitelist_users[s] = m; + global_firewall_whitelist_users[s] = m; } } // cleanup - it = global_mysql_firewall_whitelist_users.begin(); - while (it != global_mysql_firewall_whitelist_users.end()) { + it = global_firewall_whitelist_users.begin(); + while (it != global_firewall_whitelist_users.end()) { int m = it->second; if (m != WUS_NOT_FOUND) { tot_size += it->first.capacity(); @@ -2796,29 +2082,30 @@ void Query_Processor::load_mysql_firewall_users(SQLite3_result *resultset) { it++; } else { // remove the entry - it = global_mysql_firewall_whitelist_users.erase(it); + it = global_firewall_whitelist_users.erase(it); } } - global_mysql_firewall_whitelist_users_map___size = tot_size; + global_firewall_whitelist_users_map___size = tot_size; } -void Query_Processor::load_mysql_firewall_rules(SQLite3_result *resultset) { +template +void Query_Processor::load_firewall_rules(SQLite3_result *resultset) { unsigned long long tot_size = 0; - global_mysql_firewall_whitelist_rules_map___size = 0; + global_firewall_whitelist_rules_map___size = 0; //size_t rand_del_size = strlen(rand_del); int num_rows = resultset->rows_count; std::unordered_map::iterator it; if (num_rows == 0) { // we must clean it completely - for (it = global_mysql_firewall_whitelist_rules.begin() ; it != global_mysql_firewall_whitelist_rules.end(); ++it) { + for (it = global_firewall_whitelist_rules.begin() ; it != global_firewall_whitelist_rules.end(); ++it) { PtrArray * myptrarray = (PtrArray *)it->second; delete myptrarray; } - global_mysql_firewall_whitelist_rules.clear(); + global_firewall_whitelist_rules.clear(); return; } // remove all the pointer array - for (it = global_mysql_firewall_whitelist_rules.begin() ; it != global_mysql_firewall_whitelist_rules.end(); ++it) { + for (it = global_firewall_whitelist_rules.begin() ; it != global_firewall_whitelist_rules.end(); ++it) { PtrArray * myptrarray = (PtrArray *)it->second; myptrarray->reset(); } @@ -2843,19 +2130,19 @@ void Query_Processor::load_mysql_firewall_rules(SQLite3_result *resultset) { s += rand_del; s += flagIN; std::unordered_map:: iterator it2; - it2 = global_mysql_firewall_whitelist_rules.find(s); - if (it2 != global_mysql_firewall_whitelist_rules.end()) { + it2 = global_firewall_whitelist_rules.find(s); + if (it2 != global_firewall_whitelist_rules.end()) { PtrArray * myptrarray = (PtrArray *)it2->second; myptrarray->add((void *)digest_num); } else { PtrArray * myptrarray = new PtrArray(); myptrarray->add((void *)digest_num); - global_mysql_firewall_whitelist_rules[s] = (void *)myptrarray; + global_firewall_whitelist_rules[s] = (void *)myptrarray; } } // perform ordering and cleanup - it = global_mysql_firewall_whitelist_rules.begin(); - while (it != global_mysql_firewall_whitelist_rules.end()) { + it = global_firewall_whitelist_rules.begin(); + while (it != global_firewall_whitelist_rules.end()) { PtrArray * myptrarray = (PtrArray *)it->second; if (myptrarray->len) { // there are digests, sort them @@ -2867,22 +2154,24 @@ void Query_Processor::load_mysql_firewall_rules(SQLite3_result *resultset) { } else { // remove the entry delete myptrarray; - it = global_mysql_firewall_whitelist_rules.erase(it); + it = global_firewall_whitelist_rules.erase(it); } } - unsigned long long nsize = global_mysql_firewall_whitelist_rules.size(); + unsigned long long nsize = global_firewall_whitelist_rules.size(); unsigned long long oh = sizeof(std::string) + sizeof(PtrArray) + sizeof(PtrArray *); nsize *= oh; tot_size += nsize; - global_mysql_firewall_whitelist_rules_map___size = tot_size; + global_firewall_whitelist_rules_map___size = tot_size; } -void Query_Processor::save_query_rules(SQLite3_result *resultset) { +template +void Query_Processor::save_query_rules(SQLite3_result *resultset) { delete query_rules_resultset; query_rules_resultset = resultset; // save it } -fast_routing_hashmap_t Query_Processor::create_fast_routing_hashmap(SQLite3_result* resultset) { +template +fast_routing_hashmap_t Query_Processor::create_fast_routing_hashmap(SQLite3_result* resultset) { khash_t(khStrInt)* fast_routing = nullptr; char* keys_values = nullptr; unsigned long long keys_values_size = 0; @@ -2924,12 +2213,18 @@ fast_routing_hashmap_t Query_Processor::create_fast_routing_hashmap(SQLite3_resu return { resultset, resultset->get_size(), fast_routing, keys_values, keys_values_size }; } -SQLite3_result* Query_Processor::load_fast_routing(const fast_routing_hashmap_t& fast_routing_hashmap) { +template +SQLite3_result* Query_Processor::load_fast_routing(const fast_routing_hashmap_t& fast_routing_hashmap) { khash_t(khStrInt)* _rules_fast_routing = fast_routing_hashmap.rules_fast_routing; SQLite3_result* _rules_resultset = fast_routing_hashmap.rules_resultset; if (_rules_fast_routing && _rules_resultset) { - unsigned int nt = GloMTH->num_threads; + unsigned int nt = 0; + if constexpr (std::is_same_v) { + nt = GloMTH->num_threads; + } else if constexpr (std::is_same_v) { + nt = GloPTH->num_threads; + } // Replace map structures, assumed to be previously reset this->rules_fast_routing___keys_values = fast_routing_hashmap.rules_fast_routing___keys_values; this->rules_fast_routing___keys_values___size = fast_routing_hashmap.rules_fast_routing___keys_values___size; @@ -2959,9 +2254,10 @@ SQLite3_result* Query_Processor::load_fast_routing(const fast_routing_hashmap_t& // this testing function doesn't care if the user exists or not // the arguments are coming from this query: // SELECT username, schemaname, flagIN, destination_hostgroup FROM mysql_query_rules_fast_routing ORDER BY RANDOM() -int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing(char *username, char *schemaname, int flagIN) { +template +int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing(char *username, char *schemaname, int flagIN) { int ret = -1; - pthread_rwlock_rdlock(&rwlock); + rdlock(); if (rules_fast_routing) { char keybuf[256]; char * keybuf_ptr = keybuf; @@ -2979,13 +2275,14 @@ int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing(char *u free(keybuf_ptr); } } - pthread_rwlock_unlock(&rwlock); + wrunlock(); return ret; } // this testing function implement the dual search: with and without username // if the length of username is 0 , it will search for random username (that shouldn't exist!) -int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing_dual( +template +int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing_dual( khash_t(khStrInt)* _rules_fast_routing, char* username, char* schemaname, int flagIN, bool lock ) { int ret = -1; @@ -2998,93 +2295,117 @@ int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing_dual( return ret; } -void Query_Processor::get_current_mysql_firewall_whitelist(SQLite3_result **u, SQLite3_result **r, SQLite3_result **sf) { - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - if (global_mysql_firewall_whitelist_rules_runtime) { - *r = new SQLite3_result(global_mysql_firewall_whitelist_rules_runtime); +template +void Query_Processor::get_current_firewall_whitelist(SQLite3_result **u, SQLite3_result **r, SQLite3_result **sf) { + pthread_mutex_lock(&global_firewall_whitelist_mutex); + if (global_firewall_whitelist_rules_runtime) { + *r = new SQLite3_result(global_firewall_whitelist_rules_runtime); } - if (global_mysql_firewall_whitelist_users_runtime) { - *u = new SQLite3_result(global_mysql_firewall_whitelist_users_runtime); + if (global_firewall_whitelist_users_runtime) { + *u = new SQLite3_result(global_firewall_whitelist_users_runtime); } - if (global_mysql_firewall_whitelist_sqli_fingerprints_runtime) { - *sf = new SQLite3_result(global_mysql_firewall_whitelist_sqli_fingerprints_runtime); + if (global_firewall_whitelist_sqli_fingerprints_runtime) { + *sf = new SQLite3_result(global_firewall_whitelist_sqli_fingerprints_runtime); } - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_unlock(&global_firewall_whitelist_mutex); } -void Query_Processor::load_mysql_firewall(SQLite3_result *u, SQLite3_result *r, SQLite3_result *sf) { - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - if (global_mysql_firewall_whitelist_rules_runtime) { - delete global_mysql_firewall_whitelist_rules_runtime; - global_mysql_firewall_whitelist_rules_runtime = NULL; - } - global_mysql_firewall_whitelist_rules_runtime = r; - global_mysql_firewall_whitelist_rules_result___size = r->get_size(); - if (global_mysql_firewall_whitelist_users_runtime) { - delete global_mysql_firewall_whitelist_users_runtime; - global_mysql_firewall_whitelist_users_runtime = NULL; - } - global_mysql_firewall_whitelist_users_runtime = u; - if (global_mysql_firewall_whitelist_sqli_fingerprints_runtime) { - delete global_mysql_firewall_whitelist_sqli_fingerprints_runtime; - global_mysql_firewall_whitelist_sqli_fingerprints_runtime = NULL; - } - global_mysql_firewall_whitelist_sqli_fingerprints_runtime = sf; - load_mysql_firewall_users(global_mysql_firewall_whitelist_users_runtime); - load_mysql_firewall_rules(global_mysql_firewall_whitelist_rules_runtime); - load_mysql_firewall_sqli_fingerprints(global_mysql_firewall_whitelist_sqli_fingerprints_runtime); - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); +template +void Query_Processor::load_firewall(SQLite3_result *u, SQLite3_result *r, SQLite3_result *sf) { + pthread_mutex_lock(&global_firewall_whitelist_mutex); + if (global_firewall_whitelist_rules_runtime) { + delete global_firewall_whitelist_rules_runtime; + global_firewall_whitelist_rules_runtime = NULL; + } + global_firewall_whitelist_rules_runtime = r; + global_firewall_whitelist_rules_result___size = r->get_size(); + if (global_firewall_whitelist_users_runtime) { + delete global_firewall_whitelist_users_runtime; + global_firewall_whitelist_users_runtime = NULL; + } + global_firewall_whitelist_users_runtime = u; + if (global_firewall_whitelist_sqli_fingerprints_runtime) { + delete global_firewall_whitelist_sqli_fingerprints_runtime; + global_firewall_whitelist_sqli_fingerprints_runtime = NULL; + } + global_firewall_whitelist_sqli_fingerprints_runtime = sf; + load_firewall_users(global_firewall_whitelist_users_runtime); + load_firewall_rules(global_firewall_whitelist_rules_runtime); + load_firewall_sqli_fingerprints(global_firewall_whitelist_sqli_fingerprints_runtime); + pthread_mutex_unlock(&global_firewall_whitelist_mutex); return; } -unsigned long long Query_Processor::get_mysql_firewall_memory_users_table() { +template +unsigned long long Query_Processor::get_firewall_memory_users_table() { unsigned long long ret = 0; - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - ret = global_mysql_firewall_whitelist_users_map___size; - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_lock(&global_firewall_whitelist_mutex); + ret = global_firewall_whitelist_users_map___size; + pthread_mutex_unlock(&global_firewall_whitelist_mutex); return ret; } -unsigned long long Query_Processor::get_mysql_firewall_memory_users_config() { +template +unsigned long long Query_Processor::get_firewall_memory_users_config() { unsigned long long ret = 0; - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - ret = global_mysql_firewall_whitelist_users_result___size; - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_lock(&global_firewall_whitelist_mutex); + ret = global_firewall_whitelist_users_result___size; + pthread_mutex_unlock(&global_firewall_whitelist_mutex); return ret; } -unsigned long long Query_Processor::get_mysql_firewall_memory_rules_table() { +template +unsigned long long Query_Processor::get_firewall_memory_rules_table() { unsigned long long ret = 0; - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - ret = global_mysql_firewall_whitelist_rules_map___size; - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_lock(&global_firewall_whitelist_mutex); + ret = global_firewall_whitelist_rules_map___size; + pthread_mutex_unlock(&global_firewall_whitelist_mutex); return ret; } -unsigned long long Query_Processor::get_mysql_firewall_memory_rules_config() { +template +unsigned long long Query_Processor::get_firewall_memory_rules_config() { unsigned long long ret = 0; - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - ret = global_mysql_firewall_whitelist_rules_result___size; - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_lock(&global_firewall_whitelist_mutex); + ret = global_firewall_whitelist_rules_result___size; + pthread_mutex_unlock(&global_firewall_whitelist_mutex); return ret; } -SQLite3_result * Query_Processor::get_mysql_firewall_whitelist_rules() { +template +SQLite3_result* Query_Processor::get_firewall_whitelist_rules() { SQLite3_result *ret = NULL; - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - if (global_mysql_firewall_whitelist_rules_runtime) { - ret = new SQLite3_result(global_mysql_firewall_whitelist_rules_runtime); + pthread_mutex_lock(&global_firewall_whitelist_mutex); + if (global_firewall_whitelist_rules_runtime) { + ret = new SQLite3_result(global_firewall_whitelist_rules_runtime); } - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_unlock(&global_firewall_whitelist_mutex); return ret; } -SQLite3_result * Query_Processor::get_mysql_firewall_whitelist_users() { +template +SQLite3_result* Query_Processor::get_firewall_whitelist_users() { SQLite3_result *ret = NULL; - pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); - if (global_mysql_firewall_whitelist_users_runtime) { - ret = new SQLite3_result(global_mysql_firewall_whitelist_users_runtime); + pthread_mutex_lock(&global_firewall_whitelist_mutex); + if (global_firewall_whitelist_users_runtime) { + ret = new SQLite3_result(global_firewall_whitelist_users_runtime); } - pthread_mutex_unlock(&global_mysql_firewall_whitelist_mutex); + pthread_mutex_unlock(&global_firewall_whitelist_mutex); return ret; } + + +void Query_Processor_Output::get_info_json(json& j) { + j["create_new_connection"] = create_new_conn; + j["reconnect"] = reconnect; + j["sticky_conn"] = sticky_conn; + j["cache_timeout"] = cache_timeout; + j["cache_ttl"] = cache_ttl; + j["delay"] = delay; + j["destination_hostgroup"] = destination_hostgroup; + j["firewall_whitelist_mode"] = firewall_whitelist_mode; + j["multiplex"] = multiplex; + j["timeout"] = timeout; + j["retries"] = retries; + j["max_lag_ms"] = max_lag_ms; +} diff --git a/lib/c_tokenizer.cpp b/lib/c_tokenizer.cpp index 9bb3536ac2..9fc5a4c734 100644 --- a/lib/c_tokenizer.cpp +++ b/lib/c_tokenizer.cpp @@ -1,13 +1,10 @@ #include #include #include - +#include #include "c_tokenizer.h" -extern __thread int mysql_thread___query_digests_max_query_length; - -#include -#define bool char +extern __thread int mysql_thread___query_digests_max_query_length; extern __thread bool mysql_thread___query_digests_lowercase; extern __thread bool mysql_thread___query_digests_replace_null; extern __thread bool mysql_thread___query_digests_no_digits; @@ -231,17 +228,6 @@ static inline void replace_with_q_mark( } } -/** - * @brief Struct for holding all the configuration options used for query digests generation. - */ -typedef struct options { - bool lowercase; - bool replace_null; - bool replace_number; - bool keep_comment; - int grouping_limit; - int groups_grouping_limit; -} options; /** * @brief Helper functiont that initializes the supplied 'options' struct with the configuration variables @@ -249,13 +235,14 @@ typedef struct options { * * @param opts The options struct to be initialized. */ -static inline void get_options(struct options* opts) { +static inline void get_mysql_options(options* opts) { opts->lowercase = mysql_thread___query_digests_lowercase; opts->replace_null = mysql_thread___query_digests_replace_null; opts->replace_number = mysql_thread___query_digests_no_digits; opts->grouping_limit = mysql_thread___query_digests_grouping_limit; opts->groups_grouping_limit = mysql_thread___query_digests_groups_grouping_limit; opts->keep_comment = mysql_thread___query_digests_keep_comment; + opts->max_query_length = mysql_thread___query_digests_max_query_length; } /** @@ -420,11 +407,11 @@ void init_stage_1_st(struct stage_1_st* fst_stage_st) { fst_stage_st->literal_digit_st.first_digit = 1; } -static inline int get_digest_max_len(int len) { +static inline int get_digest_max_len(int len, int max_query_length) { int digest_max_len = 0; - if (len > mysql_thread___query_digests_max_query_length) { - digest_max_len = mysql_thread___query_digests_max_query_length; + if (len > max_query_length) { + digest_max_len = max_query_length; } else { digest_max_len = len; } @@ -453,7 +440,7 @@ static inline char* get_result_buffer(int len, char* buf) { * @return The next processing state. */ static __attribute__((always_inline)) inline -enum p_st get_next_st(const struct options* opts, struct shared_st* shared_st) { +enum p_st get_next_st(const options* opts, struct shared_st* shared_st) { char prev_char = shared_st->prev_char; enum p_st st = st_no_mark_found; @@ -520,7 +507,7 @@ void inc_proc_pos(shared_st* shared_st) { * @param shared_st The shared state to modify. */ static __attribute__((always_inline)) inline -void copy_next_char(shared_st* shared_st, options* opts) { +void copy_next_char(shared_st* shared_st, const options* opts) { // copy the next character; translating any space char into ' ' if (opts->lowercase==0) { *shared_st->res_cur_pos++ = !is_space_char(*shared_st->q) ? *shared_st->q : ' '; @@ -603,7 +590,7 @@ static char is_digit_string_2(shared_st* shared_st, char *f, char *t) * - 'st_no_mark_found' if the comment has completed to be parsed. */ static __attribute__((always_inline)) inline -enum p_st process_cmnt_type_1(options* opts, shared_st* shared_st, cmnt_type_1_st* c_t_1_st, char** fst_cmnt) { +enum p_st process_cmnt_type_1(const options* opts, shared_st* shared_st, cmnt_type_1_st* c_t_1_st, char** fst_cmnt) { enum p_st next_st = st_cmnt_type_1; const char* res_final_pos = shared_st->res_init_pos + shared_st->d_max_len; @@ -958,7 +945,7 @@ enum p_st process_literal_string(shared_st* shared_st, literal_string_st* str_st * - 'st_no_mark_found' if the literal number has completed to be parsed. */ static __attribute__((always_inline)) inline -enum p_st process_literal_digit(shared_st* shared_st, literal_digit_st* digit_st, options* opts) { +enum p_st process_literal_digit(shared_st* shared_st, literal_digit_st* digit_st, const options* opts) { enum p_st next_state = st_literal_number; // process the first digit @@ -1053,7 +1040,7 @@ enum p_st process_replace_null_single_chars(shared_st* shared_st, literal_null_s * @param opts Options to be used for the copying of the current char. */ static __attribute__((always_inline)) inline -enum p_st process_replace_null(shared_st* shared_st, options* opts) { +enum p_st process_replace_null(shared_st* shared_st, const options* opts) { enum p_st next_st = st_no_mark_found; char null_found = 0; @@ -1210,7 +1197,7 @@ void end_compression_stage_it(shared_st* shared_st, char* digest_end, stage_1_st * @param fst_cmnt Pointer to be updated with the found first comment, left unmodified otherwise. */ static __attribute__((always_inline)) inline -void stage_1_parsing(shared_st* shared_st, stage_1_st* stage_1_st, options* opts, char** fst_cmnt) { +void stage_1_parsing(shared_st* shared_st, stage_1_st* stage_1_st, const options* opts, char** fst_cmnt) { // state required between different iterations of special parsing states char* res_final_pos = shared_st->res_init_pos + shared_st->d_max_len - 1; cmnt_type_1_st* const cmnt_type_1_st = &stage_1_st->cmnt_type_1_st; @@ -1490,7 +1477,7 @@ void stage_2_parsing(shared_st* shared_st, stage_1_st* stage_1_st, stage_2_st* s * @param opts Options used for deciding how to perform the group collapsing. */ static __attribute__((always_inline)) inline -void stage_3_parsing(shared_st* shared_st, stage_1_st* stage_1_st, stage_3_st* stage_3_st, options* opts) { +void stage_3_parsing(shared_st* shared_st, stage_1_st* stage_1_st, stage_3_st* stage_3_st, const options* opts) { if (opts->grouping_limit == 0) { return; } // compute the 'digest_end' for the stage 3 @@ -1948,12 +1935,12 @@ void final_stage(shared_st* shared_st, stage_1_st* stage_1_st, const options* op */ char* mysql_query_digest_first_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf) { /* buffer to store first comment. */ - int d_max_len = get_digest_max_len(q_len); + int d_max_len = get_digest_max_len(q_len, mysql_thread___query_digests_max_query_length); char* res = get_result_buffer(d_max_len, buf); // global options - struct options opts; - get_options(&opts); + options opts; + get_mysql_options(&opts); // state shared between all the parsing states struct shared_st shared_st; @@ -1984,12 +1971,12 @@ char* mysql_query_digest_first_stage(const char* const q, int q_len, char** cons */ char* mysql_query_digest_second_stage(const char* const q, int q_len, char** const fst_cmnt, char* const buf) { /* buffer to store first comment. */ - int d_max_len = get_digest_max_len(q_len); + int d_max_len = get_digest_max_len(q_len, mysql_thread___query_digests_max_query_length); char* res = get_result_buffer(d_max_len, buf); // global options - struct options opts; - get_options(&opts); + options opts; + get_mysql_options(&opts); // state shared between all the parsing states struct shared_st shared_st; @@ -2031,7 +2018,7 @@ char* mysql_query_digest_second_stage(const char* const q, int q_len, char** con * * @return A pointer to the start of the supplied buffer, or the allocated memory containing the digest. */ -char* mysql_query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf) { +char* query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf, const options* opts) { #ifdef DEBUG if (buf != NULL) { memset(buf, 0, 127); @@ -2039,17 +2026,13 @@ char* mysql_query_digest_and_first_comment_2(const char* const q, int q_len, cha #endif /* buffer to store first comment. */ - int d_max_len = get_digest_max_len(q_len); + int d_max_len = get_digest_max_len(q_len, opts->max_query_length); char* res = get_result_buffer(d_max_len, buf); #ifdef DEBUG res[d_max_len] = 0; #endif - // global options - struct options opts; - get_options(&opts); - // state shared between all the parsing states struct shared_st shared_st; memset(&shared_st, 0, sizeof(struct shared_st)); @@ -2085,10 +2068,10 @@ char* mysql_query_digest_and_first_comment_2(const char* const q, int q_len, cha // collapsed. Due to this, we might want to offer a way or limit to stop the iteration and offer a // trade off between compression and performance for very big queries. while (min_digest_size == 0) { - stage_1_parsing(&shared_st, &stage_1_st, &opts, fst_cmnt); - stage_2_parsing(&shared_st, &stage_1_st, &stage_2_st, &opts); - stage_3_parsing(&shared_st, &stage_1_st, &stage_3_st, &opts); - stage_4_parsing(&shared_st, &stage_1_st, &stage_4_st, &opts); + stage_1_parsing(&shared_st, &stage_1_st, opts, fst_cmnt); + stage_2_parsing(&shared_st, &stage_1_st, &stage_2_st, opts); + stage_3_parsing(&shared_st, &stage_1_st, &stage_3_st, opts); + stage_4_parsing(&shared_st, &stage_1_st, &stage_4_st, opts); // compute the compression offset of the whole iteration shared_st.gl_c_offset = stage_1_st.pre_it_pos - shared_st.res_cur_pos; @@ -2107,11 +2090,19 @@ char* mysql_query_digest_and_first_comment_2(const char* const q, int q_len, cha } } - final_stage(&shared_st, &stage_1_st, &opts); + final_stage(&shared_st, &stage_1_st, opts); return res; } +// For TAP tests +char* mysql_query_digest_and_first_comment_2(const char* const q, int q_len, char** const fst_cmnt, char* const buf) { + // global options + options opts; + get_mysql_options(&opts); + return query_digest_and_first_comment_2(q, q_len, fst_cmnt, buf, &opts); +} + static __attribute__((always_inline)) inline enum p_st process_literal_string_space_rm(shared_st* shared_st, literal_string_st* str_st) { enum p_st next_state = st_literal_string; @@ -2340,12 +2331,12 @@ char* mysql_query_digest_and_first_comment_one_it(char* q, int q_len, char** fst } #endif - int d_max_len = get_digest_max_len(q_len); + int d_max_len = get_digest_max_len(q_len, mysql_thread___query_digests_max_query_length); char* res = get_result_buffer(d_max_len, buf); // global options - struct options opts; - get_options(&opts); + options opts; + get_mysql_options(&opts); // state shared between all the parsing states struct shared_st shared_st; @@ -2541,7 +2532,7 @@ char* mysql_query_digest_and_first_comment_one_it(char* q, int q_len, char** fst return res; } -char *mysql_query_strip_comments(char *s, int _len) { +char *query_strip_comments(char *s, int _len, bool lowercase) { int i = 0; int len = _len; char *r = (char *) malloc(len + SIZECHAR); @@ -2554,9 +2545,6 @@ char *mysql_query_strip_comments(char *s, int _len) { char fns=0; - bool lowercase=0; - lowercase=mysql_thread___query_digests_lowercase; - while(i < len) { // ================================================= @@ -2639,7 +2627,7 @@ char *mysql_query_strip_comments(char *s, int _len) { // COPY CHAR // ================================================= // convert every space char to ' ' - if (lowercase==0) { + if (lowercase==false) { *p_r++ = !is_space_char(*s) ? *s : ' '; } else { *p_r++ = !is_space_char(*s) ? (tolower(*s)) : ' '; diff --git a/lib/mysql_backend.cpp b/lib/mysql_backend.cpp index 57d68577f9..f67b836193 100644 --- a/lib/mysql_backend.cpp +++ b/lib/mysql_backend.cpp @@ -23,20 +23,9 @@ MySQL_Backend::~MySQL_Backend() { } void MySQL_Backend::reset() { - if (server_myds && server_myds->myconn) { - MySQL_Connection *mc=server_myds->myconn; - if (mysql_thread___multiplexing && (server_myds->DSS==STATE_MARIADB_GENERIC || server_myds->DSS==STATE_READY) && mc->reusable==true && mc->IsActiveTransaction()==false && mc->MultiplexDisabled()==false && mc->async_state_machine==ASYNC_IDLE) { - server_myds->myconn->last_time_used=server_myds->sess->thread->curtime; - server_myds->return_MySQL_Connection_To_Pool(); - } else { - if (server_myds->sess && server_myds->sess->session_fast_forward == false) { - server_myds->destroy_MySQL_Connection_From_Pool(true); - } else { - server_myds->destroy_MySQL_Connection_From_Pool(false); - } - } - }; + if (server_myds) { + server_myds->reset_connection(); delete server_myds; } } diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index 97936f4ec5..42013c1d59 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include "MySQL_HostGroups_Manager.h" #include "proxysql.h" #include "cpp.h" @@ -7,7 +11,7 @@ #include "MySQL_PreparedStatement.h" #include "MySQL_Data_Stream.h" -#include "query_processor.h" +#include "MySQL_Query_Processor.h" #include "MySQL_Variables.h" #include @@ -17,12 +21,12 @@ typedef int myf; // Type of MyFlags in my_funcs #define MY_KEEP_PREALLOC 1 #define MY_ALIGN(A,L) (((A) + (L) - 1) & ~((L) - 1)) #define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double)) -void ma_free_root(MA_MEM_ROOT *root, myf MyFLAGS); -void *ma_alloc_root(MA_MEM_ROOT *mem_root, size_t Size); +static void ma_free_root(MA_MEM_ROOT *root, myf MyFLAGS); +static void *ma_alloc_root(MA_MEM_ROOT *mem_root, size_t Size); #define MAX(a,b) (((a) > (b)) ? (a) : (b)) -void * ma_alloc_root(MA_MEM_ROOT *mem_root, size_t Size) +static void * ma_alloc_root(MA_MEM_ROOT *mem_root, size_t Size) { size_t get_size; void * point; @@ -75,7 +79,7 @@ void * ma_alloc_root(MA_MEM_ROOT *mem_root, size_t Size) } -void ma_free_root(MA_MEM_ROOT *root, myf MyFlags) +static void ma_free_root(MA_MEM_ROOT *root, myf MyFlags) { MA_USED_MEM *next,*old; @@ -252,6 +256,7 @@ static char * session_vars[]= { MySQL_Connection_userinfo::MySQL_Connection_userinfo() { username=NULL; password=NULL; + passtype=PASSWORD_TYPE::PRIMARY; sha1_pass=NULL; schemaname=NULL; fe_username=NULL; @@ -1278,19 +1283,11 @@ MDB_ASYNC_ST MySQL_Connection::handler(short event) { //} MySQL_Monitor::update_dns_cache_from_mysql_conn(mysql); break; - case ASYNC_CONNECT_FAILED: - // port == 0 means we are connecting to a unix socket - if (parent->port) { - MySQL_Monitor::remove_dns_record_from_dns_cache(parent->address); - } + case ASYNC_CONNECT_FAILED: MyHGM->p_update_mysql_error_counter(p_mysql_error_type::mysql, parent->myhgc->hid, parent->address, parent->port, mysql_errno(mysql)); parent->connect_error(mysql_errno(mysql)); break; case ASYNC_CONNECT_TIMEOUT: - // port == 0 means we are connecting to a unix socket - if (parent->port) { - MySQL_Monitor::remove_dns_record_from_dns_cache(parent->address); - } //proxy_error("Connect timeout on %s:%d : %llu - %llu = %llu\n", parent->address, parent->port, myds->sess->thread->curtime , myds->wait_until, myds->sess->thread->curtime - myds->wait_until); proxy_error("Connect timeout on %s:%d : exceeded by %lluus\n", parent->address, parent->port, myds->sess->thread->curtime - myds->wait_until); MyHGM->p_update_mysql_error_counter(p_mysql_error_type::mysql, parent->myhgc->hid, parent->address, parent->port, mysql_errno(mysql)); diff --git a/lib/mysql_data_stream.cpp b/lib/mysql_data_stream.cpp index a43960af4a..594bc617fc 100644 --- a/lib/mysql_data_stream.cpp +++ b/lib/mysql_data_stream.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include "proxysql.h" #include "cpp.h" #include @@ -1631,6 +1635,23 @@ bool MySQL_Data_Stream::data_in_rbio() { return false; } +void MySQL_Data_Stream::reset_connection() { + if (myconn) { + if (mysql_thread___multiplexing && (DSS == STATE_MARIADB_GENERIC || DSS == STATE_READY) && myconn->reusable == true && myconn->IsActiveTransaction() == false && myconn->MultiplexDisabled() == false && myconn->async_state_machine == ASYNC_IDLE) { + myconn->last_time_used = sess->thread->curtime; + return_MySQL_Connection_To_Pool(); + } + else { + if (sess && sess->session_fast_forward == false) { + destroy_MySQL_Connection_From_Pool(true); + } + else { + destroy_MySQL_Connection_From_Pool(false); + } + } + } +} + void MySQL_Data_Stream::get_client_myds_info_json(json& j) { json& jc1 = j["client"]; json& jc2 = j["conn"]; diff --git a/lib/proxysql_utils.cpp b/lib/proxysql_utils.cpp index fe30562ed7..b64e50d4cd 100644 --- a/lib/proxysql_utils.cpp +++ b/lib/proxysql_utils.cpp @@ -504,3 +504,19 @@ std::pair get_dollar_quote_error(const char* version) { } } } + +const nlohmann::json* get_nested_elem(const nlohmann::json& j, const vector& p) { + if (j.is_discarded()) { return nullptr; } + const nlohmann::json* next_step = &j; + + for (const auto& e : p) { + if (next_step->contains(e)) { + next_step = &next_step->at(e); + } else { + next_step = nullptr; + break; + } + } + + return next_step; +} diff --git a/lib/set_parser.cpp b/lib/set_parser.cpp index 2d28da923a..27521253fb 100644 --- a/lib/set_parser.cpp +++ b/lib/set_parser.cpp @@ -9,6 +9,15 @@ #include //#endif +#ifdef DEBUG +//#define VALGRIND_ENABLE_ERROR_REPORTING +//#define VALGRIND_DISABLE_ERROR_REPORTING +#include "valgrind.h" +#else +#define VALGRIND_ENABLE_ERROR_REPORTING +#define VALGRIND_DISABLE_ERROR_REPORTING +#endif // DEBUG + using namespace std; #define MULTI_STATEMENTS_USE "Unable to parse multi-statements command with USE statement" diff --git a/lib/sqlite3db.cpp b/lib/sqlite3db.cpp index e5652a4464..75c315e869 100644 --- a/lib/sqlite3db.cpp +++ b/lib/sqlite3db.cpp @@ -6,6 +6,15 @@ #include #include +#ifdef DEBUG +//#define VALGRIND_ENABLE_ERROR_REPORTING +//#define VALGRIND_DISABLE_ERROR_REPORTING +#include "valgrind.h" +#else +#define VALGRIND_ENABLE_ERROR_REPORTING +#define VALGRIND_DISABLE_ERROR_REPORTING +#endif // DEBUG + #define USLEEP_SQLITE_LOCKED 100 diff --git a/src/Makefile b/src/Makefile index 3b238e0361..ee7b9f50f3 100644 --- a/src/Makefile +++ b/src/Makefile @@ -89,9 +89,20 @@ EV_PATH := $(DEPS_PATH)/libev/libev/ EV_IDIR := $(EV_PATH) EV_LDIR := $(EV_PATH)/.libs +POSTGRESQL_PATH := $(DEPS_PATH)/postgresql/postgresql/src +POSTGRESQL_IDIR := $(POSTGRESQL_PATH)/include -I$(POSTGRESQL_PATH)/interfaces/libpq +POSTGRESQL_LDIR := $(POSTGRESQL_PATH)/interfaces/libpq -L$(POSTGRESQL_PATH)/common -L$(POSTGRESQL_PATH)/port -IDIRS := -I$(PROXYSQL_IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) -I$(LIBCONFIG_IDIR) -I$(LIBDAEMON_IDIR) -I$(RE2_IDIR) -L$(PCRE_IDIR) -I$(MICROHTTPD_IDIR) -I$(LIBHTTPSERVER_IDIR) -I$(LIBINJECTION_IDIR) -I$(CURL_IDIR) -I$(EV_IDIR) -I$(SSL_IDIR) -I$(PROMETHEUS_IDIR) -I$(SQLITE3_IDIR) -I$(CLICKHOUSE_CPP_IDIR) -I$(CLICKHOUSE_CPP_CDIR) -I$(0) -LDIRS := -L$(PROXYSQL_LDIR) -L$(JEMALLOC_LDIR) -L$(MARIADB_LDIR) -L$(LIBCONFIG_LDIR) -L$(LIBDAEMON_LDIR) -L$(RE2_LDIR) -L$(PCRE_LDIR) -L$(MICROHTTPD_LDIR) -L$(LIBHTTPSERVER_LDIR) -L$(LIBINJECTION_LDIR) -L$(CURL_LDIR) -L$(EV_LDIR) -L$(SSL_LDIR) -L$(PROMETHEUS_LDIR) +LIBUSUAL_PATH=$(DEPS_PATH)/libusual/libusual +LIBUSUAL_IDIR=$(LIBUSUAL_PATH) +LIBUSUAL_LDIR=$(LIBUSUAL_PATH)/.libs/ + +LIBSCRAM_PATH=$(DEPS_PATH)/libscram/ +LIBSCRAM_IDIR=$(LIBSCRAM_PATH)/include/ +LIBSCRAM_LDIR=$(LIBSCRAM_PATH)/lib/ + +IDIRS := -I$(PROXYSQL_IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) -I$(LIBCONFIG_IDIR) -I$(LIBDAEMON_IDIR) -I$(RE2_IDIR) -L$(PCRE_IDIR) -I$(MICROHTTPD_IDIR) -I$(LIBHTTPSERVER_IDIR) -I$(LIBINJECTION_IDIR) -I$(CURL_IDIR) -I$(EV_IDIR) -I$(SSL_IDIR) -I$(PROMETHEUS_IDIR) -I$(POSTGRESQL_IDIR) -I$(LIBSCRAM_IDIR) -I$(SQLITE3_IDIR) -I$(CLICKHOUSE_CPP_IDIR) -I$(CLICKHOUSE_CPP_CDIR) -I$(0) +LDIRS := -L$(PROXYSQL_LDIR) -L$(JEMALLOC_LDIR) -L$(MARIADB_LDIR) -L$(LIBCONFIG_LDIR) -L$(LIBDAEMON_LDIR) -L$(RE2_LDIR) -L$(PCRE_LDIR) -L$(MICROHTTPD_LDIR) -L$(LIBHTTPSERVER_LDIR) -L$(LIBINJECTION_LDIR) -L$(CURL_LDIR) -L$(EV_LDIR) -L$(SSL_LDIR) -L$(PROMETHEUS_LDIR) -L$(POSTGRESQL_LDIR) -L$(LIBUSUAL_LDIR) -L$(LIBSCRAM_LDIR) UNAME_S := $(shell uname -s) @@ -163,7 +174,7 @@ endif MYCXXFLAGS += $(IDIRS) $(OPTZ) $(DEBUG) $(PSQLCH) -DGITVERSION=\"$(GIT_VERSION)\" $(NOJEM) $(WGCOV) $(WASAN) -STATICMYLIBS := -Wl,-Bstatic -lconfig -lproxysql -ldaemon -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lcurl -lssl -lcrypto -lev +STATICMYLIBS := -Wl,-Bstatic -lconfig -lproxysql -ldaemon -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lcurl -lssl -lcrypto -lev -lscram -lusual -lpq -lpgcommon -lpgport ifneq ($(NOJEMALLOC),1) STATICMYLIBS += -ljemalloc endif @@ -173,7 +184,7 @@ endif MYLIBS := -Wl,--export-dynamic $(STATICMYLIBS) -Wl,-Bdynamic -lgnutls -lpthread -lm -lz -lrt -lprometheus-cpp-pull -lprometheus-cpp-core -luuid $(EXTRALINK) ifeq ($(UNAME_S),Darwin) - MYLIBS :=-lre2 -lmariadbclient -lpthread -lm -lz -liconv -lgnutls -lprometheus-cpp-pull -lprometheus-cpp-core -luuid + MYLIBS :=-lre2 -lmariadbclient -lpq -lpthread -lm -lz -liconv -lgnutls -lprometheus-cpp-pull -lprometheus-cpp-core -luuid else CURL_DIR := $(DEPS_PATH)/curl/curl IDIRS += -L$(CURL_DIR)/include diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 32b1c5358f..a310961cab 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -9,7 +9,7 @@ #include "MySQL_Logger.hpp" #include "MySQL_Data_Stream.h" #include "proxysql_utils.h" -#include "query_processor.h" +#include "MySQL_Query_Processor.h" #include "SQLite3_Server.h" #include @@ -102,7 +102,7 @@ static int __SQLite3_Server_refresh_interval=1000; extern Query_Cache *GloQC; extern MySQL_Authentication *GloMyAuth; extern ProxySQL_Admin *GloAdmin; -extern Query_Processor *GloQPro; +extern MySQL_Query_Processor* GloMyQPro; extern MySQL_Threads_Handler *GloMTH; extern MySQL_Logger *GloMyLogger; extern MySQL_Monitor *GloMyMon; @@ -357,7 +357,7 @@ vector get_hgs_info(SQLite3DB* db) { #endif -void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { +void SQLite3_Server_session_handler(MySQL_Session* sess, void *_pa, PtrSize_t *pkt) { char *error=NULL; int cols; @@ -1042,9 +1042,9 @@ static void *child_mysql(void *arg) { SQLite3_Session *sqlite_sess = new SQLite3_Session(); mysql_thr->gen_args = (void *)sqlite_sess; - GloQPro->init_thread(); + GloMyQPro->init_thread(); mysql_thr->refresh_variables(); - MySQL_Session *sess=mysql_thr->create_new_session_and_client_data_stream(client); + MySQL_Session *sess=mysql_thr->create_new_session_and_client_data_stream(client); sess->thread=mysql_thr; sess->session_type = PROXYSQL_SESSION_SQLITE; sess->handler_function=SQLite3_Server_session_handler; diff --git a/src/main.cpp b/src/main.cpp index 76f02a073b..81e192d7ff 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,4 +1,9 @@ #define MAIN_PROXY_SQLITE3 + +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include #include #include "btree_map.h" @@ -19,9 +24,12 @@ #include "MySQL_PreparedStatement.h" #include "ProxySQL_Cluster.hpp" #include "MySQL_Logger.hpp" +#include "PgSQL_Logger.hpp" #include "SQLite3_Server.h" -#include "query_processor.h" +#include "MySQL_Query_Processor.h" +#include "PgSQL_Query_Processor.h" #include "MySQL_Authentication.hpp" +#include "PgSQL_Authentication.h" #include "MySQL_LDAP_Authentication.hpp" #include "proxysql_restapi.h" #include "Web_Interface.hpp" @@ -41,6 +49,7 @@ #include #include +#include #ifdef DEBUG #include "proxy_protocol_info.h" @@ -59,6 +68,14 @@ using std::string; using std::vector; +void sleep_iter(unsigned int iter) { + usleep(50*iter); +#ifdef RUNNING_ON_VALGRIND + usleep((1000+rand()%1000)*iter); +#endif // RUNNING_ON_VALGRIND +} + + volatile create_MySQL_LDAP_Authentication_t * create_MySQL_LDAP_Authentication = NULL; void * __mysql_ldap_auth; @@ -408,7 +425,7 @@ using namespace std; //__cmd_proxysql_config_file=NULL; #define MAX_EVENTS 100 -static volatile int load_; +std::atomic load_; //__thread l_sfp *__thr_sfp=NULL; //#ifdef DEBUG @@ -427,13 +444,16 @@ int socket_fd; Query_Cache *GloQC; MySQL_Authentication *GloMyAuth; +PgSQL_Authentication* GloPgAuth; MySQL_LDAP_Authentication *GloMyLdapAuth; #ifdef PROXYSQLCLICKHOUSE ClickHouse_Authentication *GloClickHouseAuth; #endif /* PROXYSQLCLICKHOUSE */ -Query_Processor *GloQPro; +MySQL_Query_Processor* GloMyQPro; +PgSQL_Query_Processor* GloPgQPro; ProxySQL_Admin *GloAdmin; MySQL_Threads_Handler *GloMTH = NULL; +PgSQL_Threads_Handler* GloPTH = NULL; Web_Interface *GloWebInterface; MySQL_STMT_Manager_v14 *GloMyStmt; @@ -441,8 +461,9 @@ MySQL_Monitor *GloMyMon; std::thread *MyMon_thread = NULL; MySQL_Logger *GloMyLogger; +PgSQL_Logger* GloPgSQL_Logger; MySQL_Variables mysql_variables; - +PgSQL_Variables pgsql_variables; SQLite3_Server *GloSQLite3Server; #ifdef PROXYSQLCLICKHOUSE ClickHouse_Server *GloClickHouseServer; @@ -471,8 +492,9 @@ void * mysql_worker_thread_func(void *arg) { worker->init(); // worker->poll_listener_add(listen_fd); // worker->poll_listener_add(socket_fd); - __sync_fetch_and_sub(&load_,1); - do { usleep(50); } while (load_); + load_ -= 1; + unsigned int iter = 0; + do { sleep_iter(++iter); } while (load_); worker->run(); //delete worker; @@ -502,8 +524,9 @@ void * mysql_worker_thread_func_idles(void *arg) { worker->init(); // worker->poll_listener_add(listen_fd); // worker->poll_listener_add(socket_fd); - __sync_fetch_and_sub(&load_,1); - do { usleep(50); } while (load_); + load_ -= 1; + unsigned int iter = 0; + do { sleep_iter(++iter); } while (load_); worker->run(); //delete worker; @@ -516,6 +539,71 @@ void * mysql_worker_thread_func_idles(void *arg) { } #endif // IDLE_THREADS +void* pgsql_worker_thread_func(void* arg) { + + // __thr_sfp=l_mem_init(); + + pthread_attr_t thread_attr; + size_t tmp_stack_size = 0; + if (!pthread_attr_init(&thread_attr)) { + if (!pthread_attr_getstacksize(&thread_attr, &tmp_stack_size)) { + __sync_fetch_and_add(&GloVars.statuses.stack_memory_pgsql_threads, tmp_stack_size); + } + } + + proxysql_pgsql_thread_t* pgsql_thread = (proxysql_pgsql_thread_t*)arg; + PgSQL_Thread* worker = new PgSQL_Thread(); + pgsql_thread->worker = worker; + worker->init(); + // worker->poll_listener_add(listen_fd); + // worker->poll_listener_add(socket_fd); + load_ -= 1; + unsigned int iter = 0; + do { sleep_iter(++iter); } while (load_); + + worker->run(); + //delete worker; + delete worker; + pgsql_thread->worker = NULL; + // l_mem_destroy(__thr_sfp); + __sync_fetch_and_sub(&GloVars.statuses.stack_memory_pgsql_threads, tmp_stack_size); + return NULL; +} + +#ifdef IDLE_THREADS +void* pgsql_worker_thread_func_idles(void* arg) { + + pthread_attr_t thread_attr; + size_t tmp_stack_size = 0; + if (!pthread_attr_init(&thread_attr)) { + if (!pthread_attr_getstacksize(&thread_attr, &tmp_stack_size)) { + __sync_fetch_and_add(&GloVars.statuses.stack_memory_pgsql_threads, tmp_stack_size); + } + } + + // __thr_sfp=l_mem_init(); + proxysql_pgsql_thread_t* pgsql_thread = (proxysql_pgsql_thread_t*)arg; + PgSQL_Thread* worker = new PgSQL_Thread(); + pgsql_thread->worker = worker; + worker->epoll_thread = true; + worker->init(); + // worker->poll_listener_add(listen_fd); + // worker->poll_listener_add(socket_fd); + load_ -= 1; + unsigned int iter = 0; + do { sleep_iter(++iter); } while (load_); + + worker->run(); + //delete worker; + delete worker; + // l_mem_destroy(__thr_sfp); + + __sync_fetch_and_sub(&GloVars.statuses.stack_memory_pgsql_threads, tmp_stack_size); + + return NULL; +} +#endif // IDLE_THREADS + void * mysql_shared_query_cache_funct(void *arg) { GloQC->purgeHash_thread(NULL); return NULL; @@ -725,14 +813,17 @@ void ProxySQL_Main_process_global_variables(int argc, const char **argv) { void ProxySQL_Main_init_main_modules() { GloQC=NULL; - GloQPro=NULL; + GloMyQPro=NULL; GloMTH=NULL; GloMyAuth=NULL; + GloPgAuth=NULL; + GloPTH=NULL; #ifdef PROXYSQLCLICKHOUSE GloClickHouseAuth=NULL; #endif /* PROXYSQLCLICKHOUSE */ GloMyMon=NULL; GloMyLogger=NULL; + GloPgSQL_Logger = NULL; GloMyStmt=NULL; // initialize libev @@ -743,12 +834,21 @@ void ProxySQL_Main_init_main_modules() { MyHGM=new MySQL_HostGroups_Manager(); MyHGM->init(); + MySQL_Threads_Handler * _tmp_GloMTH = NULL; _tmp_GloMTH=new MySQL_Threads_Handler(); GloMTH = _tmp_GloMTH; GloMyLogger = new MySQL_Logger(); GloMyLogger->print_version(); + GloPgSQL_Logger = new PgSQL_Logger(); + GloPgSQL_Logger->print_version(); GloMyStmt=new MySQL_STMT_Manager_v14(); + + PgHGM = new PgSQL_HostGroups_Manager(); + PgHGM->init(); + PgSQL_Threads_Handler* _tmp_GloPTH = NULL; + _tmp_GloPTH = new PgSQL_Threads_Handler(); + GloPTH = _tmp_GloPTH; } @@ -771,7 +871,10 @@ void ProxySQL_Main_init_Admin_module(const bootstrap_info_t& bootstrap_info) { void ProxySQL_Main_init_Auth_module() { GloMyAuth = new MySQL_Authentication(); GloMyAuth->print_version(); + GloPgAuth = new PgSQL_Authentication(); + GloPgAuth->print_version(); GloAdmin->init_users(); + GloAdmin->init_pgsql_users(); //GloMyLdapAuth = create_MySQL_LDAP_Authentication(); if (GloMyLdapAuth) { GloMyLdapAuth->print_version(); @@ -779,10 +882,14 @@ void ProxySQL_Main_init_Auth_module() { } void ProxySQL_Main_init_Query_module() { - GloQPro = new Query_Processor(); - GloQPro->print_version(); + GloMyQPro = new MySQL_Query_Processor(); + GloMyQPro->print_version(); + GloPgQPro = new PgSQL_Query_Processor(); + GloPgQPro->print_version(); GloAdmin->init_mysql_query_rules(); GloAdmin->init_mysql_firewall(); + GloAdmin->init_pgsql_query_rules(); + GloAdmin->init_pgsql_firewall(); // if (GloWebInterface) { // GloWebInterface->print_version(); // } @@ -811,6 +918,30 @@ void ProxySQL_Main_init_MySQL_Threads_Handler_module() { } } +void ProxySQL_Main_init_PgSQL_Threads_Handler_module() { + unsigned int i; + GloPTH->init(); + //load_ = 1; + load_ += GloPTH->num_threads; +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + load_ += GloPTH->num_threads; + } + else { + proxy_warning("proxysql instance running without --idle-threads : most workloads benefit from this option\n"); + proxy_warning("proxysql instance running without --idle-threads : enabling it can potentially improve performance\n"); + } +#endif // IDLE_THREADS + for (i = 0; i < GloPTH->num_threads; i++) { + GloPTH->create_thread(i, pgsql_worker_thread_func, false); +#ifdef IDLE_THREADS + if (GloVars.global.idle_threads) { + GloPTH->create_thread(i, pgsql_worker_thread_func_idles, true); + } +#endif // IDLE_THREADS + } +} + void ProxySQL_Main_init_Query_Cache_module() { GloQC = new Query_Cache(); GloQC->print_version(); @@ -860,6 +991,13 @@ void ProxySQL_Main_join_all_threads() { GloMTH->shutdown_threads(); #ifdef DEBUG std::cerr << "GloMTH joined in "; +#endif + } + if (GloPTH) { + cpu_timer t; + GloPTH->shutdown_threads(); +#ifdef DEBUG + std::cerr << "GloPTH joined in "; #endif } if (GloQC) { @@ -912,12 +1050,20 @@ void ProxySQL_Main_shutdown_all_modules() { std::cerr << "GloQC shutdown in "; #endif } - if (GloQPro) { + if (GloMyQPro) { cpu_timer t; - delete GloQPro; - GloQPro=NULL; + delete GloMyQPro; + GloMyQPro=NULL; #ifdef DEBUG - std::cerr << "GloQPro shutdown in "; + std::cerr << "GloMyQPro shutdown in "; +#endif + } + if (GloPgQPro) { + cpu_timer t; + delete GloPgQPro; + GloPgQPro=NULL; +#ifdef DEBUG + std::cerr << "GloPgQPro shutdown in "; #endif } #ifdef PROXYSQLCLICKHOUSE @@ -952,6 +1098,14 @@ void ProxySQL_Main_shutdown_all_modules() { GloMyAuth=NULL; #ifdef DEBUG std::cerr << "GloMyAuth shutdown in "; +#endif + } + if (GloPgAuth) { + cpu_timer t; + delete GloPgAuth; + GloPgAuth = NULL; +#ifdef DEBUG + std::cerr << "GloPgAuth shutdown in "; #endif } if (GloMTH) { @@ -972,6 +1126,14 @@ void ProxySQL_Main_shutdown_all_modules() { std::cerr << "GloMyLogger shutdown in "; #endif } + if (GloPgSQL_Logger) { + cpu_timer t; + delete GloPgSQL_Logger; + GloPgSQL_Logger = NULL; +#ifdef DEBUG + std::cerr << "GloPgSQL_Logger shutdown in "; +#endif + } { #ifdef TEST_WITHASAN @@ -1115,6 +1277,14 @@ void ProxySQL_Main_init_phase2___not_started(const bootstrap_info_t& boostrap_in GloMyLogger->audit_set_datadir(GloVars.datadir); #ifdef DEBUG std::cerr << "Main phase3 : GloMyLogger initialized in "; +#endif + } + { + cpu_timer t; + GloPgSQL_Logger->events_set_datadir(GloVars.datadir); + GloPgSQL_Logger->audit_set_datadir(GloVars.datadir); +#ifdef DEBUG + std::cerr << "Main phase3 : GloPgSQL_Logger initialized in "; #endif } if (GloVars.configfile_open) { @@ -1141,6 +1311,14 @@ void ProxySQL_Main_init_phase3___start_all() { GloMyLogger->audit_set_datadir(GloVars.datadir); #ifdef DEBUG std::cerr << "Main phase3 : GloMyLogger initialized in "; +#endif + } + { + cpu_timer t; + GloPgSQL_Logger->events_set_datadir(GloVars.datadir); + GloPgSQL_Logger->audit_set_datadir(GloVars.datadir); +#ifdef DEBUG + std::cerr << "Main phase3 : GloPgSQL_Logger initialized in "; #endif } // Initialized monitor, no matter if it will be started or not @@ -1149,6 +1327,7 @@ void ProxySQL_Main_init_phase3___start_all() { { cpu_timer t; GloAdmin->init_mysql_servers(); + GloAdmin->init_pgsql_servers(); GloAdmin->init_proxysql_servers(); GloAdmin->load_scheduler_to_runtime(); #ifdef DEBUG @@ -1169,22 +1348,28 @@ void ProxySQL_Main_init_phase3___start_all() { std::cerr << "Main phase3 : MySQL Threads Handler initialized in "; #endif } + { cpu_timer t; - ProxySQL_Main_init_Query_Cache_module(); + ProxySQL_Main_init_PgSQL_Threads_Handler_module(); #ifdef DEBUG - std::cerr << "Main phase3 : Query Cache initialized in "; + std::cerr << "Main phase3 : PgSQL Threads Handler initialized in "; #endif } - do { /* nothing */ + { + cpu_timer t; + ProxySQL_Main_init_Query_Cache_module(); #ifdef DEBUG - usleep(5+rand()%10); + std::cerr << "Main phase3 : Query Cache initialized in "; #endif - } while (load_ != 1); + } + + unsigned int iter = 0; + do { sleep_iter(++iter); } while (load_ != 1); load_ = 0; __sync_fetch_and_add(&GloMTH->status_variables.threads_initialized, 1); - + __sync_fetch_and_add(&GloPTH->status_variables.threads_initialized, 1); { cpu_timer t; GloMTH->start_listeners(); @@ -1192,6 +1377,15 @@ void ProxySQL_Main_init_phase3___start_all() { std::cerr << "Main phase3 : MySQL Threads Handler listeners started in "; #endif } + + { + cpu_timer t; + GloPTH->start_listeners(); +#ifdef DEBUG + std::cerr << "Main phase3 : PgSQL Threads Handler listeners started in "; +#endif + } + if ( GloVars.global.sqlite3_server == true ) { cpu_timer t; ProxySQL_Main_init_SQLite3Server(); @@ -1380,15 +1574,15 @@ bool ProxySQL_daemonize_phase2() { * @note This function does not return if an error occurs; it exits the process. */ void call_execute_on_exit_failure() { - // Log a message indicating the attempt to call the external script - proxy_info("Trying to call external script after exit failure: %s\n", GloVars.execute_on_exit_failure ? GloVars.execute_on_exit_failure : "(null)"); - // Check if the global variable execute_on_exit_failure is NULL if (GloVars.execute_on_exit_failure == NULL) { // Exit the function if the variable is not set return; } + // Log a message indicating the attempt to call the external script + proxy_error("Trying to call external script after exit failure: %s\n", GloVars.execute_on_exit_failure); + // Fork a child process pid_t cpid; cpid = fork(); @@ -1474,16 +1668,6 @@ bool ProxySQL_daemonize_phase3() { proxy_info("ProxySQL SHA1 checksum: %s\n", binary_sha1); } call_execute_on_exit_failure(); - // automatic reload of TLS certificates after a crash , see #4658 - std::string msg; - ProxySQL_create_or_load_TLS(false, msg); - // Honor --initial after a crash , see #4659 - if (GloVars.__cmd_proxysql_initial==true) { - std::cerr << "Renaming database file " << GloVars.admindb << endl; - char *newpath=(char *)malloc(strlen(GloVars.admindb)+8); - sprintf(newpath,"%s.bak",GloVars.admindb); - rename(GloVars.admindb,newpath); // FIXME: should we check return value, or ignore whatever it successed or not? - } parent_close_error_log(); return false; } diff --git a/test/cluster/confs/proxysql01.cfg b/test/cluster/confs/proxysql01.cfg index aa3bc8c2b5..ec4c49d612 100644 --- a/test/cluster/confs/proxysql01.cfg +++ b/test/cluster/confs/proxysql01.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26001" + pgsql_ifaces="0.0.0.0:26101" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36001" } +pgsql_variables= +{ + interfaces="0.0.0.0:36101" +} + proxysql_servers = ( { diff --git a/test/cluster/confs/proxysql02.cfg b/test/cluster/confs/proxysql02.cfg index 85beb06bc9..010d987924 100644 --- a/test/cluster/confs/proxysql02.cfg +++ b/test/cluster/confs/proxysql02.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26002" + pgsql_ifaces="0.0.0.0:26102" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36002" } +pgsql_variables= +{ + interfaces="0.0.0.0:36102" +} + proxysql_servers = ( { diff --git a/test/cluster/confs/proxysql03.cfg b/test/cluster/confs/proxysql03.cfg index 4bec8f9a4c..803d416987 100644 --- a/test/cluster/confs/proxysql03.cfg +++ b/test/cluster/confs/proxysql03.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26003" + pgsql_ifaces="0.0.0.0:26103" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36003" } +pgsql_variables= +{ + interfaces="0.0.0.0:36103" +} + proxysql_servers = ( { diff --git a/test/cluster/confs/proxysql04.cfg b/test/cluster/confs/proxysql04.cfg index 37eb66274c..00548f07bb 100644 --- a/test/cluster/confs/proxysql04.cfg +++ b/test/cluster/confs/proxysql04.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26004" + pgsql_ifaces="0.0.0.0:26104" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36004" } +pgsql_variables= +{ + interfaces="0.0.0.0:36104" +} + proxysql_servers = ( { diff --git a/test/cluster/confs/proxysql05.cfg b/test/cluster/confs/proxysql05.cfg index 6a48b085e0..3a60b04412 100644 --- a/test/cluster/confs/proxysql05.cfg +++ b/test/cluster/confs/proxysql05.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26005" + pgsql_ifaces="0.0.0.0:26105" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36005" } +pgsql_variables= +{ + interfaces="0.0.0.0:36105" +} + proxysql_servers = ( { diff --git a/test/cluster/confs/proxysql06.cfg b/test/cluster/confs/proxysql06.cfg index 70728f0df5..1918813f52 100644 --- a/test/cluster/confs/proxysql06.cfg +++ b/test/cluster/confs/proxysql06.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26006" + pgsql_ifaces="0.0.0.0:26106" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36006" } +pgsql_variables= +{ + interfaces="0.0.0.0:36106" +} + proxysql_servers = ( { diff --git a/test/cluster/confs/proxysql07.cfg b/test/cluster/confs/proxysql07.cfg index d6f16179a4..c544c31685 100644 --- a/test/cluster/confs/proxysql07.cfg +++ b/test/cluster/confs/proxysql07.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26007" + pgsql_ifaces="0.0.0.0:26107" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36007" } +pgsql_variables= +{ + interfaces="0.0.0.0:36107" +} + proxysql_servers = ( { diff --git a/test/cluster/confs/proxysql08.cfg b/test/cluster/confs/proxysql08.cfg index 17d0fecb40..6ba316d1f0 100644 --- a/test/cluster/confs/proxysql08.cfg +++ b/test/cluster/confs/proxysql08.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26008" + pgsql_ifaces="0.0.0.0:26108" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36008" } +pgsql_variables= +{ + interfaces="0.0.0.0:36108" +} + proxysql_servers = ( { diff --git a/test/cluster/confs/proxysql09.cfg b/test/cluster/confs/proxysql09.cfg index cca81da812..cd65111a5b 100644 --- a/test/cluster/confs/proxysql09.cfg +++ b/test/cluster/confs/proxysql09.cfg @@ -4,6 +4,7 @@ admin_variables= { admin_credentials="admin:admin;cluster1:secret1pass" mysql_ifaces="0.0.0.0:26009" + pgsql_ifaces="0.0.0.0:26109" cluster_username="cluster1" cluster_password="secret1pass" } @@ -13,6 +14,11 @@ mysql_variables= interfaces="0.0.0.0:36009" } +pgsql_variables= +{ + interfaces="0.0.0.0:36109" +} + proxysql_servers = ( { diff --git a/test/tap/tap/SQLite3_Server.cpp b/test/tap/tap/SQLite3_Server.cpp index cc7d0d7c4d..127432eadf 100644 --- a/test/tap/tap/SQLite3_Server.cpp +++ b/test/tap/tap/SQLite3_Server.cpp @@ -8,7 +8,7 @@ #include "MySQL_Logger.hpp" #include "MySQL_Data_Stream.h" -#include "query_processor.h" +#include "MySQL_Query_Processor.h" #include "SQLite3_Server.h" #include @@ -76,7 +76,7 @@ static int testLag = 10; extern Query_Cache *GloQC; extern MySQL_Authentication *GloMyAuth; extern ProxySQL_Admin *GloAdmin; -extern Query_Processor *GloQPro; +extern MySQL_Query_Processor* GloQPro; extern MySQL_Threads_Handler *GloMTH; extern MySQL_Logger *GloMyLogger; extern MySQL_Monitor *GloMyMon; diff --git a/test/tap/tap/command_line.cpp b/test/tap/tap/command_line.cpp index ce473c77e7..c516475cd0 100644 --- a/test/tap/tap/command_line.cpp +++ b/test/tap/tap/command_line.cpp @@ -47,6 +47,27 @@ CommandLine::~CommandLine() { if (mysql_password) free(mysql_password); + if (pgsql_admin_host) + free(pgsql_admin_host); + if (pgsql_server_host) + free(pgsql_server_host); + if (pgsql_server_username) + free(pgsql_server_username); + if (pgsql_server_password) + free(pgsql_server_password); + if (pgsql_host) + free(pgsql_host); + if (pgsql_username) + free(pgsql_username); + if (pgsql_password) + free(pgsql_password); + if (pgsql_root_host) + free(pgsql_root_host); + if (pgsql_root_username) + free(pgsql_root_username); + if (pgsql_root_password) + free(pgsql_root_password); + if (workdir) free(workdir); } @@ -237,6 +258,82 @@ int CommandLine::getEnv() { replace_str_field(&this->mysql_password, value); } + { + // unprivileged test connection + value = getenv("TAP_PGSQL_HOST"); + if (value) + replace_str_field(&this->pgsql_host, value); + + value = getenv("TAP_PGSQL_PORT"); + if (value) { + env_port = strtol(value, NULL, 10); + if (env_port > 0 && env_port < 65536) + pgsql_port = env_port; + } + + value = getenv("TAP_PGSQL_USERNAME"); + if (value) + replace_str_field(&this->pgsql_username, value); + + value = getenv("TAP_PGSQL_PASSWORD"); + if (value) + replace_str_field(&this->pgsql_password, value); + + // privileged test connection + value = getenv("TAP_PGSQLROOT_HOST"); + if (value) + replace_str_field(&this->pgsql_root_host, value); + + value = getenv("TAP_PGSQLROOT_PORT"); + if (value) { + env_port = strtol(value, NULL, 10); + if (env_port > 0 && env_port < 65536) + pgsql_root_port = env_port; + } + + value = getenv("TAP_PGSQLROOT_USERNAME"); + if (value) + replace_str_field(&this->pgsql_root_username, value); + + value = getenv("TAP_PGSQLROOT_PASSWORD"); + if (value) + replace_str_field(&this->pgsql_root_password, value); + + + // pgsql proxysql admin connection + value = getenv("TAP_PGSQLADMIN_HOST"); + if (value) + replace_str_field(&this->pgsql_admin_host, value); + + value = getenv("TAP_PGSQLADMIN_PORT"); + if (value) { + env_port = strtol(value, NULL, 10); + if (env_port > 0 && env_port < 65536) + pgsql_admin_port = env_port; + } + + // admin username and password are identical for both MySQL and PostgreSQL. + + // pgsql server connection + value = getenv("TAP_PGSQLSERVER_HOST"); + if (value) + replace_str_field(&this->pgsql_server_host, value); + + value = getenv("TAP_PGSQLSERVER_PORT"); + if (value) { + env_port = strtol(value, NULL, 10); + if (env_port > 0 && env_port < 65536) + pgsql_server_port = env_port; + } + + value = getenv("TAP_PGSQLSERVER_USERNAME"); + if (value) + replace_str_field(&this->pgsql_server_username, value); + + value = getenv("TAP_PGSQLSERVER_PASSWORD"); + if (value) + replace_str_field(&this->pgsql_server_password, value); + } value = getenv("TAP_WORKDIR"); if (value) diff --git a/test/tap/tap/command_line.h b/test/tap/tap/command_line.h index b60b28cc30..f2988d2f3d 100644 --- a/test/tap/tap/command_line.h +++ b/test/tap/tap/command_line.h @@ -38,6 +38,28 @@ class CommandLine { char* mysql_username = strdup("root"); char* mysql_password = strdup("root"); + // proxysql postgresql admin connection + char* pgsql_admin_host = strdup("127.0.0.1"); + int pgsql_admin_port = 6132; + + // pgsql server connection + char* pgsql_server_host = strdup("127.0.0.1"); + int pgsql_server_port = 5432; + char* pgsql_server_username = strdup("postgres"); + char* pgsql_server_password = strdup("postgres"); + + // unpriviliged test connection + char* pgsql_host = strdup("127.0.0.1"); + int pgsql_port = 6133; + char* pgsql_username = strdup("testuser"); + char* pgsql_password = strdup("testuser"); + + // priviliged test connection + char* pgsql_root_host = strdup("127.0.0.1"); + int pgsql_root_port = 6133; + char* pgsql_root_username = strdup("postgres"); + char* pgsql_root_password = strdup("postgres"); + char* workdir = strdup("./"); uint64_t client_flags = 0; diff --git a/test/tap/tap/utils.h b/test/tap/tap/utils.h index 1aeaa8e744..e2b71a4a94 100644 --- a/test/tap/tap/utils.h +++ b/test/tap/tap/utils.h @@ -7,7 +7,7 @@ #include #include "curl/curl.h" - +#include "mysql.h" #include "sqlite3db.h" #include "json_fwd.hpp" diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index 85d57f64cb..9a78c8f5d1 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -17,6 +17,10 @@ MARIADB_PATH := $(DEPS_PATH)/mariadb-client-library/mariadb_client MARIADB_IDIR := $(MARIADB_PATH)/include MARIADB_LDIR := $(MARIADB_PATH)/libmariadb +POSTGRESQL_PATH := $(DEPS_PATH)/postgresql/postgresql/src +POSTGRESQL_IDIR := $(POSTGRESQL_PATH)/include -I$(POSTGRESQL_PATH)/interfaces/libpq +POSTGRESQL_LDIR := $(POSTGRESQL_PATH)/interfaces/libpq -L$(POSTGRESQL_PATH)/common -L$(POSTGRESQL_PATH)/port + JEMALLOC_PATH := $(DEPS_PATH)/jemalloc/jemalloc JEMALLOC_IDIR := $(JEMALLOC_PATH)/include/jemalloc JEMALLOC_LDIR := $(JEMALLOC_PATH)/lib @@ -97,6 +101,10 @@ TEST_MYSQL_IDIR := $(TEST_MYSQL_PATH)/include TEST_MYSQL_EDIR := $(TEST_MYSQL_PATH)/libbinlogevents/export/ TEST_MYSQL_LDIR := $(TEST_MYSQL_PATH)/libmysql +POSTGRESQL_PATH := $(DEPS_PATH)/postgresql/postgresql/src +POSTGRESQL_IDIR := $(POSTGRESQL_PATH)/include -I$(POSTGRESQL_PATH)/interfaces/libpq +POSTGRESQL_LDIR := $(POSTGRESQL_PATH)/interfaces/libpq -L$(POSTGRESQL_PATH)/common -L$(POSTGRESQL_PATH)/port + TEST_MYSQL8_PATH := $(TEST_DEPS_PATH)/mysql-connector-c-8.4.0/mysql-connector-c TEST_MYSQL8_IDIR := $(TEST_MYSQL8_PATH)/include TEST_MYSQL8_EDIR := $(TEST_MYSQL8_PATH)/libbinlogevents/export/ @@ -110,14 +118,15 @@ EXECUTABLE := proxysql OBJ := $(PROXYSQL_PATH)/src/obj/proxysql_global.o $(PROXYSQL_PATH)/src/obj/main.o $(PROXYSQL_PATH)/src/obj/proxy_tls.o -SOURCES := ../tap/utils.cpp - IDIRS := -I$(TAP_IDIR) -I$(RE2_IDIR) -I$(PROXYSQL_IDIR) -I$(JEMALLOC_IDIR) -I$(LIBCONFIG_IDIR) -I$(MARIADB_IDIR)\ -I$(DAEMONPATH_IDIR) -I$(MICROHTTPD_IDIR) -I$(LIBHTTPSERVER_IDIR) -I$(CURL_IDIR) -I$(EV_IDIR)\ - -I$(PROMETHEUS_IDIR) -I$(DOTENV_DYN_IDIR) -I$(SSL_IDIR) -I$(SQLITE3_IDIR) -I$(JSON_IDIR) + -I$(PROMETHEUS_IDIR) -I$(DOTENV_DYN_IDIR) -I$(SSL_IDIR) -I$(SQLITE3_IDIR) -I$(JSON_IDIR) -I$(POSTGRESQL_IDIR) + LDIRS := -L$(TAP_LDIR) -L$(RE2_LDIR) -L$(PROXYSQL_LDIR) -L$(JEMALLOC_LDIR) -L$(LIBCONFIG_LDIR) -L$(MARIADB_LDIR)\ -L$(DAEMONPATH_LDIR) -L$(MICROHTTPD_LDIR) -L$(LIBHTTPSERVER_LDIR) -L$(CURL_LDIR) -L$(EV_LDIR)\ - -L$(PROMETHEUS_LDIR) -L$(DOTENV_DYN_LDIR) -L$(SSL_LDIR) -L$(PCRE_LDIR) -L$(LIBINJECTION_LDIR) + -L$(PROMETHEUS_LDIR) -L$(DOTENV_DYN_LDIR) -L$(SSL_LDIR) -L$(PCRE_LDIR) -L$(LIBINJECTION_LDIR) -L$(POSTGRESQL_LDIR) + +#SOURCES := ../tap/utils.cpp UNAME_S := $(shell uname -s) @@ -125,11 +134,13 @@ ifeq ($(UNAME_S),Linux) LDIRS += -L$(COREDUMPER_LDIR) endif -MYLIBS := -Wl,--export-dynamic -MYLIBS += -Wl,-Bdynamic -lgnutls -ltap -lcpp_dotenv -lcurl -lssl -lcrypto -lre2 -luuid -MYLIBS += -Wl,-Bstatic -lconfig -lproxysql -ldaemon -lconfig++ -lpcrecpp -lpcre -lmariadbclient -lhttpserver\ - -lmicrohttpd -linjection -lev -lprometheus-cpp-pull -lprometheus-cpp-core -MYLIBS += -Wl,-Bdynamic -lpthread -lm -lz -lrt -ldl $(EXTRALINK) +MYLIBS_DYNAMIC_PART := -Wl,--export-dynamic -Wl,-Bdynamic -lgnutls -ltap -lcpp_dotenv -lcurl -lssl -lcrypto -luuid +MYLIBS_STATIC_PART := -Wl,-Bstatic -lconfig -lproxysql -ldaemon -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lev -lprometheus-cpp-pull -lprometheus-cpp-core +MYLIBS_PG_PART := -Wl,-Bstatic -lpq -lpgcommon -lpgport +MYLIBS_LAST_PART := -Wl,-Bdynamic -lpthread -lm -lz -lrt -ldl $(EXTRALINK) +MYLIBS := $(MYLIBS_DYNAMIC_PART) $(MYLIBS_STATIC_PART) $(MYLIBS_PG_PART) $(MYLIBS_LAST_PART) +#MYLIBS_PG := $(MYLIBS_DYNAMIC_PART) $(MYLIBS_STATIC_PART) $(MYLIBS_PG_PART) $(MYLIBS_LAST_PART) +#MYLIBS := -Wl,--export-dynamic -Wl,-Bdynamic -lssl -lcrypto -lgnutls -ltap -lcpp_dotenv -Wl,-Bstatic -lconfig -lproxysql -ldaemon -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lhttpserver -lmicrohttpd -linjection -lev -lprometheus-cpp-pull -lprometheus-cpp-core -luuid -Wl,-Bdynamic -lpthread -lm -lz -lrt -ldl $(EXTRALINK) MYLIBSJEMALLOC := -Wl,-Bstatic -ljemalloc STATIC_LIBS := $(CITYHASH_LDIR)/libcityhash.a diff --git a/test/tap/tests/admin_various_commands-t.cpp b/test/tap/tests/admin_various_commands-t.cpp index cbc2ea3c9b..003356fc13 100644 --- a/test/tap/tests/admin_various_commands-t.cpp +++ b/test/tap/tests/admin_various_commands-t.cpp @@ -79,6 +79,7 @@ int main() { { 1 , "show VARIABLES" }, { 1 , "show ALL variables" }, { 1 , "show MYSQL variables" }, + { 1 , "SHOW PGSQL VARIABLES" }, { 1 , "SHOW admin VARIABLES" }, { 3 , "sHoW DATABASES" }, { 3 , "sHoW SCHEMAS" }, diff --git a/test/tap/tests/mysql_reconnect.cpp b/test/tap/tests/mysql_reconnect.cpp index 386e7078ee..d923e29be8 100644 --- a/test/tap/tests/mysql_reconnect.cpp +++ b/test/tap/tests/mysql_reconnect.cpp @@ -12,11 +12,7 @@ #include #include -#ifdef LIBMYSQL_HELPER8 -#include "mysql/mysql.h" -#else #include "mysql.h" -#endif #include "utils.h" #include "tap.h" diff --git a/test/tap/tests/pgsql-basic_tests-t.cpp b/test/tap/tests/pgsql-basic_tests-t.cpp new file mode 100644 index 0000000000..fa94f1044f --- /dev/null +++ b/test/tap/tests/pgsql-basic_tests-t.cpp @@ -0,0 +1,653 @@ +/** + * @file pgsql-basic_tests-t.cpp + * @brief This test conducts a thorough validation of various PostgreSQL database operations. + * It begins by establishing a valid database connection and confirming successful connectivity. + * Subsequently, the test performs a series of Data Definition Language (DDL) and Data Manipulation Language (DML) operations, + * which include table creation, data insertion, selection, updates, deletions, and transactions. + */ + +#include +#include + +#include "libpq-fe.h" +#include "command_line.h" +#include "tap.h" +#include "utils.h" + + +#define PQEXEC(conn, query) ({PGresult* res = PQexec(conn, query); \ + ExecStatusType status = PQresultStatus(res); \ + if (status != PGRES_COMMAND_OK && \ + status != PGRES_TUPLES_OK) { \ + fprintf(stderr, "File %s, line %d, status %d, %s\n", \ + __FILE__, __LINE__, status, PQresultErrorMessage(res)); \ + } \ + res; \ + }) + +#define PQSENDQUERY(conn,query) ({int send_status = PQsendQuery(conn,query); \ + if (send_status != 1) { \ + fprintf(stderr, "File %s, line %d, status %d, %s\n", \ + __FILE__, __LINE__, status, PQerrorMessage(conn)); \ + } \ + send_status; \ + }) + + +CommandLine cl; + +PGconn* create_new_connection(bool with_ssl) { + std::stringstream ss; + + ss << "host=" << cl.pgsql_host << " port=" << cl.pgsql_port; + ss << " user=" << cl.pgsql_username << " password=" << cl.pgsql_password; + + if (with_ssl) { + ss << " sslmode=require"; + } else { + ss << " sslmode=disable"; + } + + PGconn* conn = PQconnectdb(ss.str().c_str()); + const bool res = (conn && PQstatus(conn) == CONNECTION_OK); + ok(res, "Connection created successfully. %s", PQerrorMessage(conn)); + + if (res) return conn; + + PQfinish(conn); + return nullptr; +} + +// Function to set up the test environment +void setup_database(PGconn* conn) { + PGresult* res; + + res = PQEXEC(conn, "DROP TABLE IF EXISTS test_table"); + PQclear(res); + + res = PQEXEC(conn, "CREATE TABLE test_table (id SERIAL PRIMARY KEY, value TEXT)"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "Created test_table"); + PQclear(res); + + res = PQEXEC(conn, "INSERT INTO test_table (value) VALUES ('test1'), ('test2'), ('test3')"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "Inserted initial records into test_table"); + PQclear(res); +} + +void test_simple_query(PGconn* conn) { + PGresult* res = PQEXEC(conn, "SELECT 1"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + ok(1, "Simple SELECT query executed successfully"); + int nFields = PQnfields(res); + int nRows = PQntuples(res); + ok(nFields == 1, "Returned one field"); + ok(nRows == 1, "Returned one row"); + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "1") == 0, "Result is 1"); + } else { + ok(0, "Simple SELECT query failed"); + } + PQclear(res); +} + +void test_insert_query(PGconn* conn) { + PGresult* res = PQEXEC(conn, "INSERT INTO test_table (value) VALUES ('test4')"); + if (PQresultStatus(res) == PGRES_COMMAND_OK) { + ok(1, "INSERT query executed successfully"); + ok(strcmp(PQcmdTuples(res), "1") == 0, "One row inserted"); + } else { + ok(0, "INSERT query failed"); + } + PQclear(res); + + // Verify insertion + res = PQEXEC(conn, "SELECT value FROM test_table WHERE value = 'test4'"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "Inserted row is present"); + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "test4") == 0, "Inserted value is correct"); + } else { + ok(0, "Failed to verify inserted row"); + } + PQclear(res); +} + +void test_update_query(PGconn* conn) { + PGresult* res = PQEXEC(conn, "UPDATE test_table SET value = 'updated' WHERE value = 'test2'"); + if (PQresultStatus(res) == PGRES_COMMAND_OK) { + ok(1, "UPDATE query executed successfully"); + ok(strcmp(PQcmdTuples(res), "1") == 0, "One row updated"); + } else { + ok(0, "UPDATE query failed"); + } + PQclear(res); + + // Verify update + res = PQEXEC(conn, "SELECT value FROM test_table WHERE value = 'updated'"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "Updated row is present"); + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "updated") == 0, "Updated value is correct"); + } else { + ok(0, "Failed to verify updated row"); + } + PQclear(res); +} + +void test_delete_query(PGconn* conn) { + PGresult* res = PQEXEC(conn, "DELETE FROM test_table WHERE value = 'test3'"); + if (PQresultStatus(res) == PGRES_COMMAND_OK) { + ok(1, "DELETE query executed successfully"); + ok(strcmp(PQcmdTuples(res), "1") == 0, "One row deleted"); + } else { + ok(0, "DELETE query failed"); + } + PQclear(res); + + // Verify deletion + res = PQEXEC(conn, "SELECT value FROM test_table WHERE value = 'test3'"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 0, "Deleted row is no longer present"); + } else { + ok(0, "Failed to verify deleted row"); + } + PQclear(res); +} + +void test_invalid_query(PGconn* conn) { + PGresult* res = PQEXEC(conn, "SELECT * FROM non_existent_table"); + ok(PQresultStatus(res) == PGRES_FATAL_ERROR, "Query on non-existent table failed as expected"); + PQclear(res); +} + +void test_transaction_commit(PGconn* conn) { + PGresult* res = PQEXEC(conn, "BEGIN"); + ok(PQtransactionStatus(conn) == PQTRANS_INTRANS, "Connection in Transaction state"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "BEGIN transaction"); + + res = PQEXEC(conn, "INSERT INTO test_table (value) VALUES ('transaction commit')"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "INSERT in transaction"); + PQclear(res); + + res = PQEXEC(conn, "COMMIT"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "COMMIT transaction"); + PQclear(res); + ok(PQtransactionStatus(conn) == PQTRANS_IDLE, "Connection in Idle state"); + + // Verify commit + res = PQEXEC(conn, "SELECT value FROM test_table WHERE value = 'transaction commit'"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "Committed row is present"); + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "transaction commit") == 0, "Committed value is correct"); + } else { + ok(0, "Failed to verify committed row"); + } + PQclear(res); +} + +void test_transaction_rollback(PGconn* conn) { + PGresult* res = PQEXEC(conn, "BEGIN"); + ok(PQtransactionStatus(conn) == PQTRANS_INTRANS, "Connection in Transaction state"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "BEGIN transaction"); + + res = PQEXEC(conn, "INSERT INTO test_table (value) VALUES ('transaction rollback')"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "INSERT in transaction"); + PQclear(res); + + res = PQEXEC(conn, "ROLLBACK"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "ROLLBACK transaction"); + PQclear(res); + ok(PQtransactionStatus(conn) == PQTRANS_IDLE, "Connection in Idle state"); + + // Verify rollback + res = PQEXEC(conn, "SELECT value FROM test_table WHERE value = 'transaction rollback'"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 0, "Rolled back row is not present"); + } else { + ok(0, "Failed to verify rolled back row"); + } + PQclear(res); +} + +void test_transaction_error(PGconn* conn) { + PGresult* res = PQEXEC(conn, "BEGIN"); + ok(PQtransactionStatus(conn) == PQTRANS_INTRANS, "Connection in Transaction state"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "BEGIN transaction"); + + res = PQEXEC(conn, "SELECT 1/0"); + ok(PQresultStatus(res) == PGRES_FATAL_ERROR, "Error result returned"); + PQclear(res); + ok(PQtransactionStatus(conn) == PQTRANS_INERROR, "Connection in Error Transaction state"); + + res = PQEXEC(conn, "ROLLBACK"); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "ROLLBACK transaction"); + PQclear(res); + ok(PQtransactionStatus(conn) == PQTRANS_IDLE, "Connection in Idle state"); + + // Verify rollback + res = PQEXEC(conn, "SELECT value FROM test_table WHERE value = 'transaction rollback'"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 0, "Rolled back row is not present"); + } else { + ok(0, "Failed to verify rolled back row"); + } + PQclear(res); +} + +void test_null_value(PGconn* conn) { + PGresult* res = PQEXEC(conn, "INSERT INTO test_table (value) VALUES (NULL)"); + if (PQresultStatus(res) == PGRES_COMMAND_OK) { + ok(1, "INSERT NULL value executed successfully"); + ok(strcmp(PQcmdTuples(res), "1") == 0, "One row inserted with NULL value"); + } else { + ok(0, "INSERT NULL value failed"); + } + PQclear(res); + + // Verify NULL insertion + res = PQEXEC(conn, "SELECT value FROM test_table WHERE value IS NULL"); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "Inserted NULL value is present"); + } else { + ok(0, "Failed to verify inserted NULL value"); + } + PQclear(res); +} + +void test_constraint_violation(PGconn* conn) { + PGresult* res = PQEXEC(conn, "INSERT INTO test_table (id, value) VALUES (1, 'duplicate id')"); + ok(PQresultStatus(res) == PGRES_FATAL_ERROR, "INSERT with duplicate ID failed as expected"); + PQclear(res); +} + +void test_multi_statement_transaction(PGconn* conn) { + PGresult* res; + int status; + + // Execute multi-statement transaction + status = PQsendQuery(conn, "BEGIN; " + "INSERT INTO test_table (value) VALUES ('multi statement'); " + "UPDATE test_table SET value = 'multi statement updated' WHERE value = 'multi statement'; " + "COMMIT;"); + ok(status == 1, "Multi-statement transaction sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + + // Check result of BEGIN + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "BEGIN executed successfully"); + PQclear(res); + + // Check result of INSERT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "INSERT executed successfully"); + PQclear(res); + + // Check result of UPDATE + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "UPDATE executed successfully"); + PQclear(res); + + // Check result of COMMIT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "COMMIT executed successfully"); + PQclear(res); + + res = PQgetResult(conn); + ok(PQtransactionStatus(conn) == PQTRANS_IDLE, "Connection in Idle state"); + + // Verify the results + status = PQsendQuery(conn, "SELECT value FROM test_table WHERE value = 'multi statement updated'"); + ok(status == 1, "Verification query sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + res = PQgetResult(conn); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "Multi-statement transaction committed correctly"); + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "multi statement updated") == 0, "Multi-statement transaction result is correct"); + } else { + ok(0, "Failed to verify multi-statement transaction"); + } + PQclear(res); + PQgetResult(conn); +} + +void test_multi_statement_transaction_with_error(PGconn* conn) { + PGresult* res; + int status; + + // Execute multi-statement transaction with an error + status = PQSENDQUERY(conn, "BEGIN; " + "INSERT INTO test_table (value) VALUES ('multi statement error'); " + "UPDATE test_table SET value = 'multi statement error updated' WHERE value = 'multi statement error'; " + "INSERT INTO test_table (non_existent_column) VALUES ('error'); " + "COMMIT;"); + ok(status == 1, "Multi-statement transaction with error sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + + // Check result of BEGIN + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "BEGIN executed successfully"); + PQclear(res); + + // Check result of INSERT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "INSERT executed successfully"); + PQclear(res); + + // Check result of UPDATE + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "UPDATE executed successfully"); + PQclear(res); + + // Check result of erroneous INSERT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_FATAL_ERROR, "Erroneous INSERT failed as expected"); + PQclear(res); + + PQgetResult(conn); + // Ensure the transaction is in error state + ok(PQtransactionStatus(conn) == PQTRANS_INERROR, "Connection in Error Transaction state"); + + // Rollback the transaction + status = PQsendQuery(conn, "ROLLBACK"); + ok(status == 1, "ROLLBACK sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "ROLLBACK executed successfully"); + PQclear(res); + + PQgetResult(conn); + + ok(PQtransactionStatus(conn) == PQTRANS_IDLE, "Connection in Idle state"); + + // Verify the results + status = PQsendQuery(conn, "SELECT value FROM test_table WHERE value = 'multi statement error' OR value = 'multi statement error updated'"); + ok(status == 1, "Verification query sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + res = PQgetResult(conn); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 0, "Multi-statement transaction with error rolled back correctly"); + } else { + ok(0, "Failed to verify rollback of multi-statement transaction with error"); + } + PQclear(res); + PQgetResult(conn); +} + +void test_multi_statement_select_insert(PGconn* conn) { + PGresult* res; + int status; + + // Execute multi-statement SELECT and INSERT + status = PQsendQuery(conn, "SELECT value FROM test_table WHERE id = 1; " + "INSERT INTO test_table (value) VALUES ('multi statement select insert');"); + ok(status == 1, "Multi-statement SELECT and INSERT sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + + // Check result of SELECT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_TUPLES_OK, "SELECT executed successfully"); + PQclear(res); + + // Check result of INSERT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "INSERT executed successfully"); + PQclear(res); + + PQgetResult(conn); + + // Verify the results + status = PQsendQuery(conn, "SELECT value FROM test_table WHERE value = 'multi statement select insert'"); + ok(status == 1, "Verification query sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + res = PQgetResult(conn); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "Multi-statement SELECT and INSERT committed correctly"); + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "multi statement select insert") == 0, "Multi-statement SELECT and INSERT result is correct"); + } else { + ok(0, "Failed to verify multi-statement SELECT and INSERT"); + } + PQclear(res); + PQgetResult(conn); +} + +void test_multi_statement_delete_update(PGconn* conn) { + PGresult* res; + int status; + + // Execute multi-statement DELETE and UPDATE + status = PQsendQuery(conn, "DELETE FROM test_table WHERE value = 'test1'; " + "UPDATE test_table SET value = 'multi statement delete update' WHERE value = 'test4';"); + ok(status == 1, "Multi-statement DELETE and UPDATE sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + + // Check result of DELETE + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "DELETE executed successfully"); + PQclear(res); + + // Check result of UPDATE + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "UPDATE executed successfully"); + PQclear(res); + + PQgetResult(conn); + + // Verify the results + status = PQsendQuery(conn, "SELECT value FROM test_table WHERE value = 'multi statement delete update'"); + ok(status == 1, "Verification query sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + res = PQgetResult(conn); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "Multi-statement DELETE and UPDATE committed correctly"); + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "multi statement delete update") == 0, "Multi-statement DELETE and UPDATE result is correct"); + } else { + ok(0, "Failed to verify multi-statement DELETE and UPDATE"); + } + PQclear(res); + PQgetResult(conn); +} + +void test_multi_statement_with_error(PGconn* conn) { + PGresult* res; + int status; + + // Execute multi-statement with an error + status = PQsendQuery(conn, "INSERT INTO test_table (value) VALUES ('multi statement error'); " + "UPDATE test_table SET value = 'multi statement error updated' WHERE value = 'multi statement error'; " + "INSERT INTO test_table (non_existent_column) VALUES ('error');"); + ok(status == 1, "Multi-statement with error sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + + // Check result of INSERT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "INSERT executed successfully"); + PQclear(res); + + // Check result of UPDATE + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "UPDATE executed successfully"); + PQclear(res); + + // Check result of erroneous INSERT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_FATAL_ERROR, "Erroneous INSERT failed as expected"); + PQclear(res); + + PQgetResult(conn); + // Verify the results + status = PQsendQuery(conn, "SELECT value FROM test_table WHERE value = 'multi statement error' OR value = 'multi statement error updated'"); + ok(status == 1, "Verification query sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + res = PQgetResult(conn); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 0, "No rows are inserted or updated"); + } else { + ok(0, "Failed to verify rows from multi-statement with error"); + } + PQclear(res); + PQgetResult(conn); +} + +void test_multi_statement_insert_select_select(PGconn* conn) { + PGresult* res; + int status; + + // Execute multi-statement INSERT, SELECT, and SELECT + status = PQsendQuery(conn, "INSERT INTO test_table (value) VALUES ('multi statement select1'), ('multi statement select2'); " + "SELECT value FROM test_table WHERE value = 'multi statement select1'; " + "SELECT value FROM test_table WHERE value = 'multi statement select2';"); + ok(status == 1, "Multi-statement INSERT and SELECTs sent"); + PQconsumeInput(conn); + while (PQisBusy(conn)) { + PQconsumeInput(conn); + } + + // Check result of the INSERT + res = PQgetResult(conn); + ok(PQresultStatus(res) == PGRES_COMMAND_OK, "INSERT executed successfully"); + PQclear(res); + + // Check result of the first SELECT + res = PQgetResult(conn); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "First SELECT executed successfully"); + if (nRows > 0) { + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "multi statement select1") == 0, "First SELECT result is correct"); + } + } else { + ok(0, "First SELECT failed"); + } + PQclear(res); + + // Check result of the second SELECT + res = PQgetResult(conn); + if (PQresultStatus(res) == PGRES_TUPLES_OK) { + int nRows = PQntuples(res); + ok(nRows == 1, "Second SELECT executed successfully"); + if (nRows > 0) { + char* result = PQgetvalue(res, 0, 0); + ok(strcmp(result, "multi statement select2") == 0, "Second SELECT result is correct"); + } + } else { + ok(0, "Second SELECT failed"); + } + PQclear(res); + PQgetResult(conn); +} + +void teardown_database(PGconn* conn) { + PGresult* res; + + res = PQEXEC(conn, "DROP TABLE IF EXISTS test_table"); + PQclear(res); +} + +void test_invalid_connection(bool with_ssl) { + + std::stringstream ss; + + ss << "host=invalid_host port=invalid_port dbname=invalid_db user=invalid_user password=invalid_password"; + + if (with_ssl) { + ss << " sslmode=require"; + } else { + ss << " sslmode=disable"; + } + + PGconn* conn = PQconnectdb(ss.str().c_str()); + ok(PQstatus(conn) == CONNECTION_BAD, "Connection failed with invalid parameters"); + PQfinish(conn); +} + +void execute_tests(bool with_ssl) { + PGconn* conn = create_new_connection(with_ssl); + + if (conn == nullptr) + return; + + setup_database(conn); + test_simple_query(conn); + test_insert_query(conn); + test_update_query(conn); + test_delete_query(conn); + test_invalid_query(conn); + test_transaction_commit(conn); + test_transaction_rollback(conn); + test_transaction_error(conn); + test_null_value(conn); + test_constraint_violation(conn); + test_multi_statement_transaction(conn); + test_multi_statement_transaction_with_error(conn); + test_multi_statement_select_insert(conn); + test_multi_statement_delete_update(conn); + test_multi_statement_with_error(conn); + test_multi_statement_insert_select_select(conn); + teardown_database(conn); + test_invalid_connection(with_ssl); + + PQfinish(conn); +} + +int main(int argc, char** argv) { + + plan(176); // Total number of tests planned + + if (cl.getEnv()) + return exit_status(); + + execute_tests(false); // without SSL + execute_tests(true); // with SSL + + return exit_status(); +} diff --git a/test/tap/tests/test_auth_methods-t.cpp b/test/tap/tests/test_auth_methods-t.cpp index d12aedcd7f..0f7bba16e0 100644 --- a/test/tap/tests/test_auth_methods-t.cpp +++ b/test/tap/tests/test_auth_methods-t.cpp @@ -223,8 +223,8 @@ using auth_reg_t = std::unordered_map; struct user_creds_t { user_def_t user_def; - mf_unique_ptr hashed_prim_pass_bin; - mf_unique_ptr hashed_addl_pass_bin; + mf_unique_ptr hashed_prim_pass_bin { nullptr }; + mf_unique_ptr hashed_addl_pass_bin { nullptr }; user_creds_t(const user_creds_t&) = delete; user_creds_t(user_creds_t&&) noexcept(false) = default; @@ -1288,9 +1288,12 @@ const vector backend_users { * @brief Minimal test cases for default backend users. */ const vector tests_creds { + { "dualpass1", MF_CHAR_("USE_PRIM_RAND_PASS") }, { "dualpass1", MF_CHAR_(nullptr) }, { "dualpass1", MF_CHAR_("") }, { "dualpass1", MF_CHAR_("inv_pass") }, + { "dualpass1", MF_CHAR_("USE_PRIM_RAND_PASS") }, + { "dualpass1", MF_CHAR_("USE_ADDL_RAND_PASS") }, { "dualpass2", MF_CHAR_(nullptr) }, { "dualpass2", MF_CHAR_("") }, @@ -1309,6 +1312,10 @@ const vector tests_creds { { "dualpass5", MF_CHAR_(nullptr) }, { "dualpass5", MF_CHAR_("") }, { "dualpass5", MF_CHAR_("inv_pass") }, + // Failure unless in RAND mode - Use fetched RAND_PASS + { "dualpass5", MF_CHAR_("USE_PRIM_RAND_PASS") }, + { "dualpass5", MF_CHAR_("USE_PRIM_RAND_PASS") }, + { "dualpass5", MF_CHAR_("USE_ADDL_RAND_PASS") }, { "dualpass6", MF_CHAR_(nullptr) }, { "dualpass6", MF_CHAR_("") }, @@ -1319,23 +1326,33 @@ const vector tests_creds { { "dualpass7", MF_CHAR_(nullptr) }, { "dualpass7", MF_CHAR_("") }, { "dualpass7", MF_CHAR_("inv_pass") }, - // { "dualpass7", MF_CHAR_("oldpass7") }, + { "dualpass7", MF_CHAR_("oldpass7") }, + // Failure unless in RAND mode - Use fetched RAND_PASS + { "dualpass7", MF_CHAR_("USE_PRIM_RAND_PASS") }, + { "dualpass7", MF_CHAR_("USE_PRIM_RAND_PASS") }, + { "dualpass7", MF_CHAR_("USE_ADDL_RAND_PASS") }, { "dualpass8", MF_CHAR_(nullptr) }, { "dualpass8", MF_CHAR_("") }, { "dualpass8", MF_CHAR_("inv_pass") }, - // { "dualpass8", MF_CHAR_("oldpass8") }, + { "dualpass8", MF_CHAR_("oldpass8") }, { "dualpass9", MF_CHAR_(nullptr) }, { "dualpass9", MF_CHAR_("") }, { "dualpass9", MF_CHAR_("inv_pass") }, - // { "dualpass9", MF_CHAR_("oldpass9") }, + { "dualpass9", MF_CHAR_("oldpass9") }, { "dualpass9", MF_CHAR_("newpass9") }, { "dualpass9", MF_CHAR_("newpass9") }, + { "dualpass9", MF_CHAR_("USE_RAND_PASS") }, { "dualpass11", MF_CHAR_(nullptr) }, { "dualpass11", MF_CHAR_("") }, { "dualpass11", MF_CHAR_("inv_pass") }, + // Failure unless in RAND mode - Use fetched RAND_PASS + { "dualpass11", MF_CHAR_("USE_ADDL_RAND_PASS") }, + // Triggers limitation on 'full_auth' again; this should be improved + { "dualpass11", MF_CHAR_("USE_ADDL_RAND_PASS") }, + { "dualpass11", MF_CHAR_("USE_PRIM_RAND_PASS") }, { "dualpass12", MF_CHAR_(nullptr) }, { "dualpass12", MF_CHAR_("") }, @@ -1364,12 +1381,16 @@ const vector tests_creds { { "dualpass17", MF_CHAR_(nullptr) }, { "dualpass17", MF_CHAR_("") }, { "dualpass17", MF_CHAR_("inv_pass") }, - // { "dualpass17", MF_CHAR_("oldpass17") }, + { "dualpass17", MF_CHAR_("oldpass17") }, + // Failure unless in RAND mode - Use fetched RAND_PASS + { "dualpass17", MF_CHAR_("USE_ADDL_RAND_PASS") }, + { "dualpass17", MF_CHAR_("USE_ADDL_RAND_PASS") }, + { "dualpass17", MF_CHAR_("USE_PRIM_RAND_PASS") }, { "dualpass18", MF_CHAR_(nullptr) }, { "dualpass18", MF_CHAR_("") }, { "dualpass18", MF_CHAR_("inv_pass") }, - // { "dualpass18", MF_CHAR_("oldpass18") }, + { "dualpass18", MF_CHAR_("oldpass18") }, { "dualpass19", MF_CHAR_(nullptr) }, { "dualpass19", MF_CHAR_("") }, @@ -1667,18 +1688,21 @@ int main(int argc, char** argv) { vector rnd_tests_creds { ::tests_creds }; - for (test_creds_t& creds : rnd_tests_creds) { creds.name = "rnd" + creds.name; - auto it = std::find_if(cbres.second.begin(), cbres.second.end(), + auto it = std::find_if(rnd_cbres.second.begin(), rnd_cbres.second.end(), [&creds] (const user_creds_t& user_creds) { return creds.name == user_creds.user_def.name; } ); - if (it != cbres.second.end()) { - creds.pass = it->user_def.prim_pass ? MF_CHAR_(it->user_def.prim_pass.get()) : nullptr; + if (it != rnd_cbres.second.end()) { + if (creds.pass && strcasecmp(creds.pass.get(), "USE_PRIM_RAND_PASS") == 0) { + creds.pass = it->user_def.prim_pass ? MF_CHAR_(it->user_def.prim_pass.get()) : nullptr; + } else if (creds.pass && strcasecmp(creds.pass.get(), "USE_ADDL_RAND_PASS") == 0) { + creds.pass = it->user_def.addl_pass ? MF_CHAR_(it->user_def.addl_pass.get()) : nullptr; + } } } diff --git a/test/tap/tests/test_warnings-t.cpp b/test/tap/tests/test_warnings-t.cpp index b111294ea6..786d13856c 100644 --- a/test/tap/tests/test_warnings-t.cpp +++ b/test/tap/tests/test_warnings-t.cpp @@ -13,7 +13,7 @@ #include "json.hpp" #include "mysql.h" -#include "mysql/mysqld_error.h" +#include "mysqld_error.h" #include "tap.h" #include "command_line.h" #include "utils.h" diff --git a/test/tap/tests_with_deps/deprecate_eof_support/Makefile b/test/tap/tests_with_deps/deprecate_eof_support/Makefile index 27062713e7..32db6d7624 100644 --- a/test/tap/tests_with_deps/deprecate_eof_support/Makefile +++ b/test/tap/tests_with_deps/deprecate_eof_support/Makefile @@ -85,6 +85,10 @@ TEST_MYSQL_IDIR := $(TEST_MYSQL_PATH)/include TEST_MYSQL_EDIR := $(TEST_MYSQL_PATH)/libbinlogevents/export TEST_MYSQL_LDIR := $(TEST_MYSQL_PATH)/libmysql +POSTGRESQL_PATH := $(DEPS_PATH)/postgresql/postgresql/src +POSTGRESQL_IDIR := $(POSTGRESQL_PATH)/include -I$(POSTGRESQL_PATH)/interfaces/libpq +POSTGRESQL_LDIR := $(POSTGRESQL_PATH)/interfaces/libpq -L$(POSTGRESQL_PATH)/common -L$(POSTGRESQL_PATH)/port + ### detect compiler support for c++11/17 CPLUSPLUS := $(shell ${CC} -std=c++17 -dM -E -x c++ /dev/null 2>/dev/null | grep -F __cplusplus | egrep -o '[0-9]{6}L') ifneq ($(CPLUSPLUS),201703L) @@ -115,10 +119,10 @@ OPT := $(STDCPP) -O2 -ggdb -Wl,--no-as-needed -Wl,-rpath,"../../tap" $(WGCOV) $( IDIRS := -I$(TAP_IDIR) -I$(RE2_IDIR) -I$(PROXYSQL_IDIR) -I$(JEMALLOC_IDIR) -I$(LIBCONFIG_IDIR)\ -I$(MICROHTTPD_IDIR) -I$(LIBHTTPSERVER_IDIR) -I$(CURL_IDIR) -I$(EV_IDIR) -I$(PROMETHEUS_IDIR)\ - -I$(DOTENV_DYN_IDIR) -I$(SSL_IDIR) -I$(SQLITE3_IDIR) -I$(JSON_IDIR) + -I$(DOTENV_DYN_IDIR) -I$(SSL_IDIR) -I$(SQLITE3_IDIR) -I$(JSON_IDIR) -I$(POSTGRESQL_IDIR) LDIRS := -L$(TAP_LDIR) -L$(RE2_LDIR) -L$(PROXYSQL_LDIR) -L$(JEMALLOC_LDIR) -L$(LIBCONFIG_LDIR)\ -L$(MICROHTTPD_LDIR) -L$(LIBHTTPSERVER_LDIR) -L$(CURL_LDIR) -L$(EV_LDIR) -L$(PROMETHEUS_LDIR)\ - -L$(DOTENV_DYN_LDIR) -L$(SSL_LDIR) -L$(SQLITE3_LDIR) -L$(PCRE_LDIR) + -L$(DOTENV_DYN_LDIR) -L$(SSL_LDIR) -L$(SQLITE3_LDIR) -L$(PCRE_LDIR) -L$(POSTGRESQL_LDIR) ### main targets diff --git a/test/tap/tests_with_deps/deprecate_eof_support/fwd_eof_query.cpp b/test/tap/tests_with_deps/deprecate_eof_support/fwd_eof_query.cpp index 5eb35f4c61..903dd54f30 100644 --- a/test/tap/tests_with_deps/deprecate_eof_support/fwd_eof_query.cpp +++ b/test/tap/tests_with_deps/deprecate_eof_support/fwd_eof_query.cpp @@ -1,3 +1,7 @@ +#include "../deps/json/json.hpp" +using json = nlohmann::json; +#define PROXYJSON + #include #include #include