Refactoring replication lag actions + Simulator (combined)

Makefile

@@ -85,6 +85,12 @@ testgalera: build_deps_debug build_lib_testgalera build_src_testgalera
 .PHONY: testgrouprep
 testgrouprep: build_deps_debug build_lib_testgrouprep build_src_testgrouprep
 
+.PHONY: testreadonly
+testreadonly: build_deps_debug build_lib_testreadonly build_src_testreadonly
+
+.PHONY: testreplicationlag
+testreplicationlag: build_deps_debug build_lib_testreplicationlag build_src_testreplicationlag
+
 .PHONY: testall
 testall: build_deps_debug build_lib_testall build_src_testall
 
@@ -140,13 +146,29 @@ build_src_testgrouprep: build_deps build_lib_testgrouprep
 build_lib_testgrouprep: build_deps_debug
 	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GROUPREP" CC=${CC} CXX=${CXX} ${MAKE}
 
+.PHONY: build_src_testreadonly
+build_src_testreadonly: build_deps build_lib_testreadonly
+	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_READONLY" CC=${CC} CXX=${CXX} ${MAKE}
+
+.PHONY: build_lib_testreadonly
+build_lib_testreadonly: build_deps_debug
+	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_READONLY" CC=${CC} CXX=${CXX} ${MAKE}
+
+.PHONY: build_src_testreplicationlag
+build_src_testreplicationlag: build_deps build_lib_testreplicationlag
+	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE}
+
+.PHONY: build_lib_testreplicationlag
+build_lib_testreplicationlag: build_deps_debug
+	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE}
+
 .PHONY: build_src_testall
 build_src_testall: build_deps build_lib_testall
-	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP" CC=${CC} CXX=${CXX} ${MAKE}
+	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_lib_testall
 build_lib_testall: build_deps_debug
-	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP" CC=${CC} CXX=${CXX} ${MAKE}
+	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_tap_test
 build_tap_test: build_src

Read_Only_Testing.md

@@ -0,0 +1,29 @@
+
+
+### Compiling
+
+To run `read_only` automated testing, ProxySQL needs to be compiled with `make testreadonly`.
+
+
+
+### shutdown mysqld
+
+When running automated testing, ProxySQL will listen on many IPs (30) an on port 3306.  
+You need to make sure that MySQL server is not running, or not listening on port 3306.
+
+
+### Running proxysql
+
+`proxysql` needs to be executed with `--sqlite3-server` .  
+For example, to run it under `gdb`: `run -f -D . --sqlite3-server`
+
+
+### Similate failover
+
+To simulate failover is enough to connect to sqlite3 server interface and update the `READONLY_STATUS` table.  
+Note that to connect it is necessary to use a user configured in `mysql_users` table.  
+To simulate a lot of failover at the same time, a query like the follow can be executed:  
+
+```
+ UPDATE READONLY_STATUS SET read_only=1; CREATE TABLE t1 AS SELECT hostname FROM READONLY_STATUS ORDER BY RANDOM() LIMIT 50; UPDATE READONLY_STATUS SET read_only=0 WHERE hostname IN (SELECT hostname FROM t1); DROP TABLE t1;
+```

include/MySQL_HostGroups_Manager.h

@@ -390,6 +390,89 @@ class MySQL_HostGroups_Manager {
 #else
 	rwlock_t rwlock;
 #endif
+
+	enum HGM_TABLES {
+		MYSQL_SERVERS = 0,
+		MYSQL_REPLICATION_HOSTGROUPS,
+		MYSQL_GROUP_REPLICATION_HOSTGROUPS,
+		MYSQL_GALERA_HOSTGROUPS,
+		MYSQL_AWS_AURORA_HOSTGROUPS,
+		MYSQL_HOSTGROUP_ATTRIBUTES,
+
+		__HGM_TABLES_SIZE
+	};
+
+	std::array<uint64_t, __HGM_TABLES_SIZE> table_resultset_checksum;
+
+	class HostGroup_Server_Mapping {
+	public:
+		enum Type {
+			WRITER = 0,
+			READER = 1
+		};
+
+		struct Node {
+			MySrvC* srv = NULL;
+			unsigned int reader_hostgroup_id = -1;
+			unsigned int writer_hostgroup_id = -1;
+			MySerStatus server_status = MYSQL_SERVER_STATUS_OFFLINE_HARD;
+		};
+
+		HostGroup_Server_Mapping() : readonly_flag(1), myHGM(NULL) { }
+		~HostGroup_Server_Mapping() = default;
+
+		// Note: copy, remove, clear method also makes changes to MyHostGroups
+		void copy(Type dest_type, Type src_type, bool update_if_exists = true);
+		void remove(Type type, size_t index);
+		void clear(Type type);
+		//
+
+		inline
+		const std::vector<Node>& get(Type type) const {
+			return mapping[type];
+		}
+
+		inline
+		void add(Type type, Node& node) {
+			mapping[type].push_back(node);
+		}
+
+		inline
+		void set(Type type, const std::vector<Node>& nodes) {
+			mapping[type] = nodes;
+		}
+
+		inline
+		void set_readonly_flag(int val) {
+			readonly_flag = val;
+		}
+
+		inline
+		int get_readonly_flag() const {
+			return readonly_flag;
+		}
+
+		inline
+		void set_HGM(MySQL_HostGroups_Manager* hgm) {
+			myHGM = hgm;
+		}
+
+	private:
+		unsigned int get_hostgroup_id(Type type, size_t index) const;
+		unsigned int get_hostgroup_id(Type type, const Node& node) const;
+		MySrvC* insert_HGM(unsigned int hostgroup_id, const MySrvC* srv);
+		void remove_HGM(MySrvC* srv);
+
+		std::array<std::vector<Node>, 2> mapping;
+		int readonly_flag;
+		MySQL_HostGroups_Manager* myHGM;
+	};
+
+	std::unordered_map<std::string, HostGroup_Server_Mapping> hostgroup_server_mapping;
+	uint64_t hgsm_mysql_servers_checksum = 0;
+	uint64_t hgsm_mysql_replication_hostgroups_checksum = 0;
+
+
 	PtrArray *MyHostGroups;
 	std::unordered_map<unsigned int, MyHGC *>MyHostGroups_map;
 
@@ -601,7 +684,7 @@ class MySQL_HostGroups_Manager {
 	int servers_add(SQLite3_result *resultset);
 	bool commit(SQLite3_result* runtime_mysql_servers = nullptr, const std::string& checksum = "", const time_t epoch = 0);
 	void commit_update_checksums_from_tables(SpookyHash& myhash, bool& init);
-	void CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName); // used by commit_update_checksums_from_tables()
+	void CUCFT1(SpookyHash& myhash, bool& init, const string& TableName, const string& ColumnName, uint64_t& raw_checksum); // used by commit_update_checksums_from_tables()
 
 	/**
 	 * @brief Store the resultset for the 'runtime_mysql_servers' table set that have been loaded to runtime.
@@ -650,9 +733,10 @@ class MySQL_HostGroups_Manager {
 	void push_MyConn_to_pool_array(MySQL_Connection **, unsigned int);
 	void destroy_MyConn_from_pool(MySQL_Connection *, bool _lock=true);	
 
-	void replication_lag_action_inner(MyHGC *, char*, unsigned int, int);
-	void replication_lag_action(int, char*, unsigned int, int);
+	void replication_lag_action_inner(MyHGC *, const char*, unsigned int, int);
+	void replication_lag_action(const std::list<std::tuple<int, std::string, unsigned int, int>>& mysql_servers);
 	void read_only_action(char *hostname, int port, int read_only);
+	void read_only_action_v2(const std::list<std::tuple<std::string,int,int>>& mysql_servers);
 	unsigned int get_servers_table_version();
 	void wait_servers_table_version(unsigned, unsigned);
 	bool shun_and_killall(char *hostname, int port);

include/SQLite3_Server.h

@@ -50,7 +50,15 @@ class SQLite3_Server {
 	std::unordered_map<std::string, group_rep_status> grouprep_map;
 	std::vector<table_def_t *> *tables_defs_grouprep;
 #endif // TEST_GROUPREP
-#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP)
+#ifdef TEST_READONLY
+	std::unordered_map<std::string, bool> readonly_map;
+	std::vector<table_def_t *> *tables_defs_readonly;
+#endif // TEST_READONLY
+#ifdef TEST_REPLICATIONLAG
+	std::unordered_map<std::string, int> replicationlag_map;
+	std::vector<table_def_t*>* tables_defs_replicationlag;
+#endif // TEST_REPLICATIONLAG
+#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) || defined(TEST_REPLICATIONLAG)
 	void insert_into_tables_defs(std::vector<table_def_t *> *, const char *table_name, const char *table_def);
 	void drop_tables_defs(std::vector<table_def_t *> *tables_defs);
 	void check_and_build_standard_tables(SQLite3DB *db, std::vector<table_def_t *> *tables_defs);
@@ -80,6 +88,22 @@ class SQLite3_Server {
 	void init_grouprep_ifaces_string(std::string& s);
 	group_rep_status grouprep_test_value(const std::string& srv_addr);
 #endif // TEST_GROUPREP
+#ifdef TEST_READONLY
+	pthread_mutex_t test_readonly_mutex;
+	void load_readonly_table(MySQL_Session *sess);
+	int readonly_test_value(char *p);
+	int readonly_map_size() {
+		return readonly_map.size();
+	}
+#endif // TEST_READONLY
+#ifdef TEST_REPLICATIONLAG
+	pthread_mutex_t test_replicationlag_mutex;
+	void load_replicationlag_table(MySQL_Session* sess);
+	int replicationlag_test_value(const char* p);
+	int replicationlag_map_size() {
+		return replicationlag_map.size();
+	}
+#endif // TEST_REPLICATIONLAG
 	SQLite3_Server();
 	~SQLite3_Server();
 	char **get_variables_list();

include/proxysql_admin.h

@@ -309,6 +309,11 @@ class ProxySQL_Admin {
 	void flush_ldap_variables___database_to_runtime(SQLite3DB *db, bool replace, const std::string& checksum = "", const time_t epoch = 0);
 
 	public:
+	/**
+	 * @brief Mutex taken by 'ProxySQL_Admin::admin_session_handler'. It's used prevent multiple
+	 *   ProxySQL_Admin 'sessions' from running in parallel, or for preventing collisions between
+	 *   modules performing operations over the internal SQLite database and 'ProxySQL_Admin' sessions.
+	 */
 	pthread_mutex_t sql_query_global_mutex;
 	struct {
 		void *opt;
@@ -345,7 +350,31 @@ class ProxySQL_Admin {
 	void init_mysql_firewall();
 	void init_proxysql_servers();
 	void save_mysql_users_runtime_to_database(bool _runtime);
-	void save_mysql_servers_runtime_to_database(bool);
+	/**
+	 * @brief Save the current MySQL servers reported by 'MySQL_HostGroups_Manager', scanning the
+	 *   current MySQL servers structures for all hostgroups, into either the
+	 *   'main.runtime_mysql_servers' or 'main.mysql_servers' table.
+	 * @param _runtime If true the servers reported by 'MySQL_HostGroups_Manager' are stored into
+	 *   'main.runtime_mysql_servers', otherwise into 'main.runtime_mysql_servers'.
+	 * @details This functions requires the caller to have locked `mysql_servers_wrlock()`, but it
+	 *   doesn't start a transaction as other function that perform several operations over the
+	 *   database. This is because, it's not required doing so, and also because if a transaction
+	 *   was started in the following fashion:
+	 *
+	 *   ```
+	 *   admindb->execute("BEGIN IMMEDIATE");
+	 *   ```
+	 *
+	 *   ProxySQL would lock in 'MySQL_HostGroups_Manager::dump_table_mysql_servers()', or in any
+	 *   other operation from 'MySQL_HostGroups_Manager' that would try to modify the database.
+	 *   Reason being is that trying to modify an attached database during a transaction. Database
+	 *   is only attached for `DEBUG` builds as part of `MySQL_Admin::init()`. Line:
+	 *
+	 *   ```
+	 *   admindb->execute("ATTACH DATABASE 'file:mem_mydb?mode=memory&cache=shared' AS myhgm");
+	 *   ```
+	 */
+	void save_mysql_servers_runtime_to_database(bool _runtime);
 	void admin_shutdown();
 	bool is_command(std::string);
 	void send_MySQL_OK(MySQL_Protocol *myprot, char *msg, int rows=0);
@@ -505,6 +534,14 @@ class ProxySQL_Admin {
 	void enable_grouprep_testing();
 #endif // TEST_GROUPREP
 
+#ifdef TEST_READONLY
+	void enable_readonly_testing();
+#endif // TEST_READONLY
+
+#ifdef TEST_REPLICATIONLAG
+	void enable_replicationlag_testing();
+#endif // TEST_REPLICATIONLAG
+
 	unsigned int ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(unsigned int, bool);
 	bool ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int *ret1, int *ret2, int cnt, int dual);
 	void ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters();