Merge branch 'eclipse:develop' into develop

.github/workflows/mosquitto-make-asan.yml

@@ -45,4 +45,4 @@ jobs:
     -
       name: make test
       run: |
-        make WITH_ASAN=yes ptest
+        make WITH_ASAN=yes test

.github/workflows/mosquitto-make.yml

@@ -40,8 +40,8 @@ jobs:
         submodules: 'true'
     -
       name: make
-      run: make ALLOC_MISMATCH_ABORT=yes
+      run: make
     -
       name: make test
       run: |
-        make ALLOC_MISMATCH_ABORT=yes ptest
+        make test

ChangeLog.txt

@@ -74,7 +74,7 @@ Broker:
   choose TLS v1.1, but this is not recommended and will be removed in a future
   version.
 - Add -q option to allow logging to be disabled at the command line.
-- Add suport for PROXY protocol v2.
+- Add suport for PROXY protocol v1 and v2.
 
 Plugins / plugin interface:
 - Add persist-sqlite plugin.
@@ -184,6 +184,9 @@ Clients:
   used by wireshark to decrypt TLS traffic for debugging purposes.
 - mosquitto_sub payload hex output can now be split by fixed field length.
 
+DB Dump:
+- Add `--json` output mode.
+
 Build:
 - Increased CMake minimal required version to 3.14, which is required for the
   preinstalled SQLite3 find module.

apps/db_dump/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_executable(mosquitto_db_dump
 	db_dump.c db_dump.h
+	json.c
 	print.c
 	stubs.c
 

apps/db_dump/Makefile

@@ -17,6 +17,7 @@ include ${R}/make/broker.mk
 
 OBJS = \
 	db_dump.o \
+	json.o \
 	print.o \
 	stubs.o
 

apps/db_dump/db_dump.c

@@ -61,6 +61,7 @@ extern uint32_t db_version;
 static int stats = 0;
 static int client_stats = 0;
 static int do_print = 1;
+static int do_json = 0;
 
 /* Counts */
 static long cfg_count = 0;
@@ -168,6 +169,9 @@ static int dump__client_chunk_process(FILE *db_fd, uint32_t length)
 		HASH_ADD_KEYPTR(hh_id, clients_by_id, cc->id, strlen(cc->id), cc);
 	}
 
+	if(do_json){
+		json_add_client(&chunk);
+	}
 	if(do_print) {
 		print__client(&chunk, length);
 	}
@@ -210,6 +214,9 @@ static int dump__client_msg_chunk_process(FILE *db_fd, uint32_t length)
 		}
 	}
 
+	if(do_json){
+		json_add_client_msg(&chunk);
+	}
 	if(do_print) {
 		print__client_msg(&chunk, length);
 	}
@@ -272,6 +279,10 @@ static int dump__base_msg_chunk_process(FILE *db_fptr, uint32_t length)
 	rc = db__message_store(&chunk.source, stored, &message_expiry_interval,
 			mosq_mo_client);
 
+	if(do_json){
+		json_add_base_msg(&chunk);
+	}
+
 	mosquitto_free(chunk.source.id);
 	mosquitto_free(chunk.source.username);
 	chunk.source.id = NULL;
@@ -326,6 +337,10 @@ static int dump__retain_chunk_process(FILE *db_fd, uint32_t length)
 		return rc;
 	}
 
+	if(do_json){
+		json_add_retained_msg(&chunk);
+	}
+
 	if(do_print) printf("\tStore ID: %" PRIu64 "\n", chunk.F.store_id);
 	return 0;
 }
@@ -358,6 +373,9 @@ static int dump__sub_chunk_process(FILE *db_fd, uint32_t length)
 		}
 	}
 
+	if(do_json){
+		json_add_subscription(&chunk);
+	}
 	if(do_print) {
 		print__sub(&chunk, length);
 	}
@@ -408,7 +426,6 @@ static void cleanup_msg_store()
 	}
 }
 
-
 #ifdef WITH_FUZZING
 int db_dump_fuzz_main(int argc, char *argv[])
 #else
@@ -434,10 +451,19 @@ int main(int argc, char *argv[])
 		client_stats = 1;
 		do_print = 0;
 		filename = argv[2];
+	}else if(argc == 3 && !strcmp(argv[1], "--json")){
+		do_print = 0;
+		do_json = 1;
+		filename = argv[2];
 	}else{
-		fprintf(stderr, "Usage: db_dump [--stats | --client-stats] <mosquitto db filename>\n");
+		fprintf(stderr, "Usage: db_dump [--stats | --client-stats | --json] <mosquitto db filename>\n");
 		return 1;
 	}
+
+	if(do_json){
+		json_init();
+	}
+
 	memset(&db, 0, sizeof(struct mosquitto_db));
 	fd = fopen(filename, "rb");
 	if(!fd){
@@ -500,6 +526,10 @@ int main(int argc, char *argv[])
 
 	fclose(fd);
 
+	if(do_json){
+		json_print();
+		json_cleanup();
+	}
 	if(stats){
 		printf("DB_CHUNK_CFG:        %ld\n", cfg_count);
 		printf("DB_CHUNK_BASE_MSG:   %ld\n", base_msg_count);

apps/db_dump/db_dump.h

@@ -25,4 +25,13 @@ void print__client_msg(struct P_client_msg *chunk, uint32_t length);
 void print__base_msg(struct P_base_msg *chunk, uint32_t length);
 void print__sub(struct P_sub *chunk, uint32_t length);
 
+void json_init(void);
+void json_print(void);
+void json_cleanup(void);
+void json_add_base_msg(struct P_base_msg *msg);
+void json_add_client(struct P_client *chunk);
+void json_add_client_msg(struct P_client_msg *chunk);
+void json_add_retained_msg(struct P_retain *msg);
+void json_add_subscription(struct P_sub *chunk);
+
 #endif

apps/db_dump/json.c

@@ -0,0 +1,170 @@
+/*
+Copyright (c) 2010-2021 Roger Light <[email protected]>
+
+All rights reserved. This program and the accompanying materials
+are made available under the terms of the Eclipse Public License 2.0
+and Eclipse Distribution License v1.0 which accompany this distribution.
+
+The Eclipse Public License is available at
+   https://www.eclipse.org/legal/epl-2.0/
+and the Eclipse Distribution License is available at
+  http://www.eclipse.org/org/documents/edl-v10.php.
+
+SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
+
+Contributors:
+   Roger Light - initial implementation and documentation.
+*/
+
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <cjson/cJSON.h>
+
+#include "db_dump.h"
+#include <mosquitto_broker_internal.h>
+#include <persist.h>
+
+#define mosquitto_malloc(A) malloc((A))
+#define mosquitto_free(A) free((A))
+#define _mosquitto_malloc(A) malloc((A))
+#define _mosquitto_free(A) free((A))
+#include <uthash.h>
+
+#include "db_dump.h"
+
+cJSON *j_tree = NULL;
+cJSON *j_base_msgs = NULL;
+cJSON *j_clients = NULL;
+cJSON *j_client_msgs = NULL;
+cJSON *j_retained_msgs = NULL;
+cJSON *j_subscriptions = NULL;
+
+void json_init(void)
+{
+	j_tree = cJSON_CreateObject();
+	if(!j_tree){
+		fprintf(stderr, "Error: Out of memory.\n");
+		exit(1);
+	}
+
+	if((j_base_msgs = cJSON_AddArrayToObject(j_tree, "base-messages")) == NULL
+			|| (j_clients = cJSON_AddArrayToObject(j_tree, "clients")) == NULL
+			|| (j_client_msgs = cJSON_AddArrayToObject(j_tree, "client-messages")) == NULL
+			|| (j_retained_msgs = cJSON_AddArrayToObject(j_tree, "retained-messages")) == NULL
+			|| (j_subscriptions = cJSON_AddArrayToObject(j_tree, "subscriptions")) == NULL
+			){
+
+		fprintf(stderr, "Error: Out of memory.\n");
+		exit(1);
+	}
+}
+
+void json_print(void)
+{
+	printf("%s\n", cJSON_Print(j_tree));
+}
+
+void json_cleanup(void)
+{
+	cJSON_Delete(j_tree);
+}
+
+void json_add_base_msg(struct P_base_msg *chunk)
+{
+	cJSON *j_base_msg = NULL;
+
+	j_base_msg = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_base_msgs, j_base_msg);
+
+	cJSON_AddNumberToObject(j_base_msg, "storeid", chunk->F.store_id);
+	cJSON_AddNumberToObject(j_base_msg, "expiry-time", chunk->F.expiry_time);
+	cJSON_AddNumberToObject(j_base_msg, "source-mid", chunk->F.source_mid);
+	cJSON_AddNumberToObject(j_base_msg, "source-port", chunk->F.source_port);
+	cJSON_AddNumberToObject(j_base_msg, "qos", chunk->F.qos);
+	cJSON_AddNumberToObject(j_base_msg, "retain", chunk->F.retain);
+	cJSON_AddStringToObject(j_base_msg, "topic", chunk->topic);
+	if(chunk->source.id){
+		cJSON_AddStringToObject(j_base_msg, "clientid", chunk->source.id);
+	}
+	if(chunk->source.username){
+		cJSON_AddStringToObject(j_base_msg, "username", chunk->source.username);
+	}
+	if(chunk->F.payloadlen > 0){
+		char *payload;
+		if(mosquitto_base64_encode(chunk->payload, chunk->F.payloadlen, &payload) == MOSQ_ERR_SUCCESS){
+			cJSON_AddStringToObject(j_base_msg, "payload", payload);
+			mosquitto_free(payload);
+		}
+	}
+	if(chunk->properties){
+		cJSON *j_props = mosquitto_properties_to_json(chunk->properties);
+		if(j_props){
+			cJSON_AddItemToObject(j_base_msg, "properties", j_props);
+		}
+	}
+}
+
+void json_add_client(struct P_client *chunk)
+{
+	cJSON *j_client;
+
+	j_client = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_clients, j_client);
+
+	cJSON_AddStringToObject(j_client, "clientid", chunk->clientid);
+	if(chunk->username){
+		cJSON_AddStringToObject(j_client, "username", chunk->username);
+	}
+	cJSON_AddNumberToObject(j_client, "session-expiry-time", chunk->F.session_expiry_time);
+	cJSON_AddNumberToObject(j_client, "session-expiry-interval", chunk->F.session_expiry_interval);
+	cJSON_AddNumberToObject(j_client, "last-mid", chunk->F.last_mid);
+	cJSON_AddNumberToObject(j_client, "listener-port", chunk->F.listener_port);
+
+}
+
+void json_add_client_msg(struct P_client_msg *chunk)
+{
+	cJSON *j_client_msg;
+
+	j_client_msg = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_client_msgs, j_client_msg);
+
+	cJSON_AddStringToObject(j_client_msg, "clientid", chunk->clientid);
+	cJSON_AddNumberToObject(j_client_msg, "storeid", chunk->subscription_identifier);
+	cJSON_AddNumberToObject(j_client_msg, "mid", chunk->F.mid);
+	cJSON_AddNumberToObject(j_client_msg, "qos", chunk->F.qos);
+	cJSON_AddNumberToObject(j_client_msg, "state", chunk->F.state);
+	cJSON_AddNumberToObject(j_client_msg, "retain-dup", chunk->F.retain_dup);
+	cJSON_AddNumberToObject(j_client_msg, "direction", chunk->F.direction);
+	cJSON_AddNumberToObject(j_client_msg, "subscription-identifier", chunk->subscription_identifier);
+}
+
+void json_add_subscription(struct P_sub *chunk)
+{
+	cJSON *j_subscription;
+
+	j_subscription = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_subscriptions, j_subscription);
+
+	cJSON_AddStringToObject(j_subscription, "clientid", chunk->clientid);
+	cJSON_AddStringToObject(j_subscription, "topic", chunk->topic);
+	cJSON_AddNumberToObject(j_subscription, "qos", chunk->F.qos);
+	cJSON_AddNumberToObject(j_subscription, "options", chunk->F.options);
+	cJSON_AddNumberToObject(j_subscription, "identifier", chunk->F.identifier);
+}
+
+void json_add_retained_msg(struct P_retain *chunk)
+{
+	cJSON *j_retained_msg;
+
+	j_retained_msg = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_retained_msgs, j_retained_msg);
+	cJSON_AddNumberToObject(j_retained_msg, "storeid", chunk->F.store_id);
+}

apps/mosquitto_ctrl/CMakeLists.txt

@@ -10,7 +10,6 @@ if(WITH_TLS)
 		../mosquitto_passwd/get_password.c ../mosquitto_passwd/get_password.h
 		options.c
 		../../common/json_help.c ../../common/json_help.h
-		../../common/password_mosq.c ../../common/password_mosq.h
 	)
 
 	target_include_directories(mosquitto_ctrl PRIVATE
@@ -46,13 +45,6 @@ if(WITH_TLS)
 		endif()
 	endif()
 
-	if(ARGON2_FOUND)
-		target_link_libraries(mosquitto_ctrl
-			PRIVATE
-				argon2
-		)
-	endif()
-
 	target_link_libraries(mosquitto_ctrl
 	  PRIVATE
 	  	common-options

apps/mosquitto_ctrl/Makefile

@@ -6,7 +6,7 @@ include ${R}/config.mk
 LOCAL_CFLAGS+=
 LOCAL_CPPFLAGS+=-I${R}/lib -I${R}/apps/mosquitto_passwd -I${R}/plugins/dynamic-security -I${R}/common
 LOCAL_LDFLAGS+=
-LOCAL_LDADD+=-lcjson -ldl ${LIBMOSQ} ${LIBMOSQ_COMMON} ${LIB_ARGON2}
+LOCAL_LDADD+=-lcjson -ldl ${LIBMOSQ} ${LIBMOSQ_COMMON}
 
 # ------------------------------------------
 #  Compile time options
@@ -34,8 +34,7 @@ OBJS= \
 
 OBJS_EXTERNAL= \
 	get_password.o \
-	json_help.o \
-	password_mosq.o
+	json_help.o
 
 EXAMPLE_OBJS= example.o
 
@@ -65,9 +64,6 @@ get_password.o : ${R}/apps/mosquitto_passwd/get_password.c ${R}/apps/mosquitto_p
 json_help.o : ${R}/common/json_help.c ${R}/common/json_help.h
 	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
 
-password_mosq.o : ${R}/common/password_mosq.c ${R}/common/password_mosq.h
-	${CROSS_COMPILE}${CC} $(LOCAL_CPPFLAGS) $(LOCAL_CFLAGS) -c $< -o $@
-
 ${R}/lib/libmosquitto.so.${SOVERSION} :
 	$(MAKE) -C ${R}/lib