db_dump: Add --json output mode.

ChangeLog.txt

@@ -184,6 +184,9 @@ Clients:
   used by wireshark to decrypt TLS traffic for debugging purposes.
 - mosquitto_sub payload hex output can now be split by fixed field length.
 
+DB Dump:
+- Add `--json` output mode.
+
 Build:
 - Increased CMake minimal required version to 3.14, which is required for the
   preinstalled SQLite3 find module.

apps/db_dump/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_executable(mosquitto_db_dump
 	db_dump.c db_dump.h
+	json.c
 	print.c
 	stubs.c
 

apps/db_dump/Makefile

@@ -17,6 +17,7 @@ include ${R}/make/broker.mk
 
 OBJS = \
 	db_dump.o \
+	json.o \
 	print.o \
 	stubs.o
 

apps/db_dump/db_dump.c

@@ -61,6 +61,7 @@ extern uint32_t db_version;
 static int stats = 0;
 static int client_stats = 0;
 static int do_print = 1;
+static int do_json = 0;
 
 /* Counts */
 static long cfg_count = 0;
@@ -168,6 +169,9 @@ static int dump__client_chunk_process(FILE *db_fd, uint32_t length)
 		HASH_ADD_KEYPTR(hh_id, clients_by_id, cc->id, strlen(cc->id), cc);
 	}
 
+	if(do_json){
+		json_add_client(&chunk);
+	}
 	if(do_print) {
 		print__client(&chunk, length);
 	}
@@ -210,6 +214,9 @@ static int dump__client_msg_chunk_process(FILE *db_fd, uint32_t length)
 		}
 	}
 
+	if(do_json){
+		json_add_client_msg(&chunk);
+	}
 	if(do_print) {
 		print__client_msg(&chunk, length);
 	}
@@ -272,6 +279,10 @@ static int dump__base_msg_chunk_process(FILE *db_fptr, uint32_t length)
 	rc = db__message_store(&chunk.source, stored, &message_expiry_interval,
 			mosq_mo_client);
 
+	if(do_json){
+		json_add_base_msg(&chunk);
+	}
+
 	mosquitto_free(chunk.source.id);
 	mosquitto_free(chunk.source.username);
 	chunk.source.id = NULL;
@@ -326,6 +337,10 @@ static int dump__retain_chunk_process(FILE *db_fd, uint32_t length)
 		return rc;
 	}
 
+	if(do_json){
+		json_add_retained_msg(&chunk);
+	}
+
 	if(do_print) printf("\tStore ID: %" PRIu64 "\n", chunk.F.store_id);
 	return 0;
 }
@@ -358,6 +373,9 @@ static int dump__sub_chunk_process(FILE *db_fd, uint32_t length)
 		}
 	}
 
+	if(do_json){
+		json_add_subscription(&chunk);
+	}
 	if(do_print) {
 		print__sub(&chunk, length);
 	}
@@ -408,7 +426,6 @@ static void cleanup_msg_store()
 	}
 }
 
-
 #ifdef WITH_FUZZING
 int db_dump_fuzz_main(int argc, char *argv[])
 #else
@@ -434,10 +451,19 @@ int main(int argc, char *argv[])
 		client_stats = 1;
 		do_print = 0;
 		filename = argv[2];
+	}else if(argc == 3 && !strcmp(argv[1], "--json")){
+		do_print = 0;
+		do_json = 1;
+		filename = argv[2];
 	}else{
-		fprintf(stderr, "Usage: db_dump [--stats | --client-stats] <mosquitto db filename>\n");
+		fprintf(stderr, "Usage: db_dump [--stats | --client-stats | --json] <mosquitto db filename>\n");
 		return 1;
 	}
+
+	if(do_json){
+		json_init();
+	}
+
 	memset(&db, 0, sizeof(struct mosquitto_db));
 	fd = fopen(filename, "rb");
 	if(!fd){
@@ -500,6 +526,10 @@ int main(int argc, char *argv[])
 
 	fclose(fd);
 
+	if(do_json){
+		json_print();
+		json_cleanup();
+	}
 	if(stats){
 		printf("DB_CHUNK_CFG:        %ld\n", cfg_count);
 		printf("DB_CHUNK_BASE_MSG:   %ld\n", base_msg_count);

apps/db_dump/db_dump.h

@@ -25,4 +25,13 @@ void print__client_msg(struct P_client_msg *chunk, uint32_t length);
 void print__base_msg(struct P_base_msg *chunk, uint32_t length);
 void print__sub(struct P_sub *chunk, uint32_t length);
 
+void json_init(void);
+void json_print(void);
+void json_cleanup(void);
+void json_add_base_msg(struct P_base_msg *msg);
+void json_add_client(struct P_client *chunk);
+void json_add_client_msg(struct P_client_msg *chunk);
+void json_add_retained_msg(struct P_retain *msg);
+void json_add_subscription(struct P_sub *chunk);
+
 #endif

apps/db_dump/json.c

@@ -0,0 +1,170 @@
+/*
+Copyright (c) 2010-2021 Roger Light <[email protected]>
+
+All rights reserved. This program and the accompanying materials
+are made available under the terms of the Eclipse Public License 2.0
+and Eclipse Distribution License v1.0 which accompany this distribution.
+
+The Eclipse Public License is available at
+   https://www.eclipse.org/legal/epl-2.0/
+and the Eclipse Distribution License is available at
+  http://www.eclipse.org/org/documents/edl-v10.php.
+
+SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
+
+Contributors:
+   Roger Light - initial implementation and documentation.
+*/
+
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <cjson/cJSON.h>
+
+#include "db_dump.h"
+#include <mosquitto_broker_internal.h>
+#include <persist.h>
+
+#define mosquitto_malloc(A) malloc((A))
+#define mosquitto_free(A) free((A))
+#define _mosquitto_malloc(A) malloc((A))
+#define _mosquitto_free(A) free((A))
+#include <uthash.h>
+
+#include "db_dump.h"
+
+cJSON *j_tree = NULL;
+cJSON *j_base_msgs = NULL;
+cJSON *j_clients = NULL;
+cJSON *j_client_msgs = NULL;
+cJSON *j_retained_msgs = NULL;
+cJSON *j_subscriptions = NULL;
+
+void json_init(void)
+{
+	j_tree = cJSON_CreateObject();
+	if(!j_tree){
+		fprintf(stderr, "Error: Out of memory.\n");
+		exit(1);
+	}
+
+	if((j_base_msgs = cJSON_AddArrayToObject(j_tree, "base-messages")) == NULL
+			|| (j_clients = cJSON_AddArrayToObject(j_tree, "clients")) == NULL
+			|| (j_client_msgs = cJSON_AddArrayToObject(j_tree, "client-messages")) == NULL
+			|| (j_retained_msgs = cJSON_AddArrayToObject(j_tree, "retained-messages")) == NULL
+			|| (j_subscriptions = cJSON_AddArrayToObject(j_tree, "subscriptions")) == NULL
+			){
+
+		fprintf(stderr, "Error: Out of memory.\n");
+		exit(1);
+	}
+}
+
+void json_print(void)
+{
+	printf("%s\n", cJSON_Print(j_tree));
+}
+
+void json_cleanup(void)
+{
+	cJSON_Delete(j_tree);
+}
+
+void json_add_base_msg(struct P_base_msg *chunk)
+{
+	cJSON *j_base_msg = NULL;
+
+	j_base_msg = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_base_msgs, j_base_msg);
+
+	cJSON_AddNumberToObject(j_base_msg, "storeid", chunk->F.store_id);
+	cJSON_AddNumberToObject(j_base_msg, "expiry-time", chunk->F.expiry_time);
+	cJSON_AddNumberToObject(j_base_msg, "source-mid", chunk->F.source_mid);
+	cJSON_AddNumberToObject(j_base_msg, "source-port", chunk->F.source_port);
+	cJSON_AddNumberToObject(j_base_msg, "qos", chunk->F.qos);
+	cJSON_AddNumberToObject(j_base_msg, "retain", chunk->F.retain);
+	cJSON_AddStringToObject(j_base_msg, "topic", chunk->topic);
+	if(chunk->source.id){
+		cJSON_AddStringToObject(j_base_msg, "clientid", chunk->source.id);
+	}
+	if(chunk->source.username){
+		cJSON_AddStringToObject(j_base_msg, "username", chunk->source.username);
+	}
+	if(chunk->F.payloadlen > 0){
+		char *payload;
+		if(mosquitto_base64_encode(chunk->payload, chunk->F.payloadlen, &payload) == MOSQ_ERR_SUCCESS){
+			cJSON_AddStringToObject(j_base_msg, "payload", payload);
+			mosquitto_free(payload);
+		}
+	}
+	if(chunk->properties){
+		cJSON *j_props = mosquitto_properties_to_json(chunk->properties);
+		if(j_props){
+			cJSON_AddItemToObject(j_base_msg, "properties", j_props);
+		}
+	}
+}
+
+void json_add_client(struct P_client *chunk)
+{
+	cJSON *j_client;
+
+	j_client = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_clients, j_client);
+
+	cJSON_AddStringToObject(j_client, "clientid", chunk->clientid);
+	if(chunk->username){
+		cJSON_AddStringToObject(j_client, "username", chunk->username);
+	}
+	cJSON_AddNumberToObject(j_client, "session-expiry-time", chunk->F.session_expiry_time);
+	cJSON_AddNumberToObject(j_client, "session-expiry-interval", chunk->F.session_expiry_interval);
+	cJSON_AddNumberToObject(j_client, "last-mid", chunk->F.last_mid);
+	cJSON_AddNumberToObject(j_client, "listener-port", chunk->F.listener_port);
+
+}
+
+void json_add_client_msg(struct P_client_msg *chunk)
+{
+	cJSON *j_client_msg;
+
+	j_client_msg = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_client_msgs, j_client_msg);
+
+	cJSON_AddStringToObject(j_client_msg, "clientid", chunk->clientid);
+	cJSON_AddNumberToObject(j_client_msg, "storeid", chunk->subscription_identifier);
+	cJSON_AddNumberToObject(j_client_msg, "mid", chunk->F.mid);
+	cJSON_AddNumberToObject(j_client_msg, "qos", chunk->F.qos);
+	cJSON_AddNumberToObject(j_client_msg, "state", chunk->F.state);
+	cJSON_AddNumberToObject(j_client_msg, "retain-dup", chunk->F.retain_dup);
+	cJSON_AddNumberToObject(j_client_msg, "direction", chunk->F.direction);
+	cJSON_AddNumberToObject(j_client_msg, "subscription-identifier", chunk->subscription_identifier);
+}
+
+void json_add_subscription(struct P_sub *chunk)
+{
+	cJSON *j_subscription;
+
+	j_subscription = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_subscriptions, j_subscription);
+
+	cJSON_AddStringToObject(j_subscription, "clientid", chunk->clientid);
+	cJSON_AddStringToObject(j_subscription, "topic", chunk->topic);
+	cJSON_AddNumberToObject(j_subscription, "qos", chunk->F.qos);
+	cJSON_AddNumberToObject(j_subscription, "options", chunk->F.options);
+	cJSON_AddNumberToObject(j_subscription, "identifier", chunk->F.identifier);
+}
+
+void json_add_retained_msg(struct P_retain *chunk)
+{
+	cJSON *j_retained_msg;
+
+	j_retained_msg = cJSON_CreateObject();
+	cJSON_AddItemToArray(j_retained_msgs, j_retained_msg);
+	cJSON_AddNumberToObject(j_retained_msg, "storeid", chunk->F.store_id);
+}

test/apps/db_dump/Makefile

@@ -6,9 +6,10 @@ check : test
 
 test-compile:
 
-test: 
+test:
 	./db-dump-client-stats.py
 	./db-dump-corrupt.py
+	./db-dump-json-v6-mqtt-v5-props.py
 	./db-dump-print-empty.py
 	./db-dump-print-v6-all.py
 	./db-dump-print-v6-mqtt-v5-props.py