From 21c1c3306562c85b4d1c1c7ec0a53337c13ec7e1 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Fri, 17 Mar 2023 16:14:34 +0100 Subject: [PATCH 01/16] Add optional RNNoise support to AudioBridge --- conf/janus.plugin.audiobridge.jcfg.sample | 1 + configure.ac | 10 + src/Makefile.am | 6 +- src/plugins/janus_audiobridge.c | 235 +++++++++++++++++++++- 4 files changed, 247 insertions(+), 5 deletions(-) diff --git a/conf/janus.plugin.audiobridge.jcfg.sample b/conf/janus.plugin.audiobridge.jcfg.sample index 39d41d5a55..294f5e19a0 100644 --- a/conf/janus.plugin.audiobridge.jcfg.sample +++ b/conf/janus.plugin.audiobridge.jcfg.sample @@ -13,6 +13,7 @@ # default_prebuffering = number of packets to buffer before decoding each particiant (default=6) # default_expectedloss = percent of packets we expect participants may miss, to help with FEC (default=0, max=20; automatically used for forwarders too) # default_bitrate = default bitrate in bps to use for the all participants (default=0, which means libopus decides; automatically used for forwarders too) +# denoise = true|false (whether denoising via RNNoise should be performed for each participant by default) # record = true|false (whether this room should be recorded, default=false) # record_file = "/path/to/recording.wav" (where to save the recording) # record_dir = "/path/to/" (path to save the recording to, makes record_file a relative path if provided) diff --git a/configure.ac b/configure.ac index f2aa615432..267ac35d5d 100644 --- a/configure.ac +++ b/configure.ac @@ -804,6 +804,16 @@ PKG_CHECK_MODULES([OGG], AC_SUBST([OGG_CFLAGS]) AC_SUBST([OGG_LIBS]) +PKG_CHECK_MODULES([RNNOISE], + [rnnoise], + [ + AC_DEFINE(HAVE_RNNOISE) + ], + [ + ]) +AC_SUBST([RNNOISE_CFLAGS]) +AC_SUBST([RNNOISE_LIBS]) + PKG_CHECK_MODULES([LUA], [lua], [ diff --git a/src/Makefile.am b/src/Makefile.am index 9b467a7f51..9f688a5348 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -397,9 +397,9 @@ plugins_libadd = \ if ENABLE_PLUGIN_AUDIOBRIDGE plugin_LTLIBRARIES += plugins/libjanus_audiobridge.la plugins_libjanus_audiobridge_la_SOURCES = plugins/janus_audiobridge.c -plugins_libjanus_audiobridge_la_CFLAGS = $(plugins_cflags) $(OPUS_CFLAGS) $(OGG_CFLAGS) $(LIBSRTP_CFLAGS) -plugins_libjanus_audiobridge_la_LDFLAGS = $(plugins_ldflags) $(OPUS_LDFLAGS) $(OPUS_LIBS) $(OGG_LDFLAGS) $(OGG_LIBS) -plugins_libjanus_audiobridge_la_LIBADD = $(plugins_libadd) $(OPUS_LIBADD) $(OGG_LIBADD) +plugins_libjanus_audiobridge_la_CFLAGS = $(plugins_cflags) $(OPUS_CFLAGS) $(OGG_CFLAGS) $(RNNOISE_CFLAGS) $(LIBSRTP_CFLAGS) +plugins_libjanus_audiobridge_la_LDFLAGS = $(plugins_ldflags) $(OPUS_LDFLAGS) $(OPUS_LIBS) $(OGG_LDFLAGS) $(OGG_LIBS) $(RNNOISE_LDFLAGS) $(RNNOISE_LIBS) +plugins_libjanus_audiobridge_la_LIBADD = $(plugins_libadd) $(OPUS_LIBADD) $(OGG_LIBADD) $(RNNOISE_LIBADD) conf_DATA += ../conf/janus.plugin.audiobridge.jcfg.sample EXTRA_DIST += ../conf/janus.plugin.audiobridge.jcfg.sample endif diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 59926fff32..704de78080 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -41,6 +41,7 @@ room-: { default_prebuffering = number of packets to buffer before decoding each participant (default=DEFAULT_PREBUFFERING) default_expectedloss = percent of packets we expect participants may miss, to help with FEC (default=0, max=20; automatically used for forwarders too) default_bitrate = default bitrate in bps to use for the all participants (default=0, which means libopus decides; automatically used for forwarders too) + denoise = true|false (whether denoising via RNNoise should be performed for each participant by default) record = true|false (whether this room should be recorded, default=false) record_file = /path/to/recording.wav (where to save the recording) record_dir = /path/to/ (path to save the recording to, makes record_file a relative path if provided) @@ -144,6 +145,7 @@ room-: { "default_prebuffering" : , "default_expectedloss" : , "default_bitrate" : , + "denoise" : , "record" : , "record_file" : "", "record_dir" : "", @@ -745,6 +747,7 @@ room-: { "expected_loss" : <0-20, a percentage of the expected loss (capped at 20%), only needed in case FEC is used; optional, default is 0 (FEC disabled even when negotiated) or the room default>, "volume" : 100 increases volume; optional, default is 100 (no volume change)>, "spatial_position" : , + "denoise" : , "secret" : "", "audio_level_average" : "", "audio_active_packets" : "", @@ -833,6 +836,7 @@ room-: { "expected_loss" : "volume" : , "spatial_position" : , + "denoise" : , "record": ", "group" : "" @@ -1034,6 +1038,10 @@ room-: { #ifdef HAVE_LIBOGG #include #endif +#ifdef HAVE_RNNOISE +#include +#endif + #include #include #include @@ -1056,8 +1064,8 @@ room-: { /* Plugin information */ -#define JANUS_AUDIOBRIDGE_VERSION 12 -#define JANUS_AUDIOBRIDGE_VERSION_STRING "0.0.12" +#define JANUS_AUDIOBRIDGE_VERSION 13 +#define JANUS_AUDIOBRIDGE_VERSION_STRING "0.0.13" #define JANUS_AUDIOBRIDGE_DESCRIPTION "This is a plugin implementing an audio conference bridge for Janus, mixing Opus streams." #define JANUS_AUDIOBRIDGE_NAME "JANUS AudioBridge plugin" #define JANUS_AUDIOBRIDGE_AUTHOR "Meetecho s.r.l." @@ -1177,6 +1185,7 @@ static struct janus_json_parameter create_parameters[] = { {"default_prebuffering", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"default_expectedloss", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"default_bitrate", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + {"denoise", JANUS_JSON_BOOL, 0}, {"groups", JSON_ARRAY, 0} }; static struct janus_json_parameter edit_parameters[] = { @@ -1214,6 +1223,7 @@ static struct janus_json_parameter join_parameters[] = { {"spatial_position", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"audio_level_average", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"audio_active_packets", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + {"denoise", JANUS_JSON_BOOL, 0}, {"record", JANUS_JSON_BOOL, 0}, {"filename", JSON_STRING, 0}, {"generate_offer", JANUS_JSON_BOOL, 0}, @@ -1245,6 +1255,7 @@ static struct janus_json_parameter configure_parameters[] = { {"volume", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, {"group", JSON_STRING, 0}, {"spatial_position", JSON_INTEGER, JANUS_JSON_PARAM_POSITIVE}, + {"denoise", JANUS_JSON_BOOL, 0}, {"record", JANUS_JSON_BOOL, 0}, {"filename", JSON_STRING, 0}, {"display", JSON_STRING, 0}, @@ -1334,6 +1345,9 @@ typedef struct janus_audiobridge_room { int32_t default_bitrate; /* Default bitrate to use for all Opus streams when encoding */ int audio_active_packets; /* Amount of packets with audio level for checkup */ int audio_level_average; /* Average audio level */ +#ifdef HAVE_RNNOISE + gboolean denoise; /* Whether we should denoise participants by default */ +#endif volatile gint record; /* Whether this room has to be recorded or not */ gchar *record_file; /* Path of the recording file (absolute or relative, depending on record_dir) */ gchar *record_dir; /* Folder to save the recording file to */ @@ -1570,6 +1584,10 @@ typedef struct janus_audiobridge_participant { int opus_complexity; /* Complexity to use in the encoder (by default, DEFAULT_COMPLEXITY) */ gboolean stereo; /* Whether stereo will be used for spatial audio */ int spatial_position; /* Panning of this participant in the mix */ +#ifdef HAVE_RNNOISE + gboolean denoise; /* Whether we should denoise this participant */ + DenoiseState *rnnoise; /* RNNoise state*/ +#endif /* RTP stuff */ GList *inbuf; /* Incoming audio from this participant, as an ordered list of packets */ GAsyncQueue *outbuf; /* Mixed audio for this participant */ @@ -1662,6 +1680,10 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant } g_async_queue_unref(participant->outbuf); } +#ifdef HAVE_RNNOISE + if(participant->rnnoise) + rnnoise_destroy(participant->rnnoise); +#endif g_free(participant->mjr_base); #ifdef HAVE_LIBOGG janus_audiobridge_file_free(participant->annc); @@ -2272,6 +2294,12 @@ int janus_audiobridge_init(janus_callbacks *callback, const char *config_path) { /* This is the callback we'll need to invoke to contact the Janus core */ gateway = callback; +#ifdef HAVE_RNNOISE + JANUS_LOG(LOG_INFO, "Denoising via RNNoise supported (%d)\n", rnnoise_get_frame_size()); +#else + JANUS_LOG(LOG_WARN, "Denoising via RNNoise NOT supported\n"); +#endif + /* Parse configuration to populate the rooms list */ if(config != NULL) { janus_config_category *config_general = janus_config_get_create(config, NULL, janus_config_type_category, "general"); @@ -2387,6 +2415,7 @@ int janus_audiobridge_init(janus_callbacks *callback, const char *config_path) { janus_config_item *default_prebuffering = janus_config_get(config, cat, janus_config_type_item, "default_prebuffering"); janus_config_item *default_expectedloss = janus_config_get(config, cat, janus_config_type_item, "default_expectedloss"); janus_config_item *default_bitrate = janus_config_get(config, cat, janus_config_type_item, "default_bitrate"); + janus_config_item *denoise = janus_config_get(config, cat, janus_config_type_item, "denoise"); janus_config_item *secret = janus_config_get(config, cat, janus_config_type_item, "secret"); janus_config_item *pin = janus_config_get(config, cat, janus_config_type_item, "pin"); janus_config_array *groups = janus_config_get(config, cat, janus_config_type_array, "groups"); @@ -2510,6 +2539,13 @@ int janus_audiobridge_init(janus_callbacks *callback, const char *config_path) { audiobridge->default_bitrate = 0; } } +#ifdef HAVE_RNNOISE + audiobridge->denoise = denoise && denoise->value && janus_is_true(denoise->value); +#else + if(denoise && denoise->value && janus_is_true(denoise->value)) { + JANUS_LOG(LOG_WARN, "RNNoise unavailable, denoising not supported\n"); + } +#endif audiobridge->room_ssrc = janus_random_uint32(); if(secret != NULL && secret->value != NULL) { audiobridge->room_secret = g_strdup(secret->value); @@ -2838,6 +2874,10 @@ json_t *janus_audiobridge_query_session(janus_plugin_session *handle) { json_object_set_new(info, "last-drop", json_integer(participant->last_drop)); if(participant->stereo) json_object_set_new(info, "spatial_position", json_integer(participant->spatial_position)); +#ifdef HAVE_RNNOISE + if(participant->denoise) + json_object_set_new(info, "denoise", json_true()); +#endif if(participant->arc && participant->arc->filename) json_object_set_new(info, "audio-recording", json_string(participant->arc->filename)); if(participant->extmap_id > 0) { @@ -2975,6 +3015,7 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s json_t *default_prebuffering = json_object_get(root, "default_prebuffering"); json_t *default_expectedloss = json_object_get(root, "default_expectedloss"); json_t *default_bitrate = json_object_get(root, "default_bitrate"); + json_t *denoise = json_object_get(root, "denoise"); json_t *groups = json_object_get(root, "groups"); json_t *record = json_object_get(root, "record"); json_t *recfile = json_object_get(root, "record_file"); @@ -3149,6 +3190,13 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s audiobridge->default_bitrate = 0; } } +#ifdef HAVE_RNNOISE + audiobridge->denoise = denoise ? json_is_true(denoise) : FALSE; +#else + if(denoise && json_is_true(denoise)) { + JANUS_LOG(LOG_WARN, "RNNoise unavailable, denoising not supported\n"); + } +#endif switch(audiobridge->sampling_rate) { case 8000: case 12000: @@ -4250,6 +4298,108 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); goto prepare_response; +#ifdef HAVE_RNNOISE + } else if(!strcasecmp(request_text, "denoise_enable") || !strcasecmp(request_text, "denoise_disable")) { + gboolean denoise = (!strcasecmp(request_text, "denoise_enable")); + JANUS_LOG(LOG_VERB, "Attempt to %s denoising for a participant in an existing AudioBridge room\n", + denoise ? "enable" : "disable"); + JANUS_VALIDATE_JSON_OBJECT(root, secret_parameters, + error_code, error_cause, TRUE, + JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); + if(error_code != 0) + goto prepare_response; + if(!string_ids) { + JANUS_VALIDATE_JSON_OBJECT(root, room_parameters, + error_code, error_cause, TRUE, + JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); + } else { + JANUS_VALIDATE_JSON_OBJECT(root, roomstr_parameters, + error_code, error_cause, TRUE, + JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); + } + if(error_code != 0) + goto prepare_response; + if(!string_ids) { + JANUS_VALIDATE_JSON_OBJECT(root, id_parameters, + error_code, error_cause, TRUE, + JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); + } else { + JANUS_VALIDATE_JSON_OBJECT(root, idstr_parameters, + error_code, error_cause, TRUE, + JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT); + } + if(error_code != 0) + goto prepare_response; + json_t *room = json_object_get(root, "room"); + json_t *id = json_object_get(root, "id"); + guint64 room_id = 0; + char room_id_num[30], *room_id_str = NULL; + if(!string_ids) { + room_id = json_integer_value(room); + g_snprintf(room_id_num, sizeof(room_id_num), "%"SCNu64, room_id); + room_id_str = room_id_num; + } else { + room_id_str = (char *)json_string_value(room); + } + janus_mutex_lock(&rooms_mutex); + janus_audiobridge_room *audiobridge = g_hash_table_lookup(rooms, + string_ids ? (gpointer)room_id_str : (gpointer)&room_id); + if(audiobridge == NULL) { + janus_mutex_unlock(&rooms_mutex); + error_code = JANUS_AUDIOBRIDGE_ERROR_NO_SUCH_ROOM; + JANUS_LOG(LOG_ERR, "No such room (%s)\n", room_id_str); + g_snprintf(error_cause, 512, "No such room (%s)", room_id_str); + goto prepare_response; + } + janus_refcount_increase(&audiobridge->ref); + janus_mutex_lock(&audiobridge->mutex); + janus_mutex_unlock(&rooms_mutex); + + /* A secret may be required for this action */ + JANUS_CHECK_SECRET(audiobridge->room_secret, root, "secret", error_code, error_cause, + JANUS_AUDIOBRIDGE_ERROR_MISSING_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_INVALID_ELEMENT, JANUS_AUDIOBRIDGE_ERROR_UNAUTHORIZED); + if(error_code != 0) { + janus_mutex_unlock(&audiobridge->mutex); + janus_refcount_decrease(&audiobridge->ref); + goto prepare_response; + } + + guint64 user_id = 0; + char user_id_num[30], *user_id_str = NULL; + if(!string_ids) { + user_id = json_integer_value(id); + g_snprintf(user_id_num, sizeof(user_id_num), "%"SCNu64, user_id); + user_id_str = user_id_num; + } else { + user_id_str = (char *)json_string_value(id); + } + janus_audiobridge_participant *participant = g_hash_table_lookup(audiobridge->participants, + string_ids ? (gpointer)user_id_str : (gpointer)&user_id); + if(participant == NULL) { + janus_mutex_unlock(&audiobridge->mutex); + janus_refcount_decrease(&audiobridge->ref); + JANUS_LOG(LOG_ERR, "No such user %s in room %s\n", user_id_str, room_id_str); + error_code = JANUS_AUDIOBRIDGE_ERROR_NO_SUCH_USER; + g_snprintf(error_cause, 512, "No such user %s in room %s", user_id_str, room_id_str); + goto prepare_response; + } + + participant->denoise = denoise; + if(participant->denoise && participant->rnnoise == NULL) { + /* Create RNNoise context */ + participant->rnnoise = rnnoise_create(NULL); + } + + /* Prepare response */ + response = json_object(); + json_object_set_new(response, "audiobridge", json_string("success")); + json_object_set_new(response, "room", string_ids ? json_string(room_id_str) : json_integer(room_id)); + + /* Done */ + janus_mutex_unlock(&audiobridge->mutex); + janus_refcount_decrease(&audiobridge->ref); + goto prepare_response; +#endif } else if(!strcasecmp(request_text, "kick")) { JANUS_LOG(LOG_VERB, "Attempt to kick a participant from an existing AudioBridge room\n"); JANUS_VALIDATE_JSON_OBJECT(root, secret_parameters, @@ -5795,6 +5945,62 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r g_free(pkt); return; } +#ifdef HAVE_RNNOISE + /* Check if we need to denoise this packet */ + if(participant->rnnoise && participant->denoise) { + /* We do: copy the samples in a float buffer, as that's what RNNoise needs */ + opus_int16 *encoded = (opus_int16 *)pkt->data; + float denoised[480]; + int i = 0, offset = 0, total = pkt->length; + if(!participant->stereo) { + /* Mono */ + while(total > 0) { + /* Denoise this audio chunk */ + for(i=0; i<480; i++) { + if((offset + i) < pkt->length) + denoised[i] = *(encoded + offset + i); + else + denoised[i] = 0; + } + /* Denoise */ + rnnoise_process_frame(participant->rnnoise, denoised, denoised); + /* Replace the audio data with the one we just denoised */ + for(i=0; i<480; i++) { + if((offset + i) < pkt->length) + *(encoded + offset + i) = denoised[i]; + } + total -= 480; + offset += 480; + } + } else { + /* Stereo (interleaved) */ + int step = 0; + while(step < 2) { + /* Denoise this audio chunk */ + for(i=0; i<480; i++) { + if((offset + i*2 + step) < pkt->length) + denoised[i] = *(encoded + offset + i*2 + step); + else + denoised[i] = 0; + } + /* Denoise */ + rnnoise_process_frame(participant->rnnoise, denoised, denoised); + /* Replace the audio data with the one we just denoised */ + for(i=0; i<480; i++) { + if((offset + i*2 + step) < pkt->length) + *(encoded + offset + i*2 + step) = denoised[i]; + } + total -= 960; + offset += 960; + if(total <= 0) { + total = pkt->length; + offset = 0; + step++; + } + } + } + } +#endif /* Enqueue the decoded frame */ janus_mutex_lock(&participant->qmutex); /* Insert packets sorting by sequence number */ @@ -6225,6 +6431,7 @@ static void *janus_audiobridge_handler(void *data) { json_t *acodec = json_object_get(root, "codec"); json_t *user_audio_level_average = json_object_get(root, "audio_level_average"); json_t *user_audio_active_packets = json_object_get(root, "audio_active_packets"); + json_t *denoise = json_object_get(root, "denoise"); json_t *record = json_object_get(root, "record"); json_t *recfile = json_object_get(root, "filename"); json_t *gen_offer = json_object_get(root, "generate_offer"); @@ -6368,6 +6575,17 @@ static void *janus_audiobridge_handler(void *data) { spatial_position = 100; participant->spatial_position = spatial_position; } +#ifdef HAVE_RNNOISE + participant->denoise = denoise ? json_is_true(denoise) : audiobridge->denoise; + if(participant->denoise && participant->rnnoise == NULL) { + /* Create RNNoise context */ + participant->rnnoise = rnnoise_create(NULL); + } +#else + if(denoise && json_is_true(denoise)) { + JANUS_LOG(LOG_WARN, "RNNoise unavailable, denoising not supported\n"); + } +#endif participant->user_audio_active_packets = json_integer_value(user_audio_active_packets); participant->user_audio_level_average = json_integer_value(user_audio_level_average); if(participant->outbuf == NULL) @@ -6692,6 +6910,7 @@ static void *janus_audiobridge_handler(void *data) { json_t *exploss = json_object_get(root, "expected_loss"); json_t *gain = json_object_get(root, "volume"); json_t *spatial = json_object_get(root, "spatial_position"); + json_t *denoise = json_object_get(root, "denoise"); json_t *record = json_object_get(root, "record"); json_t *recfile = json_object_get(root, "filename"); json_t *display = json_object_get(root, "display"); @@ -6809,6 +7028,18 @@ static void *janus_audiobridge_handler(void *data) { spatial_position = 100; participant->spatial_position = spatial_position; } +#ifdef HAVE_RNNOISE + if(denoise) + participant->denoise = json_is_true(denoise); + if(participant->denoise && participant->rnnoise == NULL) { + /* Create RNNoise context */ + participant->rnnoise = rnnoise_create(NULL); + } +#else + if(denoise && json_is_true(denoise)) { + JANUS_LOG(LOG_WARN, "RNNoise unavailable, denoising not supported\n"); + } +#endif /* Notify all other participants */ janus_mutex_lock(&rooms_mutex); janus_audiobridge_room *audiobridge = participant->room; From f52afc32e84fa8dba568c6ba334070b2e3bddcb0 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 24 May 2023 17:47:41 +0200 Subject: [PATCH 02/16] Use two separate denoisers, when dealing with stereo participants --- src/plugins/janus_audiobridge.c | 36 ++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 1cc31d4358..923f3c33f7 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -1585,8 +1585,8 @@ typedef struct janus_audiobridge_participant { gboolean stereo; /* Whether stereo will be used for spatial audio */ int spatial_position; /* Panning of this participant in the mix */ #ifdef HAVE_RNNOISE - gboolean denoise; /* Whether we should denoise this participant */ - DenoiseState *rnnoise; /* RNNoise state*/ + gboolean denoise; /* Whether we should denoise this participant */ + DenoiseState *rnnoise[2]; /* RNNoise states (we'll need two for stereo) */ #endif /* RTP stuff */ GList *inbuf; /* Incoming audio from this participant, as an ordered list of packets */ @@ -1681,8 +1681,10 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant g_async_queue_unref(participant->outbuf); } #ifdef HAVE_RNNOISE - if(participant->rnnoise) - rnnoise_destroy(participant->rnnoise); + if(participant->rnnoise[0]) + rnnoise_destroy(participant->rnnoise[0]); + if(participant->rnnoise[1]) + rnnoise_destroy(participant->rnnoise[1]); #endif g_free(participant->mjr_base); #ifdef HAVE_LIBOGG @@ -4385,9 +4387,11 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s } participant->denoise = denoise; - if(participant->denoise && participant->rnnoise == NULL) { - /* Create RNNoise context */ - participant->rnnoise = rnnoise_create(NULL); + if(participant->denoise && participant->rnnoise[0] == NULL) { + /* Create RNNoise context(s) */ + participant->rnnoise[0] = rnnoise_create(NULL); + if(participant->stereo) + participant->rnnoise[1] = rnnoise_create(NULL); } /* Prepare response */ @@ -5947,7 +5951,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r } #ifdef HAVE_RNNOISE /* Check if we need to denoise this packet */ - if(participant->rnnoise && participant->denoise) { + if(participant->rnnoise[0] && participant->denoise) { /* We do: copy the samples in a float buffer, as that's what RNNoise needs */ opus_int16 *encoded = (opus_int16 *)pkt->data; float denoised[480]; @@ -5963,7 +5967,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r denoised[i] = 0; } /* Denoise */ - rnnoise_process_frame(participant->rnnoise, denoised, denoised); + rnnoise_process_frame(participant->rnnoise[0], denoised, denoised); /* Replace the audio data with the one we just denoised */ for(i=0; i<480; i++) { if((offset + i) < pkt->length) @@ -5984,7 +5988,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r denoised[i] = 0; } /* Denoise */ - rnnoise_process_frame(participant->rnnoise, denoised, denoised); + rnnoise_process_frame(participant->rnnoise[step], denoised, denoised); /* Replace the audio data with the one we just denoised */ for(i=0; i<480; i++) { if((offset + i*2 + step) < pkt->length) @@ -6577,9 +6581,11 @@ static void *janus_audiobridge_handler(void *data) { } #ifdef HAVE_RNNOISE participant->denoise = denoise ? json_is_true(denoise) : audiobridge->denoise; - if(participant->denoise && participant->rnnoise == NULL) { + if(participant->denoise && participant->rnnoise[0] == NULL) { /* Create RNNoise context */ - participant->rnnoise = rnnoise_create(NULL); + participant->rnnoise[0] = rnnoise_create(NULL); + if(participant->stereo) + participant->rnnoise[1] = rnnoise_create(NULL); } #else if(denoise && json_is_true(denoise)) { @@ -7031,9 +7037,11 @@ static void *janus_audiobridge_handler(void *data) { #ifdef HAVE_RNNOISE if(denoise) participant->denoise = json_is_true(denoise); - if(participant->denoise && participant->rnnoise == NULL) { + if(participant->denoise && participant->rnnoise[0] == NULL) { /* Create RNNoise context */ - participant->rnnoise = rnnoise_create(NULL); + participant->rnnoise[0] = rnnoise_create(NULL); + if(participant->stereo) + participant->rnnoise[1] = rnnoise_create(NULL); } #else if(denoise && json_is_true(denoise)) { From 3c30a17f681fbd084a91eced10baa21ff91c59df Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 24 May 2023 19:37:04 +0200 Subject: [PATCH 03/16] Fixed denoising artifacts when using mono 8000/16000 --- src/plugins/janus_audiobridge.c | 67 +++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 923f3c33f7..420ebfa815 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -1608,6 +1608,7 @@ typedef struct janus_audiobridge_participant { janus_audiobridge_plainrtp_media plainrtp_media; janus_mutex pmutex; /* Opus stuff */ + uint32_t sampling_rate; /* Sampling rate to decode at */ OpusEncoder *encoder; /* Opus encoder instance */ OpusDecoder *decoder; /* Opus decoder instance */ gboolean fec; /* Opus FEC status */ @@ -5960,18 +5961,58 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r /* Mono */ while(total > 0) { /* Denoise this audio chunk */ - for(i=0; i<480; i++) { - if((offset + i) < pkt->length) - denoised[i] = *(encoded + offset + i); - else - denoised[i] = 0; + if(participant->sampling_rate == 8000) { + /* RNNoise needs 480 samples, resample */ + memset(denoised, 0, sizeof(denoised)); + for(i=0; i<160; i++) { + if((offset + i) < pkt->length) { + denoised[i*3] = encoded[offset + i]; + denoised[i*3 + 1] = 0; + denoised[i*3 + 2] = 0; + } + } + } else if(participant->sampling_rate == 16000) { + /* RNNoise needs 480 samples, resample */ + memset(denoised, 0, sizeof(denoised)); + for(i=0; i<160; i++) { + if((offset + i*2) < pkt->length) { + denoised[i*3] = encoded[offset + i*2]; + denoised[i*3 + 1] = encoded[offset + i*2 + 1]; + denoised[i*3 + 2] = 0; + } + } + } else { + /* Just copy the samples */ + for(i=0; i<480; i++) { + if((offset + i) < pkt->length) + denoised[i] = encoded[offset + i]; + else + denoised[i] = 0; + } } /* Denoise */ rnnoise_process_frame(participant->rnnoise[0], denoised, denoised); /* Replace the audio data with the one we just denoised */ - for(i=0; i<480; i++) { - if((offset + i) < pkt->length) - *(encoded + offset + i) = denoised[i]; + if(participant->sampling_rate == 8000) { + /* RNNoise generated 480 samples, resample */ + for(i=0; i<160; i++) { + if((offset + i) < pkt->length) + encoded[offset + i] = denoised[i*3]; + } + } else if(participant->sampling_rate == 16000) { + /* RNNoise generated 480 samples, resample */ + for(i=0; i<160; i++) { + if((offset + i*2) < pkt->length) { + encoded[offset + i*2] = denoised[i*3]; + encoded[offset + i*2 + 1] = denoised[i*3 + 1]; + } + } + } else { + /* Just copy the denoised samples */ + for(i=0; i<480; i++) { + if((offset + i) < pkt->length) + encoded[offset + i] = denoised[i]; + } } total -= 480; offset += 480; @@ -5979,11 +6020,11 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r } else { /* Stereo (interleaved) */ int step = 0; - while(step < 2) { + while(total > 0 && step < 2) { /* Denoise this audio chunk */ for(i=0; i<480; i++) { if((offset + i*2 + step) < pkt->length) - denoised[i] = *(encoded + offset + i*2 + step); + denoised[i] = encoded[offset + i*2 + step]; else denoised[i] = 0; } @@ -5992,10 +6033,10 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r /* Replace the audio data with the one we just denoised */ for(i=0; i<480; i++) { if((offset + i*2 + step) < pkt->length) - *(encoded + offset + i*2 + step) = denoised[i]; + encoded[offset + i*2 + step] = denoised[i]; } total -= 960; - offset += 960; + offset += 480; if(total <= 0) { total = pkt->length; offset = 0; @@ -6609,6 +6650,7 @@ static void *janus_audiobridge_handler(void *data) { /* Opus encoder */ int error = 0; if(participant->encoder == NULL) { + participant->sampling_rate = audiobridge->sampling_rate; participant->encoder = opus_encoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { @@ -7437,6 +7479,7 @@ static void *janus_audiobridge_handler(void *data) { /* Destroy the previous encoder/decoder and update the references */ if(participant->encoder) opus_encoder_destroy(participant->encoder); + participant->sampling_rate = audiobridge->sampling_rate; participant->encoder = new_encoder; if(participant->decoder) opus_decoder_destroy(participant->decoder); From a594c532560ca3ccaa3117c755b42cf314df40d1 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Thu, 25 May 2023 17:07:11 +0200 Subject: [PATCH 04/16] Fixed broken audio when denoising stereo participants --- src/plugins/janus_audiobridge.c | 77 +++++++++++++++++++++++++-------- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 420ebfa815..4e13d680c4 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -5956,7 +5956,10 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r /* We do: copy the samples in a float buffer, as that's what RNNoise needs */ opus_int16 *encoded = (opus_int16 *)pkt->data; float denoised[480]; - int i = 0, offset = 0, total = pkt->length; + int i = 0, offset = 0, read = pkt->length; + if(participant->stereo) + read *= 2; + int total = read; if(!participant->stereo) { /* Mono */ while(total > 0) { @@ -5965,7 +5968,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r /* RNNoise needs 480 samples, resample */ memset(denoised, 0, sizeof(denoised)); for(i=0; i<160; i++) { - if((offset + i) < pkt->length) { + if((offset + i) < read) { denoised[i*3] = encoded[offset + i]; denoised[i*3 + 1] = 0; denoised[i*3 + 2] = 0; @@ -5975,7 +5978,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r /* RNNoise needs 480 samples, resample */ memset(denoised, 0, sizeof(denoised)); for(i=0; i<160; i++) { - if((offset + i*2) < pkt->length) { + if((offset + i*2) < read) { denoised[i*3] = encoded[offset + i*2]; denoised[i*3 + 1] = encoded[offset + i*2 + 1]; denoised[i*3 + 2] = 0; @@ -5984,7 +5987,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r } else { /* Just copy the samples */ for(i=0; i<480; i++) { - if((offset + i) < pkt->length) + if((offset + i) < read) denoised[i] = encoded[offset + i]; else denoised[i] = 0; @@ -5996,13 +5999,13 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r if(participant->sampling_rate == 8000) { /* RNNoise generated 480 samples, resample */ for(i=0; i<160; i++) { - if((offset + i) < pkt->length) + if((offset + i) < read) encoded[offset + i] = denoised[i*3]; } } else if(participant->sampling_rate == 16000) { /* RNNoise generated 480 samples, resample */ for(i=0; i<160; i++) { - if((offset + i*2) < pkt->length) { + if((offset + i*2) < read) { encoded[offset + i*2] = denoised[i*3]; encoded[offset + i*2 + 1] = denoised[i*3 + 1]; } @@ -6010,7 +6013,7 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r } else { /* Just copy the denoised samples */ for(i=0; i<480; i++) { - if((offset + i) < pkt->length) + if((offset + i) < read) encoded[offset + i] = denoised[i]; } } @@ -6022,23 +6025,63 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r int step = 0; while(total > 0 && step < 2) { /* Denoise this audio chunk */ - for(i=0; i<480; i++) { - if((offset + i*2 + step) < pkt->length) - denoised[i] = encoded[offset + i*2 + step]; - else - denoised[i] = 0; + if(participant->sampling_rate == 8000) { + /* RNNoise needs 480 samples, resample */ + memset(denoised, 0, sizeof(denoised)); + for(i=0; i<160; i++) { + if((offset + i*2 + step) < read) { + denoised[i*3] = encoded[offset + i*2 + step]; + denoised[i*3 + 1] = 0; + denoised[i*3 + 2] = 0; + } + } + } else if(participant->sampling_rate == 16000) { + /* RNNoise needs 480 samples, resample */ + memset(denoised, 0, sizeof(denoised)); + for(i=0; i<160; i++) { + if((offset + i*4 + step + 2) < read) { + denoised[i*3] = encoded[offset + i*4 + step]; + denoised[i*3 + 1] = encoded[offset + i*4 + step + 2]; + denoised[i*3 + 2] = 0; + } + } + } else { + /* Just copy the samples */ + for(i=0; i<480; i++) { + if((offset + i*2 + step) < read) + denoised[i] = encoded[offset + i*2 + step]; + else + denoised[i] = 0; + } } /* Denoise */ rnnoise_process_frame(participant->rnnoise[step], denoised, denoised); /* Replace the audio data with the one we just denoised */ - for(i=0; i<480; i++) { - if((offset + i*2 + step) < pkt->length) - encoded[offset + i*2 + step] = denoised[i]; + if(participant->sampling_rate == 8000) { + /* RNNoise generated 480 samples, resample */ + for(i=0; i<160; i++) { + if((offset + i*2 + step) < read) + encoded[offset + i*2 + step] = denoised[i*3]; + } + } else if(participant->sampling_rate == 16000) { + /* RNNoise generated 480 samples, resample */ + for(i=0; i<160; i++) { + if((offset + i*4 + step + 2) < read) { + encoded[offset + i*4 + step] = denoised[i*3]; + encoded[offset + i*4 + step + 2] = denoised[i*3 + 1]; + } + } + } else { + /* Just copy the denoised samples */ + for(i=0; i<480; i++) { + if((offset + i*2 + step) < read) + encoded[offset + i*2 + step] = denoised[i]; + } } total -= 960; - offset += 480; + offset += 960; if(total <= 0) { - total = pkt->length; + total = read; offset = 0; step++; } From 4060993649a6f0477d42916a4f3bcba927364875 Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Fri, 26 May 2023 11:30:29 +0200 Subject: [PATCH 05/16] Move denoising code to a separate function --- src/plugins/janus_audiobridge.c | 281 ++++++++++++++++---------------- 1 file changed, 145 insertions(+), 136 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 4e13d680c4..a15f4ca138 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -1697,6 +1697,10 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant g_free(participant); } +#ifdef HAVE_RNNOISE +static void janus_audiobridge_participant_denoise(janus_audiobridge_participant *participant, char *data, int len); +#endif + static void janus_audiobridge_session_destroy(janus_audiobridge_session *session) { if(session && g_atomic_int_compare_and_exchange(&session->destroyed, 0, 1)) janus_refcount_decrease(&session->ref); @@ -5952,142 +5956,8 @@ void janus_audiobridge_incoming_rtp(janus_plugin_session *handle, janus_plugin_r } #ifdef HAVE_RNNOISE /* Check if we need to denoise this packet */ - if(participant->rnnoise[0] && participant->denoise) { - /* We do: copy the samples in a float buffer, as that's what RNNoise needs */ - opus_int16 *encoded = (opus_int16 *)pkt->data; - float denoised[480]; - int i = 0, offset = 0, read = pkt->length; - if(participant->stereo) - read *= 2; - int total = read; - if(!participant->stereo) { - /* Mono */ - while(total > 0) { - /* Denoise this audio chunk */ - if(participant->sampling_rate == 8000) { - /* RNNoise needs 480 samples, resample */ - memset(denoised, 0, sizeof(denoised)); - for(i=0; i<160; i++) { - if((offset + i) < read) { - denoised[i*3] = encoded[offset + i]; - denoised[i*3 + 1] = 0; - denoised[i*3 + 2] = 0; - } - } - } else if(participant->sampling_rate == 16000) { - /* RNNoise needs 480 samples, resample */ - memset(denoised, 0, sizeof(denoised)); - for(i=0; i<160; i++) { - if((offset + i*2) < read) { - denoised[i*3] = encoded[offset + i*2]; - denoised[i*3 + 1] = encoded[offset + i*2 + 1]; - denoised[i*3 + 2] = 0; - } - } - } else { - /* Just copy the samples */ - for(i=0; i<480; i++) { - if((offset + i) < read) - denoised[i] = encoded[offset + i]; - else - denoised[i] = 0; - } - } - /* Denoise */ - rnnoise_process_frame(participant->rnnoise[0], denoised, denoised); - /* Replace the audio data with the one we just denoised */ - if(participant->sampling_rate == 8000) { - /* RNNoise generated 480 samples, resample */ - for(i=0; i<160; i++) { - if((offset + i) < read) - encoded[offset + i] = denoised[i*3]; - } - } else if(participant->sampling_rate == 16000) { - /* RNNoise generated 480 samples, resample */ - for(i=0; i<160; i++) { - if((offset + i*2) < read) { - encoded[offset + i*2] = denoised[i*3]; - encoded[offset + i*2 + 1] = denoised[i*3 + 1]; - } - } - } else { - /* Just copy the denoised samples */ - for(i=0; i<480; i++) { - if((offset + i) < read) - encoded[offset + i] = denoised[i]; - } - } - total -= 480; - offset += 480; - } - } else { - /* Stereo (interleaved) */ - int step = 0; - while(total > 0 && step < 2) { - /* Denoise this audio chunk */ - if(participant->sampling_rate == 8000) { - /* RNNoise needs 480 samples, resample */ - memset(denoised, 0, sizeof(denoised)); - for(i=0; i<160; i++) { - if((offset + i*2 + step) < read) { - denoised[i*3] = encoded[offset + i*2 + step]; - denoised[i*3 + 1] = 0; - denoised[i*3 + 2] = 0; - } - } - } else if(participant->sampling_rate == 16000) { - /* RNNoise needs 480 samples, resample */ - memset(denoised, 0, sizeof(denoised)); - for(i=0; i<160; i++) { - if((offset + i*4 + step + 2) < read) { - denoised[i*3] = encoded[offset + i*4 + step]; - denoised[i*3 + 1] = encoded[offset + i*4 + step + 2]; - denoised[i*3 + 2] = 0; - } - } - } else { - /* Just copy the samples */ - for(i=0; i<480; i++) { - if((offset + i*2 + step) < read) - denoised[i] = encoded[offset + i*2 + step]; - else - denoised[i] = 0; - } - } - /* Denoise */ - rnnoise_process_frame(participant->rnnoise[step], denoised, denoised); - /* Replace the audio data with the one we just denoised */ - if(participant->sampling_rate == 8000) { - /* RNNoise generated 480 samples, resample */ - for(i=0; i<160; i++) { - if((offset + i*2 + step) < read) - encoded[offset + i*2 + step] = denoised[i*3]; - } - } else if(participant->sampling_rate == 16000) { - /* RNNoise generated 480 samples, resample */ - for(i=0; i<160; i++) { - if((offset + i*4 + step + 2) < read) { - encoded[offset + i*4 + step] = denoised[i*3]; - encoded[offset + i*4 + step + 2] = denoised[i*3 + 1]; - } - } - } else { - /* Just copy the denoised samples */ - for(i=0; i<480; i++) { - if((offset + i*2 + step) < read) - encoded[offset + i*2 + step] = denoised[i]; - } - } - total -= 960; - offset += 960; - if(total <= 0) { - total = read; - offset = 0; - step++; - } - } - } - } + if(participant->rnnoise[0] && participant->denoise) + janus_audiobridge_participant_denoise(participant, pkt->data, pkt->length); #endif /* Enqueue the decoded frame */ janus_mutex_lock(&participant->qmutex); @@ -9110,3 +8980,142 @@ static void *janus_audiobridge_plainrtp_relay_thread(void *data) { g_thread_unref(g_thread_self()); return NULL; } + +#ifdef HAVE_RNNOISE +static void janus_audiobridge_participant_denoise(janus_audiobridge_participant *participant, char *data, int len) { + /* Copy the samples in a float buffer, as that's what RNNoise needs */ + opus_int16 *encoded = (opus_int16 *)data; + float denoised[480]; + int i = 0, offset = 0; + if(participant->stereo) + len *= 2; + int total = len; + if(!participant->stereo) { + /* Mono */ + while(total > 0) { + /* Denoise this audio chunk */ + if(participant->sampling_rate == 8000) { + /* RNNoise needs 480 samples, resample */ + memset(denoised, 0, sizeof(denoised)); + for(i=0; i<160; i++) { + if((offset + i) < len) { + denoised[i*3] = encoded[offset + i]; + denoised[i*3 + 1] = 0; + denoised[i*3 + 2] = 0; + } + } + } else if(participant->sampling_rate == 16000) { + /* RNNoise needs 480 samples, resample */ + memset(denoised, 0, sizeof(denoised)); + for(i=0; i<160; i++) { + if((offset + i*2) < len) { + denoised[i*3] = encoded[offset + i*2]; + denoised[i*3 + 1] = encoded[offset + i*2 + 1]; + denoised[i*3 + 2] = 0; + } + } + } else { + /* Just copy the samples */ + for(i=0; i<480; i++) { + if((offset + i) < len) + denoised[i] = encoded[offset + i]; + else + denoised[i] = 0; + } + } + /* Denoise */ + rnnoise_process_frame(participant->rnnoise[0], denoised, denoised); + /* Replace the audio data with the one we just denoised */ + if(participant->sampling_rate == 8000) { + /* RNNoise generated 480 samples, resample */ + for(i=0; i<160; i++) { + if((offset + i) < len) + encoded[offset + i] = denoised[i*3]; + } + } else if(participant->sampling_rate == 16000) { + /* RNNoise generated 480 samples, resample */ + for(i=0; i<160; i++) { + if((offset + i*2) < len) { + encoded[offset + i*2] = denoised[i*3]; + encoded[offset + i*2 + 1] = denoised[i*3 + 1]; + } + } + } else { + /* Just copy the denoised samples */ + for(i=0; i<480; i++) { + if((offset + i) < len) + encoded[offset + i] = denoised[i]; + } + } + total -= 480; + offset += 480; + } + } else { + /* Stereo (interleaved) */ + int step = 0; + while(total > 0 && step < 2) { + /* Denoise this audio chunk */ + if(participant->sampling_rate == 8000) { + /* RNNoise needs 480 samples, resample */ + memset(denoised, 0, sizeof(denoised)); + for(i=0; i<160; i++) { + if((offset + i*2 + step) < len) { + denoised[i*3] = encoded[offset + i*2 + step]; + denoised[i*3 + 1] = 0; + denoised[i*3 + 2] = 0; + } + } + } else if(participant->sampling_rate == 16000) { + /* RNNoise needs 480 samples, resample */ + memset(denoised, 0, sizeof(denoised)); + for(i=0; i<160; i++) { + if((offset + i*4 + step + 2) < len) { + denoised[i*3] = encoded[offset + i*4 + step]; + denoised[i*3 + 1] = encoded[offset + i*4 + step + 2]; + denoised[i*3 + 2] = 0; + } + } + } else { + /* Just copy the samples */ + for(i=0; i<480; i++) { + if((offset + i*2 + step) < len) + denoised[i] = encoded[offset + i*2 + step]; + else + denoised[i] = 0; + } + } + /* Denoise */ + rnnoise_process_frame(participant->rnnoise[step], denoised, denoised); + /* Replace the audio data with the one we just denoised */ + if(participant->sampling_rate == 8000) { + /* RNNoise generated 480 samples, resample */ + for(i=0; i<160; i++) { + if((offset + i*2 + step) < len) + encoded[offset + i*2 + step] = denoised[i*3]; + } + } else if(participant->sampling_rate == 16000) { + /* RNNoise generated 480 samples, resample */ + for(i=0; i<160; i++) { + if((offset + i*4 + step + 2) < len) { + encoded[offset + i*4 + step] = denoised[i*3]; + encoded[offset + i*4 + step + 2] = denoised[i*3 + 1]; + } + } + } else { + /* Just copy the denoised samples */ + for(i=0; i<480; i++) { + if((offset + i*2 + step) < len) + encoded[offset + i*2 + step] = denoised[i]; + } + } + total -= 960; + offset += 960; + if(total <= 0) { + total = len; + offset = 0; + step++; + } + } + } +} +#endif From 1feba687e389ec2ab4c69dd5680d31e5f9c9d4d1 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 16 Nov 2023 19:12:58 +0100 Subject: [PATCH 06/16] Refactor denoising algorithm --- src/plugins/janus_audiobridge.c | 176 +++++++++++--------------------- 1 file changed, 59 insertions(+), 117 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 62a55e945a..9851a14b29 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -8849,139 +8849,81 @@ static void *janus_audiobridge_plainrtp_relay_thread(void *data) { #ifdef HAVE_RNNOISE static void janus_audiobridge_participant_denoise(janus_audiobridge_participant *participant, char *data, int len) { + opus_int16 *samples = (opus_int16 *)data; + int i = 0, j = 0, offset = 0; + int total = len; /* Copy the samples in a float buffer, as that's what RNNoise needs */ - opus_int16 *encoded = (opus_int16 *)data; float denoised[480]; - int i = 0, offset = 0; - if(participant->stereo) - len *= 2; - int total = len; + /* evaluate chunk size (number of samples per channel) for 10ms of audio */ + int chunk_size = participant->sampling_rate / 100; + /* evaluate the number of empty samples between samples in case of upsampling */ + int empty_samples = (480 - chunk_size) / chunk_size; if(!participant->stereo) { /* Mono */ while(total > 0) { /* Denoise this audio chunk */ - if(participant->sampling_rate == 8000) { - /* RNNoise needs 480 samples, resample */ - memset(denoised, 0, sizeof(denoised)); - for(i=0; i<160; i++) { - if((offset + i) < len) { - denoised[i*3] = encoded[offset + i]; - denoised[i*3 + 1] = 0; - denoised[i*3 + 2] = 0; - } - } - } else if(participant->sampling_rate == 16000) { - /* RNNoise needs 480 samples, resample */ - memset(denoised, 0, sizeof(denoised)); - for(i=0; i<160; i++) { - if((offset + i*2) < len) { - denoised[i*3] = encoded[offset + i*2]; - denoised[i*3 + 1] = encoded[offset + i*2 + 1]; - denoised[i*3 + 2] = 0; - } - } - } else { - /* Just copy the samples */ - for(i=0; i<480; i++) { - if((offset + i) < len) - denoised[i] = encoded[offset + i]; - else - denoised[i] = 0; - } + i = 0; + j = 0; + /* RNNoise needs 480 samples @ 48kHz */ + while(i < 480 && j < chunk_size) { + denoised[i] = samples[j + offset]; + /* Upsample with zero filling */ + if(empty_samples > 0) { + memset(&(denoised[i+1]), 0, empty_samples*sizeof(float)); + } + i = i + 1 + empty_samples; + j = j + 1; } /* Denoise */ rnnoise_process_frame(participant->rnnoise[0], denoised, denoised); /* Replace the audio data with the one we just denoised */ - if(participant->sampling_rate == 8000) { - /* RNNoise generated 480 samples, resample */ - for(i=0; i<160; i++) { - if((offset + i) < len) - encoded[offset + i] = denoised[i*3]; - } - } else if(participant->sampling_rate == 16000) { - /* RNNoise generated 480 samples, resample */ - for(i=0; i<160; i++) { - if((offset + i*2) < len) { - encoded[offset + i*2] = denoised[i*3]; - encoded[offset + i*2 + 1] = denoised[i*3 + 1]; - } - } - } else { - /* Just copy the denoised samples */ - for(i=0; i<480; i++) { - if((offset + i) < len) - encoded[offset + i] = denoised[i]; - } + i = 0; + j = 0; + /* Downsample to the original rate */ + while(i < 480 && j < chunk_size) { + samples[j + offset] = denoised[i]; + i = i + 1 + empty_samples; + j = j + 1; } - total -= 480; - offset += 480; + total -= chunk_size; + offset += chunk_size; } } else { /* Stereo (interleaved) */ - int step = 0; - while(total > 0 && step < 2) { + float denoised_alt[480]; + while(total > 0) { /* Denoise this audio chunk */ - if(participant->sampling_rate == 8000) { - /* RNNoise needs 480 samples, resample */ - memset(denoised, 0, sizeof(denoised)); - for(i=0; i<160; i++) { - if((offset + i*2 + step) < len) { - denoised[i*3] = encoded[offset + i*2 + step]; - denoised[i*3 + 1] = 0; - denoised[i*3 + 2] = 0; - } - } - } else if(participant->sampling_rate == 16000) { - /* RNNoise needs 480 samples, resample */ - memset(denoised, 0, sizeof(denoised)); - for(i=0; i<160; i++) { - if((offset + i*4 + step + 2) < len) { - denoised[i*3] = encoded[offset + i*4 + step]; - denoised[i*3 + 1] = encoded[offset + i*4 + step + 2]; - denoised[i*3 + 2] = 0; - } - } - } else { - /* Just copy the samples */ - for(i=0; i<480; i++) { - if((offset + i*2 + step) < len) - denoised[i] = encoded[offset + i*2 + step]; - else - denoised[i] = 0; - } - } - /* Denoise */ - rnnoise_process_frame(participant->rnnoise[step], denoised, denoised); + i = 0; + j = 0; + /* RNNoise needs 480 samples @ 48kHz */ + while(i < 480 && j < chunk_size) { + denoised[i] = samples[2*j + 2*offset]; + denoised_alt[i] = samples[2*j + 1 + 2*offset]; + /* Upsample with zero filling */ + if(empty_samples > 0) { + memset(&(denoised[i+1]), 0, empty_samples*sizeof(float)); + memset(&(denoised_alt[i+1]), 0, empty_samples*sizeof(float)); + } + i = i + 1 + empty_samples; + j = j + 1; + } + /* Denoise channel 1 */ + rnnoise_process_frame(participant->rnnoise[0], denoised, denoised); + /* Denoise channel 2 */ + rnnoise_process_frame(participant->rnnoise[1], denoised_alt, denoised_alt); /* Replace the audio data with the one we just denoised */ - if(participant->sampling_rate == 8000) { - /* RNNoise generated 480 samples, resample */ - for(i=0; i<160; i++) { - if((offset + i*2 + step) < len) - encoded[offset + i*2 + step] = denoised[i*3]; - } - } else if(participant->sampling_rate == 16000) { - /* RNNoise generated 480 samples, resample */ - for(i=0; i<160; i++) { - if((offset + i*4 + step + 2) < len) { - encoded[offset + i*4 + step] = denoised[i*3]; - encoded[offset + i*4 + step + 2] = denoised[i*3 + 1]; - } - } - } else { - /* Just copy the denoised samples */ - for(i=0; i<480; i++) { - if((offset + i*2 + step) < len) - encoded[offset + i*2 + step] = denoised[i]; - } - } - total -= 960; - offset += 960; - if(total <= 0) { - total = len; - offset = 0; - step++; - } + i = 0; + j = 0; + /* Downsample to the original rate */ + while(i < 480 && j < chunk_size) { + samples[2*j + 2*offset] = denoised[i]; + samples[2*j + 1 + 2*offset] = denoised_alt[i]; + i = i + 1 + empty_samples; + j = j + 1; + } + total -= chunk_size; + offset += chunk_size; } } } -#endif +#endif \ No newline at end of file From 8fc4e30ff1c25b692e777f90cae5ed7f8707382f Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 16 Nov 2023 20:45:31 +0100 Subject: [PATCH 07/16] Prefill with zeroes denoised buffers. Define FRAME_SIZE macro. Cast ptr to avoid warning. --- src/plugins/janus_audiobridge.c | 37 +++++++++++++++++---------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 9851a14b29..b183ec8443 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -8442,7 +8442,7 @@ static void *janus_audiobridge_participant_thread(void *data) { #ifdef HAVE_RNNOISE /* Check if we need to denoise this packet */ if(participant->rnnoise[0] && participant->denoise) - janus_audiobridge_participant_denoise(participant, pkt->data, pkt->length); + janus_audiobridge_participant_denoise(participant, (char *)pkt->data, pkt->length); #endif /* Queue the decoded redundant packet for the mixer */ janus_mutex_lock(&participant->qmutex); @@ -8486,7 +8486,7 @@ static void *janus_audiobridge_participant_thread(void *data) { #ifdef HAVE_RNNOISE /* Check if we need to denoise this packet */ if(participant->rnnoise[0] && participant->denoise) - janus_audiobridge_participant_denoise(participant, pkt->data, pkt->length); + janus_audiobridge_participant_denoise(participant, (char *)pkt->data, pkt->length); #endif /* Get rid of the buffered packet */ janus_audiobridge_buffer_packet_destroy(bpkt); @@ -8848,29 +8848,30 @@ static void *janus_audiobridge_plainrtp_relay_thread(void *data) { } #ifdef HAVE_RNNOISE +#define FRAME_SIZE 480 static void janus_audiobridge_participant_denoise(janus_audiobridge_participant *participant, char *data, int len) { opus_int16 *samples = (opus_int16 *)data; int i = 0, j = 0, offset = 0; int total = len; /* Copy the samples in a float buffer, as that's what RNNoise needs */ - float denoised[480]; + float denoised[FRAME_SIZE]; /* evaluate chunk size (number of samples per channel) for 10ms of audio */ int chunk_size = participant->sampling_rate / 100; /* evaluate the number of empty samples between samples in case of upsampling */ - int empty_samples = (480 - chunk_size) / chunk_size; + int empty_samples = (FRAME_SIZE - chunk_size) / chunk_size; if(!participant->stereo) { /* Mono */ while(total > 0) { + if(empty_samples > 0) { + /* Upsample with zero filling */ + memset(denoised, 0, sizeof(denoised)); + } /* Denoise this audio chunk */ i = 0; j = 0; /* RNNoise needs 480 samples @ 48kHz */ - while(i < 480 && j < chunk_size) { + while(i < FRAME_SIZE && j < chunk_size) { denoised[i] = samples[j + offset]; - /* Upsample with zero filling */ - if(empty_samples > 0) { - memset(&(denoised[i+1]), 0, empty_samples*sizeof(float)); - } i = i + 1 + empty_samples; j = j + 1; } @@ -8880,7 +8881,7 @@ static void janus_audiobridge_participant_denoise(janus_audiobridge_participant i = 0; j = 0; /* Downsample to the original rate */ - while(i < 480 && j < chunk_size) { + while(i < FRAME_SIZE && j < chunk_size) { samples[j + offset] = denoised[i]; i = i + 1 + empty_samples; j = j + 1; @@ -8890,20 +8891,20 @@ static void janus_audiobridge_participant_denoise(janus_audiobridge_participant } } else { /* Stereo (interleaved) */ - float denoised_alt[480]; + float denoised_alt[FRAME_SIZE]; while(total > 0) { + if(empty_samples > 0) { + /* Upsample with zero filling */ + memset(denoised, 0, sizeof(denoised)); + memset(denoised_alt, 0, sizeof(denoised_alt)); + } /* Denoise this audio chunk */ i = 0; j = 0; /* RNNoise needs 480 samples @ 48kHz */ - while(i < 480 && j < chunk_size) { + while(i < FRAME_SIZE && j < chunk_size) { denoised[i] = samples[2*j + 2*offset]; denoised_alt[i] = samples[2*j + 1 + 2*offset]; - /* Upsample with zero filling */ - if(empty_samples > 0) { - memset(&(denoised[i+1]), 0, empty_samples*sizeof(float)); - memset(&(denoised_alt[i+1]), 0, empty_samples*sizeof(float)); - } i = i + 1 + empty_samples; j = j + 1; } @@ -8915,7 +8916,7 @@ static void janus_audiobridge_participant_denoise(janus_audiobridge_participant i = 0; j = 0; /* Downsample to the original rate */ - while(i < 480 && j < chunk_size) { + while(i < FRAME_SIZE && j < chunk_size) { samples[2*j + 2*offset] = denoised[i]; samples[2*j + 1 + 2*offset] = denoised_alt[i]; i = i + 1 + empty_samples; From ebf2fbe9db66bb2d56301204ee21939832335667 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 16 Nov 2023 20:58:10 +0100 Subject: [PATCH 08/16] Tiny styling change in index expression --- src/plugins/janus_audiobridge.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index b183ec8443..cadeb401fe 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -8903,8 +8903,8 @@ static void janus_audiobridge_participant_denoise(janus_audiobridge_participant j = 0; /* RNNoise needs 480 samples @ 48kHz */ while(i < FRAME_SIZE && j < chunk_size) { - denoised[i] = samples[2*j + 2*offset]; - denoised_alt[i] = samples[2*j + 1 + 2*offset]; + denoised[i] = samples[2 * (j + offset)]; + denoised_alt[i] = samples[2 * (j + offset) + 1]; i = i + 1 + empty_samples; j = j + 1; } @@ -8917,8 +8917,8 @@ static void janus_audiobridge_participant_denoise(janus_audiobridge_participant j = 0; /* Downsample to the original rate */ while(i < FRAME_SIZE && j < chunk_size) { - samples[2*j + 2*offset] = denoised[i]; - samples[2*j + 1 + 2*offset] = denoised_alt[i]; + samples[2 * (j + offset)] = denoised[i]; + samples[2 * (j + offset) + 1] = denoised_alt[i]; i = i + 1 + empty_samples; j = j + 1; } From fe0b345cb7f2d30760c9e31c14b2198d82f1ae99 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Mon, 20 Nov 2023 18:52:07 +0100 Subject: [PATCH 09/16] audiobridge: fix boolean setting for the first packet received --- src/plugins/janus_audiobridge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index cadeb401fe..1d84e6b2f5 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -8400,7 +8400,6 @@ static void *janus_audiobridge_participant_thread(void *data) { if(ret == JITTER_BUFFER_OK) { bpkt = (janus_audiobridge_buffer_packet *)jbp.data; janus_mutex_unlock(&participant->qmutex); - first = FALSE; locked = FALSE; rtp = (janus_rtp_header *)bpkt->buffer; /* If this is Opus, check if there's a packet gap we should fix with FEC */ @@ -8411,6 +8410,7 @@ static void *janus_audiobridge_participant_thread(void *data) { use_fec = TRUE; } } + first = FALSE; if(!g_atomic_int_compare_and_exchange(&participant->decoding, 0, 1)) { /* This means we're cleaning up, so don't try to decode */ janus_audiobridge_buffer_packet_destroy(bpkt); From ec52fbbcf3ccfcf43392743e071215ea26eb175d Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Mon, 20 Nov 2023 19:09:50 +0100 Subject: [PATCH 10/16] audiobrige: use fec only when received packet is expected plus one --- src/plugins/janus_audiobridge.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 1d84e6b2f5..5cee6d6042 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -8405,7 +8405,7 @@ static void *janus_audiobridge_participant_thread(void *data) { /* If this is Opus, check if there's a packet gap we should fix with FEC */ use_fec = FALSE; if(!first && participant->codec == JANUS_AUDIOCODEC_OPUS && participant->fec) { - if(ntohs(rtp->seq_number) != participant->expected_seq) { + if(ntohs(rtp->seq_number) == participant->expected_seq + 1) { /* Lost a packet here? Use FEC to recover */ use_fec = TRUE; } From 23386f4035611ed33e969cd0344d1be2289f5334 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Wed, 22 Nov 2023 15:14:40 +0100 Subject: [PATCH 11/16] Use speex resampler for upsampling and downsampling --- src/plugins/janus_audiobridge.c | 251 +++++++++++++++++++++++--------- 1 file changed, 180 insertions(+), 71 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 5cee6d6042..1e29ae0238 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -1035,6 +1035,7 @@ room-: { #include #endif #include +#include #ifdef HAVE_RNNOISE #include #endif @@ -1576,8 +1577,14 @@ typedef struct janus_audiobridge_participant { gboolean stereo; /* Whether stereo will be used for spatial audio */ int spatial_position; /* Panning of this participant in the mix */ #ifdef HAVE_RNNOISE +#define FRAME_SIZE 480 + SpeexResamplerState *upsampler; /* Speex upsampler used for denoising */ + SpeexResamplerState *downsampler; /* Speex downsampler used for denoising*/ gboolean denoise; /* Whether we should denoise this participant */ DenoiseState *rnnoise[2]; /* RNNoise states (we'll need two for stereo) */ + opus_int16 *upsample_buffer; /* buffer for upsampling */ + opus_int16 *downsample_buffer; /* buffer for downsampling */ + float *denoiser_buffer[2]; /* buffer for denoising*/ #endif /* RTP stuff */ JitterBuffer *jitter; /* Jitter buffer of incoming audio packets */ @@ -1678,6 +1685,10 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant /* This participant can be destroyed, free all the resources */ g_free(participant->user_id_str); g_free(participant->display); + if(participant->upsampler) + speex_resampler_destroy(participant->upsampler); + if(participant->downsampler) + speex_resampler_destroy(participant->downsampler); if(participant->encoder) opus_encoder_destroy(participant->encoder); if(participant->decoder) @@ -1705,6 +1716,14 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant rnnoise_destroy(participant->rnnoise[0]); if(participant->rnnoise[1]) rnnoise_destroy(participant->rnnoise[1]); + if(participant->upsample_buffer) + g_free(participant->upsample_buffer); + if(participant->downsample_buffer) + g_free(participant->downsample_buffer); + if(participant->denoiser_buffer[0]) + g_free(participant->denoiser_buffer[0]); + if(participant->denoiser_buffer[1]) + g_free(participant->denoiser_buffer[1]); #endif g_free(participant->mjr_base); #ifdef HAVE_LIBOGG @@ -1718,6 +1737,8 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant #ifdef HAVE_RNNOISE static void janus_audiobridge_participant_denoise(janus_audiobridge_participant *participant, char *data, int len); +static void janus_audiobridge_participant_upsample(janus_audiobridge_participant *participant, opus_int16 *input, int *in_len, opus_int16 *output, int *out_len); +static void janus_audiobridge_participant_downsample(janus_audiobridge_participant *participant, opus_int16 *input, int *in_len, opus_int16 *output, int *out_len); #endif static void janus_audiobridge_session_destroy(janus_audiobridge_session *session) { @@ -4365,6 +4386,11 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s participant->rnnoise[0] = rnnoise_create(NULL); if(participant->stereo) participant->rnnoise[1] = rnnoise_create(NULL); + participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + participant->denoiser_buffer[0] = g_malloc(FRAME_SIZE * sizeof(float)); + if(participant->stereo) + participant->denoiser_buffer[1] = g_malloc(FRAME_SIZE * sizeof(float)); } /* Prepare response */ @@ -5898,6 +5924,12 @@ static void janus_audiobridge_hangup_media_internal(janus_plugin_session *handle /* Make sure we're not using the encoder/decoder right now, we're going to destroy them */ while(!g_atomic_int_compare_and_exchange(&participant->encoding, 0, 1)) g_usleep(5000); + if(participant->upsampler) + speex_resampler_destroy(participant->upsampler); + participant->upsampler = NULL; + if(participant->downsampler) + speex_resampler_destroy(participant->downsampler); + participant->downsampler = NULL; if(participant->encoder) opus_encoder_destroy(participant->encoder); participant->encoder = NULL; @@ -6296,6 +6328,11 @@ static void *janus_audiobridge_handler(void *data) { participant->rnnoise[0] = rnnoise_create(NULL); if(participant->stereo) participant->rnnoise[1] = rnnoise_create(NULL); + participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + participant->denoiser_buffer[0] = g_malloc(FRAME_SIZE * sizeof(float)); + if(participant->stereo) + participant->denoiser_buffer[1] = g_malloc(FRAME_SIZE * sizeof(float)); } #else if(denoise && json_is_true(denoise)) { @@ -6320,6 +6357,22 @@ static void *janus_audiobridge_handler(void *data) { int error = 0; if(participant->encoder == NULL) { participant->sampling_rate = audiobridge->sampling_rate; + if(audiobridge->sampling_rate != 48000) { + spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; + spx_uint32_t from_rate = participant->sampling_rate; + spx_uint32_t to_rate = 48000; + int quality = 10; + participant->upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); + if(participant->upsampler != NULL) + JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", from_rate, to_rate, channels, quality); + else + JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); + participant->downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); + if(participant->downsampler != NULL) + JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); + else + JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); + } participant->encoder = opus_encoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { @@ -6370,6 +6423,12 @@ static void *janus_audiobridge_handler(void *data) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); g_free(participant->display); + if(participant->upsampler) + speex_resampler_destroy(participant->upsampler); + participant->upsampler = NULL; + if(participant->downsampler) + speex_resampler_destroy(participant->downsampler); + participant->downsampler = NULL; if(participant->encoder) opus_encoder_destroy(participant->encoder); participant->encoder = NULL; @@ -6728,6 +6787,11 @@ static void *janus_audiobridge_handler(void *data) { participant->rnnoise[0] = rnnoise_create(NULL); if(participant->stereo) participant->rnnoise[1] = rnnoise_create(NULL); + participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + participant->denoiser_buffer[0] = g_malloc(FRAME_SIZE * sizeof(float)); + if(participant->stereo) + participant->denoiser_buffer[1] = g_malloc(FRAME_SIZE * sizeof(float)); } #else if(denoise && json_is_true(denoise)) { @@ -7055,12 +7119,32 @@ static void *janus_audiobridge_handler(void *data) { participant->stereo = audiobridge->spatial_audio; participant->spatial_position = 50; int error = 0; + spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; + spx_uint32_t from_rate = participant->sampling_rate; + spx_uint32_t to_rate = 48000; + int quality = 10; + SpeexResamplerState *new_upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); + if(new_upsampler != NULL) + JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", from_rate, to_rate, channels, quality); + else + JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); + SpeexResamplerState *new_downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); + if(new_downsampler != NULL) + JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); + else + JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); OpusEncoder *new_encoder = opus_encoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { if(user_id_allocated) g_free(user_id_str); janus_refcount_decrease(&audiobridge->ref); + if(new_upsampler) + speex_resampler_destroy(new_upsampler); + new_upsampler = NULL; + if(new_downsampler) + speex_resampler_destroy(new_downsampler); + new_downsampler = NULL; if(new_encoder) opus_encoder_destroy(new_encoder); new_encoder = NULL; @@ -7100,6 +7184,12 @@ static void *janus_audiobridge_handler(void *data) { if(user_id_allocated) g_free(user_id_str); janus_refcount_decrease(&audiobridge->ref); + if(new_upsampler) + speex_resampler_destroy(new_upsampler); + new_upsampler = NULL; + if(new_downsampler) + speex_resampler_destroy(new_downsampler); + new_downsampler = NULL; if(new_encoder) opus_encoder_destroy(new_encoder); new_encoder = NULL; @@ -7120,6 +7210,12 @@ static void *janus_audiobridge_handler(void *data) { } participant->reset = FALSE; /* Destroy the previous encoder/decoder and update the references */ + if(participant->upsampler) + speex_resampler_destroy(participant->upsampler); + participant->upsampler = new_upsampler; + if(participant->downsampler) + speex_resampler_destroy(participant->downsampler); + participant->downsampler = new_downsampler; if(participant->encoder) opus_encoder_destroy(participant->encoder); participant->sampling_rate = audiobridge->sampling_rate; @@ -8848,82 +8944,95 @@ static void *janus_audiobridge_plainrtp_relay_thread(void *data) { } #ifdef HAVE_RNNOISE -#define FRAME_SIZE 480 static void janus_audiobridge_participant_denoise(janus_audiobridge_participant *participant, char *data, int len) { + if(len < 0 || data == NULL) + return; + /* Opus int16 original samples */ opus_int16 *samples = (opus_int16 *)data; - int i = 0, j = 0, offset = 0; - int total = len; - /* Copy the samples in a float buffer, as that's what RNNoise needs */ - float denoised[FRAME_SIZE]; - /* evaluate chunk size (number of samples per channel) for 10ms of audio */ - int chunk_size = participant->sampling_rate / 100; - /* evaluate the number of empty samples between samples in case of upsampling */ - int empty_samples = (FRAME_SIZE - chunk_size) / chunk_size; + /* Number of original samples, should be: 160 (8kHz), 320 (16kHz), 480 (24kHz), 960 (48kHz) */ + int samples_count = len; + /* Actual length of the resampled array (double size for stereo) */ + const int samples_len = !participant->stereo ? samples_count : 2*samples_count; + + /* Should be 960 */ + int upsample_buffer_count = len * (48000/participant->sampling_rate); + /* Upsampled buffer */ + opus_int16 *upsample_buffer = samples; + + /* Downsampled data samples count is equal to original samples */ + int downsample_buffer_count = samples_count; + /* Downsampled buffer */ + opus_int16 *downsample_buffer = upsample_buffer; + + /* Upsample */ + if(participant->sampling_rate != 48000) { + upsample_buffer = participant->upsample_buffer; + janus_audiobridge_participant_upsample(participant, samples, &samples_count, upsample_buffer, &upsample_buffer_count); + } + + int i = 0, j = 0; + float *denoiser_buffer = participant->denoiser_buffer[0]; + float *denoiser_buffer_alt = participant->denoiser_buffer[1]; + + /* Denoise in chunks of 480 samples */ + if(!participant->stereo) { + for(i=0; irnnoise[0], denoiser_buffer,denoiser_buffer); + for(j=0; jrnnoise[0], denoiser_buffer, denoiser_buffer); + rnnoise_process_frame(participant->rnnoise[1], denoiser_buffer_alt, denoiser_buffer_alt); + for(j=0; jsampling_rate != 48000) { + downsample_buffer = participant->downsample_buffer; + janus_audiobridge_participant_downsample(participant, upsample_buffer, &upsample_buffer_count, downsample_buffer, &downsample_buffer_count); + } + + /* Copy denoised and downsampled data back */ + memcpy(samples, downsample_buffer, samples_len*sizeof(opus_int16)); +} + +static void janus_audiobridge_participant_upsample(janus_audiobridge_participant *participant, opus_int16 *input, int *in_len, opus_int16 *output, int *out_len) { + if(!participant->stereo) { + int err = speex_resampler_process_int(participant->upsampler, 0, (spx_int16_t *)input, (spx_uint32_t *)in_len, (spx_int16_t *)output, (spx_uint32_t *)out_len); + if(err != 0) { + //TODO + } + } else { + int err = speex_resampler_process_interleaved_int(participant->upsampler, (spx_int16_t *)input, (spx_uint32_t *)in_len, (spx_int16_t *)output, (spx_uint32_t *)out_len); + if(err != 0) { + //TODO + } + } +} +static void janus_audiobridge_participant_downsample(janus_audiobridge_participant *participant, opus_int16 *input, int *in_len, opus_int16 *output, int *out_len) { if(!participant->stereo) { - /* Mono */ - while(total > 0) { - if(empty_samples > 0) { - /* Upsample with zero filling */ - memset(denoised, 0, sizeof(denoised)); - } - /* Denoise this audio chunk */ - i = 0; - j = 0; - /* RNNoise needs 480 samples @ 48kHz */ - while(i < FRAME_SIZE && j < chunk_size) { - denoised[i] = samples[j + offset]; - i = i + 1 + empty_samples; - j = j + 1; - } - /* Denoise */ - rnnoise_process_frame(participant->rnnoise[0], denoised, denoised); - /* Replace the audio data with the one we just denoised */ - i = 0; - j = 0; - /* Downsample to the original rate */ - while(i < FRAME_SIZE && j < chunk_size) { - samples[j + offset] = denoised[i]; - i = i + 1 + empty_samples; - j = j + 1; - } - total -= chunk_size; - offset += chunk_size; + int err = speex_resampler_process_int(participant->downsampler, 0, (spx_int16_t *)input, (spx_uint32_t *)in_len, (spx_int16_t *)output, (spx_uint32_t *)out_len); + if(err != 0) { + //TODO } } else { - /* Stereo (interleaved) */ - float denoised_alt[FRAME_SIZE]; - while(total > 0) { - if(empty_samples > 0) { - /* Upsample with zero filling */ - memset(denoised, 0, sizeof(denoised)); - memset(denoised_alt, 0, sizeof(denoised_alt)); - } - /* Denoise this audio chunk */ - i = 0; - j = 0; - /* RNNoise needs 480 samples @ 48kHz */ - while(i < FRAME_SIZE && j < chunk_size) { - denoised[i] = samples[2 * (j + offset)]; - denoised_alt[i] = samples[2 * (j + offset) + 1]; - i = i + 1 + empty_samples; - j = j + 1; - } - /* Denoise channel 1 */ - rnnoise_process_frame(participant->rnnoise[0], denoised, denoised); - /* Denoise channel 2 */ - rnnoise_process_frame(participant->rnnoise[1], denoised_alt, denoised_alt); - /* Replace the audio data with the one we just denoised */ - i = 0; - j = 0; - /* Downsample to the original rate */ - while(i < FRAME_SIZE && j < chunk_size) { - samples[2 * (j + offset)] = denoised[i]; - samples[2 * (j + offset) + 1] = denoised_alt[i]; - i = i + 1 + empty_samples; - j = j + 1; - } - total -= chunk_size; - offset += chunk_size; + int err = speex_resampler_process_interleaved_int(participant->downsampler, (spx_int16_t *)input, (spx_uint32_t *)in_len, (spx_int16_t *)output, (spx_uint32_t *)out_len); + if(err != 0) { + //TODO } } } From 812cc05bfb44ea16f4565b58ad2b5b17929d953a Mon Sep 17 00:00:00 2001 From: Lorenzo Miniero Date: Wed, 22 Nov 2023 15:33:24 +0100 Subject: [PATCH 12/16] Fixed denoise not tweakable via configure requests --- src/plugins/janus_audiobridge.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 1e29ae0238..cbcd473130 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -6324,7 +6324,7 @@ static void *janus_audiobridge_handler(void *data) { #ifdef HAVE_RNNOISE participant->denoise = denoise ? json_is_true(denoise) : audiobridge->denoise; if(participant->denoise && participant->rnnoise[0] == NULL) { - /* Create RNNoise context */ + /* Create RNNoise context(s) */ participant->rnnoise[0] = rnnoise_create(NULL); if(participant->stereo) participant->rnnoise[1] = rnnoise_create(NULL); @@ -6739,7 +6739,7 @@ static void *janus_audiobridge_handler(void *data) { } participant->group = group_id; } - if(muted || display || (participant->stereo && spatial)) { + if(muted || display || (participant->stereo && spatial) || denoise) { if(muted) { participant->muted = json_is_true(muted); JANUS_LOG(LOG_VERB, "Setting muted property: %s (room %s, user %s)\n", @@ -6783,7 +6783,7 @@ static void *janus_audiobridge_handler(void *data) { if(denoise) participant->denoise = json_is_true(denoise); if(participant->denoise && participant->rnnoise[0] == NULL) { - /* Create RNNoise context */ + /* Create RNNoise context(s) */ participant->rnnoise[0] = rnnoise_create(NULL); if(participant->stereo) participant->rnnoise[1] = rnnoise_create(NULL); @@ -9036,4 +9036,4 @@ static void janus_audiobridge_participant_downsample(janus_audiobridge_participa } } } -#endif \ No newline at end of file +#endif From 36e54a6bfe06e733dc81d8e4832206c7e874ac09 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Wed, 22 Nov 2023 15:35:14 +0100 Subject: [PATCH 13/16] Do not destroy resamplers when hanging up --- src/plugins/janus_audiobridge.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index cbcd473130..1cda766d88 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -5924,12 +5924,6 @@ static void janus_audiobridge_hangup_media_internal(janus_plugin_session *handle /* Make sure we're not using the encoder/decoder right now, we're going to destroy them */ while(!g_atomic_int_compare_and_exchange(&participant->encoding, 0, 1)) g_usleep(5000); - if(participant->upsampler) - speex_resampler_destroy(participant->upsampler); - participant->upsampler = NULL; - if(participant->downsampler) - speex_resampler_destroy(participant->downsampler); - participant->downsampler = NULL; if(participant->encoder) opus_encoder_destroy(participant->encoder); participant->encoder = NULL; From 2ca13b51ed3a09a39b254338dee35451c9bf6f5b Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Wed, 22 Nov 2023 16:07:07 +0100 Subject: [PATCH 14/16] Create resamplers just once. Add missing macros. --- src/plugins/janus_audiobridge.c | 69 +++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index 1cda766d88..dfefece648 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -1685,10 +1685,6 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant /* This participant can be destroyed, free all the resources */ g_free(participant->user_id_str); g_free(participant->display); - if(participant->upsampler) - speex_resampler_destroy(participant->upsampler); - if(participant->downsampler) - speex_resampler_destroy(participant->downsampler); if(participant->encoder) opus_encoder_destroy(participant->encoder); if(participant->decoder) @@ -1716,14 +1712,18 @@ static void janus_audiobridge_participant_free(const janus_refcount *participant rnnoise_destroy(participant->rnnoise[0]); if(participant->rnnoise[1]) rnnoise_destroy(participant->rnnoise[1]); - if(participant->upsample_buffer) - g_free(participant->upsample_buffer); - if(participant->downsample_buffer) - g_free(participant->downsample_buffer); if(participant->denoiser_buffer[0]) g_free(participant->denoiser_buffer[0]); if(participant->denoiser_buffer[1]) g_free(participant->denoiser_buffer[1]); + if(participant->upsampler) + speex_resampler_destroy(participant->upsampler); + if(participant->downsampler) + speex_resampler_destroy(participant->downsampler); + if(participant->upsample_buffer) + g_free(participant->upsample_buffer); + if(participant->downsample_buffer) + g_free(participant->downsample_buffer); #endif g_free(participant->mjr_base); #ifdef HAVE_LIBOGG @@ -6320,13 +6320,30 @@ static void *janus_audiobridge_handler(void *data) { if(participant->denoise && participant->rnnoise[0] == NULL) { /* Create RNNoise context(s) */ participant->rnnoise[0] = rnnoise_create(NULL); - if(participant->stereo) - participant->rnnoise[1] = rnnoise_create(NULL); - participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); - participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); participant->denoiser_buffer[0] = g_malloc(FRAME_SIZE * sizeof(float)); - if(participant->stereo) + if(participant->stereo) { + participant->rnnoise[1] = rnnoise_create(NULL); participant->denoiser_buffer[1] = g_malloc(FRAME_SIZE * sizeof(float)); + } + if(audiobridge->sampling_rate != 48000) { + spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; + spx_uint32_t from_rate = audiobridge->sampling_rate; + spx_uint32_t to_rate = 48000; + int quality = 10; + int error; + participant->upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); + if(participant->upsampler != NULL) + JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", from_rate, to_rate, channels, quality); + else + JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); + participant->downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); + if(participant->downsampler != NULL) + JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); + else + JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); + participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + } } #else if(denoise && json_is_true(denoise)) { @@ -6351,22 +6368,6 @@ static void *janus_audiobridge_handler(void *data) { int error = 0; if(participant->encoder == NULL) { participant->sampling_rate = audiobridge->sampling_rate; - if(audiobridge->sampling_rate != 48000) { - spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; - spx_uint32_t from_rate = participant->sampling_rate; - spx_uint32_t to_rate = 48000; - int quality = 10; - participant->upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); - if(participant->upsampler != NULL) - JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", from_rate, to_rate, channels, quality); - else - JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); - participant->downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); - if(participant->downsampler != NULL) - JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); - else - JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); - } participant->encoder = opus_encoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { @@ -6417,12 +6418,14 @@ static void *janus_audiobridge_handler(void *data) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); g_free(participant->display); +#ifdef HAVE_RNNOISE if(participant->upsampler) speex_resampler_destroy(participant->upsampler); participant->upsampler = NULL; if(participant->downsampler) speex_resampler_destroy(participant->downsampler); participant->downsampler = NULL; +#endif if(participant->encoder) opus_encoder_destroy(participant->encoder); participant->encoder = NULL; @@ -7113,6 +7116,7 @@ static void *janus_audiobridge_handler(void *data) { participant->stereo = audiobridge->spatial_audio; participant->spatial_position = 50; int error = 0; +#ifdef HAVE_RNNOISE spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; spx_uint32_t from_rate = participant->sampling_rate; spx_uint32_t to_rate = 48000; @@ -7127,18 +7131,21 @@ static void *janus_audiobridge_handler(void *data) { JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); else JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); +#endif OpusEncoder *new_encoder = opus_encoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { if(user_id_allocated) g_free(user_id_str); janus_refcount_decrease(&audiobridge->ref); +#ifdef HAVE_RNNOISE if(new_upsampler) speex_resampler_destroy(new_upsampler); new_upsampler = NULL; if(new_downsampler) speex_resampler_destroy(new_downsampler); new_downsampler = NULL; +#endif if(new_encoder) opus_encoder_destroy(new_encoder); new_encoder = NULL; @@ -7178,12 +7185,14 @@ static void *janus_audiobridge_handler(void *data) { if(user_id_allocated) g_free(user_id_str); janus_refcount_decrease(&audiobridge->ref); +#ifdef HAVE_RNNOISE if(new_upsampler) speex_resampler_destroy(new_upsampler); new_upsampler = NULL; if(new_downsampler) speex_resampler_destroy(new_downsampler); new_downsampler = NULL; +#endif if(new_encoder) opus_encoder_destroy(new_encoder); new_encoder = NULL; @@ -7204,12 +7213,14 @@ static void *janus_audiobridge_handler(void *data) { } participant->reset = FALSE; /* Destroy the previous encoder/decoder and update the references */ +#ifdef HAVE_RNNOISE if(participant->upsampler) speex_resampler_destroy(participant->upsampler); participant->upsampler = new_upsampler; if(participant->downsampler) speex_resampler_destroy(participant->downsampler); participant->downsampler = new_downsampler; +#endif if(participant->encoder) opus_encoder_destroy(participant->encoder); participant->sampling_rate = audiobridge->sampling_rate; From 975502ced33de9dda20c799ae03bbb49558a6824 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Wed, 22 Nov 2023 16:18:33 +0100 Subject: [PATCH 15/16] Create resamplers later when handling changeroom --- src/plugins/janus_audiobridge.c | 62 +++++++++++++-------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index dfefece648..f12a490f4d 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -7116,36 +7116,12 @@ static void *janus_audiobridge_handler(void *data) { participant->stereo = audiobridge->spatial_audio; participant->spatial_position = 50; int error = 0; -#ifdef HAVE_RNNOISE - spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; - spx_uint32_t from_rate = participant->sampling_rate; - spx_uint32_t to_rate = 48000; - int quality = 10; - SpeexResamplerState *new_upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); - if(new_upsampler != NULL) - JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", from_rate, to_rate, channels, quality); - else - JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); - SpeexResamplerState *new_downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); - if(new_downsampler != NULL) - JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); - else - JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); -#endif OpusEncoder *new_encoder = opus_encoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); if(error != OPUS_OK) { if(user_id_allocated) g_free(user_id_str); janus_refcount_decrease(&audiobridge->ref); -#ifdef HAVE_RNNOISE - if(new_upsampler) - speex_resampler_destroy(new_upsampler); - new_upsampler = NULL; - if(new_downsampler) - speex_resampler_destroy(new_downsampler); - new_downsampler = NULL; -#endif if(new_encoder) opus_encoder_destroy(new_encoder); new_encoder = NULL; @@ -7185,14 +7161,6 @@ static void *janus_audiobridge_handler(void *data) { if(user_id_allocated) g_free(user_id_str); janus_refcount_decrease(&audiobridge->ref); -#ifdef HAVE_RNNOISE - if(new_upsampler) - speex_resampler_destroy(new_upsampler); - new_upsampler = NULL; - if(new_downsampler) - speex_resampler_destroy(new_downsampler); - new_downsampler = NULL; -#endif if(new_encoder) opus_encoder_destroy(new_encoder); new_encoder = NULL; @@ -7214,12 +7182,30 @@ static void *janus_audiobridge_handler(void *data) { participant->reset = FALSE; /* Destroy the previous encoder/decoder and update the references */ #ifdef HAVE_RNNOISE - if(participant->upsampler) - speex_resampler_destroy(participant->upsampler); - participant->upsampler = new_upsampler; - if(participant->downsampler) - speex_resampler_destroy(participant->downsampler); - participant->downsampler = new_downsampler; + spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; + spx_uint32_t from_rate = participant->sampling_rate; + spx_uint32_t to_rate = 48000; + int quality = 10; + SpeexResamplerState *new_upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); + if(new_upsampler != NULL) + JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", from_rate, to_rate, channels, quality); + else + JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); + SpeexResamplerState *new_downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); + if(new_downsampler != NULL) + JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); + else + JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); + if(new_upsampler != NULL) { + if(participant->upsampler) + speex_resampler_destroy(participant->upsampler); + participant->upsampler = new_upsampler; + } + if(new_downsampler != NULL) { + if(participant->downsampler) + speex_resampler_destroy(participant->downsampler); + participant->downsampler = new_downsampler; + } #endif if(participant->encoder) opus_encoder_destroy(participant->encoder); From 5189caf5bedc3bde5f4a30085124df216e5957a5 Mon Sep 17 00:00:00 2001 From: Alessandro Toppi Date: Thu, 23 Nov 2023 13:29:47 +0100 Subject: [PATCH 16/16] Initialize denoiser stuff only in participant thread. Use quality=8 for resamplers. Always return the denoise flag in the admin response. Add missing docs. --- src/plugins/janus_audiobridge.c | 233 ++++++++++++++++---------------- 1 file changed, 114 insertions(+), 119 deletions(-) diff --git a/src/plugins/janus_audiobridge.c b/src/plugins/janus_audiobridge.c index f12a490f4d..953d646269 100644 --- a/src/plugins/janus_audiobridge.c +++ b/src/plugins/janus_audiobridge.c @@ -942,7 +942,10 @@ room-: { "muted" : , "bitrate" : , "quality" : <0-10, Opus-related complexity to use, higher is higher quality; optional, default is 4>, - "expected_loss" : <0-20, a percentage of the expected loss (capped at 20%), only needed in case FEC is used; optional, default is 0 (FEC disabled even when negotiated) or the room default> + "expected_loss" : <0-20, a percentage of the expected loss (capped at 20%), only needed in case FEC is used; optional, default is 0 (FEC disabled even when negotiated) or the room default>, + "volume" : , + "spatial_position" : , + "denoise" : } \endverbatim * @@ -1577,14 +1580,16 @@ typedef struct janus_audiobridge_participant { gboolean stereo; /* Whether stereo will be used for spatial audio */ int spatial_position; /* Panning of this participant in the mix */ #ifdef HAVE_RNNOISE -#define FRAME_SIZE 480 +#define DENOISER_FRAME_SIZE 480 + gboolean denoise; /* Whether we should denoise this participant */ + DenoiseState *rnnoise[2]; /* RNNoise states (we'll need two for stereo) */ + uint32_t resampler_rate; /* Sampling rate of the resamplers */ + gboolean resampler_stereo; /* Whether the current resamplers are stereo */ SpeexResamplerState *upsampler; /* Speex upsampler used for denoising */ SpeexResamplerState *downsampler; /* Speex downsampler used for denoising*/ - gboolean denoise; /* Whether we should denoise this participant */ - DenoiseState *rnnoise[2]; /* RNNoise states (we'll need two for stereo) */ - opus_int16 *upsample_buffer; /* buffer for upsampling */ - opus_int16 *downsample_buffer; /* buffer for downsampling */ - float *denoiser_buffer[2]; /* buffer for denoising*/ + opus_int16 *upsample_buffer; /* Buffer for upsampling */ + opus_int16 *downsample_buffer; /* Buffer for downsampling */ + float *denoiser_buffer[2]; /* Buffer for denoising */ #endif /* RTP stuff */ JitterBuffer *jitter; /* Jitter buffer of incoming audio packets */ @@ -2885,8 +2890,7 @@ json_t *janus_audiobridge_query_session(janus_plugin_session *handle) { if(participant->stereo) json_object_set_new(info, "spatial_position", json_integer(participant->spatial_position)); #ifdef HAVE_RNNOISE - if(participant->denoise) - json_object_set_new(info, "denoise", json_true()); + json_object_set_new(info, "denoise", participant->denoise ? json_true() : json_false()); #endif if(participant->arc && participant->arc->filename) json_object_set_new(info, "audio-recording", json_string(participant->arc->filename)); @@ -4381,17 +4385,6 @@ static json_t *janus_audiobridge_process_synchronous_request(janus_audiobridge_s } participant->denoise = denoise; - if(participant->denoise && participant->rnnoise[0] == NULL) { - /* Create RNNoise context(s) */ - participant->rnnoise[0] = rnnoise_create(NULL); - if(participant->stereo) - participant->rnnoise[1] = rnnoise_create(NULL); - participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); - participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); - participant->denoiser_buffer[0] = g_malloc(FRAME_SIZE * sizeof(float)); - if(participant->stereo) - participant->denoiser_buffer[1] = g_malloc(FRAME_SIZE * sizeof(float)); - } /* Prepare response */ response = json_object(); @@ -6315,41 +6308,6 @@ static void *janus_audiobridge_handler(void *data) { spatial_position = 100; participant->spatial_position = spatial_position; } -#ifdef HAVE_RNNOISE - participant->denoise = denoise ? json_is_true(denoise) : audiobridge->denoise; - if(participant->denoise && participant->rnnoise[0] == NULL) { - /* Create RNNoise context(s) */ - participant->rnnoise[0] = rnnoise_create(NULL); - participant->denoiser_buffer[0] = g_malloc(FRAME_SIZE * sizeof(float)); - if(participant->stereo) { - participant->rnnoise[1] = rnnoise_create(NULL); - participant->denoiser_buffer[1] = g_malloc(FRAME_SIZE * sizeof(float)); - } - if(audiobridge->sampling_rate != 48000) { - spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; - spx_uint32_t from_rate = audiobridge->sampling_rate; - spx_uint32_t to_rate = 48000; - int quality = 10; - int error; - participant->upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); - if(participant->upsampler != NULL) - JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", from_rate, to_rate, channels, quality); - else - JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); - participant->downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); - if(participant->downsampler != NULL) - JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); - else - JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); - participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); - participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); - } - } -#else - if(denoise && json_is_true(denoise)) { - JANUS_LOG(LOG_WARN, "RNNoise unavailable, denoising not supported\n"); - } -#endif participant->user_audio_active_packets = json_integer_value(user_audio_active_packets); participant->user_audio_level_average = json_integer_value(user_audio_level_average); if(participant->outbuf == NULL) @@ -6418,14 +6376,6 @@ static void *janus_audiobridge_handler(void *data) { janus_mutex_unlock(&audiobridge->mutex); janus_refcount_decrease(&audiobridge->ref); g_free(participant->display); -#ifdef HAVE_RNNOISE - if(participant->upsampler) - speex_resampler_destroy(participant->upsampler); - participant->upsampler = NULL; - if(participant->downsampler) - speex_resampler_destroy(participant->downsampler); - participant->downsampler = NULL; -#endif if(participant->encoder) opus_encoder_destroy(participant->encoder); participant->encoder = NULL; @@ -6439,6 +6389,13 @@ static void *janus_audiobridge_handler(void *data) { goto error; } } +#ifdef HAVE_RNNOISE + participant->denoise = denoise ? json_is_true(denoise) : audiobridge->denoise; +#else + if(denoise && json_is_true(denoise)) { + JANUS_LOG(LOG_WARN, "RNNoise unavailable, denoising not supported\n"); + } +#endif participant->reset = FALSE; /* If this is a plain RTP participant, create the socket */ if(rtp != NULL) { @@ -6779,17 +6736,6 @@ static void *janus_audiobridge_handler(void *data) { #ifdef HAVE_RNNOISE if(denoise) participant->denoise = json_is_true(denoise); - if(participant->denoise && participant->rnnoise[0] == NULL) { - /* Create RNNoise context(s) */ - participant->rnnoise[0] = rnnoise_create(NULL); - if(participant->stereo) - participant->rnnoise[1] = rnnoise_create(NULL); - participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); - participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); - participant->denoiser_buffer[0] = g_malloc(FRAME_SIZE * sizeof(float)); - if(participant->stereo) - participant->denoiser_buffer[1] = g_malloc(FRAME_SIZE * sizeof(float)); - } #else if(denoise && json_is_true(denoise)) { JANUS_LOG(LOG_WARN, "RNNoise unavailable, denoising not supported\n"); @@ -7018,8 +6964,9 @@ static void *janus_audiobridge_handler(void *data) { json_t *bitrate = json_object_get(root, "bitrate"); json_t *quality = json_object_get(root, "quality"); json_t *exploss = json_object_get(root, "expected_loss"); + json_t *denoise = json_object_get(root, "denoise"); int volume = gain ? json_integer_value(gain) : 100; - int spatial_position = spatial ? json_integer_value(spatial) : 64; + int spatial_position = spatial ? json_integer_value(spatial) : 50; int32_t opus_bitrate = audiobridge->default_bitrate; if(bitrate) { opus_bitrate = json_integer_value(bitrate); @@ -7113,8 +7060,6 @@ static void *janus_audiobridge_handler(void *data) { if(old_audiobridge->sampling_rate != audiobridge->sampling_rate || old_audiobridge->spatial_audio != audiobridge->spatial_audio) { /* Create a new one that takes into account the sampling rate we want now */ - participant->stereo = audiobridge->spatial_audio; - participant->spatial_position = 50; int error = 0; OpusEncoder *new_encoder = opus_encoder_create(audiobridge->sampling_rate, audiobridge->spatial_audio ? 2 : 1, OPUS_APPLICATION_VOIP, &error); @@ -7181,39 +7126,19 @@ static void *janus_audiobridge_handler(void *data) { } participant->reset = FALSE; /* Destroy the previous encoder/decoder and update the references */ -#ifdef HAVE_RNNOISE - spx_uint32_t channels = !audiobridge->spatial_audio ? 1 : 2; - spx_uint32_t from_rate = participant->sampling_rate; - spx_uint32_t to_rate = 48000; - int quality = 10; - SpeexResamplerState *new_upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); - if(new_upsampler != NULL) - JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", from_rate, to_rate, channels, quality); - else - JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); - SpeexResamplerState *new_downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); - if(new_downsampler != NULL) - JANUS_LOG(LOG_INFO, "Created resampler from %d to %d (channels=%d, quality=%d)\n", to_rate, from_rate, channels, quality); - else - JANUS_LOG(LOG_ERR, "Error creating resampler %s\n", speex_resampler_strerror(error)); - if(new_upsampler != NULL) { - if(participant->upsampler) - speex_resampler_destroy(participant->upsampler); - participant->upsampler = new_upsampler; - } - if(new_downsampler != NULL) { - if(participant->downsampler) - speex_resampler_destroy(participant->downsampler); - participant->downsampler = new_downsampler; - } -#endif + while(!g_atomic_int_compare_and_exchange(&participant->encoding, 0, 1)) + g_usleep(5000); if(participant->encoder) opus_encoder_destroy(participant->encoder); participant->sampling_rate = audiobridge->sampling_rate; participant->encoder = new_encoder; + g_atomic_int_set(&participant->encoding, 0); + while(!g_atomic_int_compare_and_exchange(&participant->decoding, 0, 1)) + g_usleep(5000); if(participant->decoder) opus_decoder_destroy(participant->decoder); participant->decoder = new_decoder; + g_atomic_int_set(&participant->decoding, 0); } if(quality) opus_encoder_ctl(participant->encoder, OPUS_SET_COMPLEXITY(participant->opus_complexity)); @@ -7287,6 +7212,14 @@ static void *janus_audiobridge_handler(void *data) { participant->expected_loss = expected_loss; opus_encoder_ctl(participant->encoder, OPUS_SET_PACKET_LOSS_PERC(participant->expected_loss)); } +#ifdef HAVE_RNNOISE + /* Check if a denoiser is needed now */ + participant->denoise = denoise ? json_is_true(denoise) : audiobridge->denoise; +#else + if(denoise && json_is_true(denoise)) { + JANUS_LOG(LOG_WARN, "RNNoise unavailable, denoising not supported\n"); + } +#endif g_hash_table_insert(audiobridge->participants, string_ids ? (gpointer)g_strdup(participant->user_id_str) : (gpointer)janus_uint64_dup(participant->user_id), participant); @@ -8528,7 +8461,7 @@ static void *janus_audiobridge_participant_thread(void *data) { pkt->length = opus_decode(participant->decoder, payload, plen, (opus_int16 *)pkt->data, output_samples, 1); #ifdef HAVE_RNNOISE /* Check if we need to denoise this packet */ - if(participant->rnnoise[0] && participant->denoise) + if(participant->denoise) janus_audiobridge_participant_denoise(participant, (char *)pkt->data, pkt->length); #endif /* Queue the decoded redundant packet for the mixer */ @@ -8572,7 +8505,7 @@ static void *janus_audiobridge_participant_thread(void *data) { } #ifdef HAVE_RNNOISE /* Check if we need to denoise this packet */ - if(participant->rnnoise[0] && participant->denoise) + if(participant->denoise) janus_audiobridge_participant_denoise(participant, (char *)pkt->data, pkt->length); #endif /* Get rid of the buffered packet */ @@ -8938,15 +8871,77 @@ static void *janus_audiobridge_plainrtp_relay_thread(void *data) { static void janus_audiobridge_participant_denoise(janus_audiobridge_participant *participant, char *data, int len) { if(len < 0 || data == NULL) return; + /* Create a denoiser if we still don't have one */ + if(participant->rnnoise[0] == NULL) { + /* Create RNNoise context */ + participant->rnnoise[0] = rnnoise_create(NULL); + /* If we still don't have a denoiser, give up */ + if(participant->rnnoise[0] == NULL) + return; + /* Allocate the buffer for the denoiser */ + if(participant->denoiser_buffer[0] == NULL) + participant->denoiser_buffer[0] = g_malloc(DENOISER_FRAME_SIZE * sizeof(float)); + } + /* Check if we need a denoiser for stereo channel too */ + if(participant->stereo && participant->rnnoise[1] == NULL) { + /* Create RNNoise context */ + participant->rnnoise[1] = rnnoise_create(NULL); + /* If we still don't have a denoiser, give up */ + if(participant->rnnoise[1] == NULL) + return; + /* Allocate the buffer for the denoiser */ + if(participant->denoiser_buffer[1] == NULL) + participant->denoiser_buffer[1] = g_malloc(DENOISER_FRAME_SIZE * sizeof(float)); + } + /* Check if we need to (re)create resamplers too */ + if(participant->sampling_rate != participant->resampler_rate || + participant->stereo != participant->resampler_stereo) { + participant->resampler_rate = participant->sampling_rate; + participant->resampler_stereo = participant->stereo; + if(participant->upsampler) + speex_resampler_destroy(participant->upsampler); + participant->upsampler = NULL; + if(participant->downsampler) + speex_resampler_destroy(participant->downsampler); + participant->downsampler = NULL; + /* We need resamplers only if rate is not 48kHz */ + if(participant->resampler_rate != 48000) { + spx_uint32_t channels = !participant->resampler_stereo ? 1 : 2; + spx_uint32_t from_rate = participant->resampler_rate; + spx_uint32_t to_rate = 48000; + int quality = 8, error = 0; + participant->upsampler = speex_resampler_init(channels, from_rate, to_rate, quality, &error); + if(participant->upsampler != NULL) { + JANUS_LOG(LOG_INFO, "Created %s resampler from %d to %d (channels=%d, quality=%d)\n", + (participant->resampler_stereo ? "stereo" : "mono"), from_rate, to_rate, channels, quality); + } else { + /* We couldn't create a resampler, don't do anything */ + return; + } + participant->downsampler = speex_resampler_init(channels, to_rate, from_rate, quality, &error); + if(participant->downsampler != NULL) { + JANUS_LOG(LOG_INFO, "Created %s resampler from %d to %d (channels=%d, quality=%d)\n", + (participant->resampler_stereo ? "stereo" : "mono"), to_rate, from_rate, channels, quality); + } else { + /* We couldn't create a resampler, don't do anything */ + return; + } + if(participant->upsample_buffer == NULL) + participant->upsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + if(participant->downsample_buffer == NULL) + participant->downsample_buffer = g_malloc(2 * OPUS_SAMPLES * sizeof(opus_int16)); + } + } + /* Opus int16 original samples */ opus_int16 *samples = (opus_int16 *)data; /* Number of original samples, should be: 160 (8kHz), 320 (16kHz), 480 (24kHz), 960 (48kHz) */ int samples_count = len; /* Actual length of the resampled array (double size for stereo) */ - const int samples_len = !participant->stereo ? samples_count : 2*samples_count; + const int samples_len = !participant->resampler_stereo ? samples_count : 2*samples_count; /* Should be 960 */ - int upsample_buffer_count = len * (48000/participant->sampling_rate); + int upsample_buffer_count = len * (48000/participant->resampler_rate); /* Upsampled buffer */ opus_int16 *upsample_buffer = samples; @@ -8956,7 +8951,7 @@ static void janus_audiobridge_participant_denoise(janus_audiobridge_participant opus_int16 *downsample_buffer = upsample_buffer; /* Upsample */ - if(participant->sampling_rate != 48000) { + if(participant->resampler_rate != 48000) { upsample_buffer = participant->upsample_buffer; janus_audiobridge_participant_upsample(participant, samples, &samples_count, upsample_buffer, &upsample_buffer_count); } @@ -8966,25 +8961,25 @@ static void janus_audiobridge_participant_denoise(janus_audiobridge_participant float *denoiser_buffer_alt = participant->denoiser_buffer[1]; /* Denoise in chunks of 480 samples */ - if(!participant->stereo) { - for(i=0; iresampler_stereo) { + for(i=0; irnnoise[0], denoiser_buffer,denoiser_buffer); - for(j=0; jrnnoise[0], denoiser_buffer, denoiser_buffer); + for(j=0; jrnnoise[0], denoiser_buffer, denoiser_buffer); rnnoise_process_frame(participant->rnnoise[1], denoiser_buffer_alt, denoiser_buffer_alt); - for(j=0; jsampling_rate != 48000) { + if(participant->resampler_rate != 48000) { downsample_buffer = participant->downsample_buffer; janus_audiobridge_participant_downsample(participant, upsample_buffer, &upsample_buffer_count, downsample_buffer, &downsample_buffer_count); } @@ -9002,7 +8997,7 @@ static void janus_audiobridge_participant_denoise(janus_audiobridge_participant } static void janus_audiobridge_participant_upsample(janus_audiobridge_participant *participant, opus_int16 *input, int *in_len, opus_int16 *output, int *out_len) { - if(!participant->stereo) { + if(!participant->resampler_stereo) { int err = speex_resampler_process_int(participant->upsampler, 0, (spx_int16_t *)input, (spx_uint32_t *)in_len, (spx_int16_t *)output, (spx_uint32_t *)out_len); if(err != 0) { //TODO @@ -9015,7 +9010,7 @@ static void janus_audiobridge_participant_upsample(janus_audiobridge_participant } } static void janus_audiobridge_participant_downsample(janus_audiobridge_participant *participant, opus_int16 *input, int *in_len, opus_int16 *output, int *out_len) { - if(!participant->stereo) { + if(!participant->resampler_stereo) { int err = speex_resampler_process_int(participant->downsampler, 0, (spx_int16_t *)input, (spx_uint32_t *)in_len, (spx_int16_t *)output, (spx_uint32_t *)out_len); if(err != 0) { //TODO