From 0613b8e6ced517a70fe1a836b5074a45b2c800cf Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 20 Jan 2022 23:06:03 +0100 Subject: [PATCH 01/11] Partial changes --- libraries/plugins/src/plugins/CodecPlugin.h | 189 ++++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index cb5b857be8e..0c341a91173 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -10,14 +10,198 @@ // #pragma once +#include #include "Plugin.h" +#include + +#ifndef VIRCADIA_CODECPLUGIN_H +#define VIRCADIA_CODECPLUGIN_H class Encoder { public: + enum class EncoderBandpass { + Narrowband, Mediumband, Wideband, Superwideband, Fullband + }; + + enum class EncoderSignal { + Auto, Voice, Music + }; + + enum class EncoderApplication { + VOIP, Audio, LowDelay + }; + +/* + enum class Capabilities : int { + None = 0x00, + Lossless = 0x01, + Complexity = 0x02, + Bitrate = 0x04, + FEC = 0x08, + Bandpass = 0x10, + SignalType = 0x20, + VBR = 0x40 + }; +*/ + + virtual ~Encoder() { } virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; + + /** + * @brief Get the codec's name + * + * Returns the name of the codec, for usage in debug output and UI. + * + * @return QString Codec's name + */ + virtual const QString getName() const { return "Encoder"; } + + virtual bool isLossless() const { return false; } + + /** + * @brief Get the codec's capabilities + * + * This is a bit field of which of the possible settings have effect on this codec. + * + * For example, a codec capable of adding redundancy to compensate for packet loss will have the FEC flag set. + * + * @return Capabilities + */ + //virtual Capabilities getCapabilities() { return Capabilities::None; } + + virtual void setApplication(EncoderApplication app) { _application = app; } + virtual bool hasApplication() const { return false; } + EncoderApplication getApplication() const { return _application; } + + /** + * @brief Set the codec's complexity (CPU usage) + * + * This defines how much CPU time is spent on encoding. Lower values are expected to have + * lower quality or higher bandwidth usage, depending on the codec. + * + * @param complexity A value from 0 to 100. + */ + virtual void setComplexity(int complexity) { _complexity = qBound(complexity, 0, 100); } + virtual bool hasComplexity() const { return false; } + int getComplexity() const { return _complexity; } + + /** + * @brief Set the bitrate + * + * How many bits per second the codec is going to use. This only applies to lossy codecs. + * + * @param bitrate A rate in bits per second from 500 to 512000 + */ + virtual void setBitrate(int bitrate) { _bitrate = qBound(bitrate, 500, 512000); } + virtual bool hasBitrate() const { return false; } + int getBitrate() const { return _bitrate; } + + /** + * @brief Sets whether to use Forward Error Correction + * + * When enabled, redundant data is included in the codec's output to compensate for packet loss + * @param enabled + */ + virtual void setFEC(bool enabled) { _useFEC = enabled; } + virtual bool hasFEC() const { return false; } + bool getFEC() const { return _useFEC; } + + /** + * @brief Set how much packet loss to tolerate + * + * This is how much loss we expect to have. Higher values may consume bandwidth or lower quality to compensate. + * + * @param percent 0 to 100 + */ + virtual void setPacketLossPercent(int percent) { _packetLossPercent = qBound(percent, 0, 100); } + virtual bool hasPacketLossPercent() const { return false; } + int getPacketLossPercent() const { return _packetLossPercent; } + + /** + * @brief Set the bandpass + * + * @param bandpass + */ + virtual void setBandpass(EncoderBandpass bandpass) { _bandpass = bandpass;} + virtual bool hasBandpass() const { return false; } + EncoderBandpass getBandpass() const { return _bandpass; } + + /** + * @brief Set the type of audio signal + * + * @param signal + */ + virtual void setSignalType(EncoderSignal signal) { _signal = signal;} + virtual bool hasSignalType() const { return false; } + EncoderSignal getSignalType() const { return _signal; } + + /** + * @brief Whether to use variable bitrate + * + * @param use_vbr + */ + virtual void setVBR(bool use_vbr) { _useVBR = use_vbr; } + virtual bool hasVBR() const { return false; } + bool getVBR() const { return _useVBR; } + + QDebug operator<<(QDebug debug) { + debug << "{ Codec = " << getName(); + if ( hasComplexity() ) { + debug << "; Complexity = " << getComplexity(); + } + + if ( hasBitrate() ) { + debug << "; Bitrate = " << getBitrate(); + } + + if ( hasFEC() ) { + debug << "; FEC = " << getFEC(); + } + + if ( hasPacketLossPercent() ) { + debug << "; PacketLoss = " << getPacketLossPercent(); + } + + if ( hasBandpass() ) { + debug << "; Bandpass = " << (int)getBandpass(); + } + + if ( hasSignalType() ) { + debug << "; Signal = " << (int)getSignalType(); + } + + if ( hasVBR() ) { + debug << "; VBR = " << getVBR(); + } + + debug << "}"; + + return debug; + } + +/* + Encoder::Capabilities operator|(Encoder::Capabilities lhs, Encoder::Capabilities rhs) { + return static_cast( + static_cast::type>(lhs) | + static_cast::type>(rhs) + ); + +} + */ +protected: + int _complexity; + int _bitrate; + bool _useFEC; + bool _useVBR; + int _packetLossPercent; + EncoderBandpass _bandpass; + EncoderSignal _signal; + EncoderApplication _application; }; + + class Decoder { public: virtual ~Decoder() { } @@ -26,6 +210,9 @@ class Decoder { virtual void lostFrame(QByteArray& decodedBuffer) = 0; }; + + + class CodecPlugin : public Plugin { public: virtual Encoder* createEncoder(int sampleRate, int numChannels) = 0; @@ -33,3 +220,5 @@ class CodecPlugin : public Plugin { virtual void releaseEncoder(Encoder* encoder) = 0; virtual void releaseDecoder(Decoder* decoder) = 0; }; + +#endif From 7aae31f5aa5e28e9dfa3c938f2361605b719383d Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Wed, 26 Jan 2022 23:11:24 +0100 Subject: [PATCH 02/11] Partial implementation --- assignment-client/src/Agent.cpp | 7 +- assignment-client/src/Agent.h | 1 + assignment-client/src/audio/AudioMixer.cpp | 5 + assignment-client/src/audio/AudioMixer.h | 18 +- .../src/audio/AudioMixerClientData.cpp | 6 +- .../src/audio/AudioMixerClientData.h | 2 + .../src/scripts/EntityScriptServer.cpp | 16 +- .../src/scripts/EntityScriptServer.h | 2 + .../resources/describe-settings.json | 56 +++++- libraries/audio-client/src/AudioClient.cpp | 45 ++--- libraries/audio-client/src/AudioClient.h | 12 +- libraries/plugins/src/plugins/CodecPlugin.h | 176 +++++++++--------- plugins/opusCodec/src/OpusEncoder.cpp | 95 +++++++--- plugins/opusCodec/src/OpusEncoder.h | 28 +-- 14 files changed, 305 insertions(+), 164 deletions(-) diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 4b9b2d5095f..4efdfeb5a00 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -332,6 +332,9 @@ void Agent::selectAudioFormat(const QString& selectedCodecName) { _codec = plugin; _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + + //TODO: _codecSettings is not being set by anything -- where do we receive the settings here? + _encoder->configure(_codecSettings); qDebug() << "Selected Codec Plugin:" << _codec.get(); break; } @@ -719,7 +722,7 @@ void Agent::processAgentAvatarAudio() { if (isPlayingRecording && !_shouldMuteRecordingAudio) { _shouldMuteRecordingAudio = true; } - + auto audioData = _avatarSound->getAudioData(); nextSoundOutput = reinterpret_cast(audioData->rawData() + _numAvatarSoundSentBytes); @@ -880,7 +883,7 @@ void Agent::aboutToFinish() { { DependencyManager::get()->shutdownScripting(); } - + DependencyManager::destroy(); DependencyManager::destroy(); diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index eb58e32897c..462f71ab780 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -121,6 +121,7 @@ private slots: Encoder* _encoder { nullptr }; QTimer _avatarAudioTimer; bool _flushEncoder { false }; + std::vector _codecSettings; }; #endif // hifi_Agent_h diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 4ba58020264..f32d5c5c49c 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -57,6 +57,7 @@ QStringList AudioMixer::_codecPreferenceOrder{}; vector AudioMixer::_audioZones; vector AudioMixer::_zoneSettings; vector AudioMixer::_zoneReverbSettings; +vector AudioMixer::_codecSettings; AudioMixer::AudioMixer(ReceivedMessage& message) : ThreadedAssignment(message) @@ -219,6 +220,7 @@ void AudioMixer::handleNodeMuteRequestPacket(QSharedPointer pac // we need to set a flag so we send them the appropriate packet to mute them AudioMixerClientData* nodeData = (AudioMixerClientData*)node->getLinkedData(); nodeData->setShouldMuteClient(true); + nodeData->setCodecSettings(_codecSettings); } else { qWarning() << "Node mute packet received for unknown node " << uuidStringWithoutCurlyBraces(nodeUUID); } @@ -393,6 +395,7 @@ AudioMixerClientData* AudioMixer::getOrCreateClientData(Node* node) { node->setLinkedData(unique_ptr { new AudioMixerClientData(node->getUUID(), node->getLocalID()) }); clientData = dynamic_cast(node->getLinkedData()); connect(clientData, &AudioMixerClientData::injectorStreamFinished, this, &AudioMixer::removeHRTFsForFinishedInjector); + clientData->setCodecSettings(_codecSettings); } return clientData; @@ -806,6 +809,8 @@ void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) { } } } + + _codecSettings = Encoder::parseSettings(audioEnvGroupObject); } } diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 5b75ed54d25..382ec5cc1ab 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -24,6 +24,7 @@ #include "AudioMixerStats.h" #include "AudioMixerSlavePool.h" +#include "plugins/CodecPlugin.h" class PositionalAudioStream; class AvatarAudioStream; @@ -52,6 +53,19 @@ class AudioMixer : public ThreadedAssignment { float wetLevel; }; +/* + struct CodecSettings { + QString codec; + Encoder::Bandpass bandpass = Encoder::Bandpass::Fullband; + Encoder::ApplicationType applicationType = Encoder::ApplicationType::Audio; + Encoder::SignalType signalType = Encoder::SignalType::Auto; + int bitrate = 128000; + int complexity = 100; + bool enableFEC = 0; + int packetLossPercent = 0; + bool enableVBR = 1; + }; +*/ static int getStaticJitterFrames() { return _numStaticJitterFrames; } static bool shouldMute(float quietestFrame) { return quietestFrame > _noiseMutingThreshold; } static float getAttenuationPerDoublingInDistance() { return _attenuationPerDoublingInDistance; } @@ -59,6 +73,7 @@ class AudioMixer : public ThreadedAssignment { static const std::vector& getZoneSettings() { return _zoneSettings; } static const std::vector& getReverbSettings() { return _zoneReverbSettings; } static const std::pair negotiateCodec(std::vector codecs); + static const std::vector getCodecSettings() { return _codecSettings; } static bool shouldReplicateTo(const Node& from, const Node& to) { return to.getType() == NodeType::DownstreamAudioMixer && @@ -67,7 +82,7 @@ class AudioMixer : public ThreadedAssignment { } virtual void aboutToFinish() override; - + public slots: void run() override; void sendStatsPacket() override; @@ -149,6 +164,7 @@ private slots: static std::vector _audioZones; static std::vector _zoneSettings; static std::vector _zoneReverbSettings; + static std::vector _codecSettings; float _throttleStartTarget = 0.9f; float _throttleBackoffTarget = 0.44f; diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 470ab3f233e..513ac0748fa 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -165,7 +165,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c packet->write(message.getMessage()); } - + nodeList->sendUnreliablePacket(*packet, *downstreamNode); } }); @@ -375,7 +375,7 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { qWarning() << "Received AudioStreamStats of wrong size" << message.getBytesLeftToRead() << "instead of" << sizeof(AudioStreamStats) << "from" << message.getSourceID() << "at" << message.getSenderSockAddr(); - + return message.getPosition(); } @@ -783,6 +783,8 @@ void AudioMixerClientData::setupCodec(CodecPluginPointer codec, const QString& c if (codec) { _encoder = codec->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::STEREO); _decoder = codec->createDecoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + + _encoder->configure(_codecSettings); } auto avatarAudioStream = getAvatarAudioStream(); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index b6f26aebc0b..ff3c858b434 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -108,6 +108,7 @@ class AudioMixerClientData : public NodeData { bool shouldFlushEncoder() { return _shouldFlushEncoder; } QString getCodecName() { return _selectedCodecName; } + void setCodecSettings(const std::vector &settings) { _codecSettings = settings; } bool shouldMuteClient() { return _shouldMuteClient; } void setShouldMuteClient(bool shouldMuteClient) { _shouldMuteClient = shouldMuteClient; } @@ -201,6 +202,7 @@ public slots: QString _selectedCodecName; Encoder* _encoder{ nullptr }; // for outbound mixed stream Decoder* _decoder{ nullptr }; // for mic stream + std::vector _codecSettings; bool _shouldFlushEncoder { false }; diff --git a/assignment-client/src/scripts/EntityScriptServer.cpp b/assignment-client/src/scripts/EntityScriptServer.cpp index 4ff920516af..f4346c96036 100644 --- a/assignment-client/src/scripts/EntityScriptServer.cpp +++ b/assignment-client/src/scripts/EntityScriptServer.cpp @@ -170,6 +170,13 @@ void EntityScriptServer::handleSettings() { _maxEntityPPS = std::max(0, entityScriptServerSettings[MAX_ENTITY_PPS_OPTION].toInt()); _entityPPSPerScript = std::max(0, entityScriptServerSettings[ENTITY_PPS_PER_SCRIPT].toInt()); + static const QString AUDIO_ENV_GROUP_KEY = "audio_env"; + if ( settingsObject.contains(AUDIO_ENV_GROUP_KEY)) { + _codecSettings = Encoder::parseSettings(settingsObject[AUDIO_ENV_GROUP_KEY].toObject()); + } else { + qWarning() << "Failed to find" << AUDIO_ENV_GROUP_KEY << "key in settings. Dump: " << settingsObject; + } + qDebug() << QString("Received entity script server settings, Max Entity PPS: %1, Entity PPS Per Entity Script: %2") .arg(_maxEntityPPS).arg(_entityPPSPerScript); } @@ -288,7 +295,7 @@ void EntityScriptServer::run() { entityScriptingInterface->init(); _entityViewer.init(); - + // setup the JSON filter that asks for entities with a non-default serverScripts property QJsonObject queryJSONParameters; queryJSONParameters[EntityJSONQueryProperties::SERVER_SCRIPTS_PROPERTY] = EntityQueryFilterSymbol::NonDefault; @@ -299,7 +306,7 @@ void EntityScriptServer::run() { queryFlags[EntityJSONQueryProperties::INCLUDE_DESCENDANTS_PROPERTY] = true; queryJSONParameters[EntityJSONQueryProperties::FLAGS_PROPERTY] = queryFlags; - + // setup the JSON parameters so that OctreeQuery does not use a frustum and uses our JSON filter _entityViewer.getOctreeQuery().setJSONParameters(queryJSONParameters); @@ -376,7 +383,7 @@ void EntityScriptServer::nodeKilled(SharedNodePointer killedNode) { if (!hasAnotherEntityServer) { clear(); } - + break; } case NodeType::Agent: { @@ -438,6 +445,7 @@ void EntityScriptServer::selectAudioFormat(const QString& selectedCodecName) { if (_selectedCodecName == plugin->getName()) { _codec = plugin; _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, AudioConstants::MONO); + _encoder->configure(_codecSettings); qCDebug(entity_script_server) << "Selected Codec Plugin:" << _codec.get(); break; } @@ -579,7 +587,7 @@ void EntityScriptServer::sendStatsPacket() { } scriptEngineStats["number_running_scripts"] = numberRunningScripts; statsObject["script_engine_stats"] = scriptEngineStats; - + auto nodeList = DependencyManager::get(); QJsonObject nodesObject; diff --git a/assignment-client/src/scripts/EntityScriptServer.h b/assignment-client/src/scripts/EntityScriptServer.h index b7929eb5af3..0699ae65028 100644 --- a/assignment-client/src/scripts/EntityScriptServer.h +++ b/assignment-client/src/scripts/EntityScriptServer.h @@ -25,6 +25,7 @@ #include #include #include "../entities/EntityTreeHeadlessViewer.h" +#include "plugins/CodecPlugin.h" class EntityScriptServer : public ThreadedAssignment { Q_OBJECT @@ -90,6 +91,7 @@ private slots: QString _selectedCodecName; CodecPluginPointer _codec; Encoder* _encoder { nullptr }; + std::vector _codecSettings; }; #endif // hifi_EntityScriptServer_h diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index f6882d6a053..710562ea680 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1277,7 +1277,7 @@ { "name": "audio_env", "label": "Audio Environment", - "assignment-types": [ 0 ], + "assignment-types": [ 0, 5 ], "settings": [ { "name": "attenuation_per_doubling_in_distance", @@ -1417,10 +1417,62 @@ { "name": "codec_preference_order", "label": "Audio Codec Preference Order", - "help": "List of codec names in order of preferred usage", + "help": "List of codec names in order of preferred usage -- mod2", "placeholder": "opus, hifiAC, zlib, pcm", "default": "opus,hifiAC,zlib,pcm", "advanced": true + }, + { + "name": "codec_settings", + "label": "Audio Codec Settings", + "help": "Advanced settings for audio codecs", + "advanced": true, + "type": "table", + "numbered": false, + "content_setting": false, + "can_add_new_rows": true, + "columns" : [ + { + "name": "codec", + "label": "Codec", + "can_set": false, + "placeholder": "(codec)" + }, + { + "name": "complexity", + "label": "Complexity", + "can_set": true, + "type": "int", + "placeholder": "(percentage)" + }, + { + "name": "bitrate", + "label": "Bitrate", + "can_set": true, + "placeholder": "(bps)" + }, + { + "name": "vbr", + "label": "Variable Bitrate", + "can_set": true, + "type": "checkbox" + }, + { + "name": "fec", + "label": "Error correction", + "can_set": true, + "type": "checkbox", + "placeholder": "(percentage)" + }, + { + "name": "packet_loss", + "label": "Expected loss %", + "can_set": true, + "type": "int", + "placeholder": "(percentage)" + } + + ] } ] }, diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index eb6820421de..fedb35f0574 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -95,7 +95,7 @@ void AudioClient::setHmdAudioName(QAudio::Mode mode, const QString& name) { // thread-safe QList getAvailableDevices(QAudio::Mode mode, const QString& hmdName) { //get hmd device name prior to locking device mutex. in case of shutdown, this thread will be locked and audio client - //cannot properly shut down. + //cannot properly shut down. QString defDeviceName = defaultAudioDeviceName(mode); // NOTE: availableDevices() clobbers the Qt internal device list @@ -132,7 +132,7 @@ QList getAvailableDevices(QAudio::Mode mode, const QString& break; } } - + if (!hmdDevice.getDevice().isNull()) { newDevices.push_front(hmdDevice); } @@ -159,7 +159,7 @@ void AudioClient::checkDevices() { auto inputDevices = getAvailableDevices(QAudio::AudioInput, hmdInputName); auto outputDevices = getAvailableDevices(QAudio::AudioOutput, hmdOutputName); - + static const QMetaMethod devicesChangedSig= QMetaMethod::fromSignal(&AudioClient::devicesChanged); //only emit once the scripting interface has connected to the signal if (isSignalConnected(devicesChangedSig)) { @@ -173,7 +173,7 @@ void AudioClient::checkDevices() { _outputDevices.swap(outputDevices); emit devicesChanged(QAudio::AudioOutput, _outputDevices); } - } + } } HifiAudioDeviceInfo AudioClient::getActiveAudioDevice(QAudio::Mode mode) const { @@ -280,7 +280,7 @@ static float computeLoudness(int16_t* samples, int numSamples) { template static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float gain1) { - + // fast path for unity gain if (gain0 == 1.0f && gain1 == 1.0f) { return; @@ -295,7 +295,7 @@ static void applyGainSmoothing(float* buffer, int numFrames, float gain0, float float tStep = 1.0f / numFrames; for (int i = 0; i < numFrames; i++) { - + // evaluate poly over t=[0,1) float gain = (c3 * t + c2) * t * t + c0; t += tStep; @@ -386,7 +386,7 @@ AudioClient::AudioClient() { PacketReceiver::makeUnsourcedListenerReference(this, &AudioClient::handleSelectedAudioFormat)); auto& domainHandler = nodeList->getDomainHandler(); - connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this] { + connect(&domainHandler, &DomainHandler::disconnectedFromDomain, this, [this] { _solo.reset(); }); connect(nodeList.data(), &NodeList::nodeActivated, this, [this](SharedNodePointer node) { @@ -578,7 +578,7 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { waveInGetDevCaps(WAVE_MAPPER, &wic, sizeof(wic)); //Use the received manufacturer id to get the device's real name waveInGetDevCaps(wic.wMid, &wic, sizeof(wic)); -#if !defined(NDEBUG) +#if !defined(NDEBUG) qCDebug(audioclient) << "input device:" << wic.szPname; #endif deviceName = wic.szPname; @@ -588,7 +588,7 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { waveOutGetDevCaps(WAVE_MAPPER, &woc, sizeof(woc)); //Use the received manufacturer id to get the device's real name waveOutGetDevCaps(woc.wMid, &woc, sizeof(woc)); -#if !defined(NDEBUG) +#if !defined(NDEBUG) qCDebug(audioclient) << "output device:" << woc.szPname; #endif deviceName = woc.szPname; @@ -613,8 +613,8 @@ QString defaultAudioDeviceName(QAudio::Mode mode) { CoUninitialize(); } -#if !defined(NDEBUG) - qCDebug(audioclient) << "defaultAudioDeviceForMode mode: " << (mode == QAudio::AudioOutput ? "Output" : "Input") +#if !defined(NDEBUG) + qCDebug(audioclient) << "defaultAudioDeviceForMode mode: " << (mode == QAudio::AudioOutput ? "Output" : "Input") << " [" << deviceName << "] [" << "]"; #endif @@ -795,7 +795,7 @@ void AudioClient::start() { _desiredOutputFormat = _desiredInputFormat; _desiredOutputFormat.setChannelCount(OUTPUT_CHANNEL_COUNT); - + QString inputName; QString outputName; { @@ -803,10 +803,10 @@ void AudioClient::start() { inputName = _hmdInputName; outputName = _hmdOutputName; } - + //initialize input to the dummy device to prevent starves switchInputToAudioDevice(HifiAudioDeviceInfo()); - switchOutputToAudioDevice(defaultAudioDeviceForMode(QAudio::AudioOutput, QString())); + switchOutputToAudioDevice(defaultAudioDeviceForMode(QAudio::AudioOutput, QString())); #if defined(Q_OS_ANDROID) connect(&_checkInputTimer, &QTimer::timeout, this, &AudioClient::checkInputTimeout); @@ -1005,6 +1005,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) { _codec = plugin; _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); + _encoder->configure(_codecSettings); qCDebug(audioclient) << "Selected codec plugin:" << _codec.get(); break; } @@ -1015,7 +1016,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) { bool AudioClient::switchAudioDevice(QAudio::Mode mode, const HifiAudioDeviceInfo& deviceInfo) { auto device = deviceInfo; if (deviceInfo.getDevice().isNull()) { - qCDebug(audioclient) << __FUNCTION__ << " switching to null device :" + qCDebug(audioclient) << __FUNCTION__ << " switching to null device :" << deviceInfo.deviceName() << " : " << deviceInfo.getDevice().deviceName(); } @@ -1868,7 +1869,7 @@ void AudioClient::outputFormatChanged() { bool AudioClient::switchInputToAudioDevice(const HifiAudioDeviceInfo inputDeviceInfo, bool isShutdownRequest) { Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread"); - qCDebug(audioclient) << __FUNCTION__ << "_inputDeviceInfo: [" << _inputDeviceInfo.deviceName() << ":" << _inputDeviceInfo.getDevice().deviceName() + qCDebug(audioclient) << __FUNCTION__ << "_inputDeviceInfo: [" << _inputDeviceInfo.deviceName() << ":" << _inputDeviceInfo.getDevice().deviceName() << "-- inputDeviceInfo:" << inputDeviceInfo.deviceName() << ":" << inputDeviceInfo.getDevice().deviceName() << "]"; bool supportedFormat = false; @@ -1923,7 +1924,7 @@ bool AudioClient::switchInputToAudioDevice(const HifiAudioDeviceInfo inputDevice if (!inputDeviceInfo.getDevice().isNull()) { qCDebug(audioclient) << "The audio input device" << inputDeviceInfo.deviceName() << ":" << inputDeviceInfo.getDevice().deviceName() << "is available."; - + //do not update UI that we're changing devices if default or same device _inputDeviceInfo = inputDeviceInfo; emit deviceChanged(QAudio::AudioInput, _inputDeviceInfo); @@ -2097,13 +2098,13 @@ void AudioClient::outputNotify() { void AudioClient::noteAwakening() { qCDebug(audioclient) << "Restarting the audio devices."; - switchInputToAudioDevice(_inputDeviceInfo); + switchInputToAudioDevice(_inputDeviceInfo); switchOutputToAudioDevice(_outputDeviceInfo); } bool AudioClient::switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDeviceInfo, bool isShutdownRequest) { Q_ASSERT_X(QThread::currentThread() == thread(), Q_FUNC_INFO, "Function invoked on wrong thread"); - + qCDebug(audioclient) << __FUNCTION__ << "_outputdeviceInfo: [" << _outputDeviceInfo.deviceName() << ":" << _outputDeviceInfo.getDevice().deviceName() << "-- outputDeviceInfo:" << outputDeviceInfo.deviceName() << ":" << outputDeviceInfo.getDevice().deviceName() << "]"; bool supportedFormat = false; @@ -2143,7 +2144,7 @@ bool AudioClient::switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDevi delete[] _localOutputMixBuffer; _localOutputMixBuffer = NULL; - + _outputDeviceInfo.setDevice(QAudioDeviceInfo()); } @@ -2170,7 +2171,7 @@ bool AudioClient::switchOutputToAudioDevice(const HifiAudioDeviceInfo outputDevi if (!outputDeviceInfo.getDevice().isNull()) { qCDebug(audioclient) << "The audio output device" << outputDeviceInfo.deviceName() << ":" << outputDeviceInfo.getDevice().deviceName() << "is available."; - + //do not update UI that we're changing devices if default or same device _outputDeviceInfo = outputDeviceInfo; emit deviceChanged(QAudio::AudioOutput, _outputDeviceInfo); @@ -2398,7 +2399,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { qCDebug(audiostream, "Read %d samples from injectors (%d available, %d requested)", injectorSamplesPopped, _localInjectorsStream.samplesAvailable(), samplesRequested); } } - + // prepare injectors for the next callback _audio->_localPrepInjectorFuture = QtConcurrent::run(QThreadPool::globalInstance(), [this] { _audio->prepareLocalAudioInjectors(); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 9c25c2d4f22..2292fd2cd03 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -124,7 +124,7 @@ class AudioClient : public AbstractAudioInterface, public Dependency { AudioClient* _audio; int _unfulfilledReads; }; - + void startThread(); void negotiateAudioFormat(); void selectAudioFormat(const QString& selectedCodecName); @@ -170,7 +170,7 @@ class AudioClient : public AbstractAudioInterface, public Dependency { HifiAudioDeviceInfo getActiveAudioDevice(QAudio::Mode mode) const; QList getAudioDevices(QAudio::Mode mode) const; - + void enablePeakValues(bool enable) { _enablePeakValues = enable; } bool peakValuesAvailable() const; @@ -186,6 +186,7 @@ class AudioClient : public AbstractAudioInterface, public Dependency { void setAudioPaused(bool pause); AudioSolo& getAudioSolo() override { return _solo; } + void setCodecSettings(const std::vector &settings ) { _codecSettings = settings; } #ifdef Q_OS_WIN static QString getWinDeviceName(wchar_t* guid); @@ -225,10 +226,10 @@ public slots: void setNoiseReduction(bool isNoiseGateEnabled, bool emitSignal = true); bool isNoiseReductionEnabled() const { return _isNoiseGateEnabled; } - + void setNoiseReductionAutomatic(bool isNoiseGateAutomatic, bool emitSignal = true); bool isNoiseReductionAutomatic() const { return _isNoiseReductionAutomatic; } - + void setNoiseReductionThreshold(float noiseReductionThreshold, bool emitSignal = true); float noiseReductionThreshold() const { return _noiseReductionThreshold; } @@ -338,6 +339,7 @@ public slots: bool mixLocalAudioInjectors(float* mixBuffer); float azimuthForSource(const glm::vec3& relativePosition); float gainForSource(float distance, float volume); + std::vector _codecSettings; #ifdef Q_OS_ANDROID QTimer _checkInputTimer{ this }; @@ -527,7 +529,7 @@ public slots: #endif AudioSolo _solo; - + QFuture _localPrepInjectorFuture; QReadWriteLock _hmdNameLock; Mutex _checkDevicesMutex; diff --git a/libraries/plugins/src/plugins/CodecPlugin.h b/libraries/plugins/src/plugins/CodecPlugin.h index 0c341a91173..cce736b589a 100644 --- a/libraries/plugins/src/plugins/CodecPlugin.h +++ b/libraries/plugins/src/plugins/CodecPlugin.h @@ -13,37 +13,80 @@ #include #include "Plugin.h" #include +#include -#ifndef VIRCADIA_CODECPLUGIN_H -#define VIRCADIA_CODECPLUGIN_H +#include + +Q_DECLARE_LOGGING_CATEGORY(codec_plugin) + + +/** + * @brief Audio encoder + * + * This class is the base interface for all encoders. It provides an interface to encode audio, and exposes encoding + * settings and data about the encoder. + * + * Some settings don't apply to some encoders and will be ignored. Methods are provided to determine whether a setting will apply. + * + * Settings may be clamped or fit into certain ranges. For instance an encoder may adjust the wanted bitrate to the nearest one it supports. + * + * @warning Both set and get functions must be implemented for each setting. The class doesn't store any data internally and it's up to the + * derived class to deal with storage. + * + */ class Encoder { public: - enum class EncoderBandpass { + enum class Bandpass { Narrowband, Mediumband, Wideband, Superwideband, Fullband }; - enum class EncoderSignal { + enum class SignalType { Auto, Voice, Music }; - enum class EncoderApplication { + enum class ApplicationType { VOIP, Audio, LowDelay }; -/* - enum class Capabilities : int { - None = 0x00, - Lossless = 0x01, - Complexity = 0x02, - Bitrate = 0x04, - FEC = 0x08, - Bandpass = 0x10, - SignalType = 0x20, - VBR = 0x40 + /** + * @brief Represents the settings loaded from the domain's configuration page + * + * Encoder classes are created EntityScriptServer, Agent, AudioMixerClientData and AudioClient, + * and they need to store configuration data internally for passing it to new Encoder objects + * when they create them. + * + */ + struct CodecSettings { + QString codec; + Encoder::Bandpass bandpass = Encoder::Bandpass::Fullband; + Encoder::ApplicationType applicationType = Encoder::ApplicationType::Audio; + Encoder::SignalType signalType = Encoder::SignalType::Auto; + int bitrate = 128000; + int complexity = 100; + bool enableFEC = 0; + int packetLossPercent = 0; + bool enableVBR = 1; + + }; -*/ + /** + * @brief Parse the codec settings from a domain's config into a vector of CodecSettings. + * + * @param root A QJsonObject which contains a codec_settings dictionary with codec settings inside. + * @return std::vector + */ + static std::vector parseSettings(const QJsonObject &root); + /** + * @brief Configures a given encoder with the values parsed from the configuration + * + * This takes the whole vector as an argument, and automatically tries to find a matching + * configuration in there. + * + * @param settings A vector of CodecSettings. The function will look for a matching configuration within. + */ + void configure(const std::vector &settings); virtual ~Encoder() { } virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) = 0; @@ -51,12 +94,19 @@ class Encoder { /** * @brief Get the codec's name * - * Returns the name of the codec, for usage in debug output and UI. + * Returns the name of the codec. Used for display, debugging and configuration. + * This must match the values found within the configuration. * * @return QString Codec's name */ virtual const QString getName() const { return "Encoder"; } + /** + * @brief Whether this codec is lossless + * + * @return true Codec is lossless + * @return false Codec is lossy + */ virtual bool isLossless() const { return false; } /** @@ -70,9 +120,9 @@ class Encoder { */ //virtual Capabilities getCapabilities() { return Capabilities::None; } - virtual void setApplication(EncoderApplication app) { _application = app; } + virtual void setApplication(ApplicationType app) { } virtual bool hasApplication() const { return false; } - EncoderApplication getApplication() const { return _application; } + virtual ApplicationType getApplication() const { return ApplicationType::Audio; } /** * @brief Set the codec's complexity (CPU usage) @@ -82,9 +132,9 @@ class Encoder { * * @param complexity A value from 0 to 100. */ - virtual void setComplexity(int complexity) { _complexity = qBound(complexity, 0, 100); } + virtual void setComplexity(int complexity) { } virtual bool hasComplexity() const { return false; } - int getComplexity() const { return _complexity; } + virtual int getComplexity() const { return 0; } /** * @brief Set the bitrate @@ -93,9 +143,9 @@ class Encoder { * * @param bitrate A rate in bits per second from 500 to 512000 */ - virtual void setBitrate(int bitrate) { _bitrate = qBound(bitrate, 500, 512000); } + virtual void setBitrate(int bitrate) { } virtual bool hasBitrate() const { return false; } - int getBitrate() const { return _bitrate; } + virtual int getBitrate() const { return 0; } /** * @brief Sets whether to use Forward Error Correction @@ -103,9 +153,9 @@ class Encoder { * When enabled, redundant data is included in the codec's output to compensate for packet loss * @param enabled */ - virtual void setFEC(bool enabled) { _useFEC = enabled; } + virtual void setFEC(bool enabled) { } virtual bool hasFEC() const { return false; } - bool getFEC() const { return _useFEC; } + virtual bool getFEC() const { return false; } /** * @brief Set how much packet loss to tolerate @@ -114,93 +164,41 @@ class Encoder { * * @param percent 0 to 100 */ - virtual void setPacketLossPercent(int percent) { _packetLossPercent = qBound(percent, 0, 100); } + virtual void setPacketLossPercent(int percent) { } virtual bool hasPacketLossPercent() const { return false; } - int getPacketLossPercent() const { return _packetLossPercent; } + virtual int getPacketLossPercent() const { return 0; } /** * @brief Set the bandpass * * @param bandpass */ - virtual void setBandpass(EncoderBandpass bandpass) { _bandpass = bandpass;} + virtual void setBandpass(Bandpass bandpass) { } virtual bool hasBandpass() const { return false; } - EncoderBandpass getBandpass() const { return _bandpass; } + virtual Bandpass getBandpass() const { return Bandpass::Fullband; } /** * @brief Set the type of audio signal * * @param signal */ - virtual void setSignalType(EncoderSignal signal) { _signal = signal;} + virtual void setSignalType(SignalType signal) {} virtual bool hasSignalType() const { return false; } - EncoderSignal getSignalType() const { return _signal; } + virtual SignalType getSignalType() const { return SignalType::Auto; } /** * @brief Whether to use variable bitrate * * @param use_vbr */ - virtual void setVBR(bool use_vbr) { _useVBR = use_vbr; } + virtual void setVBR(bool use_vbr) { } virtual bool hasVBR() const { return false; } - bool getVBR() const { return _useVBR; } - - QDebug operator<<(QDebug debug) { - debug << "{ Codec = " << getName(); - if ( hasComplexity() ) { - debug << "; Complexity = " << getComplexity(); - } - - if ( hasBitrate() ) { - debug << "; Bitrate = " << getBitrate(); - } - - if ( hasFEC() ) { - debug << "; FEC = " << getFEC(); - } - - if ( hasPacketLossPercent() ) { - debug << "; PacketLoss = " << getPacketLossPercent(); - } - - if ( hasBandpass() ) { - debug << "; Bandpass = " << (int)getBandpass(); - } - - if ( hasSignalType() ) { - debug << "; Signal = " << (int)getSignalType(); - } - - if ( hasVBR() ) { - debug << "; VBR = " << getVBR(); - } - - debug << "}"; - - return debug; - } - -/* - Encoder::Capabilities operator|(Encoder::Capabilities lhs, Encoder::Capabilities rhs) { - return static_cast( - static_cast::type>(lhs) | - static_cast::type>(rhs) - ); - -} - */ -protected: - int _complexity; - int _bitrate; - bool _useFEC; - bool _useVBR; - int _packetLossPercent; - EncoderBandpass _bandpass; - EncoderSignal _signal; - EncoderApplication _application; -}; + virtual bool getVBR() const { return false; } + QDebug operator<<(QDebug debug); +}; +QDebug operator<<(QDebug &debug, const Encoder::CodecSettings &settings); class Decoder { public: @@ -220,5 +218,3 @@ class CodecPlugin : public Plugin { virtual void releaseEncoder(Encoder* encoder) = 0; virtual void releaseDecoder(Decoder* decoder) = 0; }; - -#endif diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 3408701633b..7027103f295 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -55,8 +55,8 @@ AthenaOpusEncoder::AthenaOpusEncoder(int sampleRate, int numChannels) { setBitrate(DEFAULT_BITRATE); setComplexity(DEFAULT_COMPLEXITY); - setApplication(DEFAULT_APPLICATION); - setSignal(DEFAULT_SIGNAL); + setApplication(Encoder::ApplicationType::Audio); + setSignalType(Encoder::SignalType::Auto); qCDebug(encoder) << "Opus encoder initialized, sampleRate = " << sampleRate << "; numChannels = " << numChannels; } @@ -121,16 +121,16 @@ void AthenaOpusEncoder::setBitrate(int bitrate) { } } -int AthenaOpusEncoder::getVBR() const { +bool AthenaOpusEncoder::getVBR() const { assert(_encoder); int returnValue; opus_encoder_ctl(_encoder, OPUS_GET_VBR(&returnValue)); - return returnValue; + return (bool)returnValue; } -void AthenaOpusEncoder::setVBR(int vbr) { +void AthenaOpusEncoder::setVBR(bool vbr) { assert(_encoder); - int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_VBR(vbr)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_VBR(int(vbr))); if (errorCode != OPUS_OK) { qCWarning(encoder) << "Error when setting VBR to " << vbr << ": " << errorToString(errorCode); @@ -185,35 +185,86 @@ void AthenaOpusEncoder::setBandwidth(int bandwidth) { } } -int AthenaOpusEncoder::getSignal() const { +Encoder::SignalType AthenaOpusEncoder::getSignalType() const { assert(_encoder); int signal; opus_encoder_ctl(_encoder, OPUS_GET_SIGNAL(&signal)); - return signal; + + switch(signal) { + case OPUS_AUTO: + return SignalType::Auto; + case OPUS_SIGNAL_MUSIC: + return SignalType::Music; + case OPUS_SIGNAL_VOICE: + return SignalType::Voice; + default: + qCCritical(encoder) << "Opus returned unrecognized signal type" << signal; + return SignalType::Auto; + } + } -void AthenaOpusEncoder::setSignal(int signal) { +void AthenaOpusEncoder::setSignalType(Encoder::SignalType signal) { assert(_encoder); - int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_SIGNAL(signal)); + int sig = OPUS_SIGNAL_MUSIC; + + switch(signal) { + case SignalType::Auto: + sig = OPUS_AUTO; + break; + case SignalType::Music: + sig = OPUS_SIGNAL_MUSIC; + break; + case SignalType::Voice: + sig = OPUS_SIGNAL_VOICE; + break; + } + + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_SIGNAL(sig)); if (errorCode != OPUS_OK) { - qCWarning(encoder) << "Error when setting signal to " << signal << ": " << errorToString(errorCode); + qCWarning(encoder) << "Error when setting signal to " << sig << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getApplication() const { +Encoder::ApplicationType AthenaOpusEncoder::getApplication() const { assert(_encoder); int applicationValue; opus_encoder_ctl(_encoder, OPUS_GET_APPLICATION(&applicationValue)); - return applicationValue; + + switch(applicationValue) { + case OPUS_APPLICATION_AUDIO: + return ApplicationType::Audio; + case OPUS_APPLICATION_RESTRICTED_LOWDELAY: + return ApplicationType::LowDelay; + case OPUS_APPLICATION_VOIP: + return ApplicationType::VOIP; + default: + qCCritical(encoder) << "Opus returned unrecognized application value" << applicationValue; + return ApplicationType::Audio; + } } -void AthenaOpusEncoder::setApplication(int application) { +void AthenaOpusEncoder::setApplication(Encoder::ApplicationType application) { assert(_encoder); - int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_APPLICATION(application)); + int app = OPUS_APPLICATION_AUDIO; + + switch(application) { + case ApplicationType::Audio: + app = OPUS_APPLICATION_AUDIO; + break; + case ApplicationType::LowDelay: + app = OPUS_APPLICATION_RESTRICTED_LOWDELAY; + break; + case ApplicationType::VOIP: + app = OPUS_APPLICATION_VOIP; + break; + } + + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_APPLICATION(app)); if (errorCode != OPUS_OK) { - qCWarning(encoder) << "Error when setting application to " << application << ": " << errorToString(errorCode); + qCWarning(encoder) << "Error when setting application to " << app << ": " << errorToString(errorCode); } } @@ -224,30 +275,30 @@ int AthenaOpusEncoder::getLookahead() const { return lookAhead; } -int AthenaOpusEncoder::getInbandFEC() const { +bool AthenaOpusEncoder::getFEC() const { assert(_encoder); int fec; opus_encoder_ctl(_encoder, OPUS_GET_INBAND_FEC(&fec)); - return fec; + return (bool)fec; } -void AthenaOpusEncoder::setInbandFEC(int inBandFEC) { +void AthenaOpusEncoder::setFEC(bool inBandFEC) { assert(_encoder); - int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC(inBandFEC)); + int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_INBAND_FEC((int)inBandFEC)); if (errorCode != OPUS_OK) { qCWarning(encoder) << "Error when setting inband FEC to " << inBandFEC << ": " << errorToString(errorCode); } } -int AthenaOpusEncoder::getExpectedPacketLossPercentage() const { +int AthenaOpusEncoder::getPacketLossPercent() const { assert(_encoder); int lossPercentage; opus_encoder_ctl(_encoder, OPUS_GET_PACKET_LOSS_PERC(&lossPercentage)); return lossPercentage; } -void AthenaOpusEncoder::setExpectedPacketLossPercentage(int percentage) { +void AthenaOpusEncoder::setPacketLossPercent(int percentage) { assert(_encoder); int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_PACKET_LOSS_PERC(percentage)); diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h index 10640bf409f..34a1ab73579 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -23,14 +23,14 @@ class AthenaOpusEncoder : public Encoder { virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; - int getComplexity() const; - void setComplexity(int complexity); + int getComplexity() const override; + void setComplexity(int complexity) override; - int getBitrate() const; - void setBitrate(int bitrate); + int getBitrate() const override; + void setBitrate(int bitrate) override; - int getVBR() const; - void setVBR(int vbr); + bool getVBR() const override; + void setVBR(bool vbr) override; int getVBRConstraint() const; void setVBRConstraint(int vbrConstraint); @@ -41,19 +41,19 @@ class AthenaOpusEncoder : public Encoder { int getBandwidth() const; void setBandwidth(int bandwidth); - int getSignal() const; - void setSignal(int signal); + Encoder::SignalType getSignalType() const override; + void setSignalType(Encoder::SignalType signal) override; - int getApplication() const; - void setApplication(int application); + ApplicationType getApplication() const override; + void setApplication(ApplicationType application) override; int getLookahead() const; - int getInbandFEC() const; - void setInbandFEC(int inBandFEC); + bool getFEC() const override; + void setFEC(bool inBandFEC) override; - int getExpectedPacketLossPercentage() const; - void setExpectedPacketLossPercentage(int percentage); + int getPacketLossPercent() const override; + void setPacketLossPercent(int percentage) override; int getDTX() const; void setDTX(int dtx); From b86cb4a8808497873fb4c763105e5e1f20612223 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 27 Jan 2022 19:11:49 +0100 Subject: [PATCH 03/11] Implement in zlib and PCM codecs --- plugins/pcmCodec/src/PCMCodecManager.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/pcmCodec/src/PCMCodecManager.h b/plugins/pcmCodec/src/PCMCodecManager.h index 178f7cbd9b7..85984cd1157 100644 --- a/plugins/pcmCodec/src/PCMCodecManager.h +++ b/plugins/pcmCodec/src/PCMCodecManager.h @@ -31,6 +31,8 @@ class PCMCodec : public CodecPlugin, public Encoder, public Decoder { /// Called when a plugin is no longer being used. May be called multiple times. void deactivate() override; + bool isLossless() const override { return true; } + virtual Encoder* createEncoder(int sampleRate, int numChannels) override; virtual Decoder* createDecoder(int sampleRate, int numChannels) override; virtual void releaseEncoder(Encoder* encoder) override; @@ -61,6 +63,12 @@ class zLibCodec : public CodecPlugin, public Encoder, public Decoder { bool isSupported() const override; const QString getName() const override { return NAME; } + bool isLossless() const override { return true; } + + bool hasComplexity() const override { return true; } + + void setComplexity(int complexity) override { _compressionLevel = qBound(0, complexity/11, 9); } + void init() override; void deinit() override; @@ -75,7 +83,7 @@ class zLibCodec : public CodecPlugin, public Encoder, public Decoder { virtual void releaseDecoder(Decoder* decoder) override; virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override { - encodedBuffer = qCompress(decodedBuffer); + encodedBuffer = qCompress(decodedBuffer, _compressionLevel); } virtual void decode(const QByteArray& encodedBuffer, QByteArray& decodedBuffer) override { @@ -89,6 +97,7 @@ class zLibCodec : public CodecPlugin, public Encoder, public Decoder { private: static const char* NAME; + int _compressionLevel = 6; // Default compression level for zLib }; #endif // hifi__PCMCodecManager_h From bc0c822286a86d99aedba0b8ecd208c6adb1eedf Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 27 Jan 2022 19:13:44 +0100 Subject: [PATCH 04/11] Add missing implementation file --- libraries/plugins/src/plugins/CodecPlugin.cpp | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 libraries/plugins/src/plugins/CodecPlugin.cpp diff --git a/libraries/plugins/src/plugins/CodecPlugin.cpp b/libraries/plugins/src/plugins/CodecPlugin.cpp new file mode 100644 index 00000000000..c1b0ab64485 --- /dev/null +++ b/libraries/plugins/src/plugins/CodecPlugin.cpp @@ -0,0 +1,166 @@ +// +// CodecPlugin.h +// plugins/src/plugins +// +// Created by Brad Hefta-Gaub on 6/9/2016 +// Copyright 2016 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "CodecPlugin.h" +#include + +Q_LOGGING_CATEGORY(codec_plugin, "vircadia.codec_plugin") + +std::vector Encoder::parseSettings(const QJsonObject &root) { + const QString CODEC_SETTINGS = "codec_settings"; + + std::vector list; + + if (root[CODEC_SETTINGS].isArray()) { + qCWarning(codec_plugin) << "Parsing codec settings"; + + const QJsonArray &codecSettings = root[CODEC_SETTINGS].toArray(); + + const QString CODEC = "codec"; + const QString BITRATE = "bitrate"; + const QString COMPLEXITY = "complexity"; + const QString FEC = "fec"; + const QString PACKET_LOSS = "packet_loss"; + const QString VBR = "vbr"; + + for (int i=0;i &settings) { + const QString myName = getName().trimmed().toLower(); + + for( const auto &s : settings ) { + if ( s.codec.trimmed().toLower() == myName ) { + + qCDebug(codec_plugin) << "Configuring encoder" << getName(); + + if (hasApplication()) { + setApplication(s.applicationType); + } + + if (hasBandpass()) { + setBandpass(s.bandpass); + } + + if (hasSignalType()) { + setSignalType(s.signalType); + } + + if (hasBitrate()) { + setBitrate(s.bitrate); + } + + if (hasComplexity()) { + setComplexity(s.complexity); + } + + if (hasFEC()) { + setFEC(s.enableFEC); + } + + if (hasPacketLossPercent()) { + setPacketLossPercent(s.packetLossPercent); + } + + if (hasVBR()) { + setVBR(s.enableVBR); + } + } + } +} + +QDebug operator<<(QDebug &debug, const Encoder::CodecSettings &settings) { + debug << "{ Codec =" << settings.codec; + debug << "; Complexity =" << settings.complexity; + debug << "; Bitrate =" << settings.bitrate; + debug << "; FEC =" << settings.enableFEC; + debug << "; PacketLoss =" << settings.packetLossPercent; + debug << "; Application =" << (int)settings.applicationType; + debug << "; Bandpass =" << (int)settings.bandpass; + debug << "; Signal =" << (int)settings.signalType; + debug << "; VBR =" << settings.enableVBR; + debug << "}"; + return debug; +} + + +QDebug Encoder::operator<<(QDebug debug) { + debug << "{ Codec = " << getName(); + if ( hasComplexity() ) { + debug << "; Complexity = " << getComplexity(); + } + + if ( hasBitrate() ) { + debug << "; Bitrate = " << getBitrate(); + } + + if ( hasFEC() ) { + debug << "; FEC = " << getFEC(); + } + + if ( hasPacketLossPercent() ) { + debug << "; PacketLoss = " << getPacketLossPercent(); + } + + if ( hasBandpass() ) { + debug << "; Bandpass = " << (int)getBandpass(); + } + + if ( hasSignalType() ) { + debug << "; Signal = " << (int)getSignalType(); + } + + if ( hasVBR() ) { + debug << "; VBR = " << getVBR(); + } + + debug << "}"; + + return debug; +} + From b4abd5ec8c9b42f88f6c989ed717a5ea968caa5c Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Thu, 27 Jan 2022 22:24:34 +0100 Subject: [PATCH 05/11] Pre-fill codec settings Unfortunately, int fields don't seem to be editable currently, though the checkboxes work. --- .../resources/describe-settings.json | 60 ++++++++++++++++--- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 710562ea680..0d1f6ee3e3b 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1435,43 +1435,89 @@ { "name": "codec", "label": "Codec", - "can_set": false, - "placeholder": "(codec)" + "can_set": true, + "placeholder": "(codec)", + "editable": false }, { "name": "complexity", "label": "Complexity", "can_set": true, "type": "int", - "placeholder": "(percentage)" + "placeholder": "(percentage)", + "default": 0, + "editable": true }, { "name": "bitrate", "label": "Bitrate", "can_set": true, - "placeholder": "(bps)" + "placeholder": "(bps)", + "default" : 0, + "editable": true }, { "name": "vbr", "label": "Variable Bitrate", "can_set": true, - "type": "checkbox" + "type": "checkbox", + "default": false, + "editable": true }, { "name": "fec", "label": "Error correction", "can_set": true, "type": "checkbox", - "placeholder": "(percentage)" + "default" : false, + "editable": true }, { "name": "packet_loss", "label": "Expected loss %", "can_set": true, "type": "int", - "placeholder": "(percentage)" + "placeholder": "(percentage)", + "default": 0, + "editable": true } + ], + "non-deletable-row-key": "codec", + "non-deletable-row-values": [ "pcm", "zlib", "hifiAC", "opus" ], + "default": [ + { + "codec": "pcm", + "complexity": 0, + "bitrate": 0, + "vbr": false, + "fec": false, + "packet_loss": 0 + }, + { + "codec": "zlib", + "complexity": 6, + "bitrate": 0, + "vbr": false, + "fec": false, + "packet_loss": 0 + }, + { + "codec": "hifiAC", + "complexity": 0, + "bitrate": 0, + "vbr": false, + "fec": false, + "packet_loss": 0 + }, + { + "codec": "opus", + "complexity": 10, + "bitrate": 128000, + "vbr": true, + "fec": false, + "packet_loss": 0 + } ] } ] From 76c8c1a3e98d850c93463920e22c4eee76bd2f2f Mon Sep 17 00:00:00 2001 From: ksuprynowicz Date: Fri, 28 Jan 2022 18:44:53 +0100 Subject: [PATCH 06/11] Make codec settings modifiable --- domain-server/resources/describe-settings.json | 1 + domain-server/resources/web/js/base-settings.js | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 0d1f6ee3e3b..31ea2885f0d 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1452,6 +1452,7 @@ "name": "bitrate", "label": "Bitrate", "can_set": true, + "type": "int", "placeholder": "(bps)", "default" : 0, "editable": true diff --git a/domain-server/resources/web/js/base-settings.js b/domain-server/resources/web/js/base-settings.js index d3f5baa2f94..7259f094475 100644 --- a/domain-server/resources/web/js/base-settings.js +++ b/domain-server/resources/web/js/base-settings.js @@ -138,7 +138,7 @@ function reloadSettings(callback) { Settings.pendingChanges = 0; // call the callback now that settings are loaded - if (callback) { + if (callback) { callback(true); } }).fail(function() { @@ -591,6 +591,12 @@ function makeTable(setting, keypath, setting_value) { "" + ""; + } else if (isArray && col.type === "int" && col.editable) { + html += + "" + + "" + + ""; } else { // Use a hidden input so that the values are posted. html += From 81dfff79cef7665b4a5fc7ac6c9924148d9a3b4d Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Sat, 29 Jan 2022 15:15:01 +0100 Subject: [PATCH 07/11] Implement opus value bounds and scaling --- plugins/opusCodec/src/OpusEncoder.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index 7027103f295..f7d14bd949a 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -93,11 +93,13 @@ int AthenaOpusEncoder::getComplexity() const { assert(_encoder); int returnValue; opus_encoder_ctl(_encoder, OPUS_GET_COMPLEXITY(&returnValue)); - return returnValue; + return returnValue*10; } void AthenaOpusEncoder::setComplexity(int complexity) { assert(_encoder); + complexity = qBound(0, complexity/10, 10); // The generic interface is 0 to 100, Opus is 0 to 10. + int returnValue = opus_encoder_ctl(_encoder, OPUS_SET_COMPLEXITY(complexity)); if (returnValue != OPUS_OK) { @@ -114,6 +116,7 @@ int AthenaOpusEncoder::getBitrate() const { void AthenaOpusEncoder::setBitrate(int bitrate) { assert(_encoder); + bitrate = qBound(500, bitrate, 512000); // Opus limits int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_BITRATE(bitrate)); if (errorCode != OPUS_OK) { From aa99c37c52fab12c1031c50c67a1ff1eb3f8283f Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Sun, 30 Jan 2022 18:28:35 +0100 Subject: [PATCH 08/11] Initial implementation of audio scripting interface for codecs --- libraries/audio-client/src/AudioClient.cpp | 59 +++++++++++- libraries/audio-client/src/AudioClient.h | 9 ++ libraries/audio/src/AbstractAudioInterface.h | 5 + .../src/AudioScriptingInterface.cpp | 35 ++++++- .../src/AudioScriptingInterface.h | 94 ++++++++++--------- plugins/opusCodec/src/OpusEncoder.h | 1 + 6 files changed, 154 insertions(+), 49 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index fedb35f0574..a051c350591 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -444,6 +444,48 @@ void AudioClient::setAudioPaused(bool pause) { } } + +QStringList AudioClient::getCodecs() { + QStringList codecs; + + const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (const auto& plugin : codecPlugins) { + codecs << plugin->getName(); + } + + return codecs; +} + +QString AudioClient::getCodec() { + if ( _encoder ) { + return _encoder->getName(); + } + + return ""; +} + +void AudioClient::setAllowedCodecs(const QStringList &userCodecs) { + QStringList list; + QSet knownCodecs; + + + const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); + for (const auto& plugin : codecPlugins) { + knownCodecs << plugin->getName(); + } + + for(const auto &uc : userCodecs) { + if ( knownCodecs.contains(uc)) { + list << uc; + } else { + qCWarning(audioclient) << "setAllowedCodecs called with unknown codec name" << uc; + } + } + + _allowedCodecs = list; + negotiateAudioFormat(); +} + HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, const QString& hmdName, bool isHmd=false) { HifiAudioDeviceInfo result; foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode,hmdName)) { @@ -965,12 +1007,19 @@ void AudioClient::negotiateAudioFormat() { auto negotiateFormatPacket = NLPacket::create(PacketType::NegotiateAudioFormat); const auto& codecPlugins = PluginManager::getInstance()->getCodecPlugins(); quint8 numberOfCodecs = (quint8)codecPlugins.size(); - negotiateFormatPacket->writePrimitive(numberOfCodecs); - for (const auto& plugin : codecPlugins) { - auto codecName = plugin->getName(); - negotiateFormatPacket->writeString(codecName); - } + if ( _allowedCodecs.length() ) { + negotiateFormatPacket->writePrimitive((quint8)_allowedCodecs.length()); + for (const auto& codecName : _allowedCodecs) { + negotiateFormatPacket->writeString(codecName); + } + } else { + negotiateFormatPacket->writePrimitive(numberOfCodecs); + for (const auto& plugin : codecPlugins) { + auto codecName = plugin->getName(); + negotiateFormatPacket->writeString(codecName); + } + } // grab our audio mixer from the NodeList, if it exists SharedNodePointer audioMixer = nodeList->soloNodeOfType(NodeType::AudioMixer); diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 2292fd2cd03..d1594bfe83b 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -186,6 +186,12 @@ class AudioClient : public AbstractAudioInterface, public Dependency { void setAudioPaused(bool pause); AudioSolo& getAudioSolo() override { return _solo; } + + QStringList getCodecs() override; + QString getCodec() override; + + QStringList getAllowedCodecs() override { return _allowedCodecs; } + void setAllowedCodecs(const QStringList &codecs) override; void setCodecSettings(const std::vector &settings ) { _codecSettings = settings; } #ifdef Q_OS_WIN @@ -455,6 +461,9 @@ public slots: Mutex _localAudioMutex; AudioLimiter _audioLimiter{ AudioConstants::SAMPLE_RATE, OUTPUT_CHANNEL_COUNT }; + // Negotiate only these codecs with the domain + QStringList _allowedCodecs; + // Adds Reverb void configureReverb(); void updateReverbOptions(); diff --git a/libraries/audio/src/AbstractAudioInterface.h b/libraries/audio/src/AbstractAudioInterface.h index e9e40e95f94..ed566bdb61e 100644 --- a/libraries/audio/src/AbstractAudioInterface.h +++ b/libraries/audio/src/AbstractAudioInterface.h @@ -55,6 +55,11 @@ public slots: virtual void setServerEcho(bool serverEcho) = 0; virtual void toggleServerEcho() = 0; + virtual QStringList getCodecs() = 0; + virtual QString getCodec() = 0; + virtual void setAllowedCodecs(const QStringList &codec) = 0; + virtual QStringList getAllowedCodecs() = 0; + signals: void isStereoInputChanged(bool isStereo); }; diff --git a/libraries/script-engine/src/AudioScriptingInterface.cpp b/libraries/script-engine/src/AudioScriptingInterface.cpp index a55cac292f8..dc745a4ef9d 100644 --- a/libraries/script-engine/src/AudioScriptingInterface.cpp +++ b/libraries/script-engine/src/AudioScriptingInterface.cpp @@ -29,7 +29,7 @@ void AudioScriptingInterface::setLocalAudioInterface(AbstractAudioInterface* aud disconnect(_localAudioInterface, &AbstractAudioInterface::isStereoInputChanged, this, &AudioScriptingInterface::isStereoInputChanged); } - + _localAudioInterface = audioInterface; if (_localAudioInterface) { @@ -118,3 +118,36 @@ void AudioScriptingInterface::toggleLocalEcho() { QMetaObject::invokeMethod(_localAudioInterface, "toggleLocalEcho"); } } + +QStringList AudioScriptingInterface::getCodecs() { + QStringList codecs; + if (_localAudioInterface) { + codecs = _localAudioInterface->getCodecs(); + } + + return codecs; +} + +QString AudioScriptingInterface::getCodec() { + QString codec; + if (_localAudioInterface) { + codec = _localAudioInterface->getCodec(); + } + + return codec; +} + +void AudioScriptingInterface::setAllowedCodecs(const QStringList &codecs) { + if (_localAudioInterface) { + _localAudioInterface->setAllowedCodecs(codecs); + } +} + +QStringList AudioScriptingInterface::getAllowedCodecs() { + QStringList codecs; + if (_localAudioInterface) { + codecs = _localAudioInterface->getAllowedCodecs(); + } + + return codecs; +} \ No newline at end of file diff --git a/libraries/script-engine/src/AudioScriptingInterface.h b/libraries/script-engine/src/AudioScriptingInterface.h index 6bfb7352ee8..d9d28009501 100644 --- a/libraries/script-engine/src/AudioScriptingInterface.h +++ b/libraries/script-engine/src/AudioScriptingInterface.h @@ -45,7 +45,7 @@ class AudioScriptingInterface : public QObject, public Dependency { } /*@jsdoc - * Adds avatars to the audio solo list. If the audio solo list is not empty, only audio from the avatars in the list is + * Adds avatars to the audio solo list. If the audio solo list is not empty, only audio from the avatars in the list is * played. * @function Audio.addToSoloList * @param {Uuid[]} ids - Avatar IDs to add to the solo list. @@ -54,25 +54,25 @@ class AudioScriptingInterface : public QObject, public Dependency { * // Find nearby avatars. * var RANGE = 100; // m * var nearbyAvatars = AvatarList.getAvatarsInRange(MyAvatar.position, RANGE); - * + * * // Remove own avatar from list. * var myAvatarIndex = nearbyAvatars.indexOf(MyAvatar.sessionUUID); * if (myAvatarIndex !== -1) { * nearbyAvatars.splice(myAvatarIndex, 1); * } - * + * * if (nearbyAvatars.length > 0) { * // Listen to only one of the nearby avatars. * var avatarName = AvatarList.getAvatar(nearbyAvatars[0]).displayName; * print("Listening only to " + avatarName); * Audio.addToSoloList([nearbyAvatars[0]]); - * + * * // Stop listening to only the one avatar after a short while. * Script.setTimeout(function () { * print("Finished listening only to " + avatarName); * Audio.resetSoloList(); * }, 10000); // 10s - * + * * } else { * print("No nearby avatars"); * } @@ -82,7 +82,7 @@ class AudioScriptingInterface : public QObject, public Dependency { } /*@jsdoc - * Removes avatars from the audio solo list. If the audio solo list is not empty, only audio from the avatars in the list + * Removes avatars from the audio solo list. If the audio solo list is not empty, only audio from the avatars in the list * is played. * @function Audio.removeFromSoloList * @param {Uuid[]} ids - Avatar IDs to remove from the solo list. @@ -100,44 +100,44 @@ class AudioScriptingInterface : public QObject, public Dependency { } /*@jsdoc - * Gets whether your microphone audio is echoed back to you from the server. When enabled, microphone audio is echoed only + * Gets whether your microphone audio is echoed back to you from the server. When enabled, microphone audio is echoed only * if you're unmuted or are using push-to-talk. * @function Audio.getServerEcho - * @returns {boolean} true if echoing microphone audio back to you from the server is enabled, + * @returns {boolean} true if echoing microphone audio back to you from the server is enabled, * false if it isn't. */ Q_INVOKABLE bool getServerEcho(); /*@jsdoc - * Sets whether your microphone audio is echoed back to you from the server. When enabled, microphone audio is echoed + * Sets whether your microphone audio is echoed back to you from the server. When enabled, microphone audio is echoed * only if you're unmuted or are using push-to-talk. * @function Audio.setServerEcho - * @param {boolean} serverEcho - true to enable echoing microphone back to you from the server, + * @param {boolean} serverEcho - true to enable echoing microphone back to you from the server, * false to disable. */ Q_INVOKABLE void setServerEcho(bool serverEcho); /*@jsdoc - * Toggles the echoing of microphone audio back to you from the server. When enabled, microphone audio is echoed only if + * Toggles the echoing of microphone audio back to you from the server. When enabled, microphone audio is echoed only if * you're unmuted or are using push-to-talk. * @function Audio.toggleServerEcho */ Q_INVOKABLE void toggleServerEcho(); /*@jsdoc - * Gets whether your microphone audio is echoed back to you by the client. When enabled, microphone audio is echoed + * Gets whether your microphone audio is echoed back to you by the client. When enabled, microphone audio is echoed * even if you're muted or not using push-to-talk. * @function Audio.getLocalEcho - * @returns {boolean} true if echoing microphone audio back to you from the client is enabled, + * @returns {boolean} true if echoing microphone audio back to you from the client is enabled, * false if it isn't. */ Q_INVOKABLE bool getLocalEcho(); /*@jsdoc - * Sets whether your microphone audio is echoed back to you by the client. When enabled, microphone audio is echoed + * Sets whether your microphone audio is echoed back to you by the client. When enabled, microphone audio is echoed * even if you're muted or not using push-to-talk. * @function Audio.setLocalEcho - * @parm {boolean} localEcho - true to enable echoing microphone audio back to you from the client, + * @parm {boolean} localEcho - true to enable echoing microphone audio back to you from the client, * false to disable. * @example Echo local audio for a few seconds. * Audio.setLocalEcho(true); @@ -148,12 +148,20 @@ class AudioScriptingInterface : public QObject, public Dependency { Q_INVOKABLE void setLocalEcho(bool localEcho); /*@jsdoc - * Toggles the echoing of microphone audio back to you by the client. When enabled, microphone audio is echoed even if + * Toggles the echoing of microphone audio back to you by the client. When enabled, microphone audio is echoed even if * you're muted or not using push-to-talk. * @function Audio.toggleLocalEcho */ Q_INVOKABLE void toggleLocalEcho(); + Q_INVOKABLE QStringList getCodecs(); + + Q_INVOKABLE QString getCodec(); + + Q_INVOKABLE void setAllowedCodecs(const QStringList &codecs); + + Q_INVOKABLE QStringList getAllowedCodecs(); + protected: AudioScriptingInterface() = default; @@ -161,32 +169,32 @@ class AudioScriptingInterface : public QObject, public Dependency { // these methods are protected to stop C++ callers from calling, but invokable from script /*@jsdoc - * Starts playing or "injecting" the content of an audio file. The sound is played globally (sent to the audio - * mixer) so that everyone hears it, unless the injectorOptions has localOnly set to - * true in which case only the client hears the sound played. No sound is played if sent to the audio mixer - * but the client is not connected to an audio mixer. The {@link AudioInjector} object returned by the function can be used + * Starts playing or "injecting" the content of an audio file. The sound is played globally (sent to the audio + * mixer) so that everyone hears it, unless the injectorOptions has localOnly set to + * true in which case only the client hears the sound played. No sound is played if sent to the audio mixer + * but the client is not connected to an audio mixer. The {@link AudioInjector} object returned by the function can be used * to control the playback and get information about its current state. * @function Audio.playSound - * @param {SoundObject} sound - The content of an audio file, loaded using {@link SoundCache.getSound}. See + * @param {SoundObject} sound - The content of an audio file, loaded using {@link SoundCache.getSound}. See * {@link SoundObject} for supported formats. - * @param {AudioInjector.AudioInjectorOptions} [injectorOptions={}] - Configures where and how the audio injector plays the + * @param {AudioInjector.AudioInjectorOptions} [injectorOptions={}] - Configures where and how the audio injector plays the * audio file. * @returns {AudioInjector} The audio injector that plays the audio file. * @example Play a sound. * var sound = SoundCache.getSound("https://cdn-1.vircadia.com/us-c-1/ken/samples/forest_ambiX.wav"); - * + * * function playSound() { * var injectorOptions = { * position: MyAvatar.position * }; * var injector = Audio.playSound(sound, injectorOptions); * } - * + * * function onSoundReady() { * sound.ready.disconnect(onSoundReady); * playSound(); * } - * + * * if (sound.downloaded) { * playSound(); * } else { @@ -196,18 +204,18 @@ class AudioScriptingInterface : public QObject, public Dependency { Q_INVOKABLE ScriptAudioInjector* playSound(SharedSoundPointer sound, const AudioInjectorOptions& injectorOptions = AudioInjectorOptions()); /*@jsdoc - * Starts playing the content of an audio file locally (isn't sent to the audio mixer). This is the same as calling - * {@link Audio.playSound} with {@link AudioInjector.AudioInjectorOptions} localOnly set true and + * Starts playing the content of an audio file locally (isn't sent to the audio mixer). This is the same as calling + * {@link Audio.playSound} with {@link AudioInjector.AudioInjectorOptions} localOnly set true and * the specified position. * @function Audio.playSystemSound - * @param {SoundObject} sound - The content of an audio file, which is loaded using {@link SoundCache.getSound}. See + * @param {SoundObject} sound - The content of an audio file, which is loaded using {@link SoundCache.getSound}. See * {@link SoundObject} for supported formats. * @returns {AudioInjector} The audio injector that plays the audio file. */ Q_INVOKABLE ScriptAudioInjector* playSystemSound(SharedSoundPointer sound); /*@jsdoc - * Sets whether the audio input should be used in stereo. If the audio input doesn't support stereo then setting a value + * Sets whether the audio input should be used in stereo. If the audio input doesn't support stereo then setting a value * of true has no effect. * @function Audio.setStereoInput * @param {boolean} stereo - true if the audio input should be used in stereo, otherwise false. @@ -217,57 +225,57 @@ class AudioScriptingInterface : public QObject, public Dependency { /*@jsdoc * Gets whether the audio input is used in stereo. * @function Audio.isStereoInput - * @returns {boolean} true if the audio input is used in stereo, otherwise false. + * @returns {boolean} true if the audio input is used in stereo, otherwise false. */ Q_INVOKABLE bool isStereoInput(); signals: /*@jsdoc - * Triggered when the client is muted by the mixer because their loudness value for the noise background has reached the + * Triggered when the client is muted by the mixer because their loudness value for the noise background has reached the * threshold set for the domain (in the server settings). * @function Audio.mutedByMixer - * @returns {Signal} + * @returns {Signal} */ void mutedByMixer(); /*@jsdoc - * Triggered when the client is muted by the mixer because they're within a certain radius (50m) of someone who requested + * Triggered when the client is muted by the mixer because they're within a certain radius (50m) of someone who requested * the mute through Developer > Audio > Mute Environment. * @function Audio.environmentMuted - * @returns {Signal} + * @returns {Signal} */ void environmentMuted(); /*@jsdoc * Triggered when the client receives its first packet from the audio mixer. * @function Audio.receivedFirstPacket - * @returns {Signal} + * @returns {Signal} */ void receivedFirstPacket(); /*@jsdoc * Triggered when the client is disconnected from the audio mixer. * @function Audio.disconnected - * @returns {Signal} + * @returns {Signal} */ void disconnected(); /*@jsdoc - * Triggered when the noise gate is opened. The input audio signal is no longer blocked (fully attenuated) because it has - * risen above an adaptive threshold set just above the noise floor. Only occurs if Audio.noiseReduction is + * Triggered when the noise gate is opened. The input audio signal is no longer blocked (fully attenuated) because it has + * risen above an adaptive threshold set just above the noise floor. Only occurs if Audio.noiseReduction is * true. * @function Audio.noiseGateOpened - * @returns {Signal} + * @returns {Signal} */ void noiseGateOpened(); /*@jsdoc - * Triggered when the noise gate is closed. The input audio signal is blocked (fully attenuated) because it has fallen - * below an adaptive threshold set just above the noise floor. Only occurs if Audio.noiseReduction is + * Triggered when the noise gate is closed. The input audio signal is blocked (fully attenuated) because it has fallen + * below an adaptive threshold set just above the noise floor. Only occurs if Audio.noiseReduction is * true. * @function Audio.noiseGateClosed - * @returns {Signal} + * @returns {Signal} */ void noiseGateClosed(); @@ -275,7 +283,7 @@ class AudioScriptingInterface : public QObject, public Dependency { * Triggered when a frame of audio input is processed. * @function Audio.inputReceived * @param {Int16Array} inputSamples - The audio input processed. - * @returns {Signal} + * @returns {Signal} */ void inputReceived(const QByteArray& inputSamples); diff --git a/plugins/opusCodec/src/OpusEncoder.h b/plugins/opusCodec/src/OpusEncoder.h index 34a1ab73579..792d1565a47 100644 --- a/plugins/opusCodec/src/OpusEncoder.h +++ b/plugins/opusCodec/src/OpusEncoder.h @@ -22,6 +22,7 @@ class AthenaOpusEncoder : public Encoder { virtual void encode(const QByteArray& decodedBuffer, QByteArray& encodedBuffer) override; + const QString getName() const override { return "opus"; } int getComplexity() const override; void setComplexity(int complexity) override; From cb62e3fb91b3af948671532eee1ca02065af51ef Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Mon, 31 Jan 2022 00:42:57 +0100 Subject: [PATCH 09/11] Bump minimum Opus bitrate to 2400 Experimentally verified that nothing is heard below this value. --- plugins/opusCodec/src/OpusEncoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/opusCodec/src/OpusEncoder.cpp b/plugins/opusCodec/src/OpusEncoder.cpp index f7d14bd949a..3f533e1270b 100644 --- a/plugins/opusCodec/src/OpusEncoder.cpp +++ b/plugins/opusCodec/src/OpusEncoder.cpp @@ -116,7 +116,7 @@ int AthenaOpusEncoder::getBitrate() const { void AthenaOpusEncoder::setBitrate(int bitrate) { assert(_encoder); - bitrate = qBound(500, bitrate, 512000); // Opus limits + bitrate = qBound(2400, bitrate, 512000); // Opus limits int errorCode = opus_encoder_ctl(_encoder, OPUS_SET_BITRATE(bitrate)); if (errorCode != OPUS_OK) { From 8652168ccacdbde49d2a474b18730866620a8806 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Mon, 31 Jan 2022 00:43:58 +0100 Subject: [PATCH 10/11] Implement support for VBR, FEC, and codec feature info Codec feature info doesn't actually work yet, type needs registering --- libraries/audio-client/src/AudioClient.cpp | 89 +++++++++++++++++++ libraries/audio-client/src/AudioClient.h | 17 ++++ libraries/audio/src/AbstractAudioInterface.h | 20 +++++ .../src/AudioScriptingInterface.cpp | 80 +++++++++++++++++ .../src/AudioScriptingInterface.h | 23 +++++ 5 files changed, 229 insertions(+) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index a051c350591..bce5445e32b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -486,6 +486,95 @@ void AudioClient::setAllowedCodecs(const QStringList &userCodecs) { negotiateAudioFormat(); } +QMap AudioClient::getEncoderFeatures() { + QMap features; + + if (_encoder) { + features.insert("isLossless", _encoder->isLossless()); + features.insert("hasApplication", _encoder->hasApplication()); + features.insert("hasComplexity", _encoder->hasComplexity()); + features.insert("hasBitrate", _encoder->hasBitrate()); + features.insert("hasFEC", _encoder->hasFEC()); + features.insert("hasPacketLossPercent", _encoder->hasPacketLossPercent()); + features.insert("hasBandpass", _encoder->hasBandpass()); + features.insert("hasSignalType", _encoder->hasSignalType()); + features.insert("hasVBR", _encoder->hasVBR()); + } + + return features; +} + +bool AudioClient::getEncoderVBR() { + if (_encoder) { + return _encoder->getVBR(); + } + + return false; +} + +void AudioClient::setEncoderVBR(bool enabled) { + if (_encoder) { + _encoder->setVBR(enabled); + } +} + +int AudioClient::getEncoderBitrate() { + if (_encoder) { + return _encoder->getBitrate(); + } + + return 0; +} + +void AudioClient::setEncoderBitrate(int bitrate) { + if (_encoder) { + _encoder->setBitrate(bitrate); + } +} + +int AudioClient::getEncoderComplexity() { + if (_encoder) { + return _encoder->getComplexity(); + } + + return 0; +} + +void AudioClient::setEncoderComplexity(int complexity) { + if (_encoder) { + _encoder->setComplexity(complexity); + } +} + +bool AudioClient::getEncoderFEC() { + if (_encoder) { + return _encoder->getFEC(); + } + + return false; +} + +void AudioClient::setEncoderFEC(bool enabled) { + if (_encoder) { + _encoder->setFEC(enabled); + } +} + +int AudioClient::getEncoderPacketLossPercent() { + if (_encoder) { + return _encoder->getPacketLossPercent(); + } + + return 0; +} + +void AudioClient::setEncoderPacketLossPercent(int percent) { + if (_encoder) { + _encoder->setPacketLossPercent(percent); + } +} + + HifiAudioDeviceInfo getNamedAudioDeviceForMode(QAudio::Mode mode, const QString& deviceName, const QString& hmdName, bool isHmd=false) { HifiAudioDeviceInfo result; foreach (HifiAudioDeviceInfo audioDevice, getAvailableDevices(mode,hmdName)) { diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index d1594bfe83b..913f25d1c25 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -194,6 +194,23 @@ class AudioClient : public AbstractAudioInterface, public Dependency { void setAllowedCodecs(const QStringList &codecs) override; void setCodecSettings(const std::vector &settings ) { _codecSettings = settings; } + QMap getEncoderFeatures() override; + + bool getEncoderVBR() override; + void setEncoderVBR(bool enabled) override; + + int getEncoderBitrate() override; + void setEncoderBitrate(int bitrate) override; + + int getEncoderComplexity() override; + void setEncoderComplexity(int complexity) override; + + bool getEncoderFEC() override; + void setEncoderFEC(bool enabled) override; + + int getEncoderPacketLossPercent() override; + void setEncoderPacketLossPercent(int percent) override; + #ifdef Q_OS_WIN static QString getWinDeviceName(wchar_t* guid); #endif diff --git a/libraries/audio/src/AbstractAudioInterface.h b/libraries/audio/src/AbstractAudioInterface.h index ed566bdb61e..410c5dafbc7 100644 --- a/libraries/audio/src/AbstractAudioInterface.h +++ b/libraries/audio/src/AbstractAudioInterface.h @@ -60,6 +60,26 @@ public slots: virtual void setAllowedCodecs(const QStringList &codec) = 0; virtual QStringList getAllowedCodecs() = 0; + virtual QMap getEncoderFeatures() = 0; + + virtual bool getEncoderVBR() = 0; + virtual void setEncoderVBR(bool enabled) = 0; + + virtual int getEncoderBitrate() = 0; + virtual void setEncoderBitrate(int bitrate) = 0; + + virtual int getEncoderComplexity() = 0; + virtual void setEncoderComplexity(int complexity) = 0; + + virtual bool getEncoderFEC() = 0; + virtual void setEncoderFEC(bool enabled) = 0; + + virtual int getEncoderPacketLossPercent() = 0; + virtual void setEncoderPacketLossPercent(int percent) = 0; + + + + signals: void isStereoInputChanged(bool isStereo); }; diff --git a/libraries/script-engine/src/AudioScriptingInterface.cpp b/libraries/script-engine/src/AudioScriptingInterface.cpp index dc745a4ef9d..0630a1d2110 100644 --- a/libraries/script-engine/src/AudioScriptingInterface.cpp +++ b/libraries/script-engine/src/AudioScriptingInterface.cpp @@ -150,4 +150,84 @@ QStringList AudioScriptingInterface::getAllowedCodecs() { } return codecs; +} + +QMap AudioScriptingInterface::getEncoderFeatures() { + QMap features; + + if (_localAudioInterface) { + features = _localAudioInterface->getEncoderFeatures(); + } + + return features; +} + +int AudioScriptingInterface::getEncoderBitrate() { + if (_localAudioInterface) { + return _localAudioInterface->getEncoderBitrate(); + } + + return 0; +} + +void AudioScriptingInterface::setEncoderBitrate(int bitrate) { + if (_localAudioInterface) { + _localAudioInterface->setEncoderBitrate(bitrate); + } +} + +int AudioScriptingInterface::getEncoderComplexity() { + if (_localAudioInterface) { + return _localAudioInterface->getEncoderComplexity(); + } + + return 0; +} + +void AudioScriptingInterface::setEncoderComplexity(int complexity) { + if (_localAudioInterface) { + _localAudioInterface->setEncoderComplexity(complexity); + } +} + +bool AudioScriptingInterface::getEncoderVBR() { + if (_localAudioInterface) { + return _localAudioInterface->getEncoderVBR(); + } + + return 0; +} + +void AudioScriptingInterface::setEncoderVBR(bool enabled) { + if (_localAudioInterface) { + _localAudioInterface->setEncoderVBR(enabled); + } +} + +bool AudioScriptingInterface::getEncoderFEC() { + if (_localAudioInterface) { + return _localAudioInterface->getEncoderFEC(); + } + + return 0; +} + +void AudioScriptingInterface::setEncoderFEC(bool enabled) { + if (_localAudioInterface) { + _localAudioInterface->setEncoderFEC(enabled); + } +} + +int AudioScriptingInterface::getEncoderPacketLossPercent() { + if (_localAudioInterface) { + return _localAudioInterface->getEncoderPacketLossPercent(); + } + + return 0; +} + +void AudioScriptingInterface::setEncoderPacketLossPercent(int percent) { + if (_localAudioInterface) { + _localAudioInterface->setEncoderPacketLossPercent(percent); + } } \ No newline at end of file diff --git a/libraries/script-engine/src/AudioScriptingInterface.h b/libraries/script-engine/src/AudioScriptingInterface.h index d9d28009501..69ebdd081a8 100644 --- a/libraries/script-engine/src/AudioScriptingInterface.h +++ b/libraries/script-engine/src/AudioScriptingInterface.h @@ -162,6 +162,29 @@ class AudioScriptingInterface : public QObject, public Dependency { Q_INVOKABLE QStringList getAllowedCodecs(); + Q_INVOKABLE QMap getEncoderFeatures(); + + Q_INVOKABLE int getEncoderBitrate(); + + Q_INVOKABLE void setEncoderBitrate(int bitrate); + + Q_INVOKABLE bool getEncoderVBR(); + + Q_INVOKABLE void setEncoderVBR(bool enabled); + + Q_INVOKABLE int getEncoderComplexity(); + + Q_INVOKABLE void setEncoderComplexity(int complexity); + + Q_INVOKABLE bool getEncoderFEC(); + + Q_INVOKABLE void setEncoderFEC(bool enabled); + + Q_INVOKABLE int getEncoderPacketLossPercent(); + + Q_INVOKABLE void setEncoderPacketLossPercent(int percent); + + protected: AudioScriptingInterface() = default; From 9b06c9070dd61ba97c075b0de08b71ad84374ec0 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Tue, 1 Feb 2022 00:32:46 +0100 Subject: [PATCH 11/11] Add JSDoc --- .../src/AudioScriptingInterface.h | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/libraries/script-engine/src/AudioScriptingInterface.h b/libraries/script-engine/src/AudioScriptingInterface.h index 69ebdd081a8..b4c5a70c084 100644 --- a/libraries/script-engine/src/AudioScriptingInterface.h +++ b/libraries/script-engine/src/AudioScriptingInterface.h @@ -154,34 +154,119 @@ class AudioScriptingInterface : public QObject, public Dependency { */ Q_INVOKABLE void toggleLocalEcho(); + /*@jsdoc + * Returns the list of codecs known to the system as an array + * @function Audio.getCodecs + */ Q_INVOKABLE QStringList getCodecs(); + /*@jsdoc + * Returns the currently used codec + * @function Audio.getCodec + */ Q_INVOKABLE QString getCodec(); + /*@jsdoc + * Sets the list of codecs the client will try to negotiate with the domain. + * This is a sub-set of the list returned by getCodecs. + * If this list is empty, all system codecs are accepted. + * @parm {String[]} list of codecs to accept + * @function Audio.setAllowedCodecs + */ Q_INVOKABLE void setAllowedCodecs(const QStringList &codecs); + /*@jsdoc + * Returns the list of currently accepted codecs. + * If this list is empty, all system codecs are accepted. + * @function Audio.setAllowedCodecs + */ Q_INVOKABLE QStringList getAllowedCodecs(); Q_INVOKABLE QMap getEncoderFeatures(); + /*@jsdoc + * Returns the bitrate of the current encoder + * @function Audio.getEncoderBitrate + */ Q_INVOKABLE int getEncoderBitrate(); + /*@jsdoc + * Sets the bitrate of the current encoder. + * + * @parm {int} Bit rate, in bps (eg, 128000 for 128kbps) + * @function Audio.setEncoderBitrate + */ Q_INVOKABLE void setEncoderBitrate(int bitrate); + /*@jsdoc + * Returns whether the current encoder has Variable Bit Rate enabled. + * @function Audio.getEncoderVBR + */ Q_INVOKABLE bool getEncoderVBR(); + /*@jsdoc + * Sets whether the current encoder uses Variable Bit Rate. + * + * When VBR is enabled, the encoder will use less bandwidth during times of silence and low + * audio signal complexity. + * @parm {bool} Whether VBR is abled + * @function Audio.setEncoderVBR + */ + Q_INVOKABLE void setEncoderVBR(bool enabled); + /*@jsdoc + * Returns the complexity of the current encoder. + * The complexity is a number from 0 to 100 indicating how hard the codec tries to compress the data. + * This is expected to have an effect on the amount of CPU time needed to compress the audio. + * Higher levels are more CPU intensive but produce better quality or lower bandwidth usage. + * @function Audio.getEncoderComplexity + */ Q_INVOKABLE int getEncoderComplexity(); + /*@jsdoc + * Sets the complexity of the current encoder. + * The complexity is a number from 0 to 100 indicating how hard the codec tries to compress the data. + * This is expected to have an effect on the amount of CPU time needed to compress the audio. + * Higher levels are more CPU intensive but produce better quality or lower bandwidth usage. + * @parm {int} Complexity, from 0 to 100. + * @function Audio.setEncoderComplexity + */ + Q_INVOKABLE void setEncoderComplexity(int complexity); + /*@jsdoc + * Returns whether the current encoder has Forward Error Correction enabled + * @function Audio.getEncoderFEC + */ Q_INVOKABLE bool getEncoderFEC(); + /*@jsdoc + * Sets whether the current encoder uses Forward Error Correction + * FEC compensates for data loss in audio. Enabling this on a codec that supports it should result + * in better audio quality with bad network connections. + * + * Enabling this may cost additional bandwidth, or reduce encoding quality to make room for the redundancy. + * @parm {bool} Whether FEC is enabled + * @function Audio.setEncoderFEC + */ Q_INVOKABLE void setEncoderFEC(bool enabled); + /*@jsdoc + * Returns the current encoder's expected packet loss percent + * @function Audio.getEncoderPacketLossPercent + */ Q_INVOKABLE int getEncoderPacketLossPercent(); + /*@jsdoc + * Sets whether the expected packet loss for FEC + * FEC compensates for data loss in audio. Enabling this on a codec that supports it should result + * in better audio quality with bad network connections. + * + * Enabling this may cost additional bandwidth, or reduce encoding quality to make room for the redundancy. + * @parm {int} Expected packet loss, in percent + * @function Audio.setEncoderPacketLossPercent + */ Q_INVOKABLE void setEncoderPacketLossPercent(int percent);