Skip to content

Commit

Permalink
toxav incomming audio src and fix sdl audio output sink
Browse files Browse the repository at this point in the history
  • Loading branch information
@Green-Sky
Green-Sky committed Sep 19, 2024
1 parent 38c2036 commit f6d0799
Show file tree
Hide file tree
Showing 6 changed files with 141 additions and 60 deletions.
15 changes: 1 addition & 14 deletions src/content/frame_stream2.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,7 @@ struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public F

virtual ~FrameStream2MultiSource(void) {}

//// TODO: forward args instead
//SubStreamType* aquireSubStream(size_t queue_size = 10, bool lossy = true) {
// std::lock_guard lg{_sub_stream_lock};
// return _sub_streams.emplace_back(std::make_unique<SubStreamType>(queue_size, lossy)).get();
//}
// TODO: forward args instead
std::shared_ptr<FrameStream2I<FrameType>> subscribe(void) override {
// TODO: args???
size_t queue_size = 10;
Expand All @@ -118,15 +114,6 @@ struct FrameStream2MultiSource : public FrameStream2SourceI<FrameType>, public F
return _sub_streams.emplace_back(std::make_unique<SubStreamType>(queue_size, lossy));
}

//void releaseSubStream(SubStreamType* sub) {
// std::lock_guard lg{_sub_stream_lock};
// for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {
// if (it->get() == sub) {
// _sub_streams.erase(it);
// break;
// }
// }
//}
bool unsubscribe(const std::shared_ptr<FrameStream2I<FrameType>>& sub) override {
std::lock_guard lg{_sub_stream_lock};
for (auto it = _sub_streams.begin(); it != _sub_streams.end(); it++) {
Expand Down
38 changes: 22 additions & 16 deletions src/content/sdl_audio_frame_stream2.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,33 +100,39 @@ bool SDLAudioOutputDeviceDefaultInstance::push(const AudioFrame& value) {
(value.isF32() && _last_format != SDL_AUDIO_F32) ||
(value.isS16() && _last_format != SDL_AUDIO_S16)
) {
const auto device_id = SDL_GetAudioStreamDevice(_stream.get());
SDL_FlushAudioStream(_stream.get());

const SDL_AudioSpec spec = {
static_cast<SDL_AudioFormat>((value.isF32() ? SDL_AUDIO_F32 : SDL_AUDIO_S16)),
static_cast<int>(value.channels),
static_cast<int>(value.sample_rate)
};

_stream = {
SDL_OpenAudioDeviceStream(device_id, &spec, nullptr, nullptr),
&SDL_DestroyAudioStream
};
SDL_SetAudioStreamFormat(_stream.get(), &spec, nullptr);

std::cerr << "SDLAOD: audio format changed\n";
}

// HACK
assert(value.isS16());
if (value.isS16()) {
auto data = value.getSpan<int16_t>();

auto data = value.getSpan<int16_t>();
if (data.size == 0) {
std::cerr << "empty audio frame??\n";
}

if (data.size == 0) {
std::cerr << "empty audio frame??\n";
}
if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(int16_t))) {
std::cerr << "put data error\n";
return false; // return true?
}
} else if (value.isF32()) {
auto data = value.getSpan<float>();

if (data.size == 0) {
std::cerr << "empty audio frame??\n";
}

if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(int16_t))) {
std::cerr << "put data error\n";
return false; // return true?
if (!SDL_PutAudioStreamData(_stream.get(), data.ptr, data.size * sizeof(float))) {
std::cerr << "put data error\n";
return false; // return true?
}
}

_last_sample_rate = value.sample_rate;
Expand Down
118 changes: 90 additions & 28 deletions src/debug_tox_call.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "./stream_manager.hpp"
#include "./content/sdl_video_frame_stream2.hpp"
#include "./content/sdl_audio_frame_stream2.hpp"

#include <SDL3/SDL.h>

Expand All @@ -18,20 +19,6 @@ namespace Message {
uint64_t getTimeMS();
}

namespace Components {
struct ToxAVFriendAudioSource {
};

struct ToxAVFriendAudioSink {
};

struct ToxAVFriendVideoSource {
};

struct ToxAVFriendVideoSink {
};
}

static bool isFormatPlanar(SDL_PixelFormat f) {
return
f == SDL_PIXELFORMAT_YV12 ||
Expand Down Expand Up @@ -166,20 +153,34 @@ struct PushConversionQueuedVideoStream : public QueuedFrameStream2<SDLVideoFrame
// exlusive
// TODO: replace with something better than a queue
struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
ToxAV& _toxav;

// bitrate for enabled state
uint32_t _video_bitrate {2};

uint32_t _fid;
std::shared_ptr<PushConversionQueuedVideoStream> _writer;

ToxAVCallVideoSink(uint32_t fid) : _fid(fid) {}
~ToxAVCallVideoSink(void) {}
ToxAVCallVideoSink(ToxAV& toxav, uint32_t fid) : _toxav(toxav), _fid(fid) {}
~ToxAVCallVideoSink(void) {
if (_writer) {
_writer = nullptr;
_toxav.toxavVideoSetBitRate(_fid, 0);
}
}

// sink
std::shared_ptr<FrameStream2I<SDLVideoFrame>> subscribe(void) override {
if (_writer) {
// max 1 (exclusive)
// max 1 (exclusive, composite video somewhere else)
return nullptr;
}

auto err = _toxav.toxavVideoSetBitRate(_fid, _video_bitrate);
if (err != TOXAV_ERR_BIT_RATE_SET_OK) {
return nullptr;
}

// TODO: enable video here
_writer = std::make_shared<PushConversionQueuedVideoStream>(10, true);

return _writer;
Expand All @@ -192,8 +193,11 @@ struct ToxAVCallVideoSink : public FrameStream2SinkI<SDLVideoFrame> {
}

if (sub == _writer) {
// TODO: disable video here
_writer = nullptr;

/*auto err = */_toxav.toxavVideoSetBitRate(_fid, 0);
// print warning? on error?

return true;
}

Expand All @@ -211,9 +215,28 @@ DebugToxCall::DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu)
_toxav.subscribe(this, ToxAV_Event::friend_video_frame);
}

DebugToxCall::~DebugToxCall(void) {
// destroy all calls/connections/sources/sinks here

for (auto& [fid, call] : _calls) {
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
}
if (static_cast<bool>(call.incoming_asrc)) {
call.incoming_asrc.destroy();
}
if (static_cast<bool>(call.outgoing_vsink)) {
call.outgoing_vsink.destroy();
}
if (static_cast<bool>(call.outgoing_asink)) {
call.outgoing_asink.destroy();
}
}
}

void DebugToxCall::tick(float) {
// pump sink to tox
// TODO: own thread or direct on push
// pump sinks to tox
// TODO: own thread or direct on push (requires thread save toxcore)
// TODO: pump at double the frame rate
for (const auto& [oc, vsink] : _os.registry().view<ToxAVCallVideoSink*>().each()) {
if (!vsink->_writer) {
Expand Down Expand Up @@ -258,8 +281,9 @@ float DebugToxCall::render(void) {
if (call.incoming) {
ImGui::SameLine();
if (ImGui::SmallButton("answer")) {
const auto ret = _toxav.toxavAnswer(fid, 0, 0);
//const auto ret = _toxav.toxavAnswer(fid, 0, 1); // 1mbit/s
const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s
//const auto ret = _toxav.toxavAnswer(fid, 0, 2); // 2mbit/s
//const auto ret = _toxav.toxavAnswer(fid, 0, 100); // 100mbit/s
//const auto ret = _toxav.toxavAnswer(fid, 0, 2500); // 2500mbit/s
if (ret == TOXAV_ERR_ANSWER_OK) {
Expand All @@ -268,7 +292,7 @@ float DebugToxCall::render(void) {
// create sinks
call.outgoing_vsink = {_os.registry(), _os.registry().create()};
{
auto new_vsink = std::make_unique<ToxAVCallVideoSink>(fid);
auto new_vsink = std::make_unique<ToxAVCallVideoSink>(_toxav, fid);
call.outgoing_vsink.emplace<ToxAVCallVideoSink*>(new_vsink.get());
call.outgoing_vsink.emplace<Components::FrameStream2Sink<SDLVideoFrame>>(std::move(new_vsink));
call.outgoing_vsink.emplace<Components::StreamSink>("ToxAV friend call video", std::string{entt::type_name<SDLVideoFrame>::value()});
Expand All @@ -284,6 +308,15 @@ float DebugToxCall::render(void) {
call.incoming_vsrc.emplace<Components::StreamSource>("ToxAV friend call video", std::string{entt::type_name<SDLVideoFrame>::value()});
}
}
if (call.incoming_a) {
call.incoming_asrc = {_os.registry(), _os.registry().create()};
{
auto new_asrc = std::make_unique<AudioFrameStream2MultiSource>();
call.incoming_asrc.emplace<AudioFrameStream2MultiSource*>(new_asrc.get());
call.incoming_asrc.emplace<Components::FrameStream2Source<AudioFrame>>(std::move(new_asrc));
call.incoming_asrc.emplace<Components::StreamSource>("ToxAV friend call audio", std::string{entt::type_name<AudioFrame>::value()});
}
}
}
}
} else if (call.state != TOXAV_FRIEND_CALL_STATE_FINISHED) {
Expand All @@ -297,11 +330,17 @@ float DebugToxCall::render(void) {
call.state = TOXAV_FRIEND_CALL_STATE_FINISHED;

// TODO: stream manager disconnectAll()
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
}
if (static_cast<bool>(call.incoming_asrc)) {
call.incoming_asrc.destroy();
}
if (static_cast<bool>(call.outgoing_vsink)) {
call.outgoing_vsink.destroy();
}
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
if (static_cast<bool>(call.outgoing_asink)) {
call.outgoing_asink.destroy();
}
}
}
Expand Down Expand Up @@ -341,11 +380,17 @@ bool DebugToxCall::onEvent(const Events::FriendCallState& e) {
(call.state & TOXAV_FRIEND_CALL_STATE_FINISHED) != 0 ||
(call.state & TOXAV_FRIEND_CALL_STATE_ERROR) != 0
) {
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
}
if (static_cast<bool>(call.incoming_asrc)) {
call.incoming_asrc.destroy();
}
if (static_cast<bool>(call.outgoing_vsink)) {
call.outgoing_vsink.destroy();
}
if (static_cast<bool>(call.incoming_vsrc)) {
call.incoming_vsrc.destroy();
if (static_cast<bool>(call.outgoing_asink)) {
call.outgoing_asink.destroy();
}
}

Expand All @@ -362,8 +407,25 @@ bool DebugToxCall::onEvent(const Events::FriendVideoBitrate&) {

bool DebugToxCall::onEvent(const Events::FriendAudioFrame& e) {
auto& call = _calls[e.friend_number];

if (!static_cast<bool>(call.incoming_asrc)) {
// missing src to put frame into ??
return false;
}

assert(call.incoming_asrc.all_of<AudioFrameStream2MultiSource*>());
assert(call.incoming_asrc.all_of<Components::FrameStream2Source<AudioFrame>>());

call.num_a_frames++;
return false;

call.incoming_asrc.get<AudioFrameStream2MultiSource*>()->push(AudioFrame{
0, //seq
e.sampling_rate,
e.channels,
std::vector<int16_t>(e.pcm.begin(), e.pcm.end()) // copy
});

return true;
}

bool DebugToxCall::onEvent(const Events::FriendVideoFrame& e) {
Expand Down
2 changes: 1 addition & 1 deletion src/debug_tox_call.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ class DebugToxCall : public ToxAVEventI {

public:
DebugToxCall(ObjectStore2& os, ToxAV& toxav, TextureUploaderI& tu);
~DebugToxCall(void) {}
~DebugToxCall(void);

void tick(float time_delta);
float render(void);
Expand Down
26 changes: 26 additions & 0 deletions src/stream_manager_ui.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include "./stream_manager_ui.hpp"

#include "./content/sdl_video_frame_stream2.hpp"
#include "./content/audio_stream.hpp"

#include <solanaceae/object_store/object_store.hpp>

Expand Down Expand Up @@ -58,6 +59,31 @@ void StreamManagerUI::render(void) {

ImGui::EndMenu();
}

if (ImGui::BeginMenu("connect audio", ss.frame_type_name == entt::type_name<AudioFrame>::value())) {
for (const auto& [oc_src, s_src] : _os.registry().view<Components::StreamSource>().each()) {
if (s_src.frame_type_name != ss.frame_type_name) {
continue;
}

ImGui::PushID(entt::to_integral(oc_src));

std::string source_label {"src "};
source_label += std::to_string(entt::to_integral(entt::to_entity(oc_src)));
source_label += " (";
source_label += s_src.name;
source_label += ")[";
source_label += s_src.frame_type_name;
source_label += "]";
if (ImGui::MenuItem(source_label.c_str())) {
_sm.connect<AudioFrame>(oc_src, oc);
}

ImGui::PopID();
}

ImGui::EndMenu();
}
ImGui::EndPopup();
}
ImGui::PopID();
Expand Down
2 changes: 1 addition & 1 deletion src/tox_av.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ void ToxAV::cb_audio_receive_frame(uint32_t friend_number, const int16_t pcm[],
ToxAV_Event::friend_audio_frame,
Events::FriendAudioFrame{
friend_number,
Span<int16_t>(pcm, sample_count), // TODO: is sample count *ch or /ch?
Span<int16_t>(pcm, sample_count*channels), // TODO: is sample count *ch or /ch?
channels,
sampling_rate,
}
Expand Down

0 comments on commit f6d0799

Please sign in to comment.