webrtc: support reading and publishing multichannel Opus (#3371)

bluenviron · Jun 2, 2024 · bbf3d0c · bbf3d0c
1 parent c37e895
commit bbf3d0c
Show file tree

Hide file tree

Showing 6 changed files with 346 additions and 13 deletions.
diff --git a/internal/protocols/webrtc/incoming_track.go b/internal/protocols/webrtc/incoming_track.go
@@ -93,6 +93,60 @@ var incomingVideoCodecs = []webrtc.RTPCodecParameters{
 }
 
 var incomingAudioCodecs = []webrtc.RTPCodecParameters{
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:    mimeMultiopus,
+			ClockRate:   48000,
+			Channels:    3,
+			SDPFmtpLine: "channel_mapping=0,2,1;num_streams=2;coupled_streams=1",
+		},
+		PayloadType: 112,
+	},
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:    mimeMultiopus,
+			ClockRate:   48000,
+			Channels:    4,
+			SDPFmtpLine: "channel_mapping=0,1,2,3;num_streams=2;coupled_streams=2",
+		},
+		PayloadType: 113,
+	},
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:    mimeMultiopus,
+			ClockRate:   48000,
+			Channels:    5,
+			SDPFmtpLine: "channel_mapping=0,4,1,2,3;num_streams=3;coupled_streams=2",
+		},
+		PayloadType: 114,
+	},
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:    mimeMultiopus,
+			ClockRate:   48000,
+			Channels:    6,
+			SDPFmtpLine: "channel_mapping=0,4,1,2,3,5;num_streams=4;coupled_streams=2",
+		},
+		PayloadType: 115,
+	},
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:    mimeMultiopus,
+			ClockRate:   48000,
+			Channels:    7,
+			SDPFmtpLine: "channel_mapping=0,4,1,2,3,5,6;num_streams=4;coupled_streams=4",
+		},
+		PayloadType: 116,
+	},
+	{
+		RTPCodecCapability: webrtc.RTPCodecCapability{
+			MimeType:    mimeMultiopus,
+			ClockRate:   48000,
+			Channels:    8,
+			SDPFmtpLine: "channel_mapping=0,6,1,4,5,2,3,7;num_streams=5;coupled_streams=4",
+		},
+		PayloadType: 117,
+	},
 	{
 		RTPCodecCapability: webrtc.RTPCodecCapability{
 			MimeType:    webrtc.MimeTypeOpus,
@@ -191,6 +245,12 @@ func newIncomingTrack(
 			PacketizationMode: 1,
 		}
 
+	case strings.ToLower(mimeMultiopus):
+		t.format = &format.Opus{
+			PayloadTyp:   uint8(track.PayloadType()),
+			ChannelCount: int(track.Codec().Channels),
+		}
+
 	case strings.ToLower(webrtc.MimeTypeOpus):
 		t.format = &format.Opus{
 			PayloadTyp: uint8(track.PayloadType()),
@@ -242,7 +302,7 @@ func newIncomingTrack(
 		}
 
 	default:
-		return nil, fmt.Errorf("unsupported codec: %v", track.Codec())
+		return nil, fmt.Errorf("unsupported codec: %+v", track.Codec())
 	}
 
 	// read incoming RTCP packets to make interceptors work

diff --git a/internal/protocols/webrtc/outgoing_track.go b/internal/protocols/webrtc/outgoing_track.go
@@ -8,6 +8,10 @@ import (
 	"github.com/pion/webrtc/v3"
 )
 
+const (
+	mimeMultiopus = "audio/multiopus"
+)
+
 // OutgoingTrack is a WebRTC outgoing track
 type OutgoingTrack struct {
 	Format format.Format
@@ -29,9 +33,8 @@ func (t *OutgoingTrack) codecParameters() (webrtc.RTPCodecParameters, error) {
 	case *format.VP9:
 		return webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:    webrtc.MimeTypeVP9,
-				ClockRate:   90000,
-				SDPFmtpLine: "profile-id=1",
+				MimeType:  webrtc.MimeTypeVP9,
+				ClockRate: 90000,
 			},
 			PayloadType: 96,
 		}, nil
@@ -57,14 +60,28 @@ func (t *OutgoingTrack) codecParameters() (webrtc.RTPCodecParameters, error) {
 
 	case *format.Opus:
 		if forma.ChannelCount > 2 {
-			return webrtc.RTPCodecParameters{}, fmt.Errorf("unsupported Opus channel count: %d", forma.ChannelCount)
+			return webrtc.RTPCodecParameters{
+				RTPCodecCapability: webrtc.RTPCodecCapability{
+					MimeType:  mimeMultiopus,
+					ClockRate: 48000,
+					Channels:  uint16(forma.ChannelCount),
+				},
+				PayloadType: 96,
+			}, nil
 		}
 
 		return webrtc.RTPCodecParameters{
 			RTPCodecCapability: webrtc.RTPCodecCapability{
 				MimeType:  webrtc.MimeTypeOpus,
 				ClockRate: 48000,
 				Channels:  2,
+				SDPFmtpLine: func() string {
+					s := "minptime=10;useinbandfec=1"
+					if forma.ChannelCount == 2 {
+						s += ";stereo=1;sprop-stereo=1"
+					}
+					return s
+				}(),
 			},
 			PayloadType: 96,
 		}, nil

diff --git a/internal/protocols/webrtc/peer_connection.go b/internal/protocols/webrtc/peer_connection.go
@@ -2,6 +2,7 @@ package webrtc
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strconv"
 	"sync"
@@ -261,8 +262,8 @@ func (co *PeerConnection) SetAnswer(answer *webrtc.SessionDescription) error {
 }
 
 // AddRemoteCandidate adds a remote candidate.
-func (co *PeerConnection) AddRemoteCandidate(candidate webrtc.ICECandidateInit) error {
-	return co.wr.AddICECandidate(candidate)
+func (co *PeerConnection) AddRemoteCandidate(candidate *webrtc.ICECandidateInit) error {
+	return co.wr.AddICECandidate(*candidate)
 }
 
 // CreateFullAnswer creates a full answer.
@@ -277,7 +278,7 @@ func (co *PeerConnection) CreateFullAnswer(
 
 	answer, err := co.wr.CreateAnswer(nil)
 	if err != nil {
-		if err.Error() == "unable to populate media section, RTPSender created with no codecs" {
+		if errors.Is(err, webrtc.ErrSenderWithNoCodecs) {
 			return nil, fmt.Errorf("track codecs are not supported by remote")
 		}
 		return nil, err
@@ -288,16 +289,15 @@ func (co *PeerConnection) CreateFullAnswer(
 		return nil, err
 	}
 
-	err = co.WaitGatheringDone(ctx)
+	err = co.waitGatheringDone(ctx)
 	if err != nil {
 		return nil, err
 	}
 
 	return co.wr.LocalDescription(), nil
 }
 
-// WaitGatheringDone waits until candidate gathering is complete.
-func (co *PeerConnection) WaitGatheringDone(ctx context.Context) error {
+func (co *PeerConnection) waitGatheringDone(ctx context.Context) error {
 	for {
 		select {
 		case <-co.NewLocalCandidate():