rtmp: fix crash when receiving unexpected video packets (#1459) (#1504)

internal/core/rtmp_conn.go

@@ -573,6 +573,10 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
 
 		switch tmsg := msg.(type) {
 		case *message.MsgVideo:
+			if videoFormat == nil {
+				return fmt.Errorf("received a video packet, but track is not set up")
+			}
+
 			if tmsg.H264Type == flvio.AVC_SEQHDR {
 				var conf h264conf.Conf
 				err = conf.Unmarshal(tmsg.Payload)
@@ -594,10 +598,6 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
 					c.log(logger.Warn, "%v", err)
 				}
 			} else if tmsg.H264Type == flvio.AVC_NALU {
-				if videoFormat == nil {
-					return fmt.Errorf("received a video packet, but track is not set up")
-				}
-
 				au, err := h264.AVCCUnmarshal(tmsg.Payload)
 				if err != nil {
 					c.log(logger.Warn, "unable to decode AVCC: %v", err)
@@ -608,11 +608,11 @@ func (c *rtmpConn) runPublish(ctx context.Context, u *url.URL) error {
 			}
 
 		case *message.MsgAudio:
-			if tmsg.AACType == flvio.AAC_RAW {
-				if audioFormat == nil {
-					return fmt.Errorf("received an audio packet, but track is not set up")
-				}
+			if audioFormat == nil {
+				return fmt.Errorf("received an audio packet, but track is not set up")
+			}
 
+			if tmsg.AACType == flvio.AAC_RAW {
 				err := rres.stream.writeData(audioMedia, audioFormat, &formatprocessor.DataMPEG4Audio{
 					PTS: tmsg.DTS,
 					AUs: [][]byte{tmsg.Payload},