diff --git a/error.go b/error.go
index 4df5d2a..ac3ece4 100644
--- a/error.go
+++ b/error.go
@@ -21,4 +21,6 @@ var (
 	errRFC8285TwoByteHeaderSize    = errors.New("header extension payload must be 255bytes or less for RFC 5285 two byte extensions")
 
 	errRFC3550HeaderIDRange = errors.New("header extension id must be 0 for non-RFC 5285 extensions")
+
+	errInvalidRTPPadding = errors.New("invalid RTP padding")
 )
diff --git a/packet.go b/packet.go
index af88af3..61d341d 100644
--- a/packet.go
+++ b/packet.go
@@ -215,6 +215,9 @@ func (p *Packet) Unmarshal(buf []byte) error {
 
 	end := len(buf)
 	if p.Header.Padding {
+		if end <= n {
+			return errTooSmall
+		}
 		p.PaddingSize = buf[end-1]
 		end -= int(p.PaddingSize)
 	}
@@ -475,6 +478,10 @@ func (p Packet) Marshal() (buf []byte, err error) {
 
 // MarshalTo serializes the packet and writes to the buffer.
 func (p *Packet) MarshalTo(buf []byte) (n int, err error) {
+	if p.Header.Padding && p.PaddingSize == 0 {
+		return 0, errInvalidRTPPadding
+	}
+
 	n, err = p.Header.MarshalTo(buf)
 	if err != nil {
 		return 0, err