Change on-disk format to use protobuf version 3

This commit changes the XC on-disk format to use the proto3 encoding instead
of the custom binary/gob encoding. This will help with future changes to the
on-disk format and improve accessibility from other programs.

.travis.yml

@@ -2,7 +2,6 @@ language: go
 dist: trusty
 os:
   - linux
-  - osx
 
 before_install:
   - if [ $TRAVIS_OS_NAME = linux ]; then sudo apt-get install git gnupg2; else brew install git gnupg || true; fi

backend/crypto/xc/decrypt.go

@@ -1,115 +1,133 @@
 package xc
 
 import (
-	"bytes"
 	"context"
-	"encoding/binary"
 	"fmt"
 	"time"
 
+	"github.com/gogo/protobuf/proto"
 	"github.com/justwatchcom/gopass/backend/crypto/xc/keyring"
+	"github.com/justwatchcom/gopass/backend/crypto/xc/xcpb"
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/crypto/nacl/box"
 )
 
+const (
+	maxUnlockAttempts = 3
+)
+
 // Decrypt tries to decrypt the given ciphertext and returns the plaintext
 func (x *XC) Decrypt(ctx context.Context, buf []byte) ([]byte, error) {
-	hdr, hdrLen, err := x.readHeader(buf)
-	if err != nil {
+	// unmarshal the protobuf message, the header and body are still encrypted
+	// afterwards (parts of the header are plaintext!)
+	msg := &xcpb.Message{}
+	if err := proto.Unmarshal(buf, msg); err != nil {
 		return nil, err
 	}
-	cipher := buf[hdrLen:]
-	sk, err := x.decryptSessionKey(hdr)
+
+	// try to find a suiteable decryption key in the header
+	sk, err := x.decryptSessionKey(msg.Header)
 	if err != nil {
 		return nil, err
 	}
+
+	// initialize the AEAD cipher with the session key
 	cp, err := chacha20poly1305.New(sk)
 	if err != nil {
 		return nil, err
 	}
-	return cp.Open(nil, hdr.Nonce, cipher, nil)
-}
 
-// TODO fuzz this
-func (x *XC) readHeader(buf []byte) (*Header, uint64, error) {
-	var hdrLen uint64
-	if err := binary.Read(bytes.NewReader(buf), binary.LittleEndian, &hdrLen); err != nil {
-		return nil, 0, err
-	}
-	hdrOffset := binary.Size(hdrLen)
-	hdrEnd := uint64(hdrOffset) + hdrLen
-	//fmt.Printf("Offset: %d - Len: %d - Header: %x\n", hdrOffset, hdrLen, buf[hdrOffset:hdrEnd])
-	if uint64(len(buf)) < hdrEnd {
-		return nil, 0, fmt.Errorf("invalid header (%d < %d)", len(buf), hdrEnd)
-	}
-	hdr := &Header{}
-	err := hdr.Unmarshal(buf[hdrOffset:hdrEnd])
-	return hdr, hdrEnd, err
+	// decrypt and verify the ciphertext
+	return cp.Open(nil, msg.Header.Nonce, msg.Body, nil)
 }
 
-func (x *XC) findDecryptionKey(hdr *Header) (*keyring.PrivateKey, error) {
-	for _, pk := range x.keyring.DecryptionKeys() {
-		if _, found := hdr.Recipients[pk.Fingerprint()]; found {
-			return pk, nil
+// findDecryptionKey tries to find a suiteable decryption key from the available
+// decryption keys and the recipients
+func (x *XC) findDecryptionKey(hdr *xcpb.Header) (*keyring.PrivateKey, error) {
+	for _, pk := range x.secring.KeyIDs() {
+		if _, found := hdr.Recipients[pk]; found {
+			return x.secring.Get(pk), nil
 		}
 	}
 	return nil, fmt.Errorf("no decryption key found for: %+v", hdr.Recipients)
 }
 
+// findPublicKey tries to find a given public key in the keyring
 func (x *XC) findPublicKey(needle string) (*keyring.PublicKey, error) {
-	for _, id := range x.keyring.PublicKeyIDs() {
+	for _, id := range x.pubring.KeyIDs() {
 		if id == needle {
-			return x.keyring.Get(id).PublicKey, nil
+			return x.pubring.Get(id), nil
 		}
 	}
-	return nil, fmt.Errorf("no sender found")
+	return nil, fmt.Errorf("no sender found for id '%s'", needle)
 }
 
+// decryptPrivateKey will ask the agent to unlock the private key
 func (x *XC) decryptPrivateKey(recp *keyring.PrivateKey) error {
 	fp := recp.Fingerprint()
-	for i := 0; i < 3; i++ {
-		//fmt.Printf("Trying to unlock recipient key (try %d/3)\n", i)
+
+	for i := 0; i < maxUnlockAttempts; i++ {
 		// retry asking for key in case it's wrong
 		passphrase, err := x.client.Passphrase(fp, fmt.Sprintf("Unlock private key %s", recp.Fingerprint()))
 		if err != nil {
 			return errors.Wrapf(err, "failed to get passphrase from agent: %s", err)
 		}
-		err = recp.Decrypt(passphrase)
-		if err == nil {
+
+		if err = recp.Decrypt(passphrase); err == nil {
+			// passphrase is correct, the key should now be decrypted
 			return nil
 		}
-		//fmt.Printf("Decrypt failed: %s\n", err)
+
+		// decryption failed, clear cache and wait a moment before trying again
 		if err := x.client.Remove(fp); err != nil {
 			return errors.Wrapf(err, "failed to clear cache")
 		}
 		time.Sleep(10 * time.Millisecond)
 	}
-	return fmt.Errorf("out of retries")
+
+	return fmt.Errorf("failed to unlock private key '%s' after %d retries", fp, maxUnlockAttempts)
 }
 
-func (x *XC) decryptSessionKey(hdr *Header) ([]byte, error) {
+// decryptSessionKey will attempt to find a readable recipient entry in the
+// header and decrypt it's session key
+func (x *XC) decryptSessionKey(hdr *xcpb.Header) ([]byte, error) {
+	// find a suiteable decryption key, i.e. a recipient entry which was encrypted
+	// for one of our private keys
 	recp, err := x.findDecryptionKey(hdr)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to find decryption key")
 	}
+
+	// we need the senders public key to decrypt/verify the message, since the
+	// box algorithm ties successfull decryption to successfull verification
 	sender, err := x.findPublicKey(hdr.Sender)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to find sender pub key for signature verification: %s", hdr.Sender)
 	}
+
 	// unlock recipient key
 	if err := x.decryptPrivateKey(recp); err != nil {
 		return nil, err
 	}
+
+	// this is the per recipient ciphertext, we need to decrypt it to extract
+	// the session key
 	ciphertext := hdr.Recipients[recp.Fingerprint()]
+
+	// since box works with byte arrays (or: pointers thereof) we need to copy
+	// the slice to fixed arrays
 	var nonce [24]byte
 	copy(nonce[:], ciphertext[:24])
+
 	var privKey [32]byte
 	pk := recp.PrivateKey()
 	copy(privKey[:], pk[:])
-	//fmt.Printf("[D] %s: %x - %x\n", recp.Fingerprint(), nonce, ciphertext[24:])
+
+	// now we can try to decrypt/verify the ciphertext. unfortunately box doesn't give
+	// us any diagnostic information in case it fails, i.e. we can't discern between
+	// a failed decryption and a failed verification
 	decrypted, ok := box.Open(nil, ciphertext[24:], &nonce, &sender.PublicKey, &privKey)
-	//fmt.Printf("[D] Nonce: %x - Session Key: %x - Recipient Pubkey: %x - Sender Privkey: %x\n", nonce, decrypted, recp.Public, sender.PrivateKey())
 	if !ok {
 		return nil, fmt.Errorf("failed to decrypt session key")
 	}

backend/crypto/xc/encrypt.go

@@ -1,101 +1,103 @@
 package xc
 
 import (
-	"bytes"
 	"context"
-	"encoding/binary"
 	"fmt"
 	"io"
 	"sort"
 
+	"github.com/gogo/protobuf/proto"
 	"github.com/justwatchcom/gopass/backend/crypto/xc/keyring"
+	"github.com/justwatchcom/gopass/backend/crypto/xc/xcpb"
+	"github.com/pkg/errors"
 
 	crypto_rand "crypto/rand"
 
 	"golang.org/x/crypto/chacha20poly1305"
 	"golang.org/x/crypto/nacl/box"
 )
 
+const (
+	// OnDiskVersion is the version of our on-disk format
+	OnDiskVersion = 1
+)
+
 // Encrypt encrypts the given plaintext for all the given recipients and returns the
 // ciphertext
-// TODO fuzz this
 func (x *XC) Encrypt(ctx context.Context, plaintext []byte, recipients []string) ([]byte, error) {
-	privKeys := x.keyring.DecryptionKeys()
-	if len(privKeys) < 1 {
-		return nil, fmt.Errorf("no signing keys")
+	privKeyIDs := x.secring.KeyIDs()
+	if len(privKeyIDs) < 1 {
+		return nil, fmt.Errorf("no signing keys available on our keyring")
 	}
-	privKey := privKeys[0]
+	privKey := x.secring.Get(privKeyIDs[0])
 
-	// encrypt body
+	// encrypt body (als generates a random nonce and a random session key)
 	sk, nonce, body, err := encryptBody(plaintext)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "failed to encrypt body: %s", err)
 	}
-	// encrypt per recipient
+
+	// encrypt the session key per recipient
 	header, err := x.encryptHeader(privKey, sk, nonce, recipients)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "failed to encrypt header: %s", err)
 	}
-	//fmt.Printf("Header: %x - Len: %d\n", header, len(header))
-	// write header size + header
-	buf := &bytes.Buffer{}
-
-	// write header length + header + body
-	//pos, _ := fh.Seek(0, os.SEEK_CUR)
-	//fmt.Printf("Pos: %d\n", pos)
-	if err := binary.Write(buf, binary.LittleEndian, uint64(len(header))); err != nil {
-		return nil, err
-	}
-	//pos, _ = fh.Seek(0, os.SEEK_CUR)
-	//fmt.Printf("Pos: %d\n", pos)
-	if n, err := buf.Write(header); err != nil || n < len(header) {
-		return nil, err
-	}
-	//pos, _ = fh.Seek(0, os.SEEK_CUR)
-	//fmt.Printf("Pos: %d\n", pos)
-	if n, err := buf.Write(body); err != nil || n < len(body) {
-		return nil, err
+
+	msg := &xcpb.Message{
+		Version: OnDiskVersion,
+		Header:  header,
+		Body:    body,
 	}
-	//pos, _ = fh.Seek(0, os.SEEK_CUR)
-	//fmt.Printf("Pos: %d\n", pos)
-	return buf.Bytes(), nil
+
+	return proto.Marshal(msg)
 }
 
-// TODO fuzz this
-func (x *XC) encryptHeader(signKey *keyring.PrivateKey, sk, nonce []byte, recipients []string) ([]byte, error) {
-	hdr := &Header{
+// encrypt header creates and populates a header struct with the nonce (plain)
+// and the session key encrypted per recipient
+func (x *XC) encryptHeader(signKey *keyring.PrivateKey, sk, nonce []byte, recipients []string) (*xcpb.Header, error) {
+	hdr := &xcpb.Header{
 		Sender:     signKey.Fingerprint(),
-		Recipients: make(map[string][]byte, len(recipients)),
 		Nonce:      nonce,
+		Recipients: make(map[string][]byte, len(recipients)),
+		Metadata:   make(map[string]string, 0), // metadata is plaintext!
 	}
+
 	recipients = append(recipients, signKey.Fingerprint())
 	sort.Strings(recipients)
+
 	for _, recp := range recipients {
+		// skip duplicates
+		if _, found := hdr.Recipients[recp]; found {
+			continue
+		}
+
 		r, err := x.encryptForRecipient(signKey, sk, recp)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrapf(err, "failed to encrypt session key for recipient %s: %s", recp, err)
 		}
 
-		//fmt.Printf("[E] %s: %x - %x\n", recp, r[:24], r[24:])
 		hdr.Recipients[recp] = r
 	}
-	return hdr.Marshal()
+
+	return hdr, nil
 }
 
-// TODO fuzz this
+// encryptForRecipients encrypts the given session key for the given recipient
 func (x *XC) encryptForRecipient(sender *keyring.PrivateKey, sk []byte, recipient string) ([]byte, error) {
-	//fmt.Printf("Recp: %s - Keyring: %+v\n", recipient, x.Keyring)
-	recp := x.keyring.Get(recipient).PublicKey
+	recp := x.pubring.Get(recipient)
 	if recp == nil {
 		return nil, fmt.Errorf("recipient public key not available for %s", recipient)
 	}
+
 	var recipientPublicKey [32]byte
 	copy(recipientPublicKey[:], recp.PublicKey[:])
 
 	// unlock sender key
 	if err := x.decryptPrivateKey(sender); err != nil {
 		return nil, err
 	}
+
+	// we need to copy the byte silces to byte arrays for box
 	var senderPrivateKey [32]byte
 	pk := sender.PrivateKey()
 	copy(senderPrivateKey[:], pk[:])
@@ -105,31 +107,32 @@ func (x *XC) encryptForRecipient(sender *keyring.PrivateKey, sk []byte, recipien
 		return nil, err
 	}
 
-	//fmt.Printf("[E] Nonce: %x - Session Key: %x - Recipient Pubkey: %x - Sender Privkey: %x\n", nonce, sk, recipientPublicKey, senderPrivateKey)
 	return box.Seal(nonce[:], sk, &nonce, &recipientPublicKey, &senderPrivateKey), nil
 }
 
-// TODO fuzz this
+// encryptBody generates a random session key and a nonce and encrypts the given
+// plaintext with those. it returns all three
 func encryptBody(plaintext []byte) ([]byte, []byte, []byte, error) {
 	// generate session / encryption key
 	sessionKey := make([]byte, 32)
 	if _, err := crypto_rand.Read(sessionKey); err != nil {
 		return nil, nil, nil, err
 	}
 
-	//fmt.Printf("Session Key: %x (%d)\n", sessionKey, len(sessionKey))
+	// generate a random nonce
 	nonce := make([]byte, 12)
 	if _, err := crypto_rand.Read(nonce); err != nil {
 		return nil, nil, nil, err
 	}
 
-	//fmt.Printf("Nonce: %x (%d)\n", nonce, len(nonce))
+	// initialize the AEAD with the generated session key
 	cp, err := chacha20poly1305.New(sessionKey)
 	if err != nil {
 		return nil, nil, nil, err
 	}
+
+	// encrypt the plaintext using the random nonce
 	ciphertext := cp.Seal(nil, nonce, plaintext, nil)
 
-	//fmt.Printf("Plain: '%s' - Cipher: '%s'\n", path, string(ciphertext))
 	return sessionKey, nonce, ciphertext, nil
 }