Reintroduce fsck (gopasspw#721)

Fixes gopasspw#711

commands.go

@@ -352,6 +352,18 @@ func getCommands(ctx context.Context, action *ap.Action, app *cli.App) []cli.Com
 				},
 			},
 		},
+		{
+			Name:  "fsck",
+			Usage: "Check store integrity",
+			Description: "" +
+				"Check the integrity of the given sub store or all stores if none are specified. " +
+				"Will automatically fix all issues found.",
+			Before: func(c *cli.Context) error { return action.Initialized(withGlobalFlags(ctx, c), c) },
+			Action: func(c *cli.Context) error {
+				return action.Fsck(withGlobalFlags(ctx, c), c)
+			},
+			BashComplete: action.MountsComplete,
+		},
 		{
 			Name:  "generate",
 			Usage: "Generate a new password",

commands_test.go

@@ -90,7 +90,7 @@ func TestGetCommands(t *testing.T) {
 	c := cli.NewContext(app, fs, nil)
 
 	commands := getCommands(ctx, act, app)
-	assert.Equal(t, 31, len(commands))
+	assert.Equal(t, 32, len(commands))
 
 	prefix := ""
 	testCommands(t, c, commands, prefix)

pkg/action/fsck.go

@@ -0,0 +1,33 @@
+package action
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/justwatchcom/gopass/pkg/config"
+	"github.com/justwatchcom/gopass/pkg/fsutil"
+	"github.com/justwatchcom/gopass/pkg/out"
+	"github.com/urfave/cli"
+)
+
+// Fsck checks the store integrity
+func (s *Action) Fsck(ctx context.Context, c *cli.Context) error {
+	// make sure config is in the right place
+	// we may have loaded it from one of the fallback locations
+	if err := s.cfg.Save(); err != nil {
+		return ExitError(ctx, ExitConfig, err, "failed to save config: %s", err)
+	}
+	// clean up any previous config locations
+	oldCfg := filepath.Join(config.Homedir(), ".gopass.yml")
+	if fsutil.IsFile(oldCfg) {
+		if err := os.Remove(oldCfg); err != nil {
+			out.Red(ctx, "Failed to remove old gopass config %s: %s", oldCfg, err)
+		}
+	}
+
+	if err := s.Store.Fsck(ctx, c.Args().Get(0)); err != nil {
+		return ExitError(ctx, ExitFsck, err, "fsck found errors: %s", err)
+	}
+	return nil
+}

pkg/backend/crypto.go

@@ -38,6 +38,7 @@ type Keyring interface {
 	FormatKey(ctx context.Context, id string) string
 	NameFromKey(ctx context.Context, id string) string
 	EmailFromKey(ctx context.Context, id string) string
+	Fingerprint(ctx context.Context, id string) string
 	ReadNamesFromKey(ctx context.Context, buf []byte) ([]string, error)
 
 	CreatePrivateKeyBatch(ctx context.Context, name, email, passphrase string) error

pkg/backend/crypto/gpg/cli/gpg.go

@@ -92,7 +92,11 @@ func (g *GPG) RecipientIDs(ctx context.Context, buf []byte) ([]string, error) {
 		}
 		m := splitPacket(line)
 		if keyid, found := m["keyid"]; found {
-			recp = append(recp, keyid)
+			kl, err := g.listKeys(ctx, "public", keyid)
+			if err != nil || len(kl) < 1 {
+				continue
+			}
+			recp = append(recp, kl[0].Fingerprint)
 		}
 	}
 

pkg/backend/crypto/gpg/cli/keyring.go

@@ -108,3 +108,8 @@ func (g *GPG) NameFromKey(ctx context.Context, id string) string {
 func (g *GPG) FormatKey(ctx context.Context, id string) string {
 	return g.findKey(ctx, id).Identity().ID()
 }
+
+// Fingerprint returns the full-length native fingerprint
+func (g *GPG) Fingerprint(ctx context.Context, id string) string {
+	return g.findKey(ctx, id).Fingerprint
+}

pkg/backend/crypto/gpg/openpgp/gpg.go

@@ -247,6 +247,15 @@ func (g *GPG) FormatKey(ctx context.Context, id string) string {
 	return ""
 }
 
+// Fingerprint returns the full-length native fingerprint
+func (g *GPG) Fingerprint(ctx context.Context, id string) string {
+	ent := g.findEntity(id)
+	if ent == nil || ent.PrimaryKey == nil {
+		return ""
+	}
+	return fmt.Sprintf("%x", ent.PrimaryKey.Fingerprint)
+}
+
 // Initialized returns nil
 func (g *GPG) Initialized(context.Context) error {
 	return nil

pkg/backend/crypto/plain/backend.go

@@ -174,6 +174,11 @@ func (m *Mocker) FormatKey(ctx context.Context, id string) string {
 	return id
 }
 
+// Fingerprint returns the full-length native fingerprint
+func (m *Mocker) Fingerprint(ctx context.Context, id string) string {
+	return id
+}
+
 // Initialized returns nil
 func (m *Mocker) Initialized(context.Context) error {
 	return nil

pkg/backend/crypto/xc/utils.go

@@ -111,6 +111,11 @@ func (x *XC) EmailFromKey(ctx context.Context, id string) string {
 	return id
 }
 
+// Fingerprint returns the full-length native fingerprint
+func (x *XC) Fingerprint(ctx context.Context, id string) string {
+	return id
+}
+
 // CreatePrivateKeyBatch creates a new keypair
 func (x *XC) CreatePrivateKeyBatch(ctx context.Context, name, email, passphrase string) error {
 	k, err := keyring.GenerateKeypair(passphrase)

pkg/backend/storage.go

@@ -35,4 +35,5 @@ type Storage interface {
 
 	Name() string
 	Version() semver.Version
+	Fsck(context.Context) error
 }

pkg/backend/storage/fs/fsck.go

@@ -0,0 +1,83 @@
+package fs
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/justwatchcom/gopass/pkg/fsutil"
+	"github.com/justwatchcom/gopass/pkg/out"
+)
+
+// Fsck checks the storage integrity
+func (s *Store) Fsck(ctx context.Context) error {
+	entries, err := s.List(ctx, "")
+	if err != nil {
+		return err
+	}
+	dirs := make(map[string]struct{}, len(entries))
+	for _, entry := range entries {
+		filename := filepath.Join(s.path, entry)
+		dirs[filepath.Dir(filename)] = struct{}{}
+
+		if err := s.fsckCheckFile(ctx, filename); err != nil {
+			return err
+		}
+	}
+
+	for dir := range dirs {
+		if err := s.fsckCheckDir(ctx, dir); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Store) fsckCheckFile(ctx context.Context, filename string) error {
+	fi, err := os.Stat(filename)
+	if err != nil {
+		return err
+	}
+
+	if fi.Mode().Perm()&0177 == 0 {
+		return nil
+	}
+
+	out.Yellow(ctx, "Permissions too wide: %s (%s)", filename, fi.Mode().String())
+
+	np := uint32(fi.Mode().Perm() & 0600)
+	out.Green(ctx, "Fixing permissions on %s from %s to %s", filename, fi.Mode().Perm().String(), os.FileMode(np).Perm().String())
+	if err := syscall.Chmod(filename, np); err != nil {
+		out.Red(ctx, "Failed to set permissions for %s to rw-------: %s", filename, err)
+	}
+	return nil
+}
+
+func (s *Store) fsckCheckDir(ctx context.Context, dirname string) error {
+	fi, err := os.Stat(dirname)
+	if err != nil {
+		return err
+	}
+
+	// check if any group or other perms are set,
+	// i.e. check for perms other than rwx------
+	if fi.Mode().Perm()&077 != 0 {
+		out.Yellow(ctx, "Permissions too wide %s on dir %s", fi.Mode().Perm().String(), dirname)
+
+		np := uint32(fi.Mode().Perm() & 0700)
+		out.Green(ctx, "Fixing permissions from %s to %s", fi.Mode().Perm().String(), os.FileMode(np).Perm().String())
+		if err := syscall.Chmod(dirname, np); err != nil {
+			out.Red(ctx, "Failed to set permissions for %s to rwx------: %s", dirname, err)
+		}
+	}
+	// check for empty folders
+	isEmpty, err := fsutil.IsEmptyDir(dirname)
+	if err != nil {
+		return err
+	}
+	if isEmpty {
+		out.Red(ctx, "WARNING: Folder %s is empty", dirname)
+	}
+	return nil
+}

pkg/backend/storage/fs/store.go

@@ -77,6 +77,9 @@ func (s *Store) List(ctx context.Context, prefix string) ([]string, error) {
 			return nil
 		}
 		name := strings.TrimPrefix(path, s.path+string(filepath.Separator))
+		if !strings.HasPrefix(name, prefix) {
+			return nil
+		}
 		files = append(files, name)
 		return nil
 	}); err != nil {

pkg/backend/storage/kv/consul/store.go

@@ -134,3 +134,8 @@ func (s *Store) Version() semver.Version {
 func (s *Store) Available(ctx context.Context) error {
 	return s.Set(ctx, ".test", []byte("test"))
 }
+
+// Fsck always returns nil
+func (s *Store) Fsck(ctx context.Context) error {
+	return nil
+}

pkg/backend/storage/kv/inmem/store.go

@@ -123,3 +123,8 @@ func (m *InMem) Available(ctx context.Context) error {
 	}
 	return nil
 }
+
+// Fsck always returns nil
+func (m *InMem) Fsck(ctx context.Context) error {
+	return nil
+}

pkg/store/root/fsck.go

@@ -0,0 +1,27 @@
+package root
+
+import (
+	"context"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// Fsck checks all stores/entries matching the given prefix
+func (s *Store) Fsck(ctx context.Context, path string) error {
+	for alias, sub := range s.mounts {
+		if sub == nil {
+			continue
+		}
+		if path != "" && !strings.HasPrefix(path, alias+"/") {
+			continue
+		}
+		if err := sub.Fsck(ctx, strings.TrimPrefix(path, alias+"/")); err != nil {
+			return errors.Wrapf(err, "fsck failed on sub store %s: %s", alias, err)
+		}
+	}
+	if err := s.store.Fsck(ctx, path); err != nil {
+		return errors.Wrapf(err, "fsck failed on root store: %s", err)
+	}
+	return nil
+}

pkg/store/root/list.go

@@ -69,7 +69,7 @@ func (r *Store) Tree(ctx context.Context) (tree.Tree, error) {
 		if err := root.AddMount(alias, substore.Path()); err != nil {
 			return nil, errors.Errorf("failed to add mount: %s", err)
 		}
-		sf, err := substore.List(ctx, alias)
+		sf, err := substore.List(ctx, "")
 		if err != nil {
 			return nil, errors.Errorf("failed to add file: %s", err)
 		}

pkg/store/sub/fsck.go

@@ -0,0 +1,104 @@
+package sub
+
+import (
+	"context"
+	"sort"
+
+	"github.com/justwatchcom/gopass/pkg/out"
+	"github.com/pkg/errors"
+)
+
+// Fsck checks all entries matching the given prefix
+func (s *Store) Fsck(ctx context.Context, path string) error {
+	// first let the storage backend check itself
+	if err := s.storage.Fsck(ctx); err != nil {
+		return errors.Wrapf(err, "storage backend found errors: %s", err)
+	}
+
+	// then we'll make sure all the secrets are readable by us and every
+	// valid recipient
+	names, err := s.List(ctx, path)
+	if err != nil {
+		return errors.Wrapf(err, "failed to list entries: %s", err)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		if err := s.fsckCheckEntry(ctx, name); err != nil {
+			return errors.Wrapf(err, "failed to check %s: %s", name, err)
+		}
+	}
+
+	return nil
+}
+
+func (s *Store) fsckCheckEntry(ctx context.Context, name string) error {
+	// make sure we can actually decode this secret
+	// if this fails there is no way we could fix this
+	_, err := s.Get(ctx, name)
+	if err != nil {
+		return errors.Wrapf(err, "failed to decode secret %s: %s", name, err)
+	}
+
+	// now compare the recipients this secret was encoded for and fix it if
+	// if doesn't match
+	ciphertext, err := s.storage.Get(ctx, s.passfile(name))
+	if err != nil {
+		return err
+	}
+	itemRecps, err := s.crypto.RecipientIDs(ctx, ciphertext)
+	if err != nil {
+		return err
+	}
+	perItemStoreRecps, err := s.GetRecipients(ctx, name)
+	if err != nil {
+		return err
+	}
+
+	// check itemRecps matches storeRecps
+	missing, extra := compareStringSlices(perItemStoreRecps, itemRecps)
+	if len(missing) > 0 {
+		out.Red(ctx, "Missing recipients on %s: %+v", name, missing)
+	}
+	if len(extra) > 0 {
+		out.Red(ctx, "Extra recipients on %s: %+v", name, extra)
+	}
+	if len(missing) > 0 || len(extra) > 0 {
+		sec, err := s.Get(ctx, name)
+		if err != nil {
+			return err
+		}
+		if err := s.Set(WithReason(ctx, "fsck fix recipients"), name, sec); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func compareStringSlices(want, have []string) ([]string, []string) {
+	missing := []string{}
+	extra := []string{}
+
+	wantMap := make(map[string]struct{}, len(want))
+	haveMap := make(map[string]struct{}, len(have))
+
+	for _, w := range want {
+		wantMap[w] = struct{}{}
+	}
+	for _, h := range have {
+		haveMap[h] = struct{}{}
+	}
+
+	for k := range wantMap {
+		if _, found := haveMap[k]; !found {
+			missing = append(missing, k)
+		}
+	}
+	for k := range haveMap {
+		if _, found := wantMap[k]; !found {
+			extra = append(extra, k)
+		}
+	}
+
+	return missing, extra
+}