Merge pull request #171 from cybozu-go/robust

sabakan-cryptsetup: retry sabakan API calls.

CHANGELOG.md

@@ -5,6 +5,14 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [2.4.6] - 2019-08-23
+
+### Changed
+- Update etcd to 3.3.15 and etcdutil to 1.3.3 (#171).
+- sabakan-cryptsetup: retry sabakan API calls (#171).
+
+### Changed
+
 ## [2.4.5] - 2019-08-19
 
 ### Changed
@@ -15,7 +23,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
-- [sabakan-cryptsetup] TPM 2.0 support (#164).
+- sabakan-cryptsetup: TPM 2.0 support (#164).
 
 ## [2.4.3] - 2019-07-04
 
@@ -134,7 +142,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 See [CHANGELOG-0](./CHANGELOG-0.md).
 
-[Unreleased]: https://github.com/cybozu-go/sabakan/compare/v2.4.5...HEAD
+[Unreleased]: https://github.com/cybozu-go/sabakan/compare/v2.4.6...HEAD
+[2.4.6]: https://github.com/cybozu-go/sabakan/compare/v2.4.5...v2.4.6
 [2.4.5]: https://github.com/cybozu-go/sabakan/compare/v2.4.4...v2.4.5
 [2.4.4]: https://github.com/cybozu-go/sabakan/compare/v2.4.3...v2.4.4
 [2.4.3]: https://github.com/cybozu-go/sabakan/compare/v2.4.2...v2.4.3

go.mod

@@ -4,10 +4,10 @@ require (
 	github.com/99designs/gqlgen v0.9.3
 	github.com/agnivade/levenshtein v1.0.2 // indirect
 	github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 // indirect
-	github.com/coreos/etcd v0.0.0-20190816232144-5cf5d88a18ae
+	github.com/coreos/etcd v3.3.15+incompatible
 	github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect
 	github.com/coreos/ignition v0.30.0
-	github.com/cybozu-go/etcdutil v1.3.2
+	github.com/cybozu-go/etcdutil v1.3.3
 	github.com/cybozu-go/log v1.5.0
 	github.com/cybozu-go/netutil v1.2.0
 	github.com/cybozu-go/well v1.8.1
@@ -17,6 +17,7 @@ require (
 	github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315
 	github.com/hashicorp/go-version v1.0.0
 	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2 // indirect
 	github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega v1.5.0

pkg/sabakan-cryptsetup/cmd/driver.go

@@ -16,6 +16,8 @@ import (
 	sabakan "github.com/cybozu-go/sabakan/v2/client"
 )
 
+const maxRetry = 10
+
 // Driver setup crypt devices.
 type Driver struct {
 	serial  string
@@ -139,6 +141,8 @@ func (d *Driver) setupDisk(ctx context.Context, disk Disk, tpmKek []byte) error
 		return d.formatDisk(ctx, disk, f, tpmKek)
 	}
 
+	var retries int
+RETRY:
 	ek, err := d.sabakan.CryptsGet(ctx, d.serial, md.HexID())
 	if err == nil {
 		log.Info("encryption key is found. run cryptsetup", map[string]interface{}{
@@ -152,7 +156,18 @@ func (d *Driver) setupDisk(ctx context.Context, disk Disk, tpmKek []byte) error
 		})
 		return d.formatDisk(ctx, disk, f, tpmKek)
 	}
-	return err
+
+	log.Error("failed to retrieve key from sabakan", map[string]interface{}{
+		log.FnError: err,
+		"disk":      disk.Name(),
+		"try":       retries + 1,
+	})
+	if retries == maxRetry {
+		return err
+	}
+	retries++
+	time.Sleep(time.Duration(retries) * time.Second * 2)
+	goto RETRY
 }
 
 func (d *Driver) formatDisk(ctx context.Context, disk Disk, f *os.File, tpmKek []byte) error {
@@ -183,5 +198,21 @@ func (d *Driver) formatDisk(ctx context.Context, disk Disk, f *os.File, tpmKek [
 		return err
 	}
 
-	return d.sabakan.CryptsPut(ctx, d.serial, md.HexID(), ek)
+	var retries int
+RETRY:
+	err = d.sabakan.CryptsPut(ctx, d.serial, md.HexID(), ek)
+	if err == nil {
+		return nil
+	}
+	log.Error("failed to send key to sabakan", map[string]interface{}{
+		log.FnError: err,
+		"disk":      disk.Name(),
+		"try":       retries + 1,
+	})
+	if retries == maxRetry {
+		return err
+	}
+	retries++
+	time.Sleep(time.Duration(retries) * time.Second * 2)
+	goto RETRY
 }

vendor/modules.txt

@@ -5,7 +5,7 @@ github.com/99designs/gqlgen/handler
 github.com/99designs/gqlgen/complexity
 # github.com/agnivade/levenshtein v1.0.2
 github.com/agnivade/levenshtein
-# github.com/coreos/etcd v0.0.0-20190816232144-5cf5d88a18ae
+# github.com/coreos/etcd v3.3.15+incompatible
 github.com/coreos/etcd/clientv3
 github.com/coreos/etcd/clientv3/clientv3util
 github.com/coreos/etcd/clientv3/concurrency
@@ -37,7 +37,7 @@ github.com/coreos/ignition/config/shared/validations
 github.com/coreos/ignition/config/validate/report
 # github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
 github.com/coreos/pkg/capnslog
-# github.com/cybozu-go/etcdutil v1.3.2
+# github.com/cybozu-go/etcdutil v1.3.3
 github.com/cybozu-go/etcdutil
 # github.com/cybozu-go/log v1.5.0
 github.com/cybozu-go/log

version.go

@@ -1,7 +1,7 @@
 package sabakan
 
 // Version is sabakan version
-const Version = "2.4.5"
+const Version = "2.4.6"
 
 // SchemaVersion is the schema version
 const SchemaVersion = "3"