From 38f9559080d1b3363a677a7bf4793e18f6f51d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 15 Jan 2021 20:57:32 +0100 Subject: [PATCH] Implement full-node restore option --- chain/store/store.go | 8 +++- cmd/lotus/backup.go | 107 +++++++++++++++++++++++++++++++++++++++++++ cmd/lotus/daemon.go | 21 ++++++++- 3 files changed, 133 insertions(+), 3 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index 2ca09dfe974..c2871dc8fda 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -524,9 +524,13 @@ func (cs *ChainStore) takeHeaviestTipSet(ctx context.Context, ts *types.TipSet) // FlushValidationCache removes all results of block validation from the // chain metadata store. Usually the first step after a new chain import. func (cs *ChainStore) FlushValidationCache() error { + return FlushValidationCache(cs.ds) +} + +func FlushValidationCache(ds datastore.Batching) error { log.Infof("clearing block validation cache...") - dsWalk, err := cs.ds.Query(query.Query{ + dsWalk, err := ds.Query(query.Query{ // Potential TODO: the validation cache is not a namespace on its own // but is rather constructed as prefixed-key `foo:bar` via .Instance(), which // in turn does not work with the filter, which can match only on `foo/bar` @@ -546,7 +550,7 @@ func (cs *ChainStore) FlushValidationCache() error { return xerrors.Errorf("failed to run key listing query: %w", err) } - batch, err := cs.ds.Batch() + batch, err := ds.Batch() if err != nil { return xerrors.Errorf("failed to open a DS batch: %w", err) } diff --git a/cmd/lotus/backup.go b/cmd/lotus/backup.go index aec0000c90d..5517bd9f446 100644 --- a/cmd/lotus/backup.go +++ b/cmd/lotus/backup.go @@ -1,14 +1,121 @@ package main import ( + "os" + + dstore "github.com/ipfs/go-datastore" + "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" + "golang.org/x/xerrors" + "gopkg.in/cheggaaa/pb.v1" "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/lotus/chain/store" lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/repo" ) var backupCmd = lcli.BackupCmd("repo", repo.FullNode, func(cctx *cli.Context) (lcli.BackupAPI, jsonrpc.ClientCloser, error) { return lcli.GetFullNodeAPI(cctx) }) + +func restore(cctx *cli.Context, r repo.Repo) error { + bf, err := homedir.Expand(cctx.Path("restore")) + if err != nil { + return xerrors.Errorf("expand backup file path: %w", err) + } + + st, err := os.Stat(bf) + if err != nil { + return xerrors.Errorf("stat backup file (%s): %w", bf, err) + } + + f, err := os.Open(bf) + if err != nil { + return xerrors.Errorf("opening backup file: %w", err) + } + defer f.Close() // nolint:errcheck + + lr, err := r.Lock(repo.FullNode) + if err != nil { + return err + } + defer lr.Close() // nolint:errcheck + + if cctx.IsSet("restore-config") { + log.Info("Restoring config") + + cf, err := homedir.Expand(cctx.String("restore-config")) + if err != nil { + return xerrors.Errorf("expanding config path: %w", err) + } + + _, err = os.Stat(cf) + if err != nil { + return xerrors.Errorf("stat config file (%s): %w", cf, err) + } + + var cerr error + err = lr.SetConfig(func(raw interface{}) { + rcfg, ok := raw.(*config.FullNode) + if !ok { + cerr = xerrors.New("expected miner config") + return + } + + ff, err := config.FromFile(cf, rcfg) + if err != nil { + cerr = xerrors.Errorf("loading config: %w", err) + return + } + + *rcfg = *ff.(*config.FullNode) + }) + if cerr != nil { + return cerr + } + if err != nil { + return xerrors.Errorf("setting config: %w", err) + } + + } else { + log.Warn("--restore-config NOT SET, WILL USE DEFAULT VALUES") + } + + log.Info("Restoring metadata backup") + + mds, err := lr.Datastore("/metadata") + if err != nil { + return err + } + + bar := pb.New64(st.Size()) + br := bar.NewProxyReader(f) + bar.ShowTimeLeft = true + bar.ShowPercent = true + bar.ShowSpeed = true + bar.Units = pb.U_BYTES + + bar.Start() + err = backupds.RestoreInto(br, mds) + bar.Finish() + + if err != nil { + return xerrors.Errorf("restoring metadata: %w", err) + } + + log.Info("Resetting chainstore metadata") + + chainHead := dstore.NewKey("head") + if err := mds.Delete(chainHead); err != nil { + return xerrors.Errorf("clearing chain head: %w", err) + } + if err := store.FlushValidationCache(mds); err != nil { + return xerrors.Errorf("clearing chain validation cache: %w", err) + } + + return nil +} diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index 581238d4ba5..457fb1efb84 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -144,6 +144,14 @@ var DaemonCmd = &cli.Command{ Name: "api-max-req-size", Usage: "maximum API request size accepted by the JSON RPC server", }, + &cli.PathFlag{ + Name: "restore", + Usage: "restore from backup file", + }, + &cli.PathFlag{ + Name: "restore-config", + Usage: "config file to use when restoring from backup", + }, }, Action: func(cctx *cli.Context) error { isLite := cctx.Bool("lite") @@ -203,9 +211,11 @@ var DaemonCmd = &cli.Command{ r.SetConfigPath(cctx.String("config")) } - if err := r.Init(repo.FullNode); err != nil && err != repo.ErrRepoExists { + err = r.Init(repo.FullNode) + if err != nil && err != repo.ErrRepoExists { return xerrors.Errorf("repo init error: %w", err) } + freshRepo := err != repo.ErrRepoExists if !isLite { if err := paramfetch.GetParams(lcli.ReqContext(cctx), build.ParametersJSON(), 0); err != nil { @@ -223,6 +233,15 @@ var DaemonCmd = &cli.Command{ genBytes = build.MaybeGenesis() } + if cctx.IsSet("restore") { + if !freshRepo { + return xerrors.Errorf("restoring from backup is only possible with a fresh repo!") + } + if err := restore(cctx, r); err != nil { + return xerrors.Errorf("restoring from backup: %w", err) + } + } + chainfile := cctx.String("import-chain") snapshot := cctx.String("import-snapshot") if chainfile != "" || snapshot != "" {