diff --git a/api/api_storage.go b/api/api_storage.go index 2ddbb9d1261..5d0ec73a772 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -118,7 +118,7 @@ type StorageMiner interface { // the path specified when calling CreateBackup is within the base path CreateBackup(ctx context.Context, fpath string) error - CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) } type SealRes struct { diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b1f569efbd1..3cf0647e371 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -364,7 +364,7 @@ type StorageMinerStruct struct { CreateBackup func(ctx context.Context, fpath string) error `perm:"admin"` - CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) `perm:"admin"` + CheckProvable func(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) `perm:"admin"` } } @@ -1516,8 +1516,8 @@ func (c *StorageMinerStruct) CreateBackup(ctx context.Context, fpath string) err return c.Internal.CreateBackup(ctx, fpath) } -func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorNumber]string, error) { - return c.Internal.CheckProvable(ctx, pp, sectors) +func (c *StorageMinerStruct) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) { + return c.Internal.CheckProvable(ctx, pp, sectors, expensive) } // WorkerStruct diff --git a/cmd/lotus-storage-miner/proving.go b/cmd/lotus-storage-miner/proving.go index b930476e67a..5d176a754f1 100644 --- a/cmd/lotus-storage-miner/proving.go +++ b/cmd/lotus-storage-miner/proving.go @@ -386,6 +386,10 @@ var provingCheckProvableCmd = &cli.Command{ Usage: "print only bad sectors", Value: false, }, + &cli.BoolFlag{ + Name: "slow", + Usage: "run slower checks", + }, }, Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 1 { @@ -459,7 +463,7 @@ var provingCheckProvableCmd = &cli.Command{ }) } - bad, err := sapi.CheckProvable(ctx, pf, tocheck) + bad, err := sapi.CheckProvable(ctx, pf, tocheck, cctx.Bool("slow")) if err != nil { return err } diff --git a/documentation/en/api-methods-miner.md b/documentation/en/api-methods-miner.md index ebf0973ccbf..d3f9880aaec 100644 --- a/documentation/en/api-methods-miner.md +++ b/documentation/en/api-methods-miner.md @@ -232,7 +232,8 @@ Inputs: ```json [ 8, - null + null, + true ] ``` diff --git a/extern/sector-storage/faults.go b/extern/sector-storage/faults.go index 7af81f0021f..fdd5f6b7d6c 100644 --- a/extern/sector-storage/faults.go +++ b/extern/sector-storage/faults.go @@ -2,13 +2,16 @@ package sectorstorage import ( "context" + "crypto/rand" "fmt" "os" "path/filepath" "golang.org/x/xerrors" + ffi "github.com/filecoin-project/filecoin-ffi" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/specs-actors/actors/runtime/proof" "github.com/filecoin-project/specs-storage/storage" "github.com/filecoin-project/lotus/extern/sector-storage/storiface" @@ -16,11 +19,11 @@ import ( // FaultTracker TODO: Track things more actively type FaultTracker interface { - CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) + CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) } // CheckProvable returns unprovable sectors -func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) { +func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) { var bad = make(map[abi.SectorID]string) ssize, err := pp.SectorSize() @@ -83,6 +86,49 @@ func (m *Manager) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, } } + if rg != nil { + wpp, err := sector.ProofType.RegisteredWindowPoStProof() + if err != nil { + return err + } + + var pr abi.PoStRandomness = make([]byte, abi.RandomnessLength) + _, _ = rand.Read(pr) + pr[31] &= 0x3f + + ch, err := ffi.GeneratePoStFallbackSectorChallenges(wpp, sector.ID.Miner, pr, []abi.SectorNumber{ + sector.ID.Number, + }) + if err != nil { + log.Warnw("CheckProvable Sector FAULT: generating challenges", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err) + bad[sector.ID] = fmt.Sprintf("generating fallback challenges: %s", err) + return nil + } + + commr, err := rg(ctx, sector.ID) + if err != nil { + log.Warnw("CheckProvable Sector FAULT: getting commR", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err) + bad[sector.ID] = fmt.Sprintf("getting commR: %s", err) + return nil + } + + _, err = ffi.GenerateSingleVanillaProof(ffi.PrivateSectorInfo{ + SectorInfo: proof.SectorInfo{ + SealProof: sector.ProofType, + SectorNumber: sector.ID.Number, + SealedCID: commr, + }, + CacheDirPath: lp.Cache, + PoStProofType: wpp, + SealedSectorPath: lp.Sealed, + }, ch.Challenges[sector.ID.Number]) + if err != nil { + log.Warnw("CheckProvable Sector FAULT: generating vanilla proof", "sector", sector, "sealed", lp.Sealed, "cache", lp.Cache, "err", err) + bad[sector.ID] = fmt.Sprintf("generating vanilla proof: %s", err) + return nil + } + } + return nil }() if err != nil { diff --git a/extern/sector-storage/mock/mock.go b/extern/sector-storage/mock/mock.go index 59d7d05036d..47fb2b974f2 100644 --- a/extern/sector-storage/mock/mock.go +++ b/extern/sector-storage/mock/mock.go @@ -405,7 +405,7 @@ func (mgr *SectorMgr) Remove(ctx context.Context, sector storage.SectorRef) erro return nil } -func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef) (map[abi.SectorID]string, error) { +func (mgr *SectorMgr) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, ids []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) { bad := map[abi.SectorID]string{} for _, sid := range ids { diff --git a/extern/sector-storage/storiface/ffi.go b/extern/sector-storage/storiface/ffi.go index 95d400e5248..f6b2cbdd31d 100644 --- a/extern/sector-storage/storiface/ffi.go +++ b/extern/sector-storage/storiface/ffi.go @@ -1,8 +1,11 @@ package storiface import ( + "context" "errors" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-state-types/abi" ) @@ -15,3 +18,5 @@ func (i UnpaddedByteIndex) Padded() PaddedByteIndex { } type PaddedByteIndex uint64 + +type RGetter func(ctx context.Context, id abi.SectorID) (cid.Cid, error) diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 488b80f8998..a1a6bec248d 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -544,8 +544,23 @@ func (sm *StorageMinerAPI) CreateBackup(ctx context.Context, fpath string) error return backup(sm.DS, fpath) } -func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef) (map[abi.SectorNumber]string, error) { - bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors) +func (sm *StorageMinerAPI) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []sto.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) { + var rg storiface.RGetter + if expensive { + rg = func(ctx context.Context, id abi.SectorID) (cid.Cid, error) { + si, err := sm.Miner.GetSectorInfo(id.Number) + if err != nil { + return cid.Undef, err + } + if si.CommR == nil { + return cid.Undef, xerrors.Errorf("commr is nil") + } + + return *si.CommR, nil + } + } + + bad, err := sm.StorageMgr.CheckProvable(ctx, pp, sectors, rg) if err != nil { return nil, err } diff --git a/storage/wdpost_run.go b/storage/wdpost_run.go index 87dd8ad1552..4d3d4b726bc 100644 --- a/storage/wdpost_run.go +++ b/storage/wdpost_run.go @@ -214,7 +214,7 @@ func (s *WindowPoStScheduler) checkSectors(ctx context.Context, check bitfield.B }) } - bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck) + bad, err := s.faultTracker.CheckProvable(ctx, s.proofType, tocheck, nil) if err != nil { return bitfield.BitField{}, xerrors.Errorf("checking provable sectors: %w", err) } diff --git a/storage/wdpost_run_test.go b/storage/wdpost_run_test.go index 80b4f66afe1..65b6b0183b5 100644 --- a/storage/wdpost_run_test.go +++ b/storage/wdpost_run_test.go @@ -27,6 +27,7 @@ import ( "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/extern/sector-storage/storiface" "github.com/filecoin-project/lotus/journal" ) @@ -125,7 +126,7 @@ func (m *mockProver) GenerateWindowPoSt(ctx context.Context, aid abi.ActorID, si type mockFaultTracker struct { } -func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef) (map[abi.SectorID]string, error) { +func (m mockFaultTracker) CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, rg storiface.RGetter) (map[abi.SectorID]string, error) { // Returns "bad" sectors so just return empty map meaning all sectors are good return map[abi.SectorID]string{}, nil }