feat: add power actor claim extraction

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/filecoin-project/go-fil-markets v1.0.0
 	github.com/filecoin-project/go-multistore v0.0.3
 	github.com/filecoin-project/go-state-types v0.0.0-20201013222834-41ea465f274f
-	github.com/filecoin-project/lotus v1.1.3-0.20201027132752-3d02dba5dc9b
+	github.com/filecoin-project/lotus v1.1.3-0.20201031000550-a5c05f87f15f
 	github.com/filecoin-project/specs-actors v0.9.12
 	github.com/filecoin-project/specs-actors/v2 v2.2.0
 	github.com/filecoin-project/statediff v0.0.8-0.20201027195725-7eaa5391a639

go.sum

@@ -161,6 +161,7 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
@@ -289,8 +290,8 @@ github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZO
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b h1:fkRZSPrYpk42PV3/lIXiL0LHetxde7vyYYvSsttQtfg=
 github.com/filecoin-project/go-storedcounter v0.0.0-20200421200003-1c99c62e8a5b/go.mod h1:Q0GQOBtKf1oE10eSXSlhN45kDBdGvEcVOqMiffqX+N8=
 github.com/filecoin-project/lotus v0.9.0/go.mod h1:8Qg5wmvHgCvxq2gCq9iziPlcNHl58dtzozf4StZ68Kk=
-github.com/filecoin-project/lotus v1.1.3-0.20201027132752-3d02dba5dc9b h1:A5xJDE9tJPHzMYz5Gqh+Jue1Tbs0VNXv18hZEbMdVgk=
-github.com/filecoin-project/lotus v1.1.3-0.20201027132752-3d02dba5dc9b/go.mod h1:41z5qos7adeegSp1SjylE85hu7Zey1h0SV0vpMPv2G0=
+github.com/filecoin-project/lotus v1.1.3-0.20201031000550-a5c05f87f15f h1:7mTCv5I/7kJ8sfwRjUKlTLFCa7LQYroLDkL9ekURPek=
+github.com/filecoin-project/lotus v1.1.3-0.20201031000550-a5c05f87f15f/go.mod h1:41z5qos7adeegSp1SjylE85hu7Zey1h0SV0vpMPv2G0=
 github.com/filecoin-project/specs-actors v0.9.4/go.mod h1:BStZQzx5x7TmCkLv0Bpa07U6cPKol6fd3w9KjMPZ6Z4=
 github.com/filecoin-project/specs-actors v0.9.7/go.mod h1:wM2z+kwqYgXn5Z7scV1YHLyd1Q1cy0R8HfTIWQ0BFGU=
 github.com/filecoin-project/specs-actors v0.9.12 h1:iIvk58tuMtmloFNHhAOQHG+4Gci6Lui0n7DYQGi3cJk=
@@ -700,6 +701,7 @@ github.com/ipld/go-ipld-prime-proto v0.0.0-20200922192210-9a2bfd4440a6 h1:6Mq+tZ
 github.com/ipld/go-ipld-prime-proto v0.0.0-20200922192210-9a2bfd4440a6/go.mod h1:3pHYooM9Ea65jewRwrb2u5uHZCNkNTe9ABsVB+SrkH0=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c=
 github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4=
+github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0=
 github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo=
 github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
 github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8=
@@ -718,6 +720,7 @@ github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2 h1:JVX6jT/XfzNqIjye47
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A=
 github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
 github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
 github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg=
@@ -1395,6 +1398,7 @@ github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -1504,6 +1508,7 @@ github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/
 github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=

model/actors/power/claimedpower.go

@@ -0,0 +1,45 @@
+package power
+
+import (
+	"context"
+
+	"github.com/go-pg/pg/v10"
+	"go.opentelemetry.io/otel/api/global"
+	"golang.org/x/xerrors"
+)
+
+type PowerActorClaim struct {
+	Height          int64  `pg:",pk,notnull,use_zero"`
+	MinerID         string `pg:",pk,notnull"`
+	StateRoot       string `pg:",pk,notnull"`
+	RawBytePower    string `pg:",notnull"`
+	QualityAdjPower string `pg:",notnull"`
+}
+
+func (p *PowerActorClaim) PersistWithTx(ctx context.Context, tx *pg.Tx) error {
+	ctx, span := global.Tracer("").Start(ctx, "PowerActorClaim.PersistWithTx")
+	defer span.End()
+	if _, err := tx.ModelContext(ctx, p).
+		OnConflict("do nothing").
+		Insert(); err != nil {
+		return xerrors.Errorf("persisting power actors claim: %w", err)
+	}
+	return nil
+}
+
+type PowerActorClaimList []*PowerActorClaim
+
+func (pl PowerActorClaimList) PersistWithTx(ctx context.Context, tx *pg.Tx) error {
+	ctx, span := global.Tracer("").Start(ctx, "PowerActorClaimList.PersistWithTx")
+	defer span.End()
+	if len(pl) == 0 {
+		return nil
+	}
+	if _, err := tx.ModelContext(ctx, &pl).
+		OnConflict("do nothing").
+		Insert(); err != nil {
+		return xerrors.Errorf("persisting power actor claim list: %w")
+	}
+	return nil
+
+}

model/actors/power/task.go

@@ -0,0 +1,41 @@
+package power
+
+import (
+	"context"
+
+	"github.com/go-pg/pg/v10"
+	"go.opentelemetry.io/otel/api/global"
+
+	"github.com/filecoin-project/sentinel-visor/metrics"
+)
+
+type PowerTaskResult struct {
+	ChainPowerModel *ChainPower
+	ClaimStateModel PowerActorClaimList
+}
+
+func (p *PowerTaskResult) PersistWithTx(ctx context.Context, tx *pg.Tx) error {
+	if p.ChainPowerModel != nil {
+		if err := p.ChainPowerModel.PersistWithTx(ctx, tx); err != nil {
+			return err
+		}
+	}
+	if p.ClaimStateModel != nil {
+		if err := p.ClaimStateModel.PersistWithTx(ctx, tx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (p *PowerTaskResult) Persist(ctx context.Context, db *pg.DB) error {
+	ctx, span := global.Tracer("").Start(ctx, "PowerTaskResult.Persist")
+	defer span.End()
+
+	stop := metrics.Timer(ctx, metrics.PersistDuration)
+	defer stop()
+
+	return db.RunInTransaction(ctx, func(tx *pg.Tx) error {
+		return p.PersistWithTx(ctx, tx)
+	})
+}

model/genesis/task.go

@@ -11,12 +11,14 @@ import (
 	init_ "github.com/filecoin-project/sentinel-visor/model/actors/init"
 	"github.com/filecoin-project/sentinel-visor/model/actors/market"
 	"github.com/filecoin-project/sentinel-visor/model/actors/miner"
+	"github.com/filecoin-project/sentinel-visor/model/actors/power"
 )
 
 type ProcessGenesisSingletonResult struct {
 	minerResults    miner.MinerTaskResultList
 	marketResult    *GenesisMarketTaskResult
 	initActorResult *GenesisInitActorTaskResult
+	powerResult     *power.PowerTaskResult
 }
 
 func (r *ProcessGenesisSingletonResult) Persist(ctx context.Context, db *pg.DB) error {
@@ -46,6 +48,12 @@ func (r *ProcessGenesisSingletonResult) Persist(ctx context.Context, db *pg.DB)
 				return err
 			}
 		}
+		// persist power actor
+		if r.powerResult != nil {
+			if err := r.powerResult.PersistWithTx(ctx, tx); err != nil {
+				return err
+			}
+		}
 		return nil
 	})
 }
@@ -54,6 +62,13 @@ func (r *ProcessGenesisSingletonResult) AddMiner(m *miner.MinerTaskResult) {
 	r.minerResults = append(r.minerResults, m)
 }
 
+func (r *ProcessGenesisSingletonResult) SetPower(p *power.PowerTaskResult) {
+	if r.powerResult != nil {
+		panic("Genesis Power State already set, developer error!!!")
+	}
+	r.powerResult = p
+}
+
 func (r *ProcessGenesisSingletonResult) SetMarket(m *GenesisMarketTaskResult) {
 	if r.marketResult != nil {
 		panic("Genesis Market State already set, developer error!!!")

storage/migrations/16_power_actor_claims.go

@@ -0,0 +1,26 @@
+package migrations
+
+import (
+	"github.com/go-pg/migrations/v8"
+)
+
+// Schema version 16 adds power actor claims table
+
+func init() {
+	up := batch(`
+	CREATE TABLE IF NOT EXISTS "power_actor_claims" (
+		"height" bigint not null,
+		"miner_id" text not null,
+		"state_root" text not null,
+		"raw_byte_power" text not null,
+		"quality_adj_power" text not null,
+		PRIMARY KEY ("height", "miner_id", "state_root")
+	);
+`)
+
+	down := batch(`
+	DROP TABLE IF EXISTS public.power_actor_claims;
+`)
+
+	migrations.MustRegisterTx(up, down)
+}

tasks/actorstate/actorstate_test.go

@@ -41,17 +41,19 @@ func mockTipset(minerAddr address.Address, timestamp uint64) (*types.TipSet, err
 var _ ActorStateAPI = (*MockAPI)(nil)
 
 type MockAPI struct {
-	actors map[actorKey]*types.Actor
-	bs     bstore.Blockstore
-	store  adt.Store
+	actors  map[actorKey]*types.Actor
+	tipsets map[types.TipSetKey]*types.TipSet
+	bs      bstore.Blockstore
+	store   adt.Store
 }
 
 func NewMockAPI() *MockAPI {
 	bs := bstore.NewTemporarySync()
 	return &MockAPI{
-		bs:     bs,
-		actors: make(map[actorKey]*types.Actor),
-		store:  adt.WrapStore(context.Background(), cbornode.NewCborStore(bs)),
+		bs:      bs,
+		tipsets: make(map[types.TipSetKey]*types.TipSet),
+		actors:  make(map[actorKey]*types.Actor),
+		store:   adt.WrapStore(context.Background(), cbornode.NewCborStore(bs)),
 	}
 }
 
@@ -86,6 +88,7 @@ func (m *MockAPI) ChainGetBlockMessages(ctx context.Context, msg cid.Cid) (*api.
 }
 
 func (m *MockAPI) ChainGetTipSet(ctx context.Context, tsk types.TipSetKey) (*types.TipSet, error) {
+	return m.tipsets[tsk], nil
 	blks := make([]*types.BlockHeader, len(tsk.Cids()))
 	for i, cid := range tsk.Cids() {
 		if err := m.store.Get(ctx, cid, blks[i]); err != nil {
@@ -146,6 +149,10 @@ func (m *MockAPI) setActor(tsk types.TipSetKey, addr address.Address, actor *typ
 	m.actors[key] = actor
 }
 
+func (m *MockAPI) putTipSet(ts *types.TipSet) {
+	m.tipsets[ts.Key()] = ts
+}
+
 func (m *MockAPI) createMarketState(ctx context.Context, deals map[abi.DealID]*samarket.DealState, props map[abi.DealID]*samarket.DealProposal, balances map[address.Address]balance) (cid.Cid, error) {
 	dealRootCid, err := m.createDealAMT(deals)
 	if err != nil {

tasks/actorstate/genesis.go

@@ -16,6 +16,7 @@ import (
 	initmodel "github.com/filecoin-project/sentinel-visor/model/actors/init"
 	marketmodel "github.com/filecoin-project/sentinel-visor/model/actors/market"
 	minermodel "github.com/filecoin-project/sentinel-visor/model/actors/miner"
+	powermodel "github.com/filecoin-project/sentinel-visor/model/actors/power"
 	genesismodel "github.com/filecoin-project/sentinel-visor/model/genesis"
 	"github.com/filecoin-project/sentinel-visor/storage"
 )
@@ -43,12 +44,15 @@ func (p *GenesisProcessor) ProcessGenesis(ctx context.Context, gen *types.TipSet
 	}
 
 	minerExtractor := StorageMinerExtractor{}
+	powerExtractor := StoragePowerExtractor{}
+
 	result := &genesismodel.ProcessGenesisSingletonResult{}
 	for _, addr := range genesisAddrs {
 		genesisAct, err := p.node.StateGetActor(ctx, addr, gen.Key())
 		if err != nil {
 			return xerrors.Errorf("get actor: %w", err)
 		}
+
 		switch genesisAct.Code {
 		case builtin.SystemActorCodeID:
 			// TODO
@@ -63,13 +67,27 @@ func (p *GenesisProcessor) ProcessGenesis(ctx context.Context, gen *types.TipSet
 		case builtin.AccountActorCodeID:
 			// TODO
 		case builtin.StoragePowerActorCodeID:
-			// TODO
+			res, err := powerExtractor.Extract(ctx, ActorInfo{
+				Actor:           *genesisAct,
+				Address:         addr,
+				ParentStateRoot: gen.ParentState(),
+				Epoch:           gen.Height(),
+				TipSet:          gen.Key(),
+				ParentTipSet:    gen.Parents(),
+			}, p.node)
+			if err != nil {
+				return xerrors.Errorf("power actor state: %w", err)
+			}
+			// TODO simplify the result to a slice of persistables in follow on.
+			result.SetPower(res.(*powermodel.PowerTaskResult))
+
 		case builtin.StorageMarketActorCodeID:
 			res, err := p.storageMarketState(ctx, gen)
 			if err != nil {
 				return xerrors.Errorf("storage market actor state: %w", err)
 			}
 			result.SetMarket(res)
+
 		case builtin.StorageMinerActorCodeID:
 			res, err := minerExtractor.Extract(ctx, ActorInfo{
 				Actor:           *genesisAct,
@@ -84,6 +102,7 @@ func (p *GenesisProcessor) ProcessGenesis(ctx context.Context, gen *types.TipSet
 			}
 			// TODO simplify the result to a slice of persistables in follow on.
 			result.AddMiner(res.(*minermodel.MinerTaskResult))
+
 		case builtin.PaymentChannelActorCodeID:
 			// TODO
 		case builtin.MultisigActorCodeID:

tasks/actorstate/genesis_test.go

@@ -74,7 +74,7 @@ func TestGenesisProcessor(t *testing.T) {
 
 	t.Run("miner_deal_sectors", func(t *testing.T) {
 		var count int
-		_, err := db.QueryOne(pg.Scan(&count), `SELECT COUNT(*) FROM miner_deal_sectors`)
+		_, err := db.QueryOne(pg.Scan(&count), `SELECT COUNT(*) FROM miner_sector_deals`)
 		require.NoError(t, err)
 		assert.NotEqual(t, 0, count)
 	})
@@ -128,7 +128,7 @@ func truncateGenesisTables(tb testing.TB, db *pg.DB) error {
 		"miner_states",
 		"miner_powers",
 		"miner_sector_infos",
-		"miner_deal_sectors",
+		"miner_sector_deals",
 		"market_deal_states",
 		"market_deal_proposals",
 		"id_addresses",

tasks/actorstate/miner.go

@@ -45,15 +45,6 @@ func (m StorageMinerExtractor) Extract(ctx context.Context, a ActorInfo, node Ac
 		return nil, err
 	}
 
-	// TODO add this to the power actor processing.
-	// remove when #171 closes.
-	/*
-		minerPower, err := node.StateMinerPower(ctx, a.Address, a.TipSet)
-		if err != nil {
-			return nil, xerrors.Errorf("loading miner power: %w", err)
-		}
-	*/
-
 	minerInfoModel, err := ExtractMinerInfo(a, ec)
 	if err != nil {
 		return nil, xerrors.Errorf("extracting miner info: %w", err)
@@ -289,13 +280,7 @@ func ExtractMinerSectorData(ctx context.Context, ec *MinerStateExtractionContext
 
 	sectorDealsModel := minermodel.MinerSectorDealList{}
 	if ec.IsGenesis() {
-		// load genesis miners sectors
-		mstate, err := miner.Load(node.Store(), ec.CurrActor)
-		if err != nil {
-			return nil, nil, nil, nil, err
-		}
-
-		msectors, err := mstate.LoadSectors(nil)
+		msectors, err := ec.CurrState.LoadSectors(nil)
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}