feat(gas outputs): Add Height and ActorName #270
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package migrations | ||
|
|
||
| import "github.com/go-pg/migrations/v8" | ||
|
|
||
| // Schema version 25 adds Height and ActorName to gas outputs table | ||
|
|
||
| func init() { | ||
| up := batch(` | ||
|
|
||
| DO $$ | ||
| BEGIN | ||
| IF ( | ||
| SELECT count(*) | ||
| FROM information_schema.constraint_column_usage | ||
| WHERE table_schema = 'public' | ||
| AND table_name = 'derived_gas_outputs' | ||
| AND constraint_name='derived_gas_outputs_pkey' | ||
| AND column_name IN ('height','cid','state_root') | ||
| ) != 3 -- want all three columns in the index | ||
| THEN | ||
| -- Can't change primary key while data exists | ||
| TRUNCATE TABLE public.derived_gas_outputs; | ||
|
|
||
| ALTER TABLE public.derived_gas_outputs ADD COLUMN height bigint NOT NULL; | ||
| ALTER TABLE public.derived_gas_outputs ADD COLUMN actor_name text NOT NULL; | ||
| ALTER TABLE public.derived_gas_outputs DROP CONSTRAINT derived_gas_outputs_pkey; | ||
| ALTER TABLE public.derived_gas_outputs ADD PRIMARY KEY (height,cid,state_root); | ||
| END IF; | ||
| END | ||
| $$; | ||
|
|
||
|
|
||
| `) | ||
| down := batch(` | ||
| TRUNCATE TABLE public.derived_gas_outputs; | ||
| ALTER TABLE public.derived_gas_outputs DROP CONSTRAINT derived_gas_outputs_pkey; | ||
| ALTER TABLE public.derived_gas_outputs ADD PRIMARY KEY (cid); | ||
| ALTER TABLE public.derived_gas_outputs DROP COLUMN height; | ||
| ALTER TABLE public.derived_gas_outputs DROP COLUMN actor_name; | ||
| `) | ||
|
|
||
| migrations.MustRegisterTx(up, down) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,10 +2,16 @@ package message | |
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "time" | ||
|
|
||
| "github.com/filecoin-project/go-address" | ||
| "github.com/filecoin-project/go-state-types/abi" | ||
| "github.com/filecoin-project/go-state-types/big" | ||
| "github.com/filecoin-project/lotus/chain/state" | ||
| "github.com/filecoin-project/lotus/chain/types" | ||
| "github.com/go-pg/pg/v10" | ||
| "github.com/ipfs/go-cid" | ||
| "github.com/raulk/clock" | ||
| "go.opencensus.io/tag" | ||
| "go.opentelemetry.io/otel/api/global" | ||
|
|
@@ -15,6 +21,7 @@ import ( | |
| "github.com/filecoin-project/sentinel-visor/metrics" | ||
| "github.com/filecoin-project/sentinel-visor/model/derived" | ||
| "github.com/filecoin-project/sentinel-visor/storage" | ||
| "github.com/filecoin-project/sentinel-visor/tasks/actorstate" | ||
| "github.com/filecoin-project/sentinel-visor/wait" | ||
| ) | ||
|
|
||
|
|
@@ -65,7 +72,7 @@ func (p *GasOutputsProcessor) processBatch(ctx context.Context, node lens.API) ( | |
|
|
||
| claimUntil := p.clock.Now().Add(p.leaseLength) | ||
|
|
||
| var batch []*derived.GasOutputs | ||
| var err error | ||
|
|
||
| if p.useLeases { | ||
|
|
@@ -104,7 +111,7 @@ func (p *GasOutputsProcessor) processBatch(ctx context.Context, node lens.API) ( | |
|
|
||
| errorLog := log.With("cid", item.Cid) | ||
|
|
||
| if err := p.processItem(ctx, node, item); err != nil { | ||
| // Any errors are likely to be problems using the lens, mark this tipset as failed and exit this batch | ||
| errorLog.Errorw("failed to process message", "error", err.Error()) | ||
| if err := p.storage.MarkGasOutputsMessagesComplete(ctx, item.Height, item.Cid, p.clock.Now(), err.Error()); err != nil { | ||
|
|
@@ -125,6 +132,35 @@ func (p *GasOutputsProcessor) processItem(ctx context.Context, node lens.API, it | |
| stop := metrics.Timer(ctx, metrics.ProcessingDuration) | ||
| defer stop() | ||
|
|
||
| // Note: this item will only be processed if there are receipts for | ||
| // it, which means there should be a tipset at height+1. This is only | ||
| // used to get the destination actor code, so we don't care about side | ||
| // chains. | ||
|
Comment on lines
+135
to
+138
There was a problem hiding this comment. Perhaps its worth adding a check to ensure There was a problem hiding this comment. Given that I just want to extract the dstActorCode, is it a problem if child == parent? The actor code will be the same even if it comes from a slightly wrong state. But maybe I'm missing something? There was a problem hiding this comment. I believe the destination actor may no exist in |
||
| child, err := node.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(item.Height+1), types.NewTipSetKey()) | ||
| if err != nil { | ||
| return xerrors.Errorf("Failed to load child tipset: %w", err) | ||
| } | ||
|
|
||
| st, err := state.LoadStateTree(node.Store(), child.ParentState()) | ||
| if err != nil { | ||
| return xerrors.Errorf("load state tree when gas outputs for %s: %w", item.Cid, err) | ||
| } | ||
|
|
||
| dstAddr, err := address.NewFromString(item.To) | ||
| if err != nil { | ||
| return xerrors.Errorf("parse to address failed for gas outputs in %s: %w", item.Cid, err) | ||
| } | ||
|
|
||
| var dstActorCode cid.Cid | ||
| dstActor, err := st.GetActor(dstAddr) | ||
| if err != nil { | ||
| if !errors.Is(err, types.ErrActorNotFound) { | ||
| return xerrors.Errorf("get destination actor for gas outputs %s failed: %w", item.Cid, err) | ||
| } | ||
| } else { | ||
| dstActorCode = dstActor.Code | ||
| } | ||
|
|
||
|
Comment on lines
+139
to
+163
There was a problem hiding this comment. I wish there was a better way, but this should be ok if the state is cached at some layer. There was a problem hiding this comment. #249 is much more efficient but will be a few days before it's ready. Currently checking that the data it produces matches the existing data. There was a problem hiding this comment. yeah, still need to support it in the "standard" way which I trust will become way faster with store improvement and block caching at some point. |
||
| baseFee, err := big.FromString(item.ParentBaseFee) | ||
| if err != nil { | ||
| return xerrors.Errorf("parse fee cap: %w", err) | ||
|
|
@@ -144,6 +180,7 @@ func (p *GasOutputsProcessor) processItem(ctx context.Context, node lens.API, it | |
| outputs := node.ComputeGasOutputs(item.GasUsed, item.GasLimit, baseFee, feeCap, gasPremium) | ||
| cgoStop() | ||
|
|
||
| item.ActorName = actorstate.ActorNameByCode(dstActorCode) | ||
| item.BaseFeeBurn = outputs.BaseFeeBurn.String() | ||
| item.OverEstimationBurn = outputs.OverEstimationBurn.String() | ||
| item.MinerPenalty = outputs.MinerPenalty.String() | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We will need to backfill the value for this column.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The value corresponds to the height of the message I think (not that of the receipts).
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, it's the height of the message. Backfilling is a problem. With the current version it will take some time. With the new in-order processing it takes <1s per tipset, but that's still ~211000 seconds = ~2.4 days unless we shard the processing.
An alternative is to figure out a query that can update this table by joining with messages. That will still be slow to run, several hours at least.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Custom in-order processing that just does messages and just updates this should be faster (?). That said, 2.4 days should be ok.