feat: implement vm message extraction #1027
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -79,6 +79,7 @@ var TaskLookup = map[string][]string{ | |
| ImplicitMessageTask: { | ||
| InternalMessage, | ||
| InternalParsedMessage, | ||
| VmMessage, | ||
| }, | ||
| ChainConsensusTask: { | ||
| ChainConsensus, | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -66,7 +66,7 @@ func TestMakeTaskNamesAlias(t *testing.T) { | |
| }, | ||
| { | ||
| taskAlias: tasktype.ImplicitMessageTask, | ||
| tasks: []string{tasktype.InternalMessage, tasktype.InternalParsedMessage, tasktype.VmMessage}, | ||
| }, | ||
| { | ||
| taskAlias: tasktype.ChainConsensusTask, | ||
|
|
@@ -99,9 +99,8 @@ func TestMakeAllTaskAliasNames(t *testing.T) { | |
| require.Len(t, actual, len(storage.Models)) | ||
| } | ||
|
|
||
| func TestMakeAllTaskNames(t *testing.T) { | ||
| const TotalTableTasks = 36 | ||
| actual, err := tasktype.MakeTaskNames(tasktype.AllTableTasks) | ||
| require.NoError(t, err) | ||
| // if this test fails it means a new task name was added, update the above test | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,6 +7,7 @@ import ( | |
| "fmt" | ||
| "reflect" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/filecoin-project/go-bitfield" | ||
| "github.com/filecoin-project/go-state-types/exitcode" | ||
|
|
@@ -282,6 +283,34 @@ func ParseParams(params []byte, method abi.MethodNum, actCode cid.Cid) (string, | |
| return string(b), m.Name, err | ||
| } | ||
|
|
||
| func ParseReturn(ret []byte, method abi.MethodNum, actCode cid.Cid) (string, string, error) { | ||
| m, found := ActorRegistry.Methods[actCode][method] | ||
| if !found { | ||
| return "", "", fmt.Errorf("unknown method %d for actor %s", method, actCode) | ||
| } | ||
|
|
||
| // if the actor method doesn't expect returns don't parse them | ||
| if m.Ret == reflect.TypeOf(new(abi.EmptyValue)) { | ||
| return "", m.Name, nil | ||
| } | ||
|
|
||
| p := reflect.New(m.Ret.Elem()).Interface().(cbg.CBORUnmarshaler) | ||
| if err := p.UnmarshalCBOR(bytes.NewReader(ret)); err != nil { | ||
| actorName := builtin.ActorNameByCode(actCode) | ||
| return "", m.Name, fmt.Errorf("cbor decode into %s %s:(%s.%d) failed: %v", m.Name, actorName, actCode, method, err) | ||
| } | ||
|
|
||
| b, err := MarshalWithOverrides(p, map[reflect.Type]marshaller{ | ||
| reflect.TypeOf(bitfield.BitField{}): bitfieldCountMarshaller, | ||
| }) | ||
| if err != nil { | ||
| return "", "", fmt.Errorf("failed to parse message return method: %d, actor code: %s, return: %s: %w", method, actCode, string(ret), err) | ||
| } | ||
|
|
||
| return string(b), m.Name, err | ||
|
|
||
| } | ||
|
|
||
| func MethodAndParamsForMessage(m *types.Message, destCode cid.Cid) (string, string, error) { | ||
| // Method is optional, zero means a plain value transfer | ||
| if m.Method == 0 { | ||
|
|
@@ -308,6 +337,71 @@ func MethodAndParamsForMessage(m *types.Message, destCode cid.Cid) (string, stri | |
| return method, params, nil | ||
| } | ||
|
|
||
| type MessageParamsReturn struct { | ||
| MethodName string | ||
| Params string | ||
| Return string | ||
| } | ||
|
|
||
| func MethodParamsReturnForMessage(m *MessageTrace, destCode cid.Cid) (*MessageParamsReturn, error) { | ||
| // Method is optional, zero means a plain value transfer | ||
| if m.Message.Method == 0 { | ||
| return &MessageParamsReturn{ | ||
| MethodName: "Send", | ||
| Params: "", | ||
| Return: "", | ||
| }, nil | ||
| } | ||
|
|
||
| if !destCode.Defined() { | ||
| return nil, fmt.Errorf("missing actor code") | ||
| } | ||
|
|
||
| params, _, err := ParseParams(m.Message.Params, m.Message.Method, destCode) | ||
| if err != nil { | ||
| log.Warnf("failed to parse parameters of message %s: %v", m.Message.Cid(), err) | ||
| return nil, fmt.Errorf("unknown method for actor type %s method %d: %w", destCode.String(), int64(m.Message.Method), err) | ||
| } | ||
| ret, method, err := ParseReturn(m.Receipt.Return, m.Message.Method, destCode) | ||
| if err != nil { | ||
| log.Warnf("failed to parse return of message %s: %v", m.Message.Cid(), err) | ||
| return nil, fmt.Errorf("unknown method for actor type %s method %d: %w", destCode.String(), int64(m.Message.Method), err) | ||
| } | ||
|
|
||
| return &MessageParamsReturn{ | ||
| MethodName: method, | ||
| Params: params, | ||
| Return: ret, | ||
| }, nil | ||
| } | ||
|
|
||
| func walkExecutionTrace(et *types.ExecutionTrace, trace *[]*MessageTrace) { | ||
| for _, sub := range et.Subcalls { | ||
| *trace = append(*trace, &MessageTrace{ | ||
| Message: sub.Msg, | ||
| Receipt: sub.MsgRct, | ||
| Error: sub.Error, | ||
| Duration: sub.Duration, | ||
| GasCharge: sub.GasCharges, | ||
| }) | ||
| walkExecutionTrace(&sub, trace) //nolint:scopelint,gosec | ||
| } | ||
| } | ||
|
|
||
| type MessageTrace struct { | ||
| Message *types.Message | ||
| Receipt *types.MessageReceipt | ||
| Error string | ||
| Duration time.Duration | ||
| GasCharge []*types.GasTrace | ||
| } | ||
|
|
||
| func GetChildMessagesOf(m *lens.MessageExecution) []*MessageTrace { | ||
| var out []*MessageTrace | ||
| walkExecutionTrace(&m.Ret.ExecutionTrace, &out) | ||
| return out | ||
| } | ||
|
|
||
| func ActorNameAndFamilyFromCode(c cid.Cid) (name string, family string, err error) { | ||
| if !c.Defined() { | ||
| return "", "", fmt.Errorf("cannot derive actor name from undefined CID") | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,65 @@ | ||||||
| package messages | ||||||
|
|
||||||
| import ( | ||||||
| "context" | ||||||
|
|
||||||
| "go.opencensus.io/tag" | ||||||
|
|
||||||
| "github.com/filecoin-project/lily/metrics" | ||||||
| "github.com/filecoin-project/lily/model" | ||||||
| ) | ||||||
|
|
||||||
| type VMMessage struct { | ||||||
| tableName struct{} `pg:"vm_messages"` // nolint: structcheck | ||||||
|
|
||||||
| // Height message was executed at. | ||||||
| Height int64 `pg:",pk,notnull,use_zero"` | ||||||
| // StateRoot message was applied to. | ||||||
| StateRoot string `pg:",pk,notnull"` | ||||||
| // Cid of the message. | ||||||
| Cid string `pg:",pk,notnull"` | ||||||
| // On-chain message triggering the message. | ||||||
| Source string `pg:",pk,notnull"` | ||||||
|
There was a problem hiding this comment. I don't think we need
Suggested change
There was a problem hiding this comment. Source must remain a primary key as VM message CIDs can conflict at the same height/stateroot. There was a problem hiding this comment. Isn't There was a problem hiding this comment.
Yes correct. Are you proposing we drop source as a primary key? There was a problem hiding this comment. Yes. Just remove from the PK to reduce the size of that index. We can use the standalone hash index for There was a problem hiding this comment. Okay. After some discussion w @frrist, it seems that the There was a problem hiding this comment. I would probably recommend the removal of the hash-based There was a problem hiding this comment. +1 on dropping the |
||||||
|
|
||||||
| // From sender of message. | ||||||
| From string `pg:",notnull"` | ||||||
| // To receiver of message. | ||||||
| To string `pg:",notnull"` | ||||||
| // Value attoFIL contained in message. | ||||||
| Value string `pg:"type:numeric,notnull"` | ||||||
| // Method called on To (receiver) | ||||||
| Method uint64 `pg:",use_zero"` | ||||||
|
There was a problem hiding this comment. Not null? (I know this is inconsequential given we write our own migrations, but it's good context that doesn't hurt to include and keeps consistency w other fields usage of the same tag.)
Suggested change
|
||||||
| // ActorCode of To (receiver) | ||||||
| ActorCode string `pg:",notnull"` | ||||||
| // ExitCode of message execution. | ||||||
| ExitCode int64 `pg:",use_zero"` | ||||||
|
There was a problem hiding this comment.
Suggested change
|
||||||
| // GasUsed by message. | ||||||
| GasUsed int64 `pg:",use_zero"` | ||||||
|
There was a problem hiding this comment.
Suggested change
|
||||||
| // Params contained in message. | ||||||
| Params string `pg:",type:jsonb"` | ||||||
| // Return value of message. | ||||||
| Returns string `pg:",type:jsonb"` | ||||||
| } | ||||||
|
|
||||||
| func (v *VMMessage) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { | ||||||
| ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "vm_messages")) | ||||||
| stop := metrics.Timer(ctx, metrics.PersistDuration) | ||||||
| defer stop() | ||||||
|
|
||||||
| metrics.RecordCount(ctx, metrics.PersistModel, 1) | ||||||
| return s.PersistModel(ctx, v) | ||||||
| } | ||||||
|
|
||||||
| type VMMessageList []*VMMessage | ||||||
|
|
||||||
| func (vl VMMessageList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error { | ||||||
| if len(vl) == 0 { | ||||||
| return nil | ||||||
| } | ||||||
| ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "vm_messages")) | ||||||
| stop := metrics.Timer(ctx, metrics.PersistDuration) | ||||||
| defer stop() | ||||||
|
|
||||||
| metrics.RecordCount(ctx, metrics.PersistModel, len(vl)) | ||||||
| return s.PersistModel(ctx, vl) | ||||||
| } | ||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,46 @@ | ||||||
| package v1 | ||||||
|
|
||||||
| func init() { | ||||||
| patches.Register( | ||||||
| 8, | ||||||
| ` | ||||||
| CREATE TABLE {{ .SchemaName | default "public"}}.vm_messages ( | ||||||
| height bigint NOT NULL, | ||||||
| state_root text NOT NULL, | ||||||
| cid text NOT NULL, | ||||||
| source text, | ||||||
| "from" text NOT NULL, | ||||||
| "to" text NOT NULL, | ||||||
| value numeric NOT NULL, | ||||||
| method bigint NOT NULL, | ||||||
| actor_code text NOT NULL, | ||||||
| exit_code bigint NOT NULL, | ||||||
| gas_used bigint NOT NULL, | ||||||
| params jsonb, | ||||||
| returns jsonb | ||||||
| ); | ||||||
| ALTER TABLE ONLY {{ .SchemaName | default "public"}}.vm_messages ADD CONSTRAINT vm_messages_pkey PRIMARY KEY (height, state_root, cid, source); | ||||||
frrist marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
| CREATE INDEX vm_messages_height_idx ON {{ .SchemaName | default "public"}}.vm_messages USING BRIN (height); | ||||||
|
There was a problem hiding this comment. I think BRIN indexing is dependent on write order, which is use-case specific. BTREE is the better generalized index here.
Suggested change
There was a problem hiding this comment. Indeed, will use btree instead. |
||||||
| CREATE INDEX vm_messages_cid_idx ON {{ .SchemaName | default "public"}}.vm_messages USING HASH (cid); | ||||||
| CREATE INDEX vm_messages_source_idx ON {{ .SchemaName | default "public"}}.vm_messages USING HASH (source); | ||||||
| CREATE INDEX vm_messages_from_idx ON {{ .SchemaName | default "public"}}.vm_messages USING HASH ("from"); | ||||||
| CREATE INDEX vm_messages_to_idx ON {{ .SchemaName | default "public"}}.vm_messages USING HASH ("to"); | ||||||
| CREATE INDEX vm_messages_actor_code_method_idx ON {{ .SchemaName | default "public"}}.vm_messages USING BTREE (actor_code, method); | ||||||
|
|
||||||
|
|
||||||
|
|
||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.height IS 'Height message was executed at.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.state_root IS 'CID of the parent state root at which this message was executed.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.cid IS 'CID of the message (note this CID does not appear on chain).'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.source IS 'CID of the on-chain message that caused this message to be sent.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages."from" IS 'Address of the actor that sent the message.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages."to" IS 'Address of the actor that received the message.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.value IS 'Amount of FIL (in attoFIL) transferred by this message.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.method IS 'The method number invoked on the recipient actor. Only unique to the actor the method is being invoked on. A method number of 0 is a plain token transfer - no method execution'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.actor_code IS 'The CID of the actor that received the message.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.exit_code IS 'The exit code that was returned as a result of executing the message.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.gas_used IS 'A measure of the amount of resources (or units of gas) consumed, in order to execute a message.'; | ||||||
|
There was a problem hiding this comment. Curious: Do you know what resources this would be other than gas? There was a problem hiding this comment. Reusing the description of gas here that we use elsewhere, e.g.: https://github.com/filecoin-project/lily/blob/v0.10.0/schemas/v1/base.go#L391 There was a problem hiding this comment. I suspected it was copypasta. Just wondering if there is anything other than gas that it would have been referring to and couldn't think of. |
||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.params IS 'Message parameters parsed and serialized as a JSON object.'; | ||||||
| COMMENT ON COLUMN {{ .SchemaName | default "public"}}.vm_messages.returns IS 'Result returned from executing a message parsed and serialized as a JSON object.'; | ||||||
| `, | ||||||
| ) | ||||||
| } | ||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -60,6 +60,7 @@ var Models = []interface{}{ | |
| (*messages.ParsedMessage)(nil), | ||
| (*messages.InternalMessage)(nil), | ||
| (*messages.InternalParsedMessage)(nil), | ||
| (*messages.VMMessage)(nil), | ||
|
|
||
| (*multisig.MultisigTransaction)(nil), | ||
|
|
||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't want to read the FIP to get the gist.