diff --git a/.env.example b/.env.example index b8a5089..91dbfae 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,14 @@ INDEXER_BRIDGED_TOKENS_FILE=mainnet.json # full list of files you can find in repo ./build/bridged_tokens/ INDEXER_CLASS_INTERFACES_DIR=./interfaces/ # REQUIRED +INDEXER_DATASOURCE=sequencer # REQUIRED: one of 'node' or 'sequencer' HASURA_HOST=hasura HASURA_POSTGRES_HOST=db LOG_LEVEL=info -CACHE_ENABLED=false POSTGRES_PORT=5432 POSTGRES_HOST=db POSTGRES_DB=starknet -POSTGRES_PASSWORD= # REQUIRED \ No newline at end of file +POSTGRES_PASSWORD= # REQUIRED +STARKNET_NODE_URL= # REQUIRED if INDEXER_DATASOURCE=node +STARKNET_SEQUENCER_FEEDER_GATEWAY= # REQUIRED if INDEXER_DATASOURCE=sequencer +NODE_APIKEY= # REQUIRED if your node provider has api key. It's api key. +NODE_HEADER_APIKEY= # REQUIRED if your node provider has api key. It's header name. \ No newline at end of file diff --git a/build/dipdup.yml b/build/dipdup.yml index dad6f8f..a7b3ee2 100644 --- a/build/dipdup.yml +++ b/build/dipdup.yml @@ -1,32 +1,28 @@ -version: 0.0.1 +version: 0.0.2 log_level: ${LOG_LEVEL:-info} indexer: name: ${INDEXER_NAME:-dipdup_starknet_indexer} - - sequencer: - feeder_gateway: ${STARKNET_SEQUENCER_FEEDER_GATEWAY:-https://alpha-mainnet.starknet.io/feeder_gateway} - gateway: ${STARKNET_SEQUENCER_GATEWAY:-https://alpha-mainnet.starknet.io/gateway} - requests_per_second: ${STARKNET_SEQUENCER_RPS:-3} - - node: - url: ${STARKNET_NODE_URL} - requests_per_second: ${STARKNET_NODE_RPS:-5} - + datasource: ${INDEXER_DATASOURCE:-sequencer} threads_count: ${INDEXER_THREADS_COUNT:-10} start_level: ${INDEXER_START_LEVEL:-0} timeout: ${INDEXER_REQUEST_TIMEOUT:-10} - requests_per_second: ${STARKNET_RPS:-2} class_interfaces_dir: ${INDEXER_CLASS_INTERFACES_DIR:-./interfaces/} bridged_tokens_file: ${INDEXER_BRIDGED_TOKENS_FILE:-mainnet.json} - cache_dir: ${INDEXER_CACHE_DIR} - cache: ${CACHE_ENABLED:-false} grpc: bind: ${GRPC_BIND:-127.0.0.1:7779} log: true +datasources: + node: + url: ${STARKNET_NODE_URL} + rps: ${STARKNET_NODE_RPS:-5} + sequencer: + url: ${STARKNET_SEQUENCER_FEEDER_GATEWAY:-https://alpha-mainnet.starknet.io/feeder_gateway} + rps: ${STARKNET_SEQUENCER_RPS:-3} + database: kind: postgres host: ${POSTGRES_HOST:-db} diff --git a/cmd/indexer/main.go b/cmd/indexer/main.go index 83c051f..791b673 100644 --- a/cmd/indexer/main.go +++ b/cmd/indexer/main.go @@ -103,7 +103,12 @@ func main() { } } - indexerModule := indexer.New(cfg.Indexer, postgres) + indexerModule, err := indexer.New(cfg.Indexer, postgres, cfg.DataSources) + if err != nil { + log.Panic().Err(err).Msg("creating indexer module") + cancel() + return + } grpcModule, err := grpc.NewServer( cfg.GRPC, postgres, diff --git a/go.mod b/go.mod index 1d3c60b..17b74ff 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/dipdup-io/starknet-indexer go 1.21 require ( - github.com/dipdup-io/starknet-go-api v0.0.0-20240110155038-c97c6d82a0eb + github.com/dipdup-io/starknet-go-api v0.0.0-20240117152104-ae578a76eedb github.com/dipdup-io/workerpool v0.0.4 github.com/dipdup-net/go-lib v0.3.3 github.com/dipdup-net/indexer-sdk v0.0.4 diff --git a/go.sum b/go.sum index 4dc001f..848c281 100644 --- a/go.sum +++ b/go.sum @@ -41,6 +41,10 @@ github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+ github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= github.com/dipdup-io/starknet-go-api v0.0.0-20240110155038-c97c6d82a0eb h1:IAJCAcl/bS47DVWAedA94j/EwfG4Xt/LeBLwEOonKn4= github.com/dipdup-io/starknet-go-api v0.0.0-20240110155038-c97c6d82a0eb/go.mod h1:y3KGLFQtwzUBcT0X2LMj6CxocUimr/A9XYg+j0KIRDE= +github.com/dipdup-io/starknet-go-api v0.0.0-20240116225436-db7439f09aea h1:A7y24i7t94CxDefVssoSDP2ONvehsfOSNQ8pxbcuJrE= +github.com/dipdup-io/starknet-go-api v0.0.0-20240116225436-db7439f09aea/go.mod h1:y3KGLFQtwzUBcT0X2LMj6CxocUimr/A9XYg+j0KIRDE= +github.com/dipdup-io/starknet-go-api v0.0.0-20240117152104-ae578a76eedb h1:qNK36S4zFgJ7k5cImpTUqgBCOEcckGZm3marSquZdes= +github.com/dipdup-io/starknet-go-api v0.0.0-20240117152104-ae578a76eedb/go.mod h1:y3KGLFQtwzUBcT0X2LMj6CxocUimr/A9XYg+j0KIRDE= github.com/dipdup-io/workerpool v0.0.4 h1:m58fuFY3VIPRc+trWpjw2Lsm4FvIgtjP/4VRe79r+/s= github.com/dipdup-io/workerpool v0.0.4/go.mod h1:m6YMqx7M+fORTyabHD/auKymBRpbDax0t1aPZ1i7GZA= github.com/dipdup-net/go-lib v0.3.3 h1:vTUI+sT4L+x+eiMf712Cg8EtlqUCMiN6M3vcNaPlCw8= diff --git a/internal/storage/deploy.go b/internal/storage/deploy.go index 5440753..08edf7c 100644 --- a/internal/storage/deploy.go +++ b/internal/storage/deploy.go @@ -39,7 +39,7 @@ type Deploy struct { Hash []byte `comment:"Transaction hash"` ContractAddressSalt []byte `comment:"A random salt that determines the account address"` ConstructorCalldata []string `bun:",array" comment:"Raw constructor calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Error *string `bun:"error" comment:"Reverted error"` Class Class `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"` @@ -89,12 +89,15 @@ func (d Deploy) Flat() []any { d.Hash, d.ContractAddressSalt, pq.StringArray(d.ConstructorCalldata), + nil, + d.Error, } - parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil, d.Error) - } else { - data = append(data, string(parsed), d.Error) + + if d.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[10] = string(parsed) + } } return data } diff --git a/internal/storage/deploy_account.go b/internal/storage/deploy_account.go index 2b1b48d..403e91b 100644 --- a/internal/storage/deploy_account.go +++ b/internal/storage/deploy_account.go @@ -42,7 +42,7 @@ type DeployAccount struct { MaxFee decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"` Nonce decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"` ConstructorCalldata []string `bun:",array" comment:"Raw constructor calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Error *string `bun:"error" comment:"Reverted error"` Class Class `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"` @@ -94,12 +94,14 @@ func (d DeployAccount) Flat() []any { d.MaxFee, d.Nonce, pq.StringArray(d.ConstructorCalldata), + nil, + d.Error, } - parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil, d.Error) - } else { - data = append(data, string(parsed), d.Error) + if d.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(d.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[12] = string(parsed) + } } return data } diff --git a/internal/storage/event.go b/internal/storage/event.go index 55e9701..ceb7f33 100644 --- a/internal/storage/event.go +++ b/internal/storage/event.go @@ -48,7 +48,7 @@ type Event struct { Keys []string `bun:",array" comment:"Raw event keys"` Data []string `bun:",array" comment:"Raw event data"` Name string `comment:"Event name"` - ParsedData map[string]any `comment:"Event data parsed according to contract ABI"` + ParsedData map[string]any `bun:",nullzero" comment:"Event data parsed according to contract ABI"` From Address `bun:"rel:belongs-to" hasura:"table:address,field:from_id,remote_field:id,type:oto,name:from"` Contract Address `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"` @@ -98,13 +98,14 @@ func (e Event) Flat() []any { pq.StringArray(e.Keys), pq.StringArray(e.Data), e.Name, + nil, } - parsed, err := json.MarshalWithOption(e.ParsedData, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil) - } else { - data = append(data, string(parsed)) + if e.ParsedData != nil { + parsed, err := json.MarshalWithOption(e.ParsedData, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[16] = string(parsed) + } } return data } diff --git a/internal/storage/fee.go b/internal/storage/fee.go index 1424dc1..a551f2f 100644 --- a/internal/storage/fee.go +++ b/internal/storage/fee.go @@ -57,7 +57,7 @@ type Fee struct { Entrypoint string `comment:"Entrypoint name"` Calldata []string `bun:",array" comment:"Raw calldata"` Result []string `bun:",array" comment:"Raw result"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Class Class `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"` Caller Address `bun:"rel:belongs-to" hasura:"table:address,field:caller_id,remote_field:id,type:oto,name:caller"` @@ -115,12 +115,14 @@ func (f Fee) Flat() []any { f.Entrypoint, pq.StringArray(f.Calldata), pq.StringArray(f.Result), + nil, } - parsed, err := json.MarshalWithOption(f.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil) - } else { - data = append(data, string(parsed)) + + if f.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(f.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[18] = string(parsed) + } } return data } diff --git a/internal/storage/internal.go b/internal/storage/internal.go index 3d322aa..42efd42 100644 --- a/internal/storage/internal.go +++ b/internal/storage/internal.go @@ -58,8 +58,8 @@ type Internal struct { Entrypoint string `comment:"Entrypoint name"` Result []string `bun:",array" comment:"Raw result"` Calldata []string `bun:",array" comment:"Raw calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` - ParsedResult map[string]any `comment:"Result parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` + ParsedResult map[string]any `bun:",nullzero" comment:"Result parsed according to contract ABI"` Class Class `bun:"rel:belongs-to" hasura:"table:class,field:class_id,remote_field:id,type:oto,name:class"` Caller Address `bun:"rel:belongs-to" hasura:"table:address,field:caller_id,remote_field:id,type:oto,name:caller"` @@ -120,20 +120,22 @@ func (i Internal) Flat() []any { i.Entrypoint, pq.StringArray(i.Calldata), pq.StringArray(i.Result), + nil, + nil, } - parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil) - } else { - data = append(data, string(parsed)) + if i.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[20] = string(parsed) + } } - result, err := json.MarshalWithOption(i.ParsedResult, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil) - } else { - data = append(data, string(result)) + if i.ParsedResult != nil { + result, err := json.MarshalWithOption(i.ParsedResult, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[21] = string(result) + } } return data } diff --git a/internal/storage/invoke.go b/internal/storage/invoke.go index 3a663ab..455df55 100644 --- a/internal/storage/invoke.go +++ b/internal/storage/invoke.go @@ -46,7 +46,7 @@ type Invoke struct { MaxFee decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"` Nonce decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"` CallData []string `bun:",array" comment:"Raw calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Error *string `bun:"error" comment:"Reverted error"` Contract Address `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"` @@ -98,12 +98,14 @@ func (i Invoke) Flat() []any { i.MaxFee, i.Nonce, pq.StringArray(i.CallData), + nil, + i.Error, } - parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil, i.Error) - } else { - data = append(data, string(parsed), i.Error) + if i.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(i.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[13] = string(parsed) + } } return data diff --git a/internal/storage/l1_handler.go b/internal/storage/l1_handler.go index 5e0e504..d2121ad 100644 --- a/internal/storage/l1_handler.go +++ b/internal/storage/l1_handler.go @@ -44,7 +44,7 @@ type L1Handler struct { MaxFee decimal.Decimal `bun:",type:numeric" comment:"The maximum fee that the sender is willing to pay for the transaction"` Nonce decimal.Decimal `bun:",type:numeric" comment:"The transaction nonce"` CallData []string `bun:",array" comment:"Raw calldata"` - ParsedCalldata map[string]any `comment:"Calldata parsed according to contract ABI"` + ParsedCalldata map[string]any `bun:",nullzero" comment:"Calldata parsed according to contract ABI"` Error *string `bun:"error" comment:"Reverted error"` Contract Address `bun:"rel:belongs-to" hasura:"table:address,field:contract_id,remote_field:id,type:oto,name:contract"` @@ -94,13 +94,15 @@ func (l1 L1Handler) Flat() []any { l1.MaxFee, l1.Nonce, pq.StringArray(l1.CallData), + nil, + l1.Error, } - parsed, err := json.MarshalWithOption(l1.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) - if err != nil { - data = append(data, nil, l1.Error) - } else { - data = append(data, string(parsed), l1.Error) + if l1.ParsedCalldata != nil { + parsed, err := json.MarshalWithOption(l1.ParsedCalldata, json.UnorderedMap(), json.DisableNormalizeUTF8()) + if err == nil { + data[12] = string(parsed) + } } return data } diff --git a/pkg/indexer/config/config.go b/pkg/indexer/config/config.go index 7c4ca9d..1364cc5 100644 --- a/pkg/indexer/config/config.go +++ b/pkg/indexer/config/config.go @@ -2,27 +2,11 @@ package config // Config - configuration structure for indexer type Config struct { - Name string `yaml:"name" validate:"omitempty"` - StartLevel uint64 `yaml:"start_level" validate:"omitempty"` - ThreadsCount int `yaml:"threads_count" validate:"omitempty,min=1"` - Timeout uint64 `yaml:"timeout" validate:"omitempty"` - Node *Node `yaml:"node" validate:"omitempty"` - Sequencer Sequencer `yaml:"sequencer" validate:"required"` - ClassInterfacesDir string `yaml:"class_interfaces_dir" validate:"required,dir"` - BridgedTokensFile string `yaml:"bridged_tokens_file" validate:"required,file"` - CacheDir string `yaml:"cache_dir" validate:"omitempty,dir"` - Cache bool `yaml:"cache" validate:"omitempty"` -} - -// Node - -type Node struct { - Url string `yaml:"url" validate:"omitempty,url"` - Rps int `yaml:"requests_per_second" validate:"omitempty,min=1"` -} - -// Sequencer - -type Sequencer struct { - FeederGateway string `yaml:"feeder_gateway" validate:"required,url"` - Gateway string `yaml:"gateway" validate:"required,url"` - Rps int `yaml:"requests_per_second" validate:"omitempty,min=1"` + Name string `yaml:"name" validate:"omitempty"` + StartLevel uint64 `yaml:"start_level" validate:"omitempty"` + ThreadsCount int `yaml:"threads_count" validate:"omitempty,min=1"` + Timeout uint64 `yaml:"timeout" validate:"omitempty"` + ClassInterfacesDir string `yaml:"class_interfaces_dir" validate:"required,dir"` + BridgedTokensFile string `yaml:"bridged_tokens_file" validate:"required,file"` + Datasource string `yaml:"datasource" validate:"required,oneof=sequencer node"` } diff --git a/pkg/indexer/indexer.go b/pkg/indexer/indexer.go index 956c3df..0b3f9e5 100644 --- a/pkg/indexer/indexer.go +++ b/pkg/indexer/indexer.go @@ -8,7 +8,6 @@ import ( "time" "github.com/dipdup-io/starknet-go-api/pkg/data" - "github.com/dipdup-io/starknet-go-api/pkg/sequencer" "github.com/dipdup-io/starknet-indexer/internal/starknet" models "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/internal/storage/postgres" @@ -18,6 +17,7 @@ import ( "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/generator" "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" "github.com/dipdup-io/starknet-indexer/pkg/indexer/store" + ddConfig "github.com/dipdup-net/go-lib/config" "github.com/dipdup-net/indexer-sdk/pkg/modules" sdk "github.com/dipdup-net/indexer-sdk/pkg/storage" "github.com/pkg/errors" @@ -68,7 +68,8 @@ type Indexer struct { func New( cfg config.Config, storage postgres.Storage, -) *Indexer { + datasource map[string]ddConfig.DataSource, +) (*Indexer, error) { indexer := &Indexer{ BaseModule: modules.New("indexer"), cfg: cfg, @@ -87,13 +88,18 @@ func New( proxy: storage.Proxy, state: newState(nil), cache: cache.New(storage.Address, storage.Class, storage.Proxy), - receiver: receiver.NewReceiver(cfg), rollbackManager: storage.RollbackManager, rollback: make(chan struct{}, 1), rollbackRerun: make(chan struct{}, 1), txWriteMutex: new(sync.Mutex), rollbackWait: new(sync.WaitGroup), } + rcvr, err := receiver.NewReceiver(cfg, datasource) + if err != nil { + return nil, err + } + indexer.receiver = rcvr + indexer.CreateOutput(OutputBlocks) indexer.idGenerator = generator.NewIdGenerator(storage.Address, storage.Class, indexer.cache, indexer.state.Current()) @@ -118,7 +124,7 @@ func New( storage.Transactable, ) - return indexer + return indexer, nil } // Start - @@ -301,7 +307,7 @@ func (indexer *Indexer) saveBlocks(ctx context.Context) { return case result := <-indexer.receiver.Results(): - indexer.queue[result.Block.BlockNumber] = result + indexer.queue[result.Block.Height] = result if indexer.state.Height() == 0 && !zeroBlock { if data, ok := indexer.queue[0]; ok { @@ -354,7 +360,7 @@ func (indexer *Indexer) saveBlocks(ctx context.Context) { } } -func (indexer *Indexer) handleReorg(ctx context.Context, newBlock sequencer.Block) (bool, error) { +func (indexer *Indexer) handleReorg(ctx context.Context, newBlock receiver.Block) (bool, error) { lastBlock, err := indexer.blocks.Last(ctx) if err != nil { if indexer.blocks.IsNoRows(err) { @@ -363,13 +369,12 @@ func (indexer *Indexer) handleReorg(ctx context.Context, newBlock sequencer.Bloc return false, err } - parentHash := data.Felt(newBlock.ParentHash).Bytes() - if bytes.Equal(lastBlock.Hash, parentHash) { + if bytes.Equal(lastBlock.Hash, newBlock.ParentHash) { return false, nil } log.Warn(). - Str("parent_hash_of_new_block", newBlock.ParentHash). + Hex("parent_hash_of_new_block", newBlock.ParentHash). Hex("indexer_head_block_hash", lastBlock.Hash). Msg("rollback detected by parent hash") @@ -420,16 +425,16 @@ func (indexer *Indexer) handleBlock(ctx context.Context, result receiver.Result) indexer.statusChecker.addBlock(parseResult.Block) } - delete(indexer.queue, result.Block.BlockNumber) + delete(indexer.queue, result.Block.Height) l := indexer.Log.Info(). - Uint64("height", result.Block.BlockNumber). + Uint64("height", result.Block.Height). Int("tx_count", parseResult.Block.TxCount). Time("block_time", parseResult.Block.Time). Int64("process_time_in_ms", time.Since(start).Milliseconds()). Int64("save_time_in_ms", saveTime) - if result.Block.StarknetVersion != nil { - l.Str("version", *result.Block.StarknetVersion) + if result.Block.Version != nil && *result.Block.Version != "" { + l.Str("version", *result.Block.Version) } l.Msg("indexed") diff --git a/pkg/indexer/parser/interfaces/interfaces.go b/pkg/indexer/parser/interfaces/interfaces.go index 3f56e3a..13d8124 100644 --- a/pkg/indexer/parser/interfaces/interfaces.go +++ b/pkg/indexer/parser/interfaces/interfaces.go @@ -8,17 +8,18 @@ import ( "github.com/dipdup-io/starknet-go-api/pkg/sequencer" "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // Parser - type Parser interface { - ParseDeclare(ctx context.Context, version starknetData.Felt, raw *starknetData.Declare, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Declare, *storage.Fee, error) - ParseDeployAccount(ctx context.Context, raw *starknetData.DeployAccount, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.DeployAccount, *storage.Fee, error) - ParseDeploy(ctx context.Context, raw *starknetData.Deploy, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Deploy, *storage.Fee, error) - ParseInvokeV0(ctx context.Context, raw *starknetData.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) - ParseInvokeV1(ctx context.Context, raw *starknetData.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) - ParseInvokeV3(ctx context.Context, raw *starknetData.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) - ParseL1Handler(ctx context.Context, raw *starknetData.L1Handler, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.L1Handler, *storage.Fee, error) + ParseDeclare(ctx context.Context, version starknetData.Felt, raw *starknetData.Declare, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Declare, *storage.Fee, error) + ParseDeployAccount(ctx context.Context, raw *starknetData.DeployAccount, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.DeployAccount, *storage.Fee, error) + ParseDeploy(ctx context.Context, raw *starknetData.Deploy, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Deploy, *storage.Fee, error) + ParseInvokeV0(ctx context.Context, raw *starknetData.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) + ParseInvokeV1(ctx context.Context, raw *starknetData.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) + ParseInvokeV3(ctx context.Context, raw *starknetData.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) + ParseL1Handler(ctx context.Context, raw *starknetData.L1Handler, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.L1Handler, *storage.Fee, error) } // InternalTxParser - diff --git a/pkg/indexer/parser/parser.go b/pkg/indexer/parser/parser.go index 6a5e741..d8b0639 100644 --- a/pkg/indexer/parser/parser.go +++ b/pkg/indexer/parser/parser.go @@ -2,11 +2,8 @@ package parser import ( "context" - "time" - "github.com/dipdup-io/starknet-go-api/pkg/data" starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" - "github.com/dipdup-io/starknet-go-api/pkg/encoding" "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/cache" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" @@ -24,7 +21,7 @@ func createParser( cache *cache.Cache, blocks storage.IBlock, ) (interfaces.Parser, error) { - if version == nil { + if version == nil || *version == "" { return v0.New(resolver, cache, blocks), nil } @@ -52,15 +49,15 @@ func Parse( result receiver.Result, ) (parserData.Result, error) { block := storage.Block{ - ID: result.Block.BlockNumber + 1, - Height: result.Block.BlockNumber, - Time: time.Unix(result.Block.Timestamp, 0).UTC(), - Hash: data.Felt(result.Block.BlockHash).Bytes(), - ParentHash: data.Felt(result.Block.ParentHash).Bytes(), - NewRoot: encoding.MustDecodeHex(result.Block.NewRoot), - SequencerAddress: encoding.MustDecodeHex(result.Block.SequencerAddress), - Version: result.Block.StarknetVersion, - Status: storage.NewStatus(result.Block.Status), + ID: result.Block.Height + 1, + Height: result.Block.Height, + Time: result.Block.Time, + Hash: result.Block.Hash, + ParentHash: result.Block.ParentHash, + NewRoot: result.Block.NewRoot, + SequencerAddress: result.Block.SequencerAddress, + Version: result.Block.Version, + Status: result.Block.Status, TxCount: len(result.Block.Transactions), Invoke: make([]storage.Invoke, 0), @@ -70,10 +67,7 @@ func Parse( L1Handler: make([]storage.L1Handler, 0), } - if len(result.Block.Transactions) != len(result.Trace.Traces) { - return parserData.Result{}, errors.Errorf("invalid data length") - } - if len(result.Block.Transactions) != len(result.Block.Receipts) { + if len(result.Block.Transactions) != len(result.Traces) { return parserData.Result{}, errors.Errorf("invalid data length") } @@ -100,16 +94,16 @@ func Parse( ) switch result.Block.Transactions[i].Version { case starknetData.Version0: - invoke, fee, err = p.ParseInvokeV0(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + invoke, fee, err = p.ParseInvokeV0(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) case starknetData.Version1: - invoke, fee, err = p.ParseInvokeV1(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + invoke, fee, err = p.ParseInvokeV1(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) case starknetData.Version3: - invoke, fee, err = p.ParseInvokeV3(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + invoke, fee, err = p.ParseInvokeV3(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) default: return parserData.Result{}, errors.Errorf("unknown invoke version: %s", result.Block.Transactions[i].Version) } if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s invoke version=%s", result.Block.Transactions[i].TransactionHash, result.Block.Transactions[i].Version) + return parserData.Result{}, errors.Wrapf(err, "%s invoke version=%s", result.Block.Transactions[i].Hash, result.Block.Transactions[i].Version) } invoke.Position = i block.Invoke = append(block.Invoke, invoke) @@ -117,9 +111,9 @@ func Parse( block.Fee = append(block.Fee, *fee) } case *starknetData.Declare: - tx, fee, err := p.ParseDeclare(ctx, result.Block.Transactions[i].Version, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + tx, fee, err := p.ParseDeclare(ctx, result.Block.Transactions[i].Version, typed, block, result.Block.Transactions[i], result.Traces[i]) if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s declare", result.Block.Transactions[i].TransactionHash) + return parserData.Result{}, errors.Wrapf(err, "%s declare", result.Block.Transactions[i].Hash) } tx.Position = i block.Declare = append(block.Declare, tx) @@ -127,9 +121,9 @@ func Parse( block.Fee = append(block.Fee, *fee) } case *starknetData.Deploy: - tx, fee, err := p.ParseDeploy(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + tx, fee, err := p.ParseDeploy(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s deploy", result.Block.Transactions[i].TransactionHash) + return parserData.Result{}, errors.Wrapf(err, "%s deploy", result.Block.Transactions[i].Hash) } tx.Position = i block.Deploy = append(block.Deploy, tx) @@ -137,9 +131,9 @@ func Parse( block.Fee = append(block.Fee, *fee) } case *starknetData.DeployAccount: - tx, fee, err := p.ParseDeployAccount(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + tx, fee, err := p.ParseDeployAccount(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s deploy account", result.Block.Transactions[i].TransactionHash) + return parserData.Result{}, errors.Wrapf(err, "%s deploy account", result.Block.Transactions[i].Hash) } tx.Position = i block.DeployAccount = append(block.DeployAccount, tx) @@ -147,9 +141,9 @@ func Parse( block.Fee = append(block.Fee, *fee) } case *starknetData.L1Handler: - tx, fee, err := p.ParseL1Handler(ctx, typed, block, result.Trace.Traces[i], result.Block.Receipts[i]) + tx, fee, err := p.ParseL1Handler(ctx, typed, block, result.Block.Transactions[i], result.Traces[i]) if err != nil { - return parserData.Result{}, errors.Wrapf(err, "%s l1 handler", result.Block.Transactions[i].TransactionHash) + return parserData.Result{}, errors.Wrapf(err, "%s l1 handler", result.Block.Transactions[i].Hash) } tx.Position = i block.L1Handler = append(block.L1Handler, tx) diff --git a/pkg/indexer/parser/version/v0/declare.go b/pkg/indexer/parser/version/v0/declare.go index aac233a..85d8c7e 100644 --- a/pkg/indexer/parser/version/v0/declare.go +++ b/pkg/indexer/parser/version/v0/declare.go @@ -7,10 +7,11 @@ import ( "github.com/dipdup-io/starknet-go-api/pkg/sequencer" "github.com/dipdup-io/starknet-indexer/internal/storage" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // ParseDeclare - -func (parser Parser) ParseDeclare(ctx context.Context, version data.Felt, raw *data.Declare, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Declare, *storage.Fee, error) { +func (parser Parser) ParseDeclare(ctx context.Context, version data.Felt, raw *data.Declare, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Declare, *storage.Fee, error) { tx := storage.Declare{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -102,7 +103,7 @@ func (parser Parser) ParseDeclare(ctx context.Context, version data.Felt, raw *d return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/deploy.go b/pkg/indexer/parser/version/v0/deploy.go index 1b08312..bb95713 100644 --- a/pkg/indexer/parser/version/v0/deploy.go +++ b/pkg/indexer/parser/version/v0/deploy.go @@ -11,10 +11,11 @@ import ( "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // ParseDeploy - -func (parser Parser) ParseDeploy(ctx context.Context, raw *data.Deploy, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Deploy, *storage.Fee, error) { +func (parser Parser) ParseDeploy(ctx context.Context, raw *data.Deploy, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Deploy, *storage.Fee, error) { tx := storage.Deploy{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -102,7 +103,7 @@ func (parser Parser) ParseDeploy(ctx context.Context, raw *data.Deploy, block st return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/deploy_account.go b/pkg/indexer/parser/version/v0/deploy_account.go index ab9f93e..b5ededc 100644 --- a/pkg/indexer/parser/version/v0/deploy_account.go +++ b/pkg/indexer/parser/version/v0/deploy_account.go @@ -11,11 +11,12 @@ import ( "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" "github.com/goccy/go-json" ) // ParseDeployAccount - -func (parser Parser) ParseDeployAccount(ctx context.Context, raw *data.DeployAccount, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.DeployAccount, *storage.Fee, error) { +func (parser Parser) ParseDeployAccount(ctx context.Context, raw *data.DeployAccount, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.DeployAccount, *storage.Fee, error) { tx := storage.DeployAccount{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -107,7 +108,7 @@ func (parser Parser) ParseDeployAccount(ctx context.Context, raw *data.DeployAcc return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/events.go b/pkg/indexer/parser/version/v0/events.go index 2787fdd..4a0a4fb 100644 --- a/pkg/indexer/parser/version/v0/events.go +++ b/pkg/indexer/parser/version/v0/events.go @@ -46,8 +46,8 @@ func (parser EventParser) Parse(ctx context.Context, txCtx data.TxContext, contr Height: txCtx.Height, Time: txCtx.Time, Order: event.Order, - Data: event.Data, - Keys: event.Keys, + Data: make([]string, len(event.Data)), + Keys: make([]string, len(event.Keys)), ContractID: txCtx.ContractId, Contract: txCtx.Contract, DeclareID: txCtx.DeclareID, @@ -59,6 +59,14 @@ func (parser EventParser) Parse(ctx context.Context, txCtx data.TxContext, contr InternalID: txCtx.InternalID, } + for i := 0; i < len(event.Data); i++ { + model.Data[i] = event.Data[i].String() + } + + for i := 0; i < len(event.Keys); i++ { + model.Keys[i] = event.Keys[i].String() + } + if address, err := parser.resolver.FindAddressByHash(ctx, starknetData.Felt(event.FromAddress)); err != nil { return model, err } else if address != nil { diff --git a/pkg/indexer/parser/version/v0/fee.go b/pkg/indexer/parser/version/v0/fee.go index c746e29..885417e 100644 --- a/pkg/indexer/parser/version/v0/fee.go +++ b/pkg/indexer/parser/version/v0/fee.go @@ -119,8 +119,8 @@ func (parser FeeParser) ParseInvocation(ctx context.Context, txCtx data.TxContex CallType: storage.NewCallType(feeInvocation.CallType), EntrypointType: storage.NewEntrypointType(feeInvocation.EntrypointType), Selector: feeInvocation.Selector.Bytes(), - Result: feeInvocation.Result, - Calldata: feeInvocation.Calldata, + Result: make([]string, len(feeInvocation.Result)), + Calldata: make([]string, len(feeInvocation.Calldata)), DeclareID: txCtx.DeclareID, DeployID: txCtx.DeployID, @@ -132,6 +132,12 @@ func (parser FeeParser) ParseInvocation(ctx context.Context, txCtx data.TxContex Messages: make([]storage.Message, 0), Internals: make([]storage.Internal, 0), } + for i := 0; i < len(feeInvocation.Calldata); i++ { + tx.Calldata[i] = feeInvocation.Calldata[i].String() + } + for i := 0; i < len(feeInvocation.Result); i++ { + tx.Result[i] = feeInvocation.Result[i].String() + } if class, err := parser.resolver.FindClassByHash(ctx, feeInvocation.ClassHash, tx.Height); err != nil { return nil, err @@ -207,10 +213,10 @@ func (parser FeeParser) ParseInvocation(ctx context.Context, txCtx data.TxContex } } - if len(feeInvocation.Calldata) > 0 && len(tx.Selector) > 0 { + if len(tx.Calldata) > 0 && len(tx.Selector) > 0 { if isExecute && !hasExecute { tx.Entrypoint = encoding.ExecuteEntrypoint - tx.ParsedCalldata, err = abi.DecodeExecuteCallData(feeInvocation.Calldata) + tx.ParsedCalldata, err = abi.DecodeExecuteCallData(tx.Calldata) } else { tx.ParsedCalldata, tx.Entrypoint, err = decode.CalldataBySelector(contractAbi, tx.Selector, tx.Calldata) } diff --git a/pkg/indexer/parser/version/v0/internal_tx.go b/pkg/indexer/parser/version/v0/internal_tx.go index 3e77c8f..ec9133d 100644 --- a/pkg/indexer/parser/version/v0/internal_tx.go +++ b/pkg/indexer/parser/version/v0/internal_tx.go @@ -61,8 +61,8 @@ func (parser InternalTxParser) Parse(ctx context.Context, txCtx parserData.TxCon CallType: storage.NewCallType(internal.CallType), EntrypointType: storage.NewEntrypointType(internal.EntrypointType), Selector: internal.Selector.Bytes(), - Result: internal.Result, - Calldata: internal.Calldata, + Result: make([]string, len(internal.Result)), + Calldata: make([]string, len(internal.Calldata)), DeclareID: txCtx.DeclareID, DeployID: txCtx.DeployID, @@ -75,6 +75,13 @@ func (parser InternalTxParser) Parse(ctx context.Context, txCtx parserData.TxCon Messages: make([]storage.Message, 0), Internals: make([]storage.Internal, 0), } + for i := 0; i < len(internal.Calldata); i++ { + tx.Calldata[i] = internal.Calldata[i].String() + } + + for i := 0; i < len(internal.Result); i++ { + tx.Result[i] = internal.Result[i].String() + } if class, err := parser.Resolver.FindClassByHash(ctx, internal.ClassHash, tx.Height); err != nil { return tx, err @@ -163,16 +170,16 @@ func (parser InternalTxParser) Parse(ctx context.Context, txCtx parserData.TxCon } } - if len(internal.Calldata) > 0 && !isUnknownProxy { + if len(tx.Calldata) > 0 && !isUnknownProxy { switch { case isExecute && !has: tx.Entrypoint = encoding.ExecuteEntrypoint - tx.ParsedCalldata, err = abi.DecodeExecuteCallData(internal.Calldata) + tx.ParsedCalldata, err = abi.DecodeExecuteCallData(tx.Calldata) case isChangeModules && !has: tx.Entrypoint = encoding.ChangeModulesEntrypoint - tx.ParsedCalldata, err = abi.DecodeChangeModulesCallData(internal.Calldata) + tx.ParsedCalldata, err = abi.DecodeChangeModulesCallData(tx.Calldata) default: - tx.ParsedCalldata, tx.Entrypoint, err = decode.InternalCalldata(contractAbi, tx.Selector, internal.Calldata, tx.EntrypointType) + tx.ParsedCalldata, tx.Entrypoint, err = decode.InternalCalldata(contractAbi, tx.Selector, tx.Calldata, tx.EntrypointType) } if err != nil { @@ -184,9 +191,9 @@ func (parser InternalTxParser) Parse(ctx context.Context, txCtx parserData.TxCon switch { case isExecute && !has: case isChangeModules && !has: - tx.ParsedResult, err = abi.DecodeChangeModulesResult(internal.Result) + tx.ParsedResult, err = abi.DecodeChangeModulesResult(tx.Result) default: - tx.ParsedResult, err = decode.Result(contractAbi, internal.Result, tx.Selector, tx.EntrypointType) + tx.ParsedResult, err = decode.Result(contractAbi, tx.Result, tx.Selector, tx.EntrypointType) } if err != nil { switch { diff --git a/pkg/indexer/parser/version/v0/invoke_v0.go b/pkg/indexer/parser/version/v0/invoke_v0.go index 67f461b..d3f1a6e 100644 --- a/pkg/indexer/parser/version/v0/invoke_v0.go +++ b/pkg/indexer/parser/version/v0/invoke_v0.go @@ -13,10 +13,11 @@ import ( "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/helpers" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // ParseInvokeV0 - -func (parser Parser) ParseInvokeV0(ctx context.Context, raw *data.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) { +func (parser Parser) ParseInvokeV0(ctx context.Context, raw *data.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) { tx := storage.Invoke{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -161,7 +162,7 @@ func (parser Parser) ParseInvokeV0(ctx context.Context, raw *data.Invoke, block return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/invoke_v1.go b/pkg/indexer/parser/version/v0/invoke_v1.go index 2d4010b..a771787 100644 --- a/pkg/indexer/parser/version/v0/invoke_v1.go +++ b/pkg/indexer/parser/version/v0/invoke_v1.go @@ -10,11 +10,12 @@ import ( "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" "github.com/pkg/errors" ) // ParseInvokeV1 - -func (parser Parser) ParseInvokeV1(ctx context.Context, raw *data.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) { +func (parser Parser) ParseInvokeV1(ctx context.Context, raw *data.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) { tx := storage.Invoke{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -138,7 +139,7 @@ func (parser Parser) ParseInvokeV1(ctx context.Context, raw *data.Invoke, block return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/invoke_v3.go b/pkg/indexer/parser/version/v0/invoke_v3.go index 34359de..e24808c 100644 --- a/pkg/indexer/parser/version/v0/invoke_v3.go +++ b/pkg/indexer/parser/version/v0/invoke_v3.go @@ -10,11 +10,12 @@ import ( "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" "github.com/pkg/errors" ) // ParseInvokeV1 - -func (parser Parser) ParseInvokeV3(ctx context.Context, raw *data.Invoke, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.Invoke, *storage.Fee, error) { +func (parser Parser) ParseInvokeV3(ctx context.Context, raw *data.Invoke, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.Invoke, *storage.Fee, error) { tx := storage.Invoke{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -138,7 +139,7 @@ func (parser Parser) ParseInvokeV3(ctx context.Context, raw *data.Invoke, block return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/l1_handler.go b/pkg/indexer/parser/version/v0/l1_handler.go index d6d5902..0ca9890 100644 --- a/pkg/indexer/parser/version/v0/l1_handler.go +++ b/pkg/indexer/parser/version/v0/l1_handler.go @@ -12,10 +12,11 @@ import ( "github.com/dipdup-io/starknet-indexer/pkg/indexer/decode" parserData "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/data" "github.com/dipdup-io/starknet-indexer/pkg/indexer/parser/helpers" + "github.com/dipdup-io/starknet-indexer/pkg/indexer/receiver" ) // ParseL1Handler - -func (parser Parser) ParseL1Handler(ctx context.Context, raw *data.L1Handler, block storage.Block, trace sequencer.Trace, receipts sequencer.Receipt) (storage.L1Handler, *storage.Fee, error) { +func (parser Parser) ParseL1Handler(ctx context.Context, raw *data.L1Handler, block storage.Block, receiverTx receiver.Transaction, trace sequencer.Trace) (storage.L1Handler, *storage.Fee, error) { tx := storage.L1Handler{ ID: parser.Resolver.NextTxId(), Height: block.Height, @@ -109,7 +110,7 @@ func (parser Parser) ParseL1Handler(ctx context.Context, raw *data.L1Handler, bl return tx, fee, nil } } else { - transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receipts.ActualFee) + transfer, err := parser.FeeParser.ParseActualFee(ctx, txCtx, receiverTx.ActualFee) if err != nil { return tx, nil, nil } diff --git a/pkg/indexer/parser/version/v0/message.go b/pkg/indexer/parser/version/v0/message.go index 7d747a8..00cc394 100644 --- a/pkg/indexer/parser/version/v0/message.go +++ b/pkg/indexer/parser/version/v0/message.go @@ -31,7 +31,7 @@ func (parser MessageParser) Parse(ctx context.Context, txCtx parserData.TxContex Time: txCtx.Time, Order: msg.Order, Selector: msg.Selector.String(), - Payload: msg.Payload, + Payload: make([]string, len(msg.Payload)), Nonce: msg.Nonce.Decimal(), ContractID: txCtx.ContractId, DeclareID: txCtx.DeclareID, @@ -42,6 +42,9 @@ func (parser MessageParser) Parse(ctx context.Context, txCtx parserData.TxContex FeeID: txCtx.FeeID, InternalID: txCtx.InternalID, } + for i := 0; i < len(msg.Payload); i++ { + message.Payload[i] = msg.Payload[i].String() + } if txCtx.ProxyId > 0 { message.ContractID = txCtx.ProxyId } diff --git a/pkg/indexer/receiver/api.go b/pkg/indexer/receiver/api.go new file mode 100644 index 0000000..b82091d --- /dev/null +++ b/pkg/indexer/receiver/api.go @@ -0,0 +1,44 @@ +package receiver + +import ( + "context" + "time" + + "github.com/dipdup-io/starknet-go-api/pkg/data" + starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" + starknet "github.com/dipdup-io/starknet-go-api/pkg/sequencer" + "github.com/dipdup-io/starknet-indexer/internal/storage" +) + +type API interface { + GetBlock(ctx context.Context, blockId starknetData.BlockID) (Block, error) + TraceBlock(ctx context.Context, blockId starknetData.BlockID) ([]starknet.Trace, error) + GetStateUpdate(ctx context.Context, blockId starknetData.BlockID) (starknetData.StateUpdate, error) + GetBlockStatus(ctx context.Context, height uint64) (storage.Status, error) + TransactionStatus(ctx context.Context, hash string) (storage.Status, error) + GetClass(ctx context.Context, hash string) (starknetData.Class, error) + Head(ctx context.Context) (uint64, error) +} + +type Block struct { + Height uint64 + Time time.Time + Version *string + Status storage.Status + Hash []byte + ParentHash []byte + NewRoot []byte + SequencerAddress []byte + + Transactions []Transaction + Receipts []starknet.Receipt +} + +type Transaction struct { + Type string + Version data.Felt + Hash data.Felt + ActualFee data.Felt + + Body any +} diff --git a/pkg/indexer/receiver/feeder.go b/pkg/indexer/receiver/feeder.go new file mode 100644 index 0000000..74e9fc6 --- /dev/null +++ b/pkg/indexer/receiver/feeder.go @@ -0,0 +1,109 @@ +package receiver + +import ( + "context" + "time" + + "github.com/dipdup-io/starknet-go-api/pkg/data" + starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" + "github.com/dipdup-io/starknet-go-api/pkg/encoding" + starknet "github.com/dipdup-io/starknet-go-api/pkg/sequencer" + "github.com/dipdup-io/starknet-indexer/internal/storage" + "github.com/dipdup-net/go-lib/config" + "github.com/pkg/errors" +) + +type Feeder struct { + api starknet.API +} + +func NewFeeder(cfg config.DataSource) *Feeder { + opts := make([]starknet.ApiOption, 0) + if cfg.RequestsPerSecond > 0 { + opts = append(opts, starknet.WithRateLimit(cfg.RequestsPerSecond)) + } + + return &Feeder{ + api: starknet.NewAPI("", cfg.URL, opts...), + } +} + +func (f *Feeder) GetBlock(ctx context.Context, blockId starknetData.BlockID) (block Block, err error) { + response, err := f.api.GetBlock(ctx, blockId, false) + if err != nil { + return block, err + } + + block.Height = response.BlockNumber + block.Time = time.Unix(response.Timestamp, 0).UTC() + block.Hash = data.Felt(response.BlockHash).Bytes() + block.ParentHash = data.Felt(response.ParentHash).Bytes() + block.NewRoot = encoding.MustDecodeHex(response.NewRoot) + block.SequencerAddress = encoding.MustDecodeHex(response.SequencerAddress) + block.Version = response.StarknetVersion + block.Status = storage.NewStatus(response.Status) + block.Receipts = response.Receipts + + if len(response.Transactions) != len(response.Receipts) { + return block, errors.Errorf("length arrays of txs and receipts are differ") + } + + block.Transactions = make([]Transaction, len(response.Transactions)) + + for i := range response.Transactions { + block.Transactions[i].Hash = response.Transactions[i].TransactionHash + block.Transactions[i].Type = response.Transactions[i].Type + block.Transactions[i].Version = response.Transactions[i].Version + block.Transactions[i].Body = response.Transactions[i].Body + block.Transactions[i].ActualFee = response.Receipts[i].ActualFee + } + + return +} + +func (f *Feeder) TraceBlock(ctx context.Context, blockId starknetData.BlockID) (traces []starknet.Trace, err error) { + response, err := f.api.TraceBlock(ctx, blockId) + if err != nil { + return + } + return response.Traces, nil +} + +func (f *Feeder) GetStateUpdate(ctx context.Context, blockId starknetData.BlockID) (response starknetData.StateUpdate, err error) { + return f.api.GetStateUpdate(ctx, blockId) +} + +func (f *Feeder) GetBlockStatus(ctx context.Context, height uint64) (storage.Status, error) { + response, err := f.api.GetBlock(ctx, starknetData.BlockID{Number: &height}, false) + if err != nil { + return storage.StatusUnknown, err + } + return storage.NewStatus(response.Status), nil +} + +func (f *Feeder) TransactionStatus(ctx context.Context, hash string) (storage.Status, error) { + response, err := f.api.GetTransactionStatus(ctx, hash) + if err != nil { + return storage.StatusUnknown, err + } + + return storage.NewStatus(response.Status), nil +} + +func (f *Feeder) GetClass(ctx context.Context, hash string) (starknetData.Class, error) { + blockId := starknetData.BlockID{ + String: starknetData.Latest, + } + + return f.api.GetClassByHash(ctx, blockId, hash) +} + +func (f *Feeder) Head(ctx context.Context) (uint64, error) { + response, err := f.api.GetBlock(ctx, starknetData.BlockID{ + String: starknetData.Latest, + }, true) + if err != nil { + return 0, err + } + return response.BlockNumber, nil +} diff --git a/pkg/indexer/receiver/node.go b/pkg/indexer/receiver/node.go new file mode 100644 index 0000000..2699a87 --- /dev/null +++ b/pkg/indexer/receiver/node.go @@ -0,0 +1,162 @@ +package receiver + +import ( + "context" + "os" + "time" + + "github.com/dipdup-io/starknet-go-api/pkg/data" + starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" + "github.com/dipdup-io/starknet-go-api/pkg/encoding" + starknet "github.com/dipdup-io/starknet-go-api/pkg/rpc" + "github.com/dipdup-io/starknet-go-api/pkg/sequencer" + "github.com/dipdup-io/starknet-indexer/internal/storage" + "github.com/dipdup-net/go-lib/config" +) + +type Node struct { + api starknet.API +} + +func NewNode(cfg config.DataSource) *Node { + apiKey := os.Getenv("NODE_APIKEY") + headerName := os.Getenv("NODE_HEADER_APIKEY") + return &Node{ + api: starknet.NewAPI( + cfg.URL, + starknet.WithRateLimit(cfg.RequestsPerSecond), + starknet.WithApiKey(headerName, apiKey), + ), + } +} + +func (n *Node) GetBlock(ctx context.Context, blockId starknetData.BlockID) (block Block, err error) { + response, err := n.api.GetBlockWithTxs(ctx, blockId) + if err != nil { + return + } + + block.Height = response.Result.BlockNumber + block.Time = time.Unix(response.Result.Timestamp, 0).UTC() + block.Hash = data.Felt(response.Result.BlockHash).Bytes() + block.ParentHash = data.Felt(response.Result.ParentHash).Bytes() + block.NewRoot = encoding.MustDecodeHex(response.Result.NewRoot) + block.SequencerAddress = encoding.MustDecodeHex(response.Result.SequencerAddress) + block.Version = response.Result.Version + block.Status = storage.NewStatus(response.Result.Status) + block.Transactions = make([]Transaction, len(response.Result.Transactions)) + + for i := range response.Result.Transactions { + block.Transactions[i].Hash = response.Result.Transactions[i].TransactionHash + block.Transactions[i].Type = response.Result.Transactions[i].Type + block.Transactions[i].Version = response.Result.Transactions[i].Version + block.Transactions[i].Body = response.Result.Transactions[i].Body + // TODO: resolve actual fee | block.Transactions[i].ActualFee = "" + } + + return +} + +func (n *Node) TraceBlock(ctx context.Context, block starknetData.BlockID) (traces []sequencer.Trace, err error) { + response, err := n.api.Trace(ctx, block) + if err != nil { + return + } + + traces = make([]sequencer.Trace, len(response.Result)) + for i := range response.Result { + if inv := response.Result[i].TraceRoot.ExecuteInvocation; inv != nil { + if inv.RevertReason != "" { + traces[i].RevertedError = inv.RevertReason + } else { + traces[i].FunctionInvocation = makeSeqInvocationFromNodeCall(inv) + } + } + + if inv := response.Result[i].TraceRoot.ConstructorInvocation; inv != nil { + if inv.RevertReason != "" { + traces[i].RevertedError = inv.RevertReason + } else { + traces[i].FunctionInvocation = makeSeqInvocationFromNodeCall(inv) + } + } + + traces[i].ValidateInvocation = makeSeqInvocationFromNodeCall(response.Result[i].TraceRoot.ValidateInvocation) + traces[i].FeeTransferInvocation = makeSeqInvocationFromNodeCall(response.Result[i].TraceRoot.FeeTransferInvocation) + traces[i].TransactionHash = response.Result[i].TransactionHash + } + + return +} + +func makeSeqInvocationFromNodeCall(call *starknet.Call) *sequencer.Invocation { + if call == nil { + return nil + } + + inv := &sequencer.Invocation{ + CallerAddress: call.CallerAddress, + ContractAddress: call.ContractAddress, + Calldata: call.Calldata, + CallType: call.CallType, + ClassHash: call.ClassHash, + Selector: call.EntryPointSelector, + EntrypointType: call.EntryPointType, + Result: call.Result, + Events: call.Events, + Messages: call.Messages, + InternalCalls: make([]sequencer.Invocation, len(call.Calls)), + } + + for i := range call.Calls { + internalCall := makeSeqInvocationFromNodeCall(&call.Calls[i]) + inv.InternalCalls[i] = *internalCall + } + + return inv +} + +func (n *Node) GetStateUpdate(ctx context.Context, block starknetData.BlockID) (starknetData.StateUpdate, error) { + response, err := n.api.GetStateUpdate(ctx, block) + if err != nil { + return starknetData.StateUpdate{}, err + } + return response.Result.ToStateUpdate(), nil +} + +func (n *Node) GetBlockStatus(ctx context.Context, height uint64) (storage.Status, error) { + response, err := n.api.GetBlockWithTxHashes(ctx, starknetData.BlockID{Number: &height}) + if err != nil { + return storage.StatusUnknown, err + } + return storage.NewStatus(response.Result.Status), nil +} + +func (n *Node) TransactionStatus(ctx context.Context, hash string) (storage.Status, error) { + response, err := n.api.GetTransactionStatus(ctx, hash) + if err != nil { + return storage.StatusUnknown, err + } + + return storage.NewStatus(response.Result.Finality), nil +} + +func (n *Node) GetClass(ctx context.Context, hash string) (starknetData.Class, error) { + blockId := starknetData.BlockID{ + String: starknetData.Latest, + } + + response, err := n.api.GetClass(ctx, blockId, hash) + if err != nil { + return starknetData.Class{}, err + } + return response.Result, nil +} + +func (n *Node) Head(ctx context.Context) (uint64, error) { + response, err := n.api.BlockNumber(ctx) + if err != nil { + return 0, err + } + return response.Result, nil +} diff --git a/pkg/indexer/receiver/receiver.go b/pkg/indexer/receiver/receiver.go index 1d4a202..15fe4c2 100644 --- a/pkg/indexer/receiver/receiver.go +++ b/pkg/indexer/receiver/receiver.go @@ -2,31 +2,30 @@ package receiver import ( "context" - "errors" "sync" "time" starknetData "github.com/dipdup-io/starknet-go-api/pkg/data" - starknetRpc "github.com/dipdup-io/starknet-go-api/pkg/rpc" starknet "github.com/dipdup-io/starknet-go-api/pkg/sequencer" "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/pkg/indexer/config" "github.com/dipdup-io/workerpool" + ddConfig "github.com/dipdup-net/go-lib/config" + "github.com/pkg/errors" "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) // Result - type Result struct { - Block starknet.Block - Trace starknet.TraceResponse + Block Block + Traces []starknet.Trace StateUpdate starknetData.StateUpdate } // Receiver - type Receiver struct { - api starknet.API - rpc *starknetRpc.API + api API result chan Result pool *workerpool.Pool[uint64] processing map[uint64]struct{} @@ -37,19 +36,20 @@ type Receiver struct { } // NewReceiver - -func NewReceiver(cfg config.Config) *Receiver { - opts := make([]starknet.ApiOption, 0) - if cfg.Sequencer.Rps > 0 { - opts = append(opts, starknet.WithRateLimit(cfg.Sequencer.Rps)) +func NewReceiver(cfg config.Config, ds map[string]ddConfig.DataSource) (*Receiver, error) { + dsCfg, ok := ds[cfg.Datasource] + if !ok { + return nil, errors.Errorf("unknown datasource name: %s", cfg.Datasource) } - log.Info().Bool("enabled", cfg.Cache).Str("dir", cfg.CacheDir).Msg("rpc response caching") - if cfg.Cache && cfg.CacheDir != "" { - opts = append(opts, starknet.WithCacheInFS(cfg.CacheDir)) + var api API + switch cfg.Datasource { + case "node": + api = NewNode(dsCfg) + case "sequencer": + api = NewFeeder(dsCfg) } - api := starknet.NewAPI(cfg.Sequencer.Gateway, cfg.Sequencer.FeederGateway, opts...) - receiver := &Receiver{ api: api, result: make(chan Result, cfg.ThreadsCount*2), @@ -60,18 +60,13 @@ func NewReceiver(cfg config.Config) *Receiver { wg: new(sync.WaitGroup), } - if cfg.Node != nil && cfg.Node.Url != "" { - rpc := starknetRpc.NewAPI(cfg.Node.Url, starknetRpc.WithRateLimit(cfg.Node.Rps)) - receiver.rpc = &rpc - } - if receiver.timeout == 0 { receiver.timeout = 10 * time.Second } receiver.pool = workerpool.NewPool(receiver.worker, cfg.ThreadsCount) - return receiver + return receiver, nil } // Close - @@ -134,7 +129,7 @@ func (r *Receiver) worker(ctx context.Context, height uint64) { time.Sleep(time.Second) continue } - result.Trace = response + result.Traces = response break } @@ -185,21 +180,7 @@ func (r *Receiver) Head(ctx context.Context) (uint64, error) { requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - if r.rpc != nil { - response, err := r.rpc.BlockNumber(requestCtx) - if err != nil { - return 0, err - } - return response.Result, nil - } - - response, err := r.api.GetBlock(requestCtx, starknetData.BlockID{ - String: starknetData.Latest, - }) - if err != nil { - return 0, err - } - return response.BlockNumber, nil + return r.api.Head(requestCtx) } // GetClass - @@ -207,11 +188,7 @@ func (r *Receiver) GetClass(ctx context.Context, hash string) (starknetData.Clas requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - blockId := starknetData.BlockID{ - String: starknetData.Latest, - } - - return r.api.GetClassByHash(requestCtx, blockId, hash) + return r.api.GetClass(requestCtx, hash) } // TransactionStatus - @@ -219,12 +196,7 @@ func (r *Receiver) TransactionStatus(ctx context.Context, hash string) (storage. requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - response, err := r.api.GetTransactionStatus(requestCtx, hash) - if err != nil { - return storage.StatusUnknown, err - } - - return storage.NewStatus(response.Status), nil + return r.api.TransactionStatus(requestCtx, hash) } // GetBlockStatus - @@ -232,23 +204,7 @@ func (r *Receiver) GetBlockStatus(ctx context.Context, height uint64) (storage.S requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - blockId := starknetData.BlockID{ - Number: &height, - } - - if r.rpc != nil { - response, err := r.rpc.GetBlockWithTxHashes(requestCtx, blockId) - if err != nil { - return storage.StatusUnknown, err - } - return storage.NewStatus(response.Result.Status), nil - } - - response, err := r.api.GetBlock(requestCtx, blockId) - if err != nil { - return storage.StatusUnknown, err - } - return storage.NewStatus(response.Status), nil + return r.api.GetBlockStatus(requestCtx, height) } // Results - @@ -265,14 +221,6 @@ func (r *Receiver) getStateUpdate(ctx context.Context, blockId starknetData.Bloc requestCtx, cancel := context.WithTimeout(ctx, r.timeout) defer cancel() - if r.rpc != nil { - response, err := r.rpc.GetStateUpdate(requestCtx, blockId) - if err != nil { - return starknetData.StateUpdate{}, err - } - return response.Result.ToStateUpdate(), nil - } - return r.api.GetStateUpdate(requestCtx, blockId) } diff --git a/pkg/indexer/store/sub_models.go b/pkg/indexer/store/sub_models.go index 76aafb8..d65f64b 100644 --- a/pkg/indexer/store/sub_models.go +++ b/pkg/indexer/store/sub_models.go @@ -6,6 +6,7 @@ import ( models "github.com/dipdup-io/starknet-indexer/internal/storage" "github.com/dipdup-io/starknet-indexer/internal/storage/postgres" "github.com/dipdup-net/indexer-sdk/pkg/storage" + "github.com/lib/pq" ) const copyThreashold = 25 @@ -106,17 +107,26 @@ func bulkSaveWithCopy[M models.CopiableModel](ctx context.Context, tx storage.Tr case len(arr) == 0: return nil case len(arr) < copyThreashold: - data := make([]any, len(arr)) - for i := range arr { - data[i] = &arr[i] - } - return tx.BulkSave(ctx, data) + _, err := tx.Tx().NewInsert().Model(&arr).Exec(ctx) + return err default: tableName := arr[0].TableName() - data := make([]storage.Copiable, len(arr)) + + stmt, err := tx.Tx().PrepareContext(ctx, pq.CopyIn(tableName, arr[0].Columns()...)) + if err != nil { + return err + } + for i := range arr { - data[i] = arr[i] + if _, err := stmt.ExecContext(ctx, arr[i].Flat()...); err != nil { + return err + } } - return tx.CopyFrom(ctx, tableName, data) + + if _, err := stmt.ExecContext(ctx); err != nil { + return err + } + + return stmt.Close() } }