-
Notifications
You must be signed in to change notification settings - Fork 44
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat(task): Create chainvis views and refresher #77
Merged
Merged
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| package migrations | ||
|
|
||
| import ( | ||
| "github.com/go-pg/migrations/v8" | ||
| ) | ||
|
|
||
| // Schema version 7 produces views which support queries for | ||
| // tipset visualization | ||
|
|
||
| // See https://github.com/DigitalMOB2/filecoin-lotus-explorer-playground | ||
|
|
||
| func init() { | ||
| up := batch(` | ||
| CREATE MATERIALIZED VIEW IF NOT EXISTS chain_visualizer_chain_data_view AS | ||
| SELECT | ||
| main_block.cid AS block, | ||
| bp.parent AS parent, | ||
| main_block.miner, | ||
| main_block.height, | ||
| main_block.parent_weight AS parentweight, | ||
| main_block.timestamp, | ||
| main_block.parent_state_root AS parentstateroot, | ||
| parent_block.timestamp AS parenttimestamp, | ||
| parent_block.height AS parentheight, | ||
| mp.raw_bytes_power AS parentpower, | ||
| synced.synced_at AS syncedtimestamp, | ||
| (SELECT COUNT(*) FROM block_messages WHERE block_messages.block = main_block.cid) AS messages | ||
| FROM | ||
| block_headers main_block | ||
| LEFT JOIN | ||
| block_parents bp ON bp.block = main_block.cid | ||
| LEFT JOIN | ||
| block_headers parent_block ON parent_block.cid = bp.parent | ||
| LEFT JOIN | ||
| blocks_synced synced ON synced.cid = main_block.cid | ||
| LEFT JOIN | ||
| miner_power mp ON main_block.parent_state_root = mp.state_root | ||
| WITH NO DATA; | ||
|
|
||
| CREATE MATERIALIZED VIEW IF NOT EXISTS chain_visualizer_orphans_view AS | ||
| SELECT | ||
| block_headers.cid AS block, | ||
| block_headers.miner, | ||
| block_headers.height, | ||
| block_headers.parent_weight AS parentweight, | ||
| block_headers.timestamp, | ||
| block_headers.parent_state_root AS parentstateroot, | ||
| block_parents.parent AS parent | ||
| FROM | ||
| block_headers | ||
| LEFT JOIN | ||
| block_parents ON block_headers.cid = block_parents.parent | ||
| WHERE | ||
| block_parents.block IS NULL | ||
| WITH NO DATA; | ||
|
|
||
| CREATE MATERIALIZED VIEW IF NOT EXISTS chain_visualizer_blocks_with_parents_view AS | ||
| SELECT | ||
| block, | ||
| parent, | ||
| b.miner, | ||
| b.height, | ||
| b.timestamp | ||
| FROM | ||
| block_parents | ||
| INNER JOIN | ||
| block_headers b ON block_parents.block = b.cid | ||
| WITH NO DATA; | ||
|
|
||
| CREATE MATERIALIZED VIEW IF NOT EXISTS chain_visualizer_blocks_view AS | ||
| SELECT * FROM block_headers | ||
| WITH NO DATA; | ||
| `) | ||
| down := batch(` | ||
| DROP MATERIALIZED VIEW IF EXISTS chain_visualizer_blocks_view; | ||
| DROP MATERIALIZED VIEW IF EXISTS chain_visualizer_blocks_with_parents_view; | ||
| DROP MATERIALIZED VIEW IF EXISTS chain_visualizer_orphans_view; | ||
| DROP MATERIALIZED VIEW IF EXISTS chain_visualizer_chain_data_view; | ||
| `) | ||
| migrations.MustRegisterTx(up, down) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package views | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| "golang.org/x/xerrors" | ||
|
|
||
| "github.com/filecoin-project/sentinel-visor/storage" | ||
| "github.com/filecoin-project/sentinel-visor/wait" | ||
| ) | ||
|
|
||
| var chainVisViews = []string{ | ||
| "chain_visualizer_blocks_view", | ||
| "chain_visualizer_blocks_with_parents_view", | ||
| "chain_visualizer_chain_data_view", | ||
| "chain_visualizer_orphans_view", | ||
| } | ||
|
|
||
| func NewChainVisRefresher(d *storage.Database, refreshRate time.Duration) *ChainVisRefresher { | ||
| return &ChainVisRefresher{ | ||
| db: d, | ||
| refreshRate: refreshRate, | ||
| } | ||
| } | ||
|
|
||
| // ChainVisRefresher is a task which refreshes a set of views that support | ||
| // chain visualization queries at a specific refreshRate | ||
| type ChainVisRefresher struct { | ||
| db *storage.Database | ||
| refreshRate time.Duration | ||
| } | ||
|
|
||
| // Run starts regularly refreshing until context is done or an error occurs | ||
| func (r *ChainVisRefresher) Run(ctx context.Context) error { | ||
| if r.refreshRate == 0 { | ||
| return nil | ||
| } | ||
| return wait.RepeatUntil(ctx, r.refreshRate, r.refreshView) | ||
| } | ||
|
|
||
| func (r *ChainVisRefresher) refreshView(ctx context.Context) (bool, error) { | ||
| for _, v := range chainVisViews { | ||
| _, err := r.db.DB.ExecContext(ctx, fmt.Sprintf("REFRESH MATERIALIZED VIEW %s;", v)) | ||
| if err != nil { | ||
| return true, xerrors.Errorf("refresh %s: %w", v, err) | ||
| } | ||
| } | ||
| return false, nil | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Minor nit: would be cleaner to have these views in a slice that is looped over