From bce28f4780c0587c343d153818bf787b7adedd74 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 10 Jan 2020 20:37:33 +0800 Subject: [PATCH 1/5] perf: add index on block timestamp, status and event type --- app/models/dao_event.rb | 4 +++- ...0110123617_add_index_to_block_timestamp_on_dao_events.rb | 6 ++++++ db/schema.rb | 4 +++- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 db/migrate/20200110123617_add_index_to_block_timestamp_on_dao_events.rb diff --git a/app/models/dao_event.rb b/app/models/dao_event.rb index ba7d31fe0..2c1ef357f 100644 --- a/app/models/dao_event.rb +++ b/app/models/dao_event.rb @@ -26,5 +26,7 @@ class DaoEvent < ApplicationRecord # # Indexes # -# index_dao_events_on_block_id (block_id) +# index_dao_events_on_block_id (block_id) +# index_dao_events_on_block_timestamp (block_timestamp) +# index_dao_events_on_status_and_event_type (status,event_type) # diff --git a/db/migrate/20200110123617_add_index_to_block_timestamp_on_dao_events.rb b/db/migrate/20200110123617_add_index_to_block_timestamp_on_dao_events.rb new file mode 100644 index 000000000..bfae71a1f --- /dev/null +++ b/db/migrate/20200110123617_add_index_to_block_timestamp_on_dao_events.rb @@ -0,0 +1,6 @@ +class AddIndexToBlockTimestampOnDaoEvents < ActiveRecord::Migration[6.0] + def change + add_index :dao_events, :block_timestamp + add_index :dao_events, [:status, :event_type] + end +end diff --git a/db/schema.rb b/db/schema.rb index 5f543aa4c..bcb30a30f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_01_03_051008) do +ActiveRecord::Schema.define(version: 2020_01_10_123617) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -197,6 +197,8 @@ t.datetime "updated_at", precision: 6, null: false t.decimal "block_timestamp", precision: 30 t.index ["block_id"], name: "index_dao_events_on_block_id" + t.index ["block_timestamp"], name: "index_dao_events_on_block_timestamp" + t.index ["status", "event_type"], name: "index_dao_events_on_status_and_event_type" end create_table "epoch_statistics", force: :cascade do |t| From 00ca74ce0f53786ca8f1d551529d702b093c5c3f Mon Sep 17 00:00:00 2001 From: shaojunda Date: Fri, 10 Jan 2020 21:12:44 +0800 Subject: [PATCH 2/5] chore: add some useful scopes --- app/models/address.rb | 2 ++ app/models/cell_output.rb | 5 +++++ app/models/ckb_transaction.rb | 2 ++ app/models/dao_event.rb | 2 ++ 4 files changed, 11 insertions(+) diff --git a/app/models/address.rb b/app/models/address.rb index 8d28aaefc..5d99bdabf 100644 --- a/app/models/address.rb +++ b/app/models/address.rb @@ -9,6 +9,8 @@ class Address < ApplicationRecord validates :balance, :cell_consumed, :ckb_transactions_count, :interest, :dao_deposit, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true scope :visible, -> { where(visible: true) } + scope :created_after, ->(block_timestamp) { where("block_timestamp >= ?", block_timestamp) } + scope :created_before, ->(block_timestamp) { where("block_timestamp <= ?", block_timestamp) } after_commit :flush_cache diff --git a/app/models/cell_output.rb b/app/models/cell_output.rb index cb9d14cb5..f5d81face 100644 --- a/app/models/cell_output.rb +++ b/app/models/cell_output.rb @@ -16,6 +16,11 @@ class CellOutput < ApplicationRecord attribute :tx_hash, :ckb_hash + scope :consumed_before, -> (block_timestamp) { where("consumed_block_timestamp <= ?", block_timestamp) } + scope :unconsumed_at, -> (block_timestamp) { where("consumed_block_timestamp > ? or consumed_block_timestamp is null", block_timestamp) } + scope :generated_after, -> (block_timestamp) { where("block_timestamp >= ?", block_timestamp) } + scope :generated_before, -> (block_timestamp) { where("block_timestamp <= ?", block_timestamp) } + after_commit :flush_cache def address_hash diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index ebeb768e8..bdbdf1834 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -18,6 +18,8 @@ class CkbTransaction < ApplicationRecord scope :recent, -> { order(block_timestamp: :desc) } scope :cellbase, -> { where(is_cellbase: true) } scope :normal, -> { where(is_cellbase: false) } + scope :created_after, -> (block_timestamp) { where("block_timestamp >= ?", block_timestamp) } + scope :created_before, -> (block_timestamp) { where("block_timestamp <= ?", block_timestamp) } after_commit :flush_cache before_destroy :recover_dead_cell diff --git a/app/models/dao_event.rb b/app/models/dao_event.rb index 2c1ef357f..e26949a39 100644 --- a/app/models/dao_event.rb +++ b/app/models/dao_event.rb @@ -6,6 +6,8 @@ class DaoEvent < ApplicationRecord belongs_to :block belongs_to :ckb_transaction belongs_to :address + + scope :created_before, ->(block_timestamp) { where("block_timestamp <= ?", block_timestamp) } end # == Schema Information From 35b6181c58974b8a28a86565515a403fde74e225 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Sat, 11 Jan 2020 20:15:06 +0800 Subject: [PATCH 3/5] refactor: use scope replace where statement and add from scratch param --- .../charts/daily_statistic_generator.rb | 54 +++++++++++++------ 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb index 597f125ad..1400d49ba 100644 --- a/app/services/charts/daily_statistic_generator.rb +++ b/app/services/charts/daily_statistic_generator.rb @@ -2,21 +2,21 @@ module Charts class DailyStatisticGenerator MILLISECONDS_IN_DAY = BigDecimal(24 * 60 * 60 * 1000) - def initialize(datetime = nil) + def initialize(datetime = nil, from_scratch = false) raise "datetime must be a Time" if datetime.present? && !datetime.is_a?(Time) @datetime = datetime + @from_scratch = from_scratch end def call - daily_ckb_transactions_count = CkbTransaction.where("block_timestamp >= ? and block_timestamp <= ?", started_at, ended_at).count - addresses_count = Address.where("block_timestamp <= ?", ended_at).count + daily_ckb_transactions_count = CkbTransaction.created_after(started_at).created_before(ended_at).count cell_outputs = CellOutput.where.not(cell_type: "normal") - current_tip_block = Block.where("timestamp <= ?", ended_at).recent.first mining_reward = Block.where("timestamp <= ?", ended_at).sum(:secondary_reward) deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs) estimated_apc = DaoContract.default_contract.estimated_apc(current_tip_block.fraction_epoch) block_timestamp = Block.created_after(started_at).created_before(ended_at).recent.pick(:timestamp) + addresses_count = processed_addresses_count daily_statistic = ::DailyStatistic.find_or_create_by!(created_at_unixtimestamp: to_be_counted_date.to_i) daily_statistic.update(block_timestamp: block_timestamp, transactions_count: daily_ckb_transactions_count, addresses_count: addresses_count, total_dao_deposit: total_dao_deposit, @@ -28,14 +28,35 @@ def call private + attr_reader :datetime, :from_scratch + + def processed_addresses_count + if from_scratch + Address.created_before(ended_at).count + else + Address.created_after(started_at).created_before(ended_at).count + latest_daily_statistic.addresses_count.to_i + end + end + + def current_tip_block + @current_tip_block ||= + begin + if from_scratch + Block.created_before(ended_at).recent.first + else + Block.created_after(started_at).created_before(ended_at).recent.first + end + end + end + def total_dao_deposit - deposit_amount = DaoEvent.processed.where("block_timestamp <= ?", ended_at).where(event_type: "deposit_to_dao").sum(:value) - withdraw_amount = DaoEvent.processed.where("block_timestamp <= ?", ended_at).where(event_type: "withdraw_from_dao").sum(:value) + deposit_amount = DaoEvent.processed.deposit_to_dao.created_before(ended_at).sum(:value) + withdraw_amount = DaoEvent.processed.withdraw_from_dao.created_before(ended_at).sum(:value) deposit_amount - withdraw_amount end def dao_depositors_count - DaoEvent.processed.where(event_type: "new_dao_depositor").where("block_timestamp <= ?", ended_at).count - DaoEvent.processed.where(event_type: "take_away_all_deposit").where("block_timestamp <= ?", ended_at).count + DaoEvent.processed.new_dao_depositor.created_before(ended_at).count - DaoEvent.processed.take_away_all_deposit.created_before(ended_at).count end def to_be_counted_date @@ -50,8 +71,6 @@ def ended_at @ended_at ||= time_in_milliseconds(to_be_counted_date.end_of_day) end - attr_reader :datetime - def unclaimed_compensation(cell_outputs, current_tip_block) @unclaimed_compensation ||= begin @@ -62,14 +81,14 @@ def unclaimed_compensation(cell_outputs, current_tip_block) def claimed_compensation(cell_outputs) @claimed_compensation ||= begin - cell_outputs.where("consumed_block_timestamp <= ?", ended_at).nervos_dao_withdrawing.dead.reduce(0) do |memo, nervos_dao_withdrawing_cell| + cell_outputs.nervos_dao_withdrawing.consumed_before(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) end end end def phase1_dao_interests(cell_outputs) - cell_outputs.where("block_timestamp <= ?", ended_at).nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell| + cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell) end end @@ -77,7 +96,7 @@ def phase1_dao_interests(cell_outputs) def unmade_dao_interests(cell_outputs, current_tip_block) @unmade_dao_interests ||= begin - cell_outputs.where("block_timestamp <= ?", ended_at).nervos_dao_deposit.live.reduce(0) do |memo, cell_output| + cell_outputs.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, cell_output| dao = cell_output.block.dao tip_dao = current_tip_block.dao parse_dao = CkbUtils.parse_dao(dao) @@ -90,17 +109,16 @@ def unmade_dao_interests(cell_outputs, current_tip_block) def average_deposit_time(cell_outputs) interest_bearing_deposits = 0 uninterest_bearing_deposits = 0 - sum_interest_bearing = cell_outputs.where("block_timestamp <= ?", ended_at).nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell| + sum_interest_bearing = cell_outputs.nervos_dao_withdrawing.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_withdrawing_cell| nervos_dao_withdrawing_cell_generated_tx = nervos_dao_withdrawing_cell.generated_by nervos_dao_deposit_cell = nervos_dao_withdrawing_cell_generated_tx.cell_inputs.order(:id)[nervos_dao_withdrawing_cell.cell_index].previous_cell_output interest_bearing_deposits += nervos_dao_deposit_cell.capacity memo + nervos_dao_deposit_cell.capacity * (nervos_dao_withdrawing_cell.block_timestamp - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY end - sum_uninterest_bearing = cell_outputs.where("block_timestamp <= ?", ended_at).nervos_dao_deposit.live.reduce(0) do |memo, nervos_dao_deposit_cell| - current_time = time_in_milliseconds(Time.current) + sum_uninterest_bearing = cell_outputs.nervos_dao_deposit.generated_before(ended_at).unconsumed_at(ended_at).reduce(0) do |memo, nervos_dao_deposit_cell| uninterest_bearing_deposits += nervos_dao_deposit_cell.capacity - memo + nervos_dao_deposit_cell.capacity * (current_time - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY + memo + nervos_dao_deposit_cell.capacity * (ended_at - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY end (sum_interest_bearing + sum_uninterest_bearing) / (interest_bearing_deposits + uninterest_bearing_deposits) @@ -114,5 +132,9 @@ def treasury_amount(cell_outputs, current_tip_block) def time_in_milliseconds(time) (time.to_f * 1000).floor end + + def latest_daily_statistic + ::DailyStatistic.order(created_at_unixtimestamp: :desc).first || OpenStruct.new(addresses_count: 0, total_dao_deposit: 0, dao_depositors_count: 0, unclaimed_compensation: 0, claimed_compensation: 0, average_deposit_time: 0, mining_reward: 0, deposit_compensation: 0, treasury_amount: 0) + end end end From f23276e2d144c61196a312cdb52d2ec7e5178fa0 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Sun, 12 Jan 2020 19:42:42 +0800 Subject: [PATCH 4/5] chore: adjust test --- test/services/charts/daily_statistic_generator_test.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/services/charts/daily_statistic_generator_test.rb b/test/services/charts/daily_statistic_generator_test.rb index 73f76c82b..768e6c310 100644 --- a/test/services/charts/daily_statistic_generator_test.rb +++ b/test/services/charts/daily_statistic_generator_test.rb @@ -3,9 +3,9 @@ module Charts class DailyStatisticGeneratorTest < ActiveSupport::TestCase test "should create daily statistic record" do - block = create(:block, dao: "0xaff1568bbe49672f8a02516252ab2300df8c9e15dad428000035a1d671700007") + block = create(:block, dao: "0xaff1568bbe49672f8a02516252ab2300df8c9e15dad428000035a1d671700007", timestamp: (Time.current - 1.day).end_of_day.to_i * 1000) tx = create(:ckb_transaction, block: block) - create(:cell_output, cell_type: "nervos_dao_deposit", generated_by: tx, ckb_transaction: tx, block: block, capacity: 10**8 * 1000, block_timestamp: (Time.current - 1.day).end_of_day.strftime("%Q")) + create(:cell_output, cell_type: "nervos_dao_deposit", generated_by: tx, ckb_transaction: tx, block: block, capacity: 10**8 * 1000, block_timestamp: (Time.current - 1.day).end_of_day.to_i * 1000) assert_difference -> { ::DailyStatistic.count }, 1 do Charts::DailyStatisticGenerator.new.call end From 6ff92fb3d64895bdef949b6148b3a61f8d1f6ccc Mon Sep 17 00:00:00 2001 From: shaojunda Date: Mon, 13 Jan 2020 21:43:12 +0800 Subject: [PATCH 5/5] Bump version to v0.9.1 --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5874be76..b1ee16ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,45 @@ +# [0.9.1](https://github.com/shaojunda/ckb-explorer/compare/v0.8.4...v0.9.0) (2020-01-13) + + +### Features + +* add block list serializer ([8d729d7](https://github.com/shaojunda/ckb-explorer/commit/8d729d7)) +* add block statistic generator service ([db91614](https://github.com/shaojunda/ckb-explorer/commit/db91614)) +* add block timestamp to dao event ([a40453b](https://github.com/shaojunda/ckb-explorer/commit/a40453b)) +* add capacity_involved column to ckb transaction ([7bcd7a5](https://github.com/shaojunda/ckb-explorer/commit/7bcd7a5)) +* add chart forked event processor ([83927bf](https://github.com/shaojunda/ckb-explorer/commit/83927bf)) +* add consumed block timestamp to cell output ([0f4986f](https://github.com/shaojunda/ckb-explorer/commit/0f4986f)) +* add epoch statistic generator service ([a99926c](https://github.com/shaojunda/ckb-explorer/commit/a99926c)) +* add estimated_apc to dao contract ([cb31d1c](https://github.com/shaojunda/ckb-explorer/commit/cb31d1c)) +* add forked event model ([2cf8bff](https://github.com/shaojunda/ckb-explorer/commit/2cf8bff)) +* add hash rate to epoch statistic ([e0746df](https://github.com/shaojunda/ckb-explorer/commit/e0746df)) +* add index action to ckb transactions controller ([e71fef7](https://github.com/shaojunda/ckb-explorer/commit/e71fef7)) +* add live_cell_changes to block and ckb_transaction ([2f04c62](https://github.com/shaojunda/ckb-explorer/commit/2f04c62)) +* add live_cell_changes to forked blocks ([a2d42b2](https://github.com/shaojunda/ckb-explorer/commit/a2d42b2)) +* add miner reward to block ([efe97fa](https://github.com/shaojunda/ckb-explorer/commit/efe97fa)) +* add more field to daily statistics ([3912139](https://github.com/shaojunda/ckb-explorer/commit/3912139)) +* add pagination to ckb transactions controller ([2251ba2](https://github.com/shaojunda/ckb-explorer/commit/2251ba2)) +* add ratio scale ([3b68742](https://github.com/shaojunda/ckb-explorer/commit/3b68742)) +* calculate estimated apc ([4a4f04f](https://github.com/shaojunda/ckb-explorer/commit/4a4f04f)) +* create forked event when forked ([ce61a1f](https://github.com/shaojunda/ckb-explorer/commit/ce61a1f)) +* implement ckb transactions index action ([26e5a36](https://github.com/shaojunda/ckb-explorer/commit/26e5a36)) +* regenerate block statistic data when block forked ([c5a444e](https://github.com/shaojunda/ckb-explorer/commit/c5a444e)) +* return hash_rate ([354ebf1](https://github.com/shaojunda/ckb-explorer/commit/354ebf1)) +* save capacity involved to ckb_transaction ([61fc5ba](https://github.com/shaojunda/ckb-explorer/commit/61fc5ba)) +* save hash rate on epoch statistic worker ([48be8e4](https://github.com/shaojunda/ckb-explorer/commit/48be8e4)) +* save live_cell_changes to block ([76f3efc](https://github.com/shaojunda/ckb-explorer/commit/76f3efc)) +* save live_cell_changes to ckb_transaction ([0c5589f](https://github.com/shaojunda/ckb-explorer/commit/0c5589f)) +* show more attributes on dao contract ([6369a86](https://github.com/shaojunda/ckb-explorer/commit/6369a86)) +* use ckb transaction list serializer ([7d12caa](https://github.com/shaojunda/ckb-explorer/commit/7d12caa)) + + +### Performance Improvements + +* add index on block timestamp, status and event type ([bce28f4](https://github.com/shaojunda/ckb-explorer/commit/bce28f4)) +* use redis pipeline and use delete replace delete_matched ([da666e0](https://github.com/shaojunda/ckb-explorer/commit/da666e0)) + + + # [0.9.0](https://github.com/shaojunda/ckb-explorer/compare/v0.8.3...v0.9.0) (2020-01-02)