From a18ef6a11dfa80a2158181e15538fe9ffa3dfa93 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 12 Dec 2019 21:48:26 +0800 Subject: [PATCH 01/77] test: ckb transactions index action media type --- .../v1/ckb_transactions_controller_test.rb | 42 +++++++++++++++++++ test/test_helper.rb | 3 -- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb index f033810d5..d20c0b595 100644 --- a/test/controllers/api/v1/ckb_transactions_controller_test.rb +++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb @@ -128,6 +128,48 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest assert_equal 15, json["data"].dig("attributes", "display_outputs").count assert_equal [false], json["data"].dig("attributes", "display_outputs").map { |input| input.key?("from_cellbase") }.uniq end + + test "should get success code when call index" do + valid_get api_v1_ckb_transactions_url + + assert_response :success + end + + test "should set right content type when call index" do + valid_get api_v1_ckb_transactions_url + + assert_equal "application/vnd.api+json", response.media_type + end + + test "should respond with 415 Unsupported Media Type when call index and Content-Type is wrong" do + get api_v1_ckb_transactions_url, headers: { "Content-Type": "text/plain" } + + assert_equal 415, response.status + end + + test "should respond with error object when call index and Content-Type is wrong" do + error_object = Api::V1::Exceptions::WrongContentTypeError.new + response_json = RequestErrorSerializer.new([error_object], message: error_object.title).serialized_json + + get api_v1_ckb_transactions_url, headers: { "Content-Type": "text/plain" } + + assert_equal response_json, response.body + end + + test "should respond with 406 Not Acceptable when call index and Accept is wrong" do + get api_v1_ckb_transactions_url, headers: { "Content-Type": "application/vnd.api+json", "Accept": "application/json" } + + assert_equal 406, response.status + end + + test "should respond with error object when call index and Accept is wrong" do + error_object = Api::V1::Exceptions::WrongAcceptError.new + response_json = RequestErrorSerializer.new([error_object], message: error_object.title).serialized_json + + get api_v1_ckb_transactions_url, headers: { "Content-Type": "application/vnd.api+json", "Accept": "application/json" } + + assert_equal response_json, response.body + end end end end diff --git a/test/test_helper.rb b/test/test_helper.rb index e0a4a7932..89241dce5 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -260,9 +260,6 @@ class TestCase # fixtures :all include FactoryBot::Syntax::Methods include ::RequestHelpers - if ENV["CI"] != "true" - parallelize(workers: 2, with: :threads) - end # Add more helper methods to be used by all tests here... def before_setup From e71fef76e4185bf7f6eb673e1c348ff231ec64a1 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 12 Dec 2019 21:48:53 +0800 Subject: [PATCH 02/77] feat: add index action to ckb transactions controller --- app/controllers/api/v1/ckb_transactions_controller.rb | 4 ++++ config/routes.rb | 1 + 2 files changed, 5 insertions(+) diff --git a/app/controllers/api/v1/ckb_transactions_controller.rb b/app/controllers/api/v1/ckb_transactions_controller.rb index fb3027641..2279d7c99 100644 --- a/app/controllers/api/v1/ckb_transactions_controller.rb +++ b/app/controllers/api/v1/ckb_transactions_controller.rb @@ -3,6 +3,10 @@ module V1 class CkbTransactionsController < ApplicationController before_action :validate_query_params, only: :show + def index + render json: {} + end + def show ckb_transaction = CkbTransaction.cached_find(params[:id]) diff --git a/config/routes.rb b/config/routes.rb index 027ebd8de..70f5744c9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -17,6 +17,7 @@ resources :block_transactions, only: :show resources :addresses, only: :show get "/transactions/:id", to: "ckb_transactions#show", as: "ckb_transaction" + get "/transactions", to: "ckb_transactions#index", as: "ckb_transactions" resources :cell_input_lock_scripts, only: :show resources :cell_input_type_scripts, only: :show resources :cell_input_data, only: :show From 63480f9394dede9c710e2deaec708ffc751852cc Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 12 Dec 2019 21:59:41 +0800 Subject: [PATCH 03/77] test: ckb transactions controller's index action --- .env | 1 + .../v1/ckb_transactions_controller_test.rb | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/.env b/.env index 24bc86fef..4c47524db 100644 --- a/.env +++ b/.env @@ -29,3 +29,4 @@ BLOCK_PROCESS_LOOP_INTERVAL="1" DEFAULT_EPOCH_REWARD="1_917_808_21917808" CKB_NET_MODE="mainnet" HOMEPAGE_BLOCK_RECORDS_COUNT="15" +HOMEPAGE_TRANSACTIONS_RECORDS_COUNT="15" diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb index d20c0b595..2c0e7609a 100644 --- a/test/controllers/api/v1/ckb_transactions_controller_test.rb +++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb @@ -170,6 +170,62 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest assert_equal response_json, response.body end + + test "should get serialized objects" do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 15, block: block) + + ckb_transactions = CkbTransaction.recent.limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) + + valid_get api_v1_ckb_transactions_url + + assert_equal CkbTransactionSerializer.new(ckb_transactions).serialized_json, response.body + end + + test "serialized objects should in reverse order of timestamp" do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 15, block: block) + + valid_get api_v1_ckb_transactions_url + + first_ckb_transaction = json["data"].first + last_ckb_transaction = json["data"].last + + assert_operator first_ckb_transaction.dig("attributes", "block_timestamp"), :>, last_ckb_transaction.dig("attributes", "block_timestamp") + end + + test "should contain right keys in the serialized object" do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 15, block: block) + + valid_get api_v1_ckb_transactions_url + + response_ckb_transaction = json["data"].first + assert_equal %w(block_number transaction_hash block_timestamp transaction_fee version display_inputs display_outputs is_cellbase income witnesses cell_deps header_deps).sort, response_ckb_transaction["attributes"].keys.sort + end + + test "should return the corresponding number of ckb transactions " do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 30, block: block) + + valid_get api_v1_ckb_transactions_url + + ckb_transactions = CkbTransaction.order(block_timestamp: :desc).limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) + response_ckb_transaction = CkbTransactionSerializer.new(ckb_transactions).serialized_json + assert_equal response_ckb_transaction, response.body + assert_equal 15, json["data"].size + end + + test "should return empty array when there is no ckb_transactions" do + ckb_transactions = CkbTransaction.order(block_timestamp: :desc).limit(15) + + valid_get api_v1_ckb_transactions_url + + response_ckb_transaction = CkbTransactionSerializer.new(ckb_transactions).serialized_json + + assert_equal [], json["data"] + assert_equal response_ckb_transaction, response.body + end end end end From 26e5a36ed807986bf9d6f28667580b375d993529 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Thu, 12 Dec 2019 22:04:49 +0800 Subject: [PATCH 04/77] feat: implement ckb transactions index action --- app/controllers/api/v1/ckb_transactions_controller.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/controllers/api/v1/ckb_transactions_controller.rb b/app/controllers/api/v1/ckb_transactions_controller.rb index 2279d7c99..8e28c9028 100644 --- a/app/controllers/api/v1/ckb_transactions_controller.rb +++ b/app/controllers/api/v1/ckb_transactions_controller.rb @@ -4,7 +4,8 @@ class CkbTransactionsController < ApplicationController before_action :validate_query_params, only: :show def index - render json: {} + ckb_transactions = CkbTransaction.recent.limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) + render json: CkbTransactionSerializer.new(ckb_transactions) end def show From 8d729d755d15b89ebcca39395c483d6fc5869ad9 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 16:46:27 +0800 Subject: [PATCH 05/77] feat: add block list serializer --- app/controllers/api/v1/blocks_controller.rb | 2 +- app/serializers/block_list_serializer.rb | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 app/serializers/block_list_serializer.rb diff --git a/app/controllers/api/v1/blocks_controller.rb b/app/controllers/api/v1/blocks_controller.rb index 233c70567..9f0844f70 100644 --- a/app/controllers/api/v1/blocks_controller.rb +++ b/app/controllers/api/v1/blocks_controller.rb @@ -13,7 +13,7 @@ def index options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: @page, page_size: @page_size).call end - render json: BlockSerializer.new(blocks, options) + render json: BlockListSerializer.new(blocks, options) end def show diff --git a/app/serializers/block_list_serializer.rb b/app/serializers/block_list_serializer.rb new file mode 100644 index 000000000..96c5b7171 --- /dev/null +++ b/app/serializers/block_list_serializer.rb @@ -0,0 +1,18 @@ +class BlockListSerializer + include FastJsonapi::ObjectSerializer + + attributes :miner_hash + + attribute :number do |object| + object.number.to_s + end + attribute :timestamp do |object| + object.timestamp.to_s + end + attribute :reward do |object| + object.reward.to_s + end + attribute :transactions_count do |object| + object.ckb_transactions_count.to_s + end +end From 93f437b2e965844f074ddeb2f8fc933a70ec6efa Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 16:46:38 +0800 Subject: [PATCH 06/77] chore: adjust tests --- .../controllers/api/v1/blocks_controller_test.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/controllers/api/v1/blocks_controller_test.rb b/test/controllers/api/v1/blocks_controller_test.rb index 7cb72d1f1..c7f1d6fcc 100644 --- a/test/controllers/api/v1/blocks_controller_test.rb +++ b/test/controllers/api/v1/blocks_controller_test.rb @@ -52,7 +52,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_blocks_url - assert_equal BlockSerializer.new(blocks, {}).serialized_json, response.body + assert_equal BlockListSerializer.new(blocks, {}).serialized_json, response.body end test "serialized objects should in reverse order of timestamp" do @@ -72,7 +72,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_blocks_url response_block = json["data"].first - assert_equal %w(block_hash number transactions_count proposals_count uncles_count uncle_block_hashes reward total_transaction_fee cell_consumed total_cell_capacity miner_hash timestamp difficulty version nonce epoch start_number length transactions_root reward_status received_tx_fee received_tx_fee_status block_index_in_epoch).sort, response_block["attributes"].keys.sort + assert_equal %w(number transactions_count reward miner_hash timestamp ).sort, response_block["attributes"].keys.sort end test "should return error object when page param is invalid" do @@ -123,10 +123,10 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_blocks_url, params: { page: page, page_size: page_size } - block_hashes = Block.recent.map(&:block_hash) - search_result_block_hashes = json["data"].map { |ckb_transaction| ckb_transaction.dig("attributes", "block_hash") } + block_hashes = Block.recent.map(&:number).map(&:to_s) + search_result_block_numbers = json["data"].map { |block| block.dig("attributes", "number")} - assert_equal block_hashes, search_result_block_hashes + assert_equal block_hashes, search_result_block_numbers end test "should return corresponding page's records when page is set and page_size is not set" do @@ -136,7 +136,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_blocks_url, params: { page: page } - response_blocks = BlockSerializer.new(blocks, {}).serialized_json + response_blocks = BlockListSerializer.new(blocks, {}).serialized_json assert_equal response_blocks, response.body assert_equal 15, json["data"].size @@ -148,7 +148,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_blocks_url, params: { page_size: 12 } blocks = Block.order(timestamp: :desc).limit(ENV["HOMEPAGE_BLOCK_RECORDS_COUNT"].to_i) - response_blocks = BlockSerializer.new(blocks, {}).serialized_json + response_blocks = BlockListSerializer.new(blocks, {}).serialized_json assert_equal response_blocks, response.body assert_equal 15, json["data"].size end @@ -162,7 +162,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_blocks_url, params: { page: page, page_size: page_size } options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: page, page_size: page_size).call - response_blocks = BlockSerializer.new(blocks, options).serialized_json + response_blocks = BlockListSerializer.new(blocks, options).serialized_json assert_equal response_blocks, response.body end From 2f04c6222892f284fc4d70443e3d0d268666e346 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 20:50:47 +0800 Subject: [PATCH 07/77] feat: add live_cell_changes to block and ckb_transaction --- app/models/block.rb | 1 + app/models/ckb_transaction.rb | 29 ++++++++++--------- ...ll_changes_to_block_and_ckb_transaction.rb | 6 ++++ db/schema.rb | 4 ++- 4 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 db/migrate/20191225110229_add_live_cell_changes_to_block_and_ckb_transaction.rb diff --git a/app/models/block.rb b/app/models/block.rb index feef47fe9..2515f1b5d 100644 --- a/app/models/block.rb +++ b/app/models/block.rb @@ -131,6 +131,7 @@ def invalid! # length :decimal(30, ) default(0) # uncles_count :integer # compact_target :decimal(20, ) +# live_cell_changes :integer # # Indexes # diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index a16904a91..e019f4d6e 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -117,20 +117,21 @@ def recover_dead_cell # # Table name: ckb_transactions # -# id :bigint not null, primary key -# tx_hash :binary -# deps :jsonb -# block_id :bigint -# block_number :decimal(30, ) -# block_timestamp :decimal(30, ) -# transaction_fee :decimal(30, ) -# version :integer -# created_at :datetime not null -# updated_at :datetime not null -# is_cellbase :boolean default(FALSE) -# header_deps :binary -# cell_deps :jsonb -# witnesses :jsonb +# id :bigint not null, primary key +# tx_hash :binary +# deps :jsonb +# block_id :bigint +# block_number :decimal(30, ) +# block_timestamp :decimal(30, ) +# transaction_fee :decimal(30, ) +# version :integer +# created_at :datetime not null +# updated_at :datetime not null +# is_cellbase :boolean default(FALSE) +# witnesses :jsonb +# header_deps :binary +# cell_deps :jsonb +# live_cell_changes :integer # # Indexes # diff --git a/db/migrate/20191225110229_add_live_cell_changes_to_block_and_ckb_transaction.rb b/db/migrate/20191225110229_add_live_cell_changes_to_block_and_ckb_transaction.rb new file mode 100644 index 000000000..78c6c585d --- /dev/null +++ b/db/migrate/20191225110229_add_live_cell_changes_to_block_and_ckb_transaction.rb @@ -0,0 +1,6 @@ +class AddLiveCellChangesToBlockAndCkbTransaction < ActiveRecord::Migration[6.0] + def change + add_column :blocks, :live_cell_changes, :integer + add_column :ckb_transactions, :live_cell_changes, :integer + end +end diff --git a/db/schema.rb b/db/schema.rb index 2592edadd..e8f94824b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_12_16_080206) do +ActiveRecord::Schema.define(version: 2019_12_25_110229) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -89,6 +89,7 @@ t.decimal "length", precision: 30, default: "0" t.integer "uncles_count" t.decimal "compact_target", precision: 20 + t.integer "live_cell_changes" t.index ["block_hash"], name: "index_blocks_on_block_hash", unique: true t.index ["number"], name: "index_blocks_on_number" t.index ["timestamp"], name: "index_blocks_on_timestamp" @@ -147,6 +148,7 @@ t.jsonb "witnesses" t.binary "header_deps" t.jsonb "cell_deps" + t.integer "live_cell_changes" t.index ["block_id", "block_timestamp"], name: "index_ckb_transactions_on_block_id_and_block_timestamp" t.index ["is_cellbase"], name: "index_ckb_transactions_on_is_cellbase" t.index ["tx_hash", "block_id"], name: "index_ckb_transactions_on_tx_hash_and_block_id", unique: true From 9cbd02f59ae94d751ca2a4c9058134f0d6c723af Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:07:13 +0800 Subject: [PATCH 08/77] test: live cell changes on transaction --- test/models/ckb_sync/node_data_processor_test.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index a9b474eb3..5e178d560 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -238,6 +238,22 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end end + test "#process_block created ckb_transaction's live cell changes should equal to outputs count minus inputs count" do + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do + node_block = fake_node_block("0x3307186493c5da8b91917924253a5ffd35231151649d0c7e2941aa8801815063") + block = create(:block, :with_block_hash) + ckb_transaction1 = create(:ckb_transaction, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", block: block) + ckb_transaction2 = create(:ckb_transaction, tx_hash: "0x598315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", block: block) + create(:cell_output, ckb_transaction: ckb_transaction1, cell_index: 1, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction2, block: block, capacity: 4 * 10**8) + create(:cell_output, ckb_transaction: ckb_transaction2, cell_index: 2, tx_hash: "0x598315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction1, block: block) + node_block_transactions = node_block.transactions + + local_block = node_data_processor.process_block(node_block) + expected_live_cell_changes = node_block_transactions.map { |transaction| transaction.outputs.count - transaction.inputs.count } + assert_equal expected_live_cell_changes, local_block.ckb_transactions.map(&:live_cell_changes) + end + end + test "#process_block should create cell_inputs" do VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do node_block = CkbSync::Api.instance.get_block_by_number(DEFAULT_NODE_BLOCK_NUMBER) From 0c5589f32c57a6261cf13f03c23b9614fb8461c6 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:07:45 +0800 Subject: [PATCH 09/77] feat: save live_cell_changes to ckb_transaction --- app/models/ckb_sync/node_data_processor.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index b756c475a..8a13c4ec2 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -337,7 +337,8 @@ def build_ckb_transaction(local_block, transaction, transaction_index) block_timestamp: local_block.timestamp, transaction_fee: 0, witnesses: transaction.witnesses, - is_cellbase: transaction_index.zero? + is_cellbase: transaction_index.zero?, + live_cell_changes: transaction.outputs.count - transaction.inputs.count ) end From 60ca75362b33a29dcf8d9515fdcf8ea38ef6c4d3 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:12:43 +0800 Subject: [PATCH 10/77] test: live_cell_changes on block --- test/models/ckb_sync/node_data_processor_test.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 5e178d560..51d200904 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -254,6 +254,20 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end end + test "#process_block created block's live cell changes should equal to sum of ckb_transaction's live cell changes" do + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do + node_block = fake_node_block("0x3307186493c5da8b91917924253a5ffd35231151649d0c7e2941aa8801815063") + block = create(:block, :with_block_hash) + ckb_transaction1 = create(:ckb_transaction, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", block: block) + ckb_transaction2 = create(:ckb_transaction, tx_hash: "0x598315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", block: block) + create(:cell_output, ckb_transaction: ckb_transaction1, cell_index: 1, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction2, block: block, capacity: 4 * 10**8) + create(:cell_output, ckb_transaction: ckb_transaction2, cell_index: 2, tx_hash: "0x598315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction1, block: block) + + local_block = node_data_processor.process_block(node_block) + assert_equal local_block.live_cell_changes, local_block.ckb_transactions.sum(&:live_cell_changes) + end + end + test "#process_block should create cell_inputs" do VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do node_block = CkbSync::Api.instance.get_block_by_number(DEFAULT_NODE_BLOCK_NUMBER) From 76f3efcc4d5274ce534bdacb874f43fc2279bcaa Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:12:58 +0800 Subject: [PATCH 11/77] feat: save live_cell_changes to block --- app/models/ckb_sync/node_data_processor.rb | 1 + app/serializers/block_list_serializer.rb | 3 +++ test/controllers/api/v1/blocks_controller_test.rb | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index 8a13c4ec2..c8743b6ce 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -29,6 +29,7 @@ def process_block(node_block) ckb_transactions = build_ckb_transactions(local_block, node_block.transactions, outputs, new_dao_depositor_events) local_block.ckb_transactions_count = ckb_transactions.size + local_block.live_cell_changes = ckb_transactions.sum(&:live_cell_changes) CkbTransaction.import!(ckb_transactions, recursive: true, batch_size: 3500, validate: false) input_capacities = ckb_transactions.reject(&:is_cellbase).pluck(:id).to_h { |id| [id, []] } update_tx_fee_related_data(local_block, input_capacities) diff --git a/app/serializers/block_list_serializer.rb b/app/serializers/block_list_serializer.rb index 96c5b7171..322c21d04 100644 --- a/app/serializers/block_list_serializer.rb +++ b/app/serializers/block_list_serializer.rb @@ -15,4 +15,7 @@ class BlockListSerializer attribute :transactions_count do |object| object.ckb_transactions_count.to_s end + attribute :live_cell_changes do |object| + object.live_cell_changes.to_s + end end diff --git a/test/controllers/api/v1/blocks_controller_test.rb b/test/controllers/api/v1/blocks_controller_test.rb index c7f1d6fcc..1257abb79 100644 --- a/test/controllers/api/v1/blocks_controller_test.rb +++ b/test/controllers/api/v1/blocks_controller_test.rb @@ -72,7 +72,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_blocks_url response_block = json["data"].first - assert_equal %w(number transactions_count reward miner_hash timestamp ).sort, response_block["attributes"].keys.sort + assert_equal %w(number transactions_count reward miner_hash timestamp live_cell_changes).sort, response_block["attributes"].keys.sort end test "should return error object when page param is invalid" do From 7bcd7a5f0492204982abd9ed7b42f00ff3545978 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:15:03 +0800 Subject: [PATCH 12/77] feat: add capacity_involved column to ckb transaction --- app/models/ckb_transaction.rb | 1 + db/schema.rb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb index e019f4d6e..9aa8d6cbd 100644 --- a/app/models/ckb_transaction.rb +++ b/app/models/ckb_transaction.rb @@ -132,6 +132,7 @@ def recover_dead_cell # header_deps :binary # cell_deps :jsonb # live_cell_changes :integer +# capacity_involved :decimal(30, ) # # Indexes # diff --git a/db/schema.rb b/db/schema.rb index e8f94824b..3a05c2c84 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_12_25_110229) do +ActiveRecord::Schema.define(version: 2019_12_25_131337) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -149,6 +149,7 @@ t.binary "header_deps" t.jsonb "cell_deps" t.integer "live_cell_changes" + t.decimal "capacity_involved", precision: 30 t.index ["block_id", "block_timestamp"], name: "index_ckb_transactions_on_block_id_and_block_timestamp" t.index ["is_cellbase"], name: "index_ckb_transactions_on_is_cellbase" t.index ["tx_hash", "block_id"], name: "index_ckb_transactions_on_tx_hash_and_block_id", unique: true From 8aeec5c44ed9838f70601f85549c614efa14960e Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:39:59 +0800 Subject: [PATCH 13/77] test: capacity involved should equal to sum of inputs capacity --- test/models/ckb_sync/node_data_processor_test.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb index 51d200904..24fb54ff7 100644 --- a/test/models/ckb_sync/node_data_processor_test.rb +++ b/test/models/ckb_sync/node_data_processor_test.rb @@ -254,6 +254,21 @@ class NodeDataProcessorTest < ActiveSupport::TestCase end end + test "#process_block created ckb_transaction's capacity_involved should equal to outputs count minus inputs count" do + VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do + node_block = fake_node_block("0x3307186493c5da8b91917924253a5ffd35231151649d0c7e2941aa8801815063") + block = create(:block, :with_block_hash) + ckb_transaction1 = create(:ckb_transaction, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", block: block) + ckb_transaction2 = create(:ckb_transaction, tx_hash: "0x598315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", block: block) + create(:cell_output, ckb_transaction: ckb_transaction1, cell_index: 1, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction2, block: block) + create(:cell_output, ckb_transaction: ckb_transaction2, cell_index: 2, tx_hash: "0x598315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction1, block: block) + local_block = node_data_processor.process_block(node_block) + expected_capacity_involved = local_block.ckb_transactions.map(&:capacity_involved) + + assert_equal expected_capacity_involved, local_block.ckb_transactions.map { |transaction| transaction.inputs.sum(:capacity) } + end + end + test "#process_block created block's live cell changes should equal to sum of ckb_transaction's live cell changes" do VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do node_block = fake_node_block("0x3307186493c5da8b91917924253a5ffd35231151649d0c7e2941aa8801815063") From 61fc5baf7754f7c84cc9ccf6825bf1b99b8ebaa8 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:43:07 +0800 Subject: [PATCH 14/77] feat: save capacity involved to ckb_transaction --- app/models/ckb_sync/node_data_processor.rb | 3 ++- ...191225131337_add_capacity_involved_to_ckb_transactions.rb | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20191225131337_add_capacity_involved_to_ckb_transactions.rb diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb index c8743b6ce..dba8f19cf 100644 --- a/app/models/ckb_sync/node_data_processor.rb +++ b/app/models/ckb_sync/node_data_processor.rb @@ -502,10 +502,11 @@ def calculate_tx_fee(local_block, ckb_transactions, input_capacities, outputs) txs = [] ckb_transactions.each do |ckb_transaction| update_transaction_fee(ckb_transaction, input_capacities[ckb_transaction.id].sum, output_capacities[ckb_transaction.id].sum) + ckb_transaction.capacity_involved = input_capacities[ckb_transaction.id].sum txs << ckb_transaction end - CkbTransaction.import!(txs, validate: false, on_duplicate_key_update: [:transaction_fee]) + CkbTransaction.import!(txs, validate: false, on_duplicate_key_update: [:transaction_fee, :capacity_involved]) local_block.total_transaction_fee = local_block.ckb_transactions.sum(:transaction_fee) local_block.save! rescue ActiveRecord::RecordInvalid diff --git a/db/migrate/20191225131337_add_capacity_involved_to_ckb_transactions.rb b/db/migrate/20191225131337_add_capacity_involved_to_ckb_transactions.rb new file mode 100644 index 000000000..7cca9dc36 --- /dev/null +++ b/db/migrate/20191225131337_add_capacity_involved_to_ckb_transactions.rb @@ -0,0 +1,5 @@ +class AddCapacityInvolvedToCkbTransactions < ActiveRecord::Migration[6.0] + def change + add_column :ckb_transactions, :capacity_involved, :decimal, precision: 30, scale: 0 + end +end From 7d12caa3c52707421dc28265aeb2f54c5132cca6 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:52:35 +0800 Subject: [PATCH 15/77] feat: use ckb transaction list serializer --- .../api/v1/ckb_transactions_controller.rb | 2 +- .../ckb_transaction_list_serializer.rb | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 app/serializers/ckb_transaction_list_serializer.rb diff --git a/app/controllers/api/v1/ckb_transactions_controller.rb b/app/controllers/api/v1/ckb_transactions_controller.rb index 8e28c9028..d33c33d5f 100644 --- a/app/controllers/api/v1/ckb_transactions_controller.rb +++ b/app/controllers/api/v1/ckb_transactions_controller.rb @@ -5,7 +5,7 @@ class CkbTransactionsController < ApplicationController def index ckb_transactions = CkbTransaction.recent.limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) - render json: CkbTransactionSerializer.new(ckb_transactions) + render json: CkbTransactionListSerializer.new(ckb_transactions) end def show diff --git a/app/serializers/ckb_transaction_list_serializer.rb b/app/serializers/ckb_transaction_list_serializer.rb new file mode 100644 index 000000000..84d4174e1 --- /dev/null +++ b/app/serializers/ckb_transaction_list_serializer.rb @@ -0,0 +1,18 @@ +class CkbTransactionListSerializer + include FastJsonapi::ObjectSerializer + + attribute :transaction_hash, &:tx_hash + + attribute :block_number do |object| + object.block_number.to_s + end + attribute :block_timestamp do |object| + object.block_timestamp.to_s + end + attribute :capacity_involved do |object| + object.capacity_involved.to_s + end + attribute :live_cell_changes do |object| + object.live_cell_changes.to_s + end +end From 18fa0f35f741dcbdeebce85e17d56c3ee778ba46 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 21:52:46 +0800 Subject: [PATCH 16/77] chore: adjust tests --- .../api/v1/ckb_transactions_controller_test.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb index 2c0e7609a..e3605b69d 100644 --- a/test/controllers/api/v1/ckb_transactions_controller_test.rb +++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb @@ -179,7 +179,7 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_ckb_transactions_url - assert_equal CkbTransactionSerializer.new(ckb_transactions).serialized_json, response.body + assert_equal CkbTransactionListSerializer.new(ckb_transactions).serialized_json, response.body end test "serialized objects should in reverse order of timestamp" do @@ -201,7 +201,7 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_ckb_transactions_url response_ckb_transaction = json["data"].first - assert_equal %w(block_number transaction_hash block_timestamp transaction_fee version display_inputs display_outputs is_cellbase income witnesses cell_deps header_deps).sort, response_ckb_transaction["attributes"].keys.sort + assert_equal %w(block_number transaction_hash block_timestamp capacity_involved live_cell_changes).sort, response_ckb_transaction["attributes"].keys.sort end test "should return the corresponding number of ckb transactions " do @@ -211,7 +211,7 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_ckb_transactions_url ckb_transactions = CkbTransaction.order(block_timestamp: :desc).limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) - response_ckb_transaction = CkbTransactionSerializer.new(ckb_transactions).serialized_json + response_ckb_transaction = CkbTransactionListSerializer.new(ckb_transactions).serialized_json assert_equal response_ckb_transaction, response.body assert_equal 15, json["data"].size end @@ -221,7 +221,7 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest valid_get api_v1_ckb_transactions_url - response_ckb_transaction = CkbTransactionSerializer.new(ckb_transactions).serialized_json + response_ckb_transaction = CkbTransactionListSerializer.new(ckb_transactions).serialized_json assert_equal [], json["data"] assert_equal response_ckb_transaction, response.body From 7c8dbedaec3dc44b90eba82a39d14bfcbfd0db0b Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 22:09:29 +0800 Subject: [PATCH 17/77] test: ckb transactions index page params --- .../v1/ckb_transactions_controller_test.rb | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb index e3605b69d..a459ca627 100644 --- a/test/controllers/api/v1/ckb_transactions_controller_test.rb +++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb @@ -226,6 +226,41 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest assert_equal [], json["data"] assert_equal response_ckb_transaction, response.body end + + test "should return error object when page param is invalid" do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 15, block: block) + error_object = Api::V1::Exceptions::PageParamError.new + response_json = RequestErrorSerializer.new([error_object], message: error_object.title).serialized_json + + valid_get api_v1_ckb_transactions_url, params: { page: "aaa" } + + assert_equal response_json, response.body + end + + test "should return error object when page size param is invalid" do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 15, block: block) + error_object = Api::V1::Exceptions::PageSizeParamError.new + response_json = RequestErrorSerializer.new([error_object], message: error_object.title).serialized_json + + valid_get api_v1_ckb_transactions_url, params: { page_size: "aaa" } + + assert_equal response_json, response.body + end + + test "should return error object when page and page size param are invalid" do + errors = [] + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 15, block: block) + errors << Api::V1::Exceptions::PageParamError.new + errors << Api::V1::Exceptions::PageSizeParamError.new + response_json = RequestErrorSerializer.new(errors, message: errors.first.title).serialized_json + + valid_get api_v1_ckb_transactions_url, params: { page: "bbb", page_size: "aaa" } + + assert_equal response_json, response.body + end end end end From 2b4fa1a464b87d6fe69710861173885810760f7d Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 22:25:46 +0800 Subject: [PATCH 18/77] test: ckb transactions index pagination --- .../v1/ckb_transactions_controller_test.rb | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb index a459ca627..0315258d0 100644 --- a/test/controllers/api/v1/ckb_transactions_controller_test.rb +++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb @@ -261,6 +261,55 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest assert_equal response_json, response.body end + + test "should return 15 records when page and page_size are not set" do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 20, block: block) + + valid_get api_v1_ckb_transactions_url + + assert_equal 15, json["data"].size + end + + test "should return corresponding page's records when page is set and page_size is not set" do + page = 2 + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 20, block: block) + ckb_transactions = CkbTransaction.order(block_timestamp: :desc).limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) + + valid_get api_v1_ckb_transactions_url, params: { page: page } + + response_ckb_transactions = CkbTransactionListSerializer.new(ckb_transactions, {}).serialized_json + + assert_equal response_ckb_transactions, response.body + assert_equal 15, json["data"].size + end + + test "should return the corresponding number of ckb_transactions when page is not set and page_size is set" do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 20, block: block) + + valid_get api_v1_ckb_transactions_url, params: { page_size: 12 } + + ckb_transactions = CkbTransaction.order(block_timestamp: :desc).limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) + response_ckb_transactions = CkbTransactionListSerializer.new(ckb_transactions, {}).serialized_json + assert_equal response_ckb_transactions, response.body + assert_equal 15, json["data"].size + end + + test "should return the corresponding blocks when page and page_size are set" do + block = create(:block, :with_block_hash) + create_list(:ckb_transaction, 15, block: block) + page = 2 + page_size = 5 + ckb_transactions = CkbTransaction.order(block_timestamp: :desc).page(page).per(page_size) + + valid_get api_v1_ckb_transactions_url, params: { page: page, page_size: page_size } + + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size).call + response_ckb_transactions = CkbTransactionListSerializer.new(ckb_transactions, options).serialized_json + assert_equal response_ckb_transactions, response.body + end end end end From 2251ba25b83a404c038a11c7672d1de3ccef8d0a Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 22:26:10 +0800 Subject: [PATCH 19/77] feat: add pagination to ckb transactions controller --- .../api/v1/ckb_transactions_controller.rb | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/app/controllers/api/v1/ckb_transactions_controller.rb b/app/controllers/api/v1/ckb_transactions_controller.rb index d33c33d5f..531e29616 100644 --- a/app/controllers/api/v1/ckb_transactions_controller.rb +++ b/app/controllers/api/v1/ckb_transactions_controller.rb @@ -2,10 +2,18 @@ module Api module V1 class CkbTransactionsController < ApplicationController before_action :validate_query_params, only: :show + before_action :validate_pagination_params, :pagination_params, only: :index def index - ckb_transactions = CkbTransaction.recent.limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) - render json: CkbTransactionListSerializer.new(ckb_transactions) + if from_home_page? + ckb_transactions = CkbTransaction.recent.limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i) + options = {} + else + ckb_transactions = CkbTransaction.recent.page(@page).per(@page_size) + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size).call + end + + render json: CkbTransactionListSerializer.new(ckb_transactions, options) end def show @@ -18,6 +26,15 @@ def show private + def from_home_page? + params[:page].blank? || params[:page_size].blank? + end + + def pagination_params + @page = params[:page] || 1 + @page_size = params[:page_size] || CkbTransaction.default_per_page + end + def validate_query_params validator = Validations::CkbTransaction.new(params) From d4a5887c7f77f3c9740ac8b129fee731c3410191 Mon Sep 17 00:00:00 2001 From: shaojunda Date: Wed, 25 Dec 2019 22:44:28 +0800 Subject: [PATCH 20/77] chore: update api doc --- doc/api.raml | 94 +++++++++++++++++++++++++++++++++++++++++++++ public/api_doc.html | 2 +- 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/doc/api.raml b/doc/api.raml index aef3558b2..60610d770 100644 --- a/doc/api.raml +++ b/doc/api.raml @@ -596,6 +596,27 @@ types: } } + ckb_transaction_list_attributes: { + type: "object", + properties: { + "transaction_hash": { + type: "string" + }, + "block_number": { + type: "string" + }, + "block_timestamp": { + type: "string" + }, + "capacity_involved": { + type: "string" + }, + "live_cell_changes": { + type: "string" + } + } + } + index_statistic_info: { type: object, properties: { @@ -662,6 +683,15 @@ types: } } + ckb_transaction_list_response: { + type: object, + properties: { + "data": { + "type": "ckb_transaction_list_attributes" + } + } + } + blocks_response: { type: object, properties: { @@ -1039,6 +1069,70 @@ types: } /transactions: + get: + description: Returns blocks in reverse chronological order. + queryParameters: + page_size: + type: integer + default: 10 + page: + type: integer + default: 1 + responses: + 200: + body: + application/vnd.api+json: + type: ckb_transaction_list_response + example: | + { + "data": [ + { + "id": "519759", + "type": "ckb_transaction_list", + "attributes": { + "transaction_hash": "0x51c4008aecf7b32762f8de7b1e0db97b0b99af6dfbedacddca4723e4e3469735", + "block_number": "355801", + "block_timestamp": "1577284662673", + "capacity_involved": "", + "live_cell_changes": "0" + } + }, + { + "id": "519760", + "type": "ckb_transaction_list", + "attributes": { + "transaction_hash": "0x93dcd049c02c94a4f8b83ed86e6dfdd662bb381585f6294e5cc316f301ed688d", + "block_number": "355801", + "block_timestamp": "1577284662673", + "capacity_involved": "", + "live_cell_changes": "0" + } + }, + { + "id": "519757", + "type": "ckb_transaction_list", + "attributes": { + "transaction_hash": "0xfe642e494030e77c46753f7ac2d29c38d6780e38c1a3a7accb6b4adf415a511b", + "block_number": "355800", + "block_timestamp": "1577284656255", + "capacity_involved": "", + "live_cell_changes": "0" + } + }, + ] + "meta": { + "total": 100, + "page_size": 15, + }, + links: { + "self": "https://explorer.nervos.org/api/v1/transactions?page_size=15", + "first": "https://explorer.nervos.org/api/v1/transactions?page=1&page_size=10", + "prev": "https://explorer.nervos.org/api/v1/transactions?page=2&page_size=10", + "next": "https://explorer.nervos.org/api/v1/transactions?page=2&page_size=15", + "last": "https://explorer.nervos.org/api/v1/transactions?page=28909&page_size=15" + } + } + /{tx_hash}: uriParameters: tx_hash: diff --git a/public/api_doc.html b/public/api_doc.html index cc2c3f507..e71610e37 100644 --- a/public/api_doc.html +++ b/public/api_doc.html @@ -323,7 +323,7 @@ } ] } -

/transactions

get

get

Returns a transaction by transaction hash

Returns a transaction by transaction hash

URI Parameters

  • tx_hash: required(string - minLength: 66 - maxLength: 66)

    The transaction hash

HTTP status code 200

Body

Media type: application/vnd.api+json

Type: object

Properties
  • data: required(object)
    • id: required(string)
    • type: required(string)
    • attributes: required(object)
      • block_number: required(string)
      • transaction_hash: required(string)
      • block_timestamp: required(string)
      • transaction_fee": required(string)
      • is_cellbase: required(boolean)
      • income: required(string)
      • version: required(string)
      • display_inputs: required(array of display_input)

        Items: display_input

        • from_cellbase: required(boolean)
        • id: required(string)
        • address_hash: required(string)
        • capacity: required(integer)
        • target_block_number: (integer)
        • generated_tx_hash: required(string)
        • cell_type: required(string)
        • started_block_number: (integer)
        • ended_block_number: (integer)
        • interest: (integer)
        • dao_type_hash: required(string)
      • display_outputs: required(array of display_output)

        Items: display_output

        • id: required(string)
        • address_hash: required(string)
        • capacity: required(integer)
        • target_block_number: (integer)
        • base_reward: (integer)
        • commit_reward: (integer)
        • proposal_reward: (integer)
        • secondary_reward: (integer)
        • consumed_by_tx_hash: (string)
        • status: required(one of live, dead)
        • cell_type": required(string)
        • dao_type_hash: required(string)
      • witnesses: required(array of string)
      • cell_deps: required(array of cell_dep)

        Items: cell_dep

        • dep_type: required(one of dep_group, code)
        • out_point: required(object)
          • index: required(integer)
          • tx_hash: required(string)
      • header_deps: required(array of string)

Example:

{
+

/transactions

get

get

Returns blocks in reverse chronological order.

get

get

Returns a transaction by transaction hash

Returns a transaction by transaction hash

URI Parameters

  • tx_hash: required(string - minLength: 66 - maxLength: 66)

    The transaction hash

HTTP status code 200

Body

Media type: application/vnd.api+json

Type: object

Properties
  • data: required(object)
    • id: required(string)
    • type: required(string)
    • attributes: required(object)
      • block_number: required(string)
      • transaction_hash: required(string)
      • block_timestamp: required(string)
      • transaction_fee": required(string)
      • is_cellbase: required(boolean)
      • income: required(string)
      • version: required(string)
      • display_inputs: required(array of display_input)

        Items: display_input

        • from_cellbase: required(boolean)
        • id: required(string)
        • address_hash: required(string)
        • capacity: required(integer)
        • target_block_number: (integer)
        • generated_tx_hash: required(string)
        • cell_type: required(string)
        • started_block_number: (integer)
        • ended_block_number: (integer)
        • interest: (integer)
        • dao_type_hash: required(string)
      • display_outputs: required(array of display_output)

        Items: display_output

        • id: required(string)
        • address_hash: required(string)
        • capacity: required(integer)
        • target_block_number: (integer)
        • base_reward: (integer)
        • commit_reward: (integer)
        • proposal_reward: (integer)
        • secondary_reward: (integer)
        • consumed_by_tx_hash: (string)
        • status: required(one of live, dead)
        • cell_type": required(string)
        • dao_type_hash: required(string)
      • witnesses: required(array of string)
      • cell_deps: required(array of cell_dep)

        Items: cell_dep

        • dep_type: required(one of dep_group, code)
        • out_point: required(object)
          • index: required(integer)
          • tx_hash: required(string)
      • header_deps: required(array of string)

Example:

{
   "data": {
     "id": 11,
     "type": "transaction",

From adfd9c6f9808f5b53324881396b3326e0800f1de Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 23:29:09 +0800
Subject: [PATCH 21/77] chore: just return normal tx

---
 app/controllers/api/v1/ckb_transactions_controller.rb | 4 ++--
 app/models/ckb_transaction.rb                         | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/app/controllers/api/v1/ckb_transactions_controller.rb b/app/controllers/api/v1/ckb_transactions_controller.rb
index 531e29616..507bf8c62 100644
--- a/app/controllers/api/v1/ckb_transactions_controller.rb
+++ b/app/controllers/api/v1/ckb_transactions_controller.rb
@@ -6,10 +6,10 @@ class CkbTransactionsController < ApplicationController
 
       def index
         if from_home_page?
-          ckb_transactions = CkbTransaction.recent.limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i)
+          ckb_transactions = CkbTransaction.recent.normal.limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i)
           options = {}
         else
-          ckb_transactions = CkbTransaction.recent.page(@page).per(@page_size)
+          ckb_transactions = CkbTransaction.recent.normal.page(@page).per(@page_size)
           options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size).call
         end
 
diff --git a/app/models/ckb_transaction.rb b/app/models/ckb_transaction.rb
index 9aa8d6cbd..ebeb768e8 100644
--- a/app/models/ckb_transaction.rb
+++ b/app/models/ckb_transaction.rb
@@ -17,6 +17,7 @@ class CkbTransaction < ApplicationRecord
 
   scope :recent, -> { order(block_timestamp: :desc) }
   scope :cellbase, -> { where(is_cellbase: true) }
+  scope :normal, -> { where(is_cellbase: false) }
 
   after_commit :flush_cache
   before_destroy :recover_dead_cell

From 8d359535399ffa3146c72f31559a03aa313ac8b0 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 23:29:39 +0800
Subject: [PATCH 22/77] test: index action just return normal txs

---
 test/controllers/api/v1/ckb_transactions_controller_test.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb
index 0315258d0..14a149190 100644
--- a/test/controllers/api/v1/ckb_transactions_controller_test.rb
+++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb
@@ -293,7 +293,9 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest
 
         ckb_transactions = CkbTransaction.order(block_timestamp: :desc).limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i)
         response_ckb_transactions = CkbTransactionListSerializer.new(ckb_transactions, {}).serialized_json
+
         assert_equal response_ckb_transactions, response.body
+        assert_equal [false], CkbTransaction.where(id: json["data"].map { |tx| tx.dig("id") }).pluck(:is_cellbase).uniq
         assert_equal 15, json["data"].size
       end
 

From ffd14364a8abe04c3e8b053fe276084d857028ac Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 23:29:55 +0800
Subject: [PATCH 23/77] chore: update api doc

---
 doc/api.raml        | 94 ++++++++++++++++++++++-----------------------
 public/api_doc.html | 50 +++++++++++++++++++++++-
 2 files changed, 96 insertions(+), 48 deletions(-)

diff --git a/doc/api.raml b/doc/api.raml
index 60610d770..153c531d4 100644
--- a/doc/api.raml
+++ b/doc/api.raml
@@ -1084,54 +1084,54 @@ types:
           application/vnd.api+json:
             type: ckb_transaction_list_response
             example: |
-            {
-              "data": [
-                 {
-                    "id": "519759",
-                    "type": "ckb_transaction_list",
-                    "attributes": {
-                      "transaction_hash": "0x51c4008aecf7b32762f8de7b1e0db97b0b99af6dfbedacddca4723e4e3469735",
-                      "block_number": "355801",
-                      "block_timestamp": "1577284662673",
-                      "capacity_involved": "",
-                      "live_cell_changes": "0"
-                    }
-                 },
-                 {
-                   "id": "519760",
-                   "type": "ckb_transaction_list",
-                   "attributes": {
-                     "transaction_hash": "0x93dcd049c02c94a4f8b83ed86e6dfdd662bb381585f6294e5cc316f301ed688d",
-                     "block_number": "355801",
-                     "block_timestamp": "1577284662673",
-                     "capacity_involved": "",
-                     "live_cell_changes": "0"
-                   }
-                 },
-                  {
-                    "id": "519757",
-                    "type": "ckb_transaction_list",
-                    "attributes": {
-                      "transaction_hash": "0xfe642e494030e77c46753f7ac2d29c38d6780e38c1a3a7accb6b4adf415a511b",
-                      "block_number": "355800",
-                      "block_timestamp": "1577284656255",
-                      "capacity_involved": "",
-                      "live_cell_changes": "0"
-                    }
-                 },
-              ]
-              "meta": {
-                "total": 100,
-                "page_size": 15,
-              },
-              links: {
-                "self": "https://explorer.nervos.org/api/v1/transactions?page_size=15",
-                "first": "https://explorer.nervos.org/api/v1/transactions?page=1&page_size=10",
-                "prev": "https://explorer.nervos.org/api/v1/transactions?page=2&page_size=10",
-                "next": "https://explorer.nervos.org/api/v1/transactions?page=2&page_size=15",
-                "last": "https://explorer.nervos.org/api/v1/transactions?page=28909&page_size=15"
+              {
+                "data": [
+                   {
+                      "id": "519759",
+                      "type": "ckb_transaction_list",
+                      "attributes": {
+                        "transaction_hash": "0x51c4008aecf7b32762f8de7b1e0db97b0b99af6dfbedacddca4723e4e3469735",
+                        "block_number": "355801",
+                        "block_timestamp": "1577284662673",
+                        "capacity_involved": "",
+                        "live_cell_changes": "0"
+                      }
+                   },
+                   {
+                     "id": "519760",
+                     "type": "ckb_transaction_list",
+                     "attributes": {
+                       "transaction_hash": "0x93dcd049c02c94a4f8b83ed86e6dfdd662bb381585f6294e5cc316f301ed688d",
+                       "block_number": "355801",
+                       "block_timestamp": "1577284662673",
+                       "capacity_involved": "",
+                       "live_cell_changes": "0"
+                     }
+                   },
+                    {
+                      "id": "519757",
+                      "type": "ckb_transaction_list",
+                      "attributes": {
+                        "transaction_hash": "0xfe642e494030e77c46753f7ac2d29c38d6780e38c1a3a7accb6b4adf415a511b",
+                        "block_number": "355800",
+                        "block_timestamp": "1577284656255",
+                        "capacity_involved": "",
+                        "live_cell_changes": "0"
+                      }
+                   },
+                ]
+                "meta": {
+                  "total": 100,
+                  "page_size": 15,
+                },
+                links: {
+                  "self": "https://explorer.nervos.org/api/v1/transactions?page_size=15",
+                  "first": "https://explorer.nervos.org/api/v1/transactions?page=1&page_size=10",
+                  "prev": "https://explorer.nervos.org/api/v1/transactions?page=2&page_size=10",
+                  "next": "https://explorer.nervos.org/api/v1/transactions?page=2&page_size=15",
+                  "last": "https://explorer.nervos.org/api/v1/transactions?page=28909&page_size=15"
+                }
               }
-            }
 
   /{tx_hash}:
     uriParameters:
diff --git a/public/api_doc.html b/public/api_doc.html
index e71610e37..c25317f74 100644
--- a/public/api_doc.html
+++ b/public/api_doc.html
@@ -323,7 +323,55 @@
     }
   ]
 }
-

/transactions

get

get

Returns blocks in reverse chronological order.

get

get

Returns a transaction by transaction hash

Returns a transaction by transaction hash

URI Parameters

  • tx_hash: required(string - minLength: 66 - maxLength: 66)

    The transaction hash

HTTP status code 200

Body

Media type: application/vnd.api+json

Type: object

Properties
  • data: required(object)
    • id: required(string)
    • type: required(string)
    • attributes: required(object)
      • block_number: required(string)
      • transaction_hash: required(string)
      • block_timestamp: required(string)
      • transaction_fee": required(string)
      • is_cellbase: required(boolean)
      • income: required(string)
      • version: required(string)
      • display_inputs: required(array of display_input)

        Items: display_input

        • from_cellbase: required(boolean)
        • id: required(string)
        • address_hash: required(string)
        • capacity: required(integer)
        • target_block_number: (integer)
        • generated_tx_hash: required(string)
        • cell_type: required(string)
        • started_block_number: (integer)
        • ended_block_number: (integer)
        • interest: (integer)
        • dao_type_hash: required(string)
      • display_outputs: required(array of display_output)

        Items: display_output

        • id: required(string)
        • address_hash: required(string)
        • capacity: required(integer)
        • target_block_number: (integer)
        • base_reward: (integer)
        • commit_reward: (integer)
        • proposal_reward: (integer)
        • secondary_reward: (integer)
        • consumed_by_tx_hash: (string)
        • status: required(one of live, dead)
        • cell_type": required(string)
        • dao_type_hash: required(string)
      • witnesses: required(array of string)
      • cell_deps: required(array of cell_dep)

        Items: cell_dep

        • dep_type: required(one of dep_group, code)
        • out_point: required(object)
          • index: required(integer)
          • tx_hash: required(string)
      • header_deps: required(array of string)

Example:

{
+

/transactions

get

get

Returns blocks in reverse chronological order.

get

get

Returns a transaction by transaction hash

Returns a transaction by transaction hash

URI Parameters

  • tx_hash: required(string - minLength: 66 - maxLength: 66)

    The transaction hash

HTTP status code 200

Body

Media type: application/vnd.api+json

Type: object

Properties
  • data: required(object)
    • id: required(string)
    • type: required(string)
    • attributes: required(object)
      • block_number: required(string)
      • transaction_hash: required(string)
      • block_timestamp: required(string)
      • transaction_fee": required(string)
      • is_cellbase: required(boolean)
      • income: required(string)
      • version: required(string)
      • display_inputs: required(array of display_input)

        Items: display_input

        • from_cellbase: required(boolean)
        • id: required(string)
        • address_hash: required(string)
        • capacity: required(integer)
        • target_block_number: (integer)
        • generated_tx_hash: required(string)
        • cell_type: required(string)
        • started_block_number: (integer)
        • ended_block_number: (integer)
        • interest: (integer)
        • dao_type_hash: required(string)
      • display_outputs: required(array of display_output)

        Items: display_output

        • id: required(string)
        • address_hash: required(string)
        • capacity: required(integer)
        • target_block_number: (integer)
        • base_reward: (integer)
        • commit_reward: (integer)
        • proposal_reward: (integer)
        • secondary_reward: (integer)
        • consumed_by_tx_hash: (string)
        • status: required(one of live, dead)
        • cell_type": required(string)
        • dao_type_hash: required(string)
      • witnesses: required(array of string)
      • cell_deps: required(array of cell_dep)

        Items: cell_dep

        • dep_type: required(one of dep_group, code)
        • out_point: required(object)
          • index: required(integer)
          • tx_hash: required(string)
      • header_deps: required(array of string)

Example:

{
   "data": {
     "id": 11,
     "type": "transaction",

From d2cf72e0ea69153655808335924a476e3659164c Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 23:34:44 +0800
Subject: [PATCH 24/77] chore: transactions list use ckb transaction serializer

---
 app/controllers/api/v1/ckb_transactions_controller.rb | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/controllers/api/v1/ckb_transactions_controller.rb b/app/controllers/api/v1/ckb_transactions_controller.rb
index 507bf8c62..3735233ad 100644
--- a/app/controllers/api/v1/ckb_transactions_controller.rb
+++ b/app/controllers/api/v1/ckb_transactions_controller.rb
@@ -7,13 +7,12 @@ class CkbTransactionsController < ApplicationController
       def index
         if from_home_page?
           ckb_transactions = CkbTransaction.recent.normal.limit(ENV["HOMEPAGE_TRANSACTIONS_RECORDS_COUNT"].to_i)
-          options = {}
+          render json: CkbTransactionListSerializer.new(ckb_transactions)
         else
           ckb_transactions = CkbTransaction.recent.normal.page(@page).per(@page_size)
           options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size).call
+          render json: CkbTransactionSerializer.new(ckb_transactions, options)
         end
-
-        render json: CkbTransactionListSerializer.new(ckb_transactions, options)
       end
 
       def show

From a92046a3230caad940a3f82abe72a8e05b12c070 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 23:34:56 +0800
Subject: [PATCH 25/77] chore: adjust test

---
 test/controllers/api/v1/ckb_transactions_controller_test.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb
index 14a149190..65af4cf9e 100644
--- a/test/controllers/api/v1/ckb_transactions_controller_test.rb
+++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb
@@ -309,7 +309,7 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest
         valid_get api_v1_ckb_transactions_url, params: { page: page, page_size: page_size }
 
         options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size).call
-        response_ckb_transactions = CkbTransactionListSerializer.new(ckb_transactions, options).serialized_json
+        response_ckb_transactions = CkbTransactionSerializer.new(ckb_transactions, options).serialized_json
         assert_equal response_ckb_transactions, response.body
       end
     end

From be4474cd4c8aeec6d649da01c2b014dd5c9729d4 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 23:50:07 +0800
Subject: [PATCH 26/77] chore: just save capacity_involved to normal tx

---
 app/models/ckb_sync/node_data_processor.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb
index dba8f19cf..a614d7262 100644
--- a/app/models/ckb_sync/node_data_processor.rb
+++ b/app/models/ckb_sync/node_data_processor.rb
@@ -502,7 +502,7 @@ def calculate_tx_fee(local_block, ckb_transactions, input_capacities, outputs)
       txs = []
       ckb_transactions.each do |ckb_transaction|
         update_transaction_fee(ckb_transaction, input_capacities[ckb_transaction.id].sum, output_capacities[ckb_transaction.id].sum)
-        ckb_transaction.capacity_involved = input_capacities[ckb_transaction.id].sum
+        ckb_transaction.capacity_involved = input_capacities[ckb_transaction.id].sum unless ckb_transaction.is_cellbase
         txs << ckb_transaction
       end
 

From 1c1bfc9796ee019a54512af0a40d07a5ee38529e Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 23:50:19 +0800
Subject: [PATCH 27/77] chore: adjust test

---
 test/models/ckb_sync/node_data_processor_test.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb
index 24fb54ff7..af9b588e1 100644
--- a/test/models/ckb_sync/node_data_processor_test.rb
+++ b/test/models/ckb_sync/node_data_processor_test.rb
@@ -263,9 +263,9 @@ class NodeDataProcessorTest < ActiveSupport::TestCase
         create(:cell_output, ckb_transaction: ckb_transaction1, cell_index: 1, tx_hash: "0x498315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction2, block: block)
         create(:cell_output, ckb_transaction: ckb_transaction2, cell_index: 2, tx_hash: "0x598315db9c7ba144cca74d2e9122ac9b3a3da1641b2975ae321d91ec34f1c0e3", generated_by: ckb_transaction1, block: block)
         local_block = node_data_processor.process_block(node_block)
-        expected_capacity_involved = local_block.ckb_transactions.map(&:capacity_involved)
+        expected_capacity_involved = local_block.ckb_transactions.normal.map(&:capacity_involved)
 
-        assert_equal expected_capacity_involved, local_block.ckb_transactions.map { |transaction| transaction.inputs.sum(:capacity) }
+        assert_equal expected_capacity_involved, local_block.ckb_transactions.normal.map { |transaction| transaction.inputs.sum(:capacity) }
       end
     end
 

From a2d42b23ed8f1b04aec7ab0bf8b2dbbf3d15f639 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 23:51:03 +0800
Subject: [PATCH 28/77] feat: add live_cell_changes to forked blocks

---
 app/models/forked_block.rb                                   | 1 +
 .../20191225153746_add_live_cell_chagnes_to_forked_block.rb  | 5 +++++
 db/schema.rb                                                 | 3 ++-
 3 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20191225153746_add_live_cell_chagnes_to_forked_block.rb

diff --git a/app/models/forked_block.rb b/app/models/forked_block.rb
index 82db53546..5485c0ecf 100644
--- a/app/models/forked_block.rb
+++ b/app/models/forked_block.rb
@@ -51,4 +51,5 @@ class ForkedBlock < ApplicationRecord
 #  length                     :decimal(30, )    default(0)
 #  uncles_count               :integer
 #  compact_target             :decimal(20, )
+#  live_cell_changes          :integer
 #
diff --git a/db/migrate/20191225153746_add_live_cell_chagnes_to_forked_block.rb b/db/migrate/20191225153746_add_live_cell_chagnes_to_forked_block.rb
new file mode 100644
index 000000000..0194aceea
--- /dev/null
+++ b/db/migrate/20191225153746_add_live_cell_chagnes_to_forked_block.rb
@@ -0,0 +1,5 @@
+class AddLiveCellChagnesToForkedBlock < ActiveRecord::Migration[6.0]
+  def change
+    add_column :forked_blocks, :live_cell_changes, :integer
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 3a05c2c84..9cb1fdfc2 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_12_25_131337) do
+ActiveRecord::Schema.define(version: 2019_12_25_153746) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -233,6 +233,7 @@
     t.decimal "length", precision: 30, default: "0"
     t.integer "uncles_count"
     t.decimal "compact_target", precision: 20
+    t.integer "live_cell_changes"
   end
 
   create_table "lock_scripts", force: :cascade do |t|

From 67a01ec2c73db2b362d225a369df18e8ebaaacbb Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 11:39:15 +0800
Subject: [PATCH 29/77] chore: use task to update block's live cell changes

---
 .../update_block_live_cell_changes.rake       | 18 ++++++++++++++++++
 ...ve_cell_changes_and_capacity_involved.rake | 19 +++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 lib/tasks/migration/update_block_live_cell_changes.rake
 create mode 100644 lib/tasks/migration/update_tx_live_cell_changes_and_capacity_involved.rake

diff --git a/lib/tasks/migration/update_block_live_cell_changes.rake b/lib/tasks/migration/update_block_live_cell_changes.rake
new file mode 100644
index 000000000..6c1aa764f
--- /dev/null
+++ b/lib/tasks/migration/update_block_live_cell_changes.rake
@@ -0,0 +1,18 @@
+namespace :migration do
+  task update_block_live_cell_changes: :environment do
+    progress_bar = ProgressBar.create({ total: Block.count, format: "%e %B %p%% %c/%C" })
+    columns = [:id, :live_cell_changes]
+    Block.order(:id).find_in_batches(batch_size: 4000) do |blocks|
+      values =
+        blocks.map do |block|
+          progress_bar.increment
+          live_cell_changes = block.ckb_transactions.sum(&:live_cell_changes)
+          [block.id, live_cell_changes]
+        end
+
+      Block.import columns, values, validate:false, on_duplicate_key_update: [:live_cell_changes]
+    end
+
+    puts "done"
+  end
+end
diff --git a/lib/tasks/migration/update_tx_live_cell_changes_and_capacity_involved.rake b/lib/tasks/migration/update_tx_live_cell_changes_and_capacity_involved.rake
new file mode 100644
index 000000000..48194b413
--- /dev/null
+++ b/lib/tasks/migration/update_tx_live_cell_changes_and_capacity_involved.rake
@@ -0,0 +1,19 @@
+namespace :migration do
+  task update_tx_live_cell_changes_and_capacity_involved: :environment do
+    progress_bar = ProgressBar.create({ total: CkbTransaction.count, format: "%e %B %p%% %c/%C" })
+    columns = [:id, :block_id, :live_cell_changes, :capacity_involved]
+    CkbTransaction.order(:id).find_in_batches(batch_size: 4000) do |ckb_transactions|
+      values =
+        ckb_transactions.map do |ckb_transaction|
+          progress_bar.increment
+          live_cell_changes = ckb_transaction.outputs.count - ckb_transaction.inputs.count
+          capacity_involved = ckb_transaction.inputs.sum(:capacity)
+          [ckb_transaction.id, ckb_transaction.block_id, live_cell_changes, capacity_involved]
+        end
+
+      CkbTransaction.import! columns, values, validate:false, on_duplicate_key_update: [:live_cell_changes, :capacity_involved]
+    end
+
+    puts "done"
+  end
+end

From 2563832bffcfb503f1d16555c9c35b367e0742f4 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 15:12:25 +0800
Subject: [PATCH 30/77] chore: adjust test

---
 app/controllers/api/v1/ckb_transactions_controller.rb       | 2 +-
 test/controllers/api/v1/ckb_transactions_controller_test.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/controllers/api/v1/ckb_transactions_controller.rb b/app/controllers/api/v1/ckb_transactions_controller.rb
index 3735233ad..ff7a048b4 100644
--- a/app/controllers/api/v1/ckb_transactions_controller.rb
+++ b/app/controllers/api/v1/ckb_transactions_controller.rb
@@ -11,7 +11,7 @@ def index
         else
           ckb_transactions = CkbTransaction.recent.normal.page(@page).per(@page_size)
           options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: @page, page_size: @page_size).call
-          render json: CkbTransactionSerializer.new(ckb_transactions, options)
+          render json: CkbTransactionListSerializer.new(ckb_transactions, options)
         end
       end
 
diff --git a/test/controllers/api/v1/ckb_transactions_controller_test.rb b/test/controllers/api/v1/ckb_transactions_controller_test.rb
index 65af4cf9e..14a149190 100644
--- a/test/controllers/api/v1/ckb_transactions_controller_test.rb
+++ b/test/controllers/api/v1/ckb_transactions_controller_test.rb
@@ -309,7 +309,7 @@ class CkbTransactionsControllerTest < ActionDispatch::IntegrationTest
         valid_get api_v1_ckb_transactions_url, params: { page: page, page_size: page_size }
 
         options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: ckb_transactions, page: page, page_size: page_size).call
-        response_ckb_transactions = CkbTransactionSerializer.new(ckb_transactions, options).serialized_json
+        response_ckb_transactions = CkbTransactionListSerializer.new(ckb_transactions, options).serialized_json
         assert_equal response_ckb_transactions, response.body
       end
     end

From 13c933573be5ecdb7112e3d346698107953726f0 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 09:52:41 +0800
Subject: [PATCH 31/77] refactor: extract live cell changes method

---
 app/models/ckb_sync/node_data_processor.rb                  | 6 +++++-
 .../update_tx_live_cell_changes_and_capacity_involved.rake  | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb
index a614d7262..6de92719c 100644
--- a/app/models/ckb_sync/node_data_processor.rb
+++ b/app/models/ckb_sync/node_data_processor.rb
@@ -339,10 +339,14 @@ def build_ckb_transaction(local_block, transaction, transaction_index)
         transaction_fee: 0,
         witnesses: transaction.witnesses,
         is_cellbase: transaction_index.zero?,
-        live_cell_changes: transaction.outputs.count - transaction.inputs.count
+        live_cell_changes: live_cell_changes(transaction, transaction_index)
       )
     end
 
+    def live_cell_changes(transaction, transaction_index)
+      transaction_index.zero? ? 1 : transaction.outputs.count - transaction.inputs.count
+    end
+
     def build_cell_inputs(node_inputs, ckb_transaction)
       node_inputs.each do |node_input|
         build_cell_input(ckb_transaction, node_input)
diff --git a/lib/tasks/migration/update_tx_live_cell_changes_and_capacity_involved.rake b/lib/tasks/migration/update_tx_live_cell_changes_and_capacity_involved.rake
index 48194b413..ec8dfdd87 100644
--- a/lib/tasks/migration/update_tx_live_cell_changes_and_capacity_involved.rake
+++ b/lib/tasks/migration/update_tx_live_cell_changes_and_capacity_involved.rake
@@ -6,7 +6,7 @@ namespace :migration do
       values =
         ckb_transactions.map do |ckb_transaction|
           progress_bar.increment
-          live_cell_changes = ckb_transaction.outputs.count - ckb_transaction.inputs.count
+          live_cell_changes = ckb_transaction.is_cellbase ? 1 : ckb_transaction.outputs.count - ckb_transaction.inputs.count
           capacity_involved = ckb_transaction.inputs.sum(:capacity)
           [ckb_transaction.id, ckb_transaction.block_id, live_cell_changes, capacity_involved]
         end

From feaa5a65e3fe1f992e2c2a28c4f6132f4fc589af Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 13:01:33 +0800
Subject: [PATCH 32/77] chore: adjust test

---
 test/models/ckb_sync/node_data_processor_test.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb
index af9b588e1..a3aff6b96 100644
--- a/test/models/ckb_sync/node_data_processor_test.rb
+++ b/test/models/ckb_sync/node_data_processor_test.rb
@@ -249,8 +249,8 @@ class NodeDataProcessorTest < ActiveSupport::TestCase
         node_block_transactions = node_block.transactions
 
         local_block = node_data_processor.process_block(node_block)
-        expected_live_cell_changes = node_block_transactions.map { |transaction| transaction.outputs.count - transaction.inputs.count }
-        assert_equal expected_live_cell_changes, local_block.ckb_transactions.map(&:live_cell_changes)
+        expected_live_cell_changes = node_block_transactions.each_with_index.map { |transaction, index| index.zero? ? 1 : transaction.outputs.count - transaction.inputs.count }
+        assert_equal expected_live_cell_changes, local_block.ckb_transactions.order(:id).map(&:live_cell_changes)
       end
     end
 

From e0746dfc452319a09206ecd2eba081cf34554b69 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 16:51:33 +0800
Subject: [PATCH 33/77] feat: add hash rate to epoch statistic

---
 app/models/epoch_statistic.rb                                | 1 +
 .../20191226084920_add_hash_rate_to_epoch_statistic.rb       | 5 +++++
 db/schema.rb                                                 | 3 ++-
 3 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20191226084920_add_hash_rate_to_epoch_statistic.rb

diff --git a/app/models/epoch_statistic.rb b/app/models/epoch_statistic.rb
index b0e6c5cd7..3c5eea851 100644
--- a/app/models/epoch_statistic.rb
+++ b/app/models/epoch_statistic.rb
@@ -12,6 +12,7 @@ class EpochStatistic < ApplicationRecord
 #  epoch_number :decimal(30, )
 #  created_at   :datetime         not null
 #  updated_at   :datetime         not null
+#  hash_rate    :string
 #
 # Indexes
 #
diff --git a/db/migrate/20191226084920_add_hash_rate_to_epoch_statistic.rb b/db/migrate/20191226084920_add_hash_rate_to_epoch_statistic.rb
new file mode 100644
index 000000000..8a400beb1
--- /dev/null
+++ b/db/migrate/20191226084920_add_hash_rate_to_epoch_statistic.rb
@@ -0,0 +1,5 @@
+class AddHashRateToEpochStatistic < ActiveRecord::Migration[6.0]
+  def change
+    add_column :epoch_statistics, :hash_rate, :string
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 9cb1fdfc2..704f980d0 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_12_25_153746) do
+ActiveRecord::Schema.define(version: 2019_12_26_084920) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -195,6 +195,7 @@
     t.decimal "epoch_number", precision: 30
     t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
+    t.string "hash_rate"
     t.index ["epoch_number"], name: "index_epoch_statistics_on_epoch_number", unique: true
   end
 

From 48be8e428ceee100ac650e5c9a1c9de420d9eac1 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 16:52:00 +0800
Subject: [PATCH 34/77] feat: save hash rate on epoch statistic worker

---
 app/workers/charts/epoch_statistic.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/workers/charts/epoch_statistic.rb b/app/workers/charts/epoch_statistic.rb
index 76e254421..eea9be9af 100644
--- a/app/workers/charts/epoch_statistic.rb
+++ b/app/workers/charts/epoch_statistic.rb
@@ -14,8 +14,12 @@ def perform
       uncles_count = Block.where(epoch: target_epoch_number).sum(:uncles_count)
       uncle_rate = uncles_count / blocks_count.to_d
       difficulty = Block.where(epoch: target_epoch_number).first.difficulty
+      first_block_in_epoch = Block.where(epoch: target_epoch_number).order(:number).first
+      last_lock_in_epoch = Block.where(epoch: target_epoch_number).order(:number).last
+      block_time = last_lock_in_epoch.timestamp - first_block_in_epoch.timestamp
+      hash_rate = difficulty * first_block_in_epoch.length / block_time
 
-      ::EpochStatistic.create(epoch_number: target_epoch_number, difficulty: difficulty, uncle_rate: uncle_rate)
+      ::EpochStatistic.create(epoch_number: target_epoch_number, difficulty: difficulty, uncle_rate: uncle_rate, hash_rate: hash_rate)
     end
   end
 end

From a8de860b5f4ebb822fe08eab876af75bff9fa876 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 17:10:06 +0800
Subject: [PATCH 35/77] chore: add task to fill hash rate

---
 .../fill_hash_rate_to_epoch_statistic.rake    | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake

diff --git a/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake b/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake
new file mode 100644
index 000000000..4cb862130
--- /dev/null
+++ b/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake
@@ -0,0 +1,19 @@
+namespace :migration do
+  task fill_hash_rate_to_epoch_static: :environment do
+    max_epoch_number = Block.maximum(:epoch)
+    progress_bar = ProgressBar.create({ total: max_epoch_number + 1, format: "%e %B %p%% %c/%C" })
+    columns = [:epoch_number, :hash_rate]
+    values =
+      (0..max_epoch_number).map do |epoch_number|
+        first_block_in_epoch = Block.where(epoch: epoch_number).order(:number).first
+        last_lock_in_epoch = Block.where(epoch: epoch_number).order(:number).last
+        block_time = last_lock_in_epoch.timestamp - first_block_in_epoch.timestamp
+        hash_rate = first_block_in_epoch.difficulty * first_block_in_epoch.length / block_time
+        progress_bar.increment
+        [epoch_number, hash_rate]
+      end
+    ::EpochStatistic.import(columns, values, validate: false, on_duplicate_key_update: [:hash_rate])
+
+    puts "done"
+  end
+end

From 354ebf18c206add908084bcdfa37f80be645ae80 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 17:13:52 +0800
Subject: [PATCH 36/77] feat: return hash_rate

---
 app/models/epoch_statistic.rb                 | 2 +-
 app/serializers/epoch_statistic_serializer.rb | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/models/epoch_statistic.rb b/app/models/epoch_statistic.rb
index 3c5eea851..591b47a88 100644
--- a/app/models/epoch_statistic.rb
+++ b/app/models/epoch_statistic.rb
@@ -1,5 +1,5 @@
 class EpochStatistic < ApplicationRecord
-  VALID_INDICATORS = %w(difficulty uncle_rate).freeze
+  VALID_INDICATORS = %w(difficulty uncle_rate hash_rate).freeze
 end
 
 # == Schema Information
diff --git a/app/serializers/epoch_statistic_serializer.rb b/app/serializers/epoch_statistic_serializer.rb
index 6f1d65f01..ba5e2ce9a 100644
--- a/app/serializers/epoch_statistic_serializer.rb
+++ b/app/serializers/epoch_statistic_serializer.rb
@@ -12,4 +12,8 @@ class EpochStatisticSerializer
   attribute :uncle_rate, if: Proc.new { |_record, params|
     params && params[:indicator].include?("uncle_rate")
   }
+
+  attribute :hash_rate, if: Proc.new { |_record, params|
+    params && params[:indicator].include?("hash_rate")
+  }
 end

From a6b3ee6cf98ad947cdebf2bce3861bf4cd6ed32c Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 17:16:07 +0800
Subject: [PATCH 37/77] test: hash_rate on epoch statistic

---
 .../migration/fill_hash_rate_to_epoch_statistic.rake  |  4 ++--
 .../api/v1/epoch_statistics_controller_test.rb        | 11 +++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake b/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake
index 4cb862130..f9dc1de6c 100644
--- a/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake
+++ b/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake
@@ -8,11 +8,11 @@ namespace :migration do
         first_block_in_epoch = Block.where(epoch: epoch_number).order(:number).first
         last_lock_in_epoch = Block.where(epoch: epoch_number).order(:number).last
         block_time = last_lock_in_epoch.timestamp - first_block_in_epoch.timestamp
-        hash_rate = first_block_in_epoch.difficulty * first_block_in_epoch.length / block_time
+        hash_rate = BigDecimal(first_block_in_epoch.difficulty * first_block_in_epoch.length) / block_time
         progress_bar.increment
         [epoch_number, hash_rate]
       end
-    ::EpochStatistic.import(columns, values, validate: false, on_duplicate_key_update: [:hash_rate])
+    ::EpochStatistic.import(columns, values, validate: false, on_duplicate_key_update: { conflict_target: [:epoch_number], columns: [:hash_rate] })
 
     puts "done"
   end
diff --git a/test/controllers/api/v1/epoch_statistics_controller_test.rb b/test/controllers/api/v1/epoch_statistics_controller_test.rb
index a13cdadaa..31422b57c 100644
--- a/test/controllers/api/v1/epoch_statistics_controller_test.rb
+++ b/test/controllers/api/v1/epoch_statistics_controller_test.rb
@@ -50,6 +50,17 @@ class EpochStatisticsControllerTest < ActionDispatch::IntegrationTest
         assert_equal EpochStatisticSerializer.new(block_statistic_data, { params: { indicator: "difficulty-uncle_rate" } }).serialized_json, response.body
       end
 
+      test "should return difficulty, hash_rate and epoch number" do
+        (1..15).each do |number|
+          EpochStatistic.create(epoch_number: number)
+        end
+        block_statistic_data = EpochStatistic.order(epoch_number: :desc).reverse
+        valid_get api_v1_epoch_statistic_url("difficulty-hash_rate")
+
+        assert_equal [%w(difficulty hash_rate epoch_number).sort], json.dig("data").map { |item| item.dig("attributes").keys.sort }.uniq
+        assert_equal EpochStatisticSerializer.new(block_statistic_data, { params: { indicator: "difficulty-hash_rate" } }).serialized_json, response.body
+      end
+
       test "should respond with error object when indicator name is invalid" do
         error_object = Api::V1::Exceptions::IndicatorNameInvalidError.new
         response_json = RequestErrorSerializer.new([error_object], message: error_object.title).serialized_json

From 7384f7dcac29c97f89b825d79f10afbe70677b09 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 13:50:46 +0800
Subject: [PATCH 38/77] chore: use current epoch length

---
 app/workers/charts/epoch_statistic.rb                      | 3 ++-
 lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/workers/charts/epoch_statistic.rb b/app/workers/charts/epoch_statistic.rb
index eea9be9af..35929ff4f 100644
--- a/app/workers/charts/epoch_statistic.rb
+++ b/app/workers/charts/epoch_statistic.rb
@@ -17,7 +17,8 @@ def perform
       first_block_in_epoch = Block.where(epoch: target_epoch_number).order(:number).first
       last_lock_in_epoch = Block.where(epoch: target_epoch_number).order(:number).last
       block_time = last_lock_in_epoch.timestamp - first_block_in_epoch.timestamp
-      hash_rate = difficulty * first_block_in_epoch.length / block_time
+      epoch_length = Block.where(epoch: target_epoch_number).count
+      hash_rate = difficulty * epoch_length / block_time
 
       ::EpochStatistic.create(epoch_number: target_epoch_number, difficulty: difficulty, uncle_rate: uncle_rate, hash_rate: hash_rate)
     end
diff --git a/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake b/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake
index f9dc1de6c..87db34238 100644
--- a/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake
+++ b/lib/tasks/migration/fill_hash_rate_to_epoch_statistic.rake
@@ -8,7 +8,8 @@ namespace :migration do
         first_block_in_epoch = Block.where(epoch: epoch_number).order(:number).first
         last_lock_in_epoch = Block.where(epoch: epoch_number).order(:number).last
         block_time = last_lock_in_epoch.timestamp - first_block_in_epoch.timestamp
-        hash_rate = BigDecimal(first_block_in_epoch.difficulty * first_block_in_epoch.length) / block_time
+        epoch_length = Block.where(epoch: epoch_number).count
+        hash_rate = BigDecimal(first_block_in_epoch.difficulty * epoch_length) / block_time
         progress_bar.increment
         [epoch_number, hash_rate]
       end

From 3912139c8ff6c39b6f2e3ed9c63cae8d43518030 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 19 Dec 2019 10:15:37 +0800
Subject: [PATCH 39/77] feat: add more field to daily statistics

---
 app/models/daily_statistic.rb                        |  8 ++++++++
 ...91219014039_add_more_field_to_daily_statistics.rb | 12 ++++++++++++
 db/schema.rb                                         |  8 ++++++++
 3 files changed, 28 insertions(+)
 create mode 100644 db/migrate/20191219014039_add_more_field_to_daily_statistics.rb

diff --git a/app/models/daily_statistic.rb b/app/models/daily_statistic.rb
index 0c77b8c67..d097220c3 100644
--- a/app/models/daily_statistic.rb
+++ b/app/models/daily_statistic.rb
@@ -14,4 +14,12 @@ class DailyStatistic < ApplicationRecord
 #  created_at_unixtimestamp :integer
 #  created_at               :datetime         not null
 #  updated_at               :datetime         not null
+#  dao_depositors_count     :string           default("0")
+#  unclaimed_compensation   :string           default("0")
+#  claimed_compensation     :string           default("0")
+#  average_deposit_time     :string           default("0")
+#  estimated_apc            :string           default("0")
+#  mining_reward            :string           default("0")
+#  deposit_compensation     :string           default("0")
+#  treasury_amount          :string           default("0")
 #
diff --git a/db/migrate/20191219014039_add_more_field_to_daily_statistics.rb b/db/migrate/20191219014039_add_more_field_to_daily_statistics.rb
new file mode 100644
index 000000000..4a7ffd1ce
--- /dev/null
+++ b/db/migrate/20191219014039_add_more_field_to_daily_statistics.rb
@@ -0,0 +1,12 @@
+class AddMoreFieldToDailyStatistics < ActiveRecord::Migration[6.0]
+  def change
+    add_column :daily_statistics, :dao_depositors_count, :string, default: "0"
+    add_column :daily_statistics, :unclaimed_compensation, :string, default: "0"
+    add_column :daily_statistics, :claimed_compensation, :string, default: "0"
+    add_column :daily_statistics, :average_deposit_time, :string, default: "0"
+    add_column :daily_statistics, :estimated_apc, :string, default: "0"
+    add_column :daily_statistics, :mining_reward, :string, default: "0"
+    add_column :daily_statistics, :deposit_compensation, :string, default: "0"
+    add_column :daily_statistics, :treasury_amount, :string, default: "0"
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 704f980d0..ef83bfe7f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -163,6 +163,14 @@
     t.integer "created_at_unixtimestamp"
     t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
+    t.string "dao_depositors_count", default: "0"
+    t.string "unclaimed_compensation", default: "0"
+    t.string "claimed_compensation", default: "0"
+    t.string "average_deposit_time", default: "0"
+    t.string "estimated_apc", default: "0"
+    t.string "mining_reward", default: "0"
+    t.string "deposit_compensation", default: "0"
+    t.string "treasury_amount", default: "0"
   end
 
   create_table "dao_contracts", force: :cascade do |t|

From a40453b5cd3e07eb983ec7f7e438a08274353e14 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 19 Dec 2019 10:16:44 +0800
Subject: [PATCH 40/77] feat: add block timestamp to dao event

---
 app/models/dao_event.rb                                      | 1 +
 .../20191219020035_add_block_timestamp_to_dao_events.rb      | 5 +++++
 db/schema.rb                                                 | 1 +
 3 files changed, 7 insertions(+)
 create mode 100644 db/migrate/20191219020035_add_block_timestamp_to_dao_events.rb

diff --git a/app/models/dao_event.rb b/app/models/dao_event.rb
index 98cdd2380..ba7d31fe0 100644
--- a/app/models/dao_event.rb
+++ b/app/models/dao_event.rb
@@ -22,6 +22,7 @@ class DaoEvent < ApplicationRecord
 #  status             :integer          default("pending")
 #  created_at         :datetime         not null
 #  updated_at         :datetime         not null
+#  block_timestamp    :decimal(30, )
 #
 # Indexes
 #
diff --git a/db/migrate/20191219020035_add_block_timestamp_to_dao_events.rb b/db/migrate/20191219020035_add_block_timestamp_to_dao_events.rb
new file mode 100644
index 000000000..93a6ae036
--- /dev/null
+++ b/db/migrate/20191219020035_add_block_timestamp_to_dao_events.rb
@@ -0,0 +1,5 @@
+class AddBlockTimestampToDaoEvents < ActiveRecord::Migration[6.0]
+  def change
+    add_column :dao_events, :block_timestamp, :decimal, precision: 30, scale: 0
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index ef83bfe7f..1637ea4da 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -194,6 +194,7 @@
     t.integer "status", limit: 2, default: 0
     t.datetime "created_at", precision: 6, null: false
     t.datetime "updated_at", precision: 6, null: false
+    t.decimal "block_timestamp", precision: 30
     t.index ["block_id"], name: "index_dao_events_on_block_id"
   end
 

From 49ca0757187d1bfb0bd1daa7b8972940a3a4d87f Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 19 Dec 2019 10:17:38 +0800
Subject: [PATCH 41/77] chore: fill block timestamp to dao event

---
 .../fill_block_timestamp_to_dao_event.rake    | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 lib/tasks/migration/fill_block_timestamp_to_dao_event.rake

diff --git a/lib/tasks/migration/fill_block_timestamp_to_dao_event.rake b/lib/tasks/migration/fill_block_timestamp_to_dao_event.rake
new file mode 100644
index 000000000..8e468b2d1
--- /dev/null
+++ b/lib/tasks/migration/fill_block_timestamp_to_dao_event.rake
@@ -0,0 +1,20 @@
+namespace :migration do
+  task fill_block_timestamp_to_dao_event: :environment do
+    dao_events = DaoEvent.processed
+    progress_bar = ProgressBar.create({
+      total: dao_events.count,
+      format: "%e %B %p%% %c/%C"
+    })
+
+    values =
+      dao_events.map do |dao_event|
+        progress_bar.increment
+        [dao_event.id, dao_event.block.timestamp]
+      end
+
+    columns = [:id, :block_timestamp]
+    DaoEvent.import columns, values, on_duplicate_key_update: [:block_timestamp]
+
+    puts "done"
+  end
+end

From 22bcc35928b165d832418a28824c019ba8e322d8 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 19 Dec 2019 13:57:17 +0800
Subject: [PATCH 42/77] chore: fill new fields to daily statistic on worker

---
 app/workers/charts/daily_statistic.rb | 73 ++++++++++++++++++++++++++-
 1 file changed, 72 insertions(+), 1 deletion(-)

diff --git a/app/workers/charts/daily_statistic.rb b/app/workers/charts/daily_statistic.rb
index 00adb8844..91cd49d3e 100644
--- a/app/workers/charts/daily_statistic.rb
+++ b/app/workers/charts/daily_statistic.rb
@@ -5,6 +5,8 @@ class DailyStatistic
     sidekiq_options retry: false
     sidekiq_options queue: "critical"
 
+    MILLISECONDS_IN_DAY = BigDecimal.new(24 * 60 * 60 * 1000)
+
     def perform(datetime = nil)
       to_be_counted_date = datetime.presence || DateTime.now - 1.day
       started_at = to_be_counted_date.beginning_of_day.strftime("%Q")
@@ -13,11 +15,80 @@ def perform(datetime = nil)
       addresses_count = Address.where("block_timestamp <= ?", ended_at).count
       deposit_cells = CellOutput.where(cell_type: "nervos_dao_deposit").where("block_timestamp <= ?", ended_at)
       total_dao_deposit = datetime.blank? ? deposit_cells.where(status: "live").sum(:capacity) : deposit_cells.sum(:capacity)
+      dao_depositors_count = DaoEvent.processed.where("block_timestamp <= ?", ended_at).count
+      cell_outputs = CellOutput.where("block_timestamp <= ?", ended_at)
+      current_tip_block = Block.where("timestamp <= ?", ended_at).recent.first
+      mining_reward = Block.where("timestamp <= ?", ended_at).sum(:secondary_reward)
+      deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs)
 
       block_timestamp = Block.created_after(started_at).created_before(ended_at).recent.pick(:timestamp)
       daily_statistic = ::DailyStatistic.create_or_find_by!(block_timestamp: block_timestamp)
       daily_statistic.update(created_at_unixtimestamp: to_be_counted_date.to_i, transactions_count: daily_ckb_transactions_count,
-                             addresses_count: addresses_count, total_dao_deposit: total_dao_deposit)
+                             addresses_count: addresses_count, total_dao_deposit: total_dao_deposit,
+                             dao_depositors_count: dao_depositors_count, unclaimed_compensation: unclaimed_compensation(cell_outputs, current_tip_block),
+                             claimed_compensation: claimed_compensation(cell_outputs), average_deposit_time: average_deposit_time(cell_outputs),
+                             mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount(cell_outputs, current_tip_block))
+    end
+
+    private
+
+    def unclaimed_compensation(cell_outputs, current_tip_block)
+      @unclaimed_compensation ||=
+        begin
+          phase1_dao_interests(cell_outputs) + unmade_dao_interests(cell_outputs, current_tip_block)
+        end
+    end
+
+    def claimed_compensation(cell_outputs)
+      @claimed_compensation ||=
+        begin
+          cell_outputs.nervos_dao_withdrawing.dead.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+            memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell)
+          end
+        end
+    end
+
+    def phase1_dao_interests(cell_outputs)
+      cell_outputs.nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+        memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell)
+      end
+    end
+
+    def unmade_dao_interests(cell_outputs, current_tip_block)
+      @unmade_dao_interests ||=
+        begin
+          cell_outputs.nervos_dao_deposit.live.reduce(0) do |memo, cell_output|
+            dao = cell_output.block.dao
+            tip_dao = current_tip_block.dao
+            parse_dao = CkbUtils.parse_dao(dao)
+            tip_parse_dao = CkbUtils.parse_dao(tip_dao)
+            memo + cell_output.capacity * tip_parse_dao.ar_i / parse_dao.ar_i
+          end
+        end
+    end
+
+    def average_deposit_time(cell_outputs)
+      interest_bearing_deposits = 0
+      uninterest_bearing_deposits = 0
+      sum_interest_bearing = cell_outputs.nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+        nervos_dao_withdrawing_cell_generated_tx = nervos_dao_withdrawing_cell.generated_by
+        nervos_dao_deposit_cell = nervos_dao_withdrawing_cell_generated_tx.cell_inputs.order(:id)[nervos_dao_withdrawing_cell.cell_index].previous_cell_output
+        interest_bearing_deposits += nervos_dao_deposit_cell.capacity
+        memo + nervos_dao_deposit_cell.capacity * (nervos_dao_withdrawing_cell.block_timestamp - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
+      end
+      sum_uninterest_bearing = cell_outputs.nervos_dao_deposit.live.reduce(0) do |memo, nervos_dao_deposit_cell|
+        current_time = DateTime.now.strftime("%Q").to_i
+        uninterest_bearing_deposits += nervos_dao_deposit_cell.capacity
+
+        memo + nervos_dao_deposit_cell.capacity * (current_time - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
+      end
+
+      (sum_interest_bearing + sum_uninterest_bearing) / (interest_bearing_deposits + uninterest_bearing_deposits)
+    end
+
+    def treasury_amount(cell_outputs, current_tip_block)
+      parse_dao = CkbUtils.parse_dao(current_tip_block.dao)
+      parse_dao.s_i - unmade_dao_interests(cell_outputs, current_tip_block)
     end
   end
 end

From cb31d1c9341c39f7233b3f1ece7714eb8e220fe1 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Fri, 20 Dec 2019 18:06:23 +0800
Subject: [PATCH 43/77] feat: add estimated_apc to dao contract

---
 app/models/dao_contract.rb            | 58 +++++++++++++++++++++++++++
 app/workers/charts/daily_statistic.rb |  2 +-
 2 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/app/models/dao_contract.rb b/app/models/dao_contract.rb
index 20c62bdba..1ee6bca91 100644
--- a/app/models/dao_contract.rb
+++ b/app/models/dao_contract.rb
@@ -1,6 +1,13 @@
 class DaoContract < ApplicationRecord
   validates :total_deposit, :interest_granted, :deposit_transactions_count, :withdraw_transactions_count, :depositors_count, :total_depositors_count, presence: true, numericality: { greater_than_or_equal_to: 0 }
   CONTRACT_NAME = "nervos_dao".freeze
+  GENESIS_ISSUANCE = 336 * 10**8
+  ANNUAL_PRIMARY_ISSUANCE_BASE = GENESIS_ISSUANCE / 8
+  PRIMARY_ISSUANCE_PER_YEAR_BASE = BigDecimal(42 * 10**8)
+  EPOCHS_IN_ONE_NATURAL_YEAR = 2190
+  YEARS_IN_PERIOD = 4
+  EPOCHS_IN_PERIOD = BigDecimal(EPOCHS_IN_ONE_NATURAL_YEAR * YEARS_IN_PERIOD)
+  SECONDARY_ISSUANCE_PER_EPOCH = BigDecimal(1344 * 10**6) / 2190
 
   def self.default_contract
     find_or_create_by(id: 1)
@@ -10,6 +17,57 @@ def ckb_transactions
     ckb_transaction_ids = CellOutput.nervos_dao_deposit.pluck("generated_by_id") + CellOutput.nervos_dao_withdrawing.pluck("generated_by_id") + CellOutput.nervos_dao_withdrawing.pluck("consumed_by_id").compact
     CkbTransaction.where(id: ckb_transaction_ids.uniq)
   end
+
+  def estimated_apc(deposit_epoch, deposited_epochs = EPOCHS_IN_ONE_NATURAL_YEAR)
+    start_epoch_number = deposit_epoch.number
+    end_epoch_number = start_epoch_number + deposited_epochs - 1
+    checkpoint_start =(start_epoch_number / EPOCHS_IN_PERIOD).ceil * EPOCHS_IN_PERIOD
+    checkpoint_end = (end_epoch_number / EPOCHS_IN_PERIOD).floor * EPOCHS_IN_PERIOD
+    checkpoints_size = (checkpoint_end - checkpoint_start) / EPOCHS_IN_PERIOD + 1
+    checkpoints = checkpoints_size.to_i.times.map { |index| (index * EPOCHS_IN_PERIOD + checkpoint_start - 1).to_i }
+
+    checkpoints.unshift(start_epoch_number.to_i) if checkpoints.empty? || checkpoints[0] > start_epoch_number
+    checkpoints.push(end_epoch_number.to_i) if checkpoints.last < end_epoch_number
+    end_epoch_numbers = checkpoints[1..-1]
+    rates = end_epoch_numbers.each_with_index.map do |inner_end_epoch_number, index|
+      epoch_index = deposit_epoch.index * 1800 / deposit_epoch.length
+      start_epoch = OpenStruct.new(number: checkpoints[index], index: epoch_index, length: 1800)
+      end_epoch = OpenStruct.new(number: inner_end_epoch_number, index: epoch_index, length: 1800)
+      rate(start_epoch, end_epoch)
+    end
+
+    rates.reduce(1) { |memo, rate| memo * (1 + rate) } - 1
+  end
+
+  private
+
+  def alpha(start_epoch_number)
+    i = ((start_epoch_number + 1) / EPOCHS_IN_PERIOD).floor
+    p = PRIMARY_ISSUANCE_PER_YEAR_BASE / 2 ** i / 2190
+    p / SECONDARY_ISSUANCE_PER_EPOCH
+  end
+
+  def rate(start_epoch, end_epoch)
+    alpha = alpha(start_epoch.number)
+    sn = SECONDARY_ISSUANCE_PER_EPOCH * ((end_epoch.number + end_epoch.index / end_epoch.length) - (start_epoch.number + start_epoch.index / start_epoch.length))
+    Math.log(1 + (alpha + 1) * sn / total_issuance(start_epoch)) / (alpha + 1)
+  end
+
+  def total_issuance(start_epoch)
+    primary_issuance(start_epoch) + secondary_issuance(start_epoch)
+  end
+
+  def primary_issuance(start_epoch)
+    epochs = (start_epoch.number / EPOCHS_IN_PERIOD).floor
+    epochs.times.reduce(GENESIS_ISSUANCE) { |memo, item| memo + (ANNUAL_PRIMARY_ISSUANCE_BASE * YEARS_IN_PERIOD) / 2 ** item } \
+      + (ANNUAL_PRIMARY_ISSUANCE_BASE * ((start_epoch.number + 1 - epochs * EPOCHS_IN_PERIOD) / EPOCHS_IN_ONE_NATURAL_YEAR)) / 2 ** epochs
+  end
+
+  def secondary_issuance(start_epoch)
+    deposit_fraction = start_epoch.number + (start_epoch.index / start_epoch.length)
+    epochs = deposit_fraction > 0 ? (deposit_fraction + 1) : deposit_fraction
+    epochs * SECONDARY_ISSUANCE_PER_EPOCH
+  end
 end
 
 # == Schema Information
diff --git a/app/workers/charts/daily_statistic.rb b/app/workers/charts/daily_statistic.rb
index 91cd49d3e..860e4d46c 100644
--- a/app/workers/charts/daily_statistic.rb
+++ b/app/workers/charts/daily_statistic.rb
@@ -5,7 +5,7 @@ class DailyStatistic
     sidekiq_options retry: false
     sidekiq_options queue: "critical"
 
-    MILLISECONDS_IN_DAY = BigDecimal.new(24 * 60 * 60 * 1000)
+    MILLISECONDS_IN_DAY = BigDecimal(24 * 60 * 60 * 1000)
 
     def perform(datetime = nil)
       to_be_counted_date = datetime.presence || DateTime.now - 1.day

From 3b6874280a761aa62d91127025c6752315b40bec Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Sat, 21 Dec 2019 13:30:03 +0800
Subject: [PATCH 44/77] feat: add ratio scale

---
 app/models/dao_contract.rb | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/app/models/dao_contract.rb b/app/models/dao_contract.rb
index 1ee6bca91..bf0b57dfb 100644
--- a/app/models/dao_contract.rb
+++ b/app/models/dao_contract.rb
@@ -21,13 +21,19 @@ def ckb_transactions
   def estimated_apc(deposit_epoch, deposited_epochs = EPOCHS_IN_ONE_NATURAL_YEAR)
     start_epoch_number = deposit_epoch.number
     end_epoch_number = start_epoch_number + deposited_epochs - 1
-    checkpoint_start =(start_epoch_number / EPOCHS_IN_PERIOD).ceil * EPOCHS_IN_PERIOD
-    checkpoint_end = (end_epoch_number / EPOCHS_IN_PERIOD).floor * EPOCHS_IN_PERIOD
+    scaled_end_epoch_number = end_epoch_number
+    ratio = (end_epoch_number - start_epoch_number) / EPOCHS_IN_ONE_NATURAL_YEAR
+    if ratio < 1
+      scaled_end_epoch_number = start_epoch_number + EPOCHS_IN_ONE_NATURAL_YEAR - 1
+      ratio = 1
+    end
+    checkpoint_start = ((start_epoch_number + 1) / EPOCHS_IN_PERIOD).ceil * EPOCHS_IN_PERIOD
+    checkpoint_end = ((scaled_end_epoch_number + 1) / EPOCHS_IN_PERIOD).floor * EPOCHS_IN_PERIOD
     checkpoints_size = (checkpoint_end - checkpoint_start) / EPOCHS_IN_PERIOD + 1
     checkpoints = checkpoints_size.to_i.times.map { |index| (index * EPOCHS_IN_PERIOD + checkpoint_start - 1).to_i }
 
     checkpoints.unshift(start_epoch_number.to_i) if checkpoints.empty? || checkpoints[0] > start_epoch_number
-    checkpoints.push(end_epoch_number.to_i) if checkpoints.last < end_epoch_number
+    checkpoints.push(scaled_end_epoch_number.to_i) if checkpoints.last < scaled_end_epoch_number
     end_epoch_numbers = checkpoints[1..-1]
     rates = end_epoch_numbers.each_with_index.map do |inner_end_epoch_number, index|
       epoch_index = deposit_epoch.index * 1800 / deposit_epoch.length
@@ -35,8 +41,8 @@ def estimated_apc(deposit_epoch, deposited_epochs = EPOCHS_IN_ONE_NATURAL_YEAR)
       end_epoch = OpenStruct.new(number: inner_end_epoch_number, index: epoch_index, length: 1800)
       rate(start_epoch, end_epoch)
     end
-
-    rates.reduce(1) { |memo, rate| memo * (1 + rate) } - 1
+    rate = rates.reduce(1) { |memo, rate| memo * (1 + rate) } - 1
+    (rate * 100) / ratio
   end
 
   private

From 4a4f04f1c1cf5b6930af94ac1d11cf66df3142f5 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Sat, 21 Dec 2019 13:49:27 +0800
Subject: [PATCH 45/77] feat: calculate estimated apc

---
 app/models/block.rb                   | 4 ++++
 app/workers/charts/daily_statistic.rb | 5 +++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/models/block.rb b/app/models/block.rb
index 2515f1b5d..1cb486514 100644
--- a/app/models/block.rb
+++ b/app/models/block.rb
@@ -60,6 +60,10 @@ def block_index_in_epoch
      number - start_number
   end
 
+  def fraction_epoch
+    OpenStruct.new(number: epoch, index: block_index_in_epoch, length: length)
+  end
+
   def self.find_block!(query_key)
     cached_find(query_key) || raise(Api::V1::Exceptions::BlockNotFoundError)
   end
diff --git a/app/workers/charts/daily_statistic.rb b/app/workers/charts/daily_statistic.rb
index 860e4d46c..2ca5fd390 100644
--- a/app/workers/charts/daily_statistic.rb
+++ b/app/workers/charts/daily_statistic.rb
@@ -20,14 +20,15 @@ def perform(datetime = nil)
       current_tip_block = Block.where("timestamp <= ?", ended_at).recent.first
       mining_reward = Block.where("timestamp <= ?", ended_at).sum(:secondary_reward)
       deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs)
-
+      estimated_apc = DaoContract.default_contract.estimated_apc(current_tip_block.fraction_epoch)
       block_timestamp = Block.created_after(started_at).created_before(ended_at).recent.pick(:timestamp)
       daily_statistic = ::DailyStatistic.create_or_find_by!(block_timestamp: block_timestamp)
       daily_statistic.update(created_at_unixtimestamp: to_be_counted_date.to_i, transactions_count: daily_ckb_transactions_count,
                              addresses_count: addresses_count, total_dao_deposit: total_dao_deposit,
                              dao_depositors_count: dao_depositors_count, unclaimed_compensation: unclaimed_compensation(cell_outputs, current_tip_block),
                              claimed_compensation: claimed_compensation(cell_outputs), average_deposit_time: average_deposit_time(cell_outputs),
-                             mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount(cell_outputs, current_tip_block))
+                             mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount(cell_outputs, current_tip_block),
+                             estimated_apc: estimated_apc)
     end
 
     private

From 87f2f1a88d43ce72964d7d675301f120084daa01 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Sat, 21 Dec 2019 13:58:35 +0800
Subject: [PATCH 46/77] test: estimated apc

---
 test/models/dao_contract_test.rb | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb
index 54e9c0d32..f5886f069 100644
--- a/test/models/dao_contract_test.rb
+++ b/test/models/dao_contract_test.rb
@@ -27,4 +27,27 @@ class DaoContractTest < ActiveSupport::TestCase
     expected_attributes = %w(created_at deposit_transactions_count depositors_count id interest_granted total_deposit total_depositors_count updated_at withdraw_transactions_count)
     assert_equal expected_attributes, dao_contract.attributes.keys.sort
   end
+
+  test "estimated apc when deposit period is less than one year" do
+    dao_contract = DaoContract.default_contract
+    expected_estimated_apc = 3.7
+    deposit_epoch = OpenStruct.new(number: 0, index:0, length: 1800)
+    assert_equal expected_estimated_apc, dao_contract.estimated_apc(deposit_epoch, 2190 * 0.19).round(2)
+  end
+
+  test "estimated apc when deposit period is one year cross period" do
+    dao_contract = DaoContract.default_contract
+    expected_estimated_apc = 2.44
+    deposit_epoch = OpenStruct.new(number: 2190 * 3.5, index:0, length: 1800)
+
+    assert_equal expected_estimated_apc, dao_contract.estimated_apc(deposit_epoch).round(2)
+  end
+
+  test "estimated apc when deposit period is more than four year" do
+    dao_contract = DaoContract.default_contract
+    expected_estimated_apc = 2.94
+    deposit_epoch = OpenStruct.new(number: 0, index:0, length: 1800)
+
+    assert_equal expected_estimated_apc, dao_contract.estimated_apc(deposit_epoch, 2190 * 5.5).round(2)
+  end
 end

From 6369a86e657e268be3fef6b2ece3cad4ed6a982b Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Sun, 22 Dec 2019 10:12:55 +0800
Subject: [PATCH 47/77] feat: show more attributes on dao contract

---
 app/models/dao_contract.rb                 | 48 ++++++++++++++++++++++
 app/serializers/dao_contract_serializer.rb | 38 ++++++++++++-----
 2 files changed, 76 insertions(+), 10 deletions(-)

diff --git a/app/models/dao_contract.rb b/app/models/dao_contract.rb
index bf0b57dfb..5e0a35e8e 100644
--- a/app/models/dao_contract.rb
+++ b/app/models/dao_contract.rb
@@ -45,8 +45,56 @@ def estimated_apc(deposit_epoch, deposited_epochs = EPOCHS_IN_ONE_NATURAL_YEAR)
     (rate * 100) / ratio
   end
 
+  def deposit_changes
+    total_deposit - latest_daily_statistic.total_dao_deposit.to_i
+  end
+
+  def depositor_changes
+    depositors_count - latest_daily_statistic.dao_depositors_count.to_i
+  end
+
+  def unclaimed_compensation_changes
+    latest_daily_statistic.unclaimed_compensation.to_i - penultimate_daily_statistic.unclaimed_compensation.to_i
+  end
+
+  def claimed_compensation_changes
+    latest_daily_statistic.claimed_compensation.to_i - penultimate_daily_statistic.claimed_compensation.to_i
+  end
+
+  def unclaimed_compensation
+    latest_daily_statistic.unclaimed_compensation
+  end
+
+  def claimed_compensation
+    latest_daily_statistic.claimed_compensation
+  end
+
+  def average_deposit_time
+    latest_daily_statistic.average_deposit_time
+  end
+
+  def mining_reward
+    latest_daily_statistic.mining_reward
+  end
+
+  def deposit_compensation
+    latest_daily_statistic.deposit_compensation
+  end
+
+  def treasury_amount
+    latest_daily_statistic.treasury_amount
+  end
+
   private
 
+  def latest_daily_statistic
+    @latest_daily_statistic ||= DailyStatistic.order(id: :desc).first || OpenStruct.new(total_dao_deposit: 0, dao_depositors_count: 0, unclaimed_compensation: 0, claimed_compensation: 0, average_deposit_time: 0, mining_reward: 0, deposit_compensation: 0, treasury_amount: 0)
+  end
+
+  def penultimate_daily_statistic
+    @penultimate_daily_statistic ||= DailyStatistic.order(id: :desc).second || OpenStruct.new(total_dao_deposit: 0, dao_depositors_count: 0, unclaimed_compensation: 0, claimed_compensation: 0, average_deposit_time: 0, mining_reward: 0, deposit_compensation: 0, treasury_amount: 0)
+  end
+
   def alpha(start_epoch_number)
     i = ((start_epoch_number + 1) / EPOCHS_IN_PERIOD).floor
     p = PRIMARY_ISSUANCE_PER_YEAR_BASE / 2 ** i / 2190
diff --git a/app/serializers/dao_contract_serializer.rb b/app/serializers/dao_contract_serializer.rb
index 911323c51..8e6eedd42 100644
--- a/app/serializers/dao_contract_serializer.rb
+++ b/app/serializers/dao_contract_serializer.rb
@@ -7,19 +7,37 @@ class DaoContractSerializer
   attribute :total_deposit do |object|
     object.total_deposit.to_s
   end
-  attribute :interest_granted do |object|
-    object.interest_granted.to_s
+  attribute :depositors_count do |object|
+    object.depositors_count.to_s
   end
-  attribute :deposit_transactions_count do |object|
-    object.deposit_transactions_count.to_s
+  attribute :deposit_changes do |object|
+    object.deposit_changes.to_s
   end
-  attribute :withdraw_transactions_count do |object|
-    object.withdraw_transactions_count.to_s
+  attribute :unclaimed_compensation_changes do |object|
+    object.unclaimed_compensation_changes.to_s
   end
-  attribute :depositors_count do |object|
-    object.depositors_count.to_s
+  attribute :claimed_compensation_changes do |object|
+    object.claimed_compensation_changes.to_s
+  end
+  attribute :depositor_changes do |object|
+    object.depositor_changes.to_s
+  end
+  attribute :unclaimed_compensation do |object|
+    object.unclaimed_compensation.to_s
+  end
+  attribute :claimed_compensation do |object|
+    object.claimed_compensation.to_s
+  end
+  attribute :average_deposit_time do |object|
+    object.average_deposit_time.to_s
+  end
+  attribute :mining_reward do |object|
+    object.mining_reward.to_s
+  end
+  attribute :deposit_compensation do |object|
+    object.deposit_compensation.to_s
   end
-  attribute :total_depositors_count do |object|
-    object.total_depositors_count.to_s
+  attribute :treasury_amount do |object|
+    object.treasury_amount.to_s
   end
 end

From ec14b45fdc4d59ae581423e582f3a4b0d02276ab Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Sun, 22 Dec 2019 15:59:31 +0800
Subject: [PATCH 48/77] chore: deduct interest

---
 app/models/dao_contract.rb                 | 14 +++++++++-----
 app/serializers/dao_contract_serializer.rb |  6 +++---
 app/workers/charts/daily_statistic.rb      |  4 ++--
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/app/models/dao_contract.rb b/app/models/dao_contract.rb
index 5e0a35e8e..82e9721a4 100644
--- a/app/models/dao_contract.rb
+++ b/app/models/dao_contract.rb
@@ -18,7 +18,7 @@ def ckb_transactions
     CkbTransaction.where(id: ckb_transaction_ids.uniq)
   end
 
-  def estimated_apc(deposit_epoch, deposited_epochs = EPOCHS_IN_ONE_NATURAL_YEAR)
+  def estimated_apc(deposit_epoch = tip_block_fraction_epoch, deposited_epochs = EPOCHS_IN_ONE_NATURAL_YEAR)
     start_epoch_number = deposit_epoch.number
     end_epoch_number = start_epoch_number + deposited_epochs - 1
     scaled_end_epoch_number = end_epoch_number
@@ -46,19 +46,19 @@ def estimated_apc(deposit_epoch, deposited_epochs = EPOCHS_IN_ONE_NATURAL_YEAR)
   end
 
   def deposit_changes
-    total_deposit - latest_daily_statistic.total_dao_deposit.to_i
+    total_deposit - latest_daily_statistic.total_dao_deposit.to_d
   end
 
   def depositor_changes
-    depositors_count - latest_daily_statistic.dao_depositors_count.to_i
+    depositors_count - latest_daily_statistic.dao_depositors_count.to_d
   end
 
   def unclaimed_compensation_changes
-    latest_daily_statistic.unclaimed_compensation.to_i - penultimate_daily_statistic.unclaimed_compensation.to_i
+    latest_daily_statistic.unclaimed_compensation.to_d - penultimate_daily_statistic.unclaimed_compensation.to_d
   end
 
   def claimed_compensation_changes
-    latest_daily_statistic.claimed_compensation.to_i - penultimate_daily_statistic.claimed_compensation.to_i
+    latest_daily_statistic.claimed_compensation.to_d - penultimate_daily_statistic.claimed_compensation.to_d
   end
 
   def unclaimed_compensation
@@ -87,6 +87,10 @@ def treasury_amount
 
   private
 
+  def tip_block_fraction_epoch
+    Block.recent.first.fraction_epoch
+  end
+
   def latest_daily_statistic
     @latest_daily_statistic ||= DailyStatistic.order(id: :desc).first || OpenStruct.new(total_dao_deposit: 0, dao_depositors_count: 0, unclaimed_compensation: 0, claimed_compensation: 0, average_deposit_time: 0, mining_reward: 0, deposit_compensation: 0, treasury_amount: 0)
   end
diff --git a/app/serializers/dao_contract_serializer.rb b/app/serializers/dao_contract_serializer.rb
index 8e6eedd42..ef7080e21 100644
--- a/app/serializers/dao_contract_serializer.rb
+++ b/app/serializers/dao_contract_serializer.rb
@@ -1,9 +1,6 @@
 class DaoContractSerializer
   include FastJsonapi::ObjectSerializer
   set_type :nervos_dao
-  attribute :dao_type_hash do
-    ENV["DAO_TYPE_HASH"]
-  end
   attribute :total_deposit do |object|
     object.total_deposit.to_s
   end
@@ -40,4 +37,7 @@ class DaoContractSerializer
   attribute :treasury_amount do |object|
     object.treasury_amount.to_s
   end
+  attribute :estimated_apc do |object|
+    object.estimated_apc.to_s
+  end
 end
diff --git a/app/workers/charts/daily_statistic.rb b/app/workers/charts/daily_statistic.rb
index 2ca5fd390..3b38601cc 100644
--- a/app/workers/charts/daily_statistic.rb
+++ b/app/workers/charts/daily_statistic.rb
@@ -22,7 +22,7 @@ def perform(datetime = nil)
       deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs)
       estimated_apc = DaoContract.default_contract.estimated_apc(current_tip_block.fraction_epoch)
       block_timestamp = Block.created_after(started_at).created_before(ended_at).recent.pick(:timestamp)
-      daily_statistic = ::DailyStatistic.create_or_find_by!(block_timestamp: block_timestamp)
+      daily_statistic = ::DailyStatistic.find_or_create_by!(block_timestamp: block_timestamp)
       daily_statistic.update(created_at_unixtimestamp: to_be_counted_date.to_i, transactions_count: daily_ckb_transactions_count,
                              addresses_count: addresses_count, total_dao_deposit: total_dao_deposit,
                              dao_depositors_count: dao_depositors_count, unclaimed_compensation: unclaimed_compensation(cell_outputs, current_tip_block),
@@ -63,7 +63,7 @@ def unmade_dao_interests(cell_outputs, current_tip_block)
             tip_dao = current_tip_block.dao
             parse_dao = CkbUtils.parse_dao(dao)
             tip_parse_dao = CkbUtils.parse_dao(tip_dao)
-            memo + cell_output.capacity * tip_parse_dao.ar_i / parse_dao.ar_i
+            memo + (cell_output.capacity * tip_parse_dao.ar_i / parse_dao.ar_i) - cell_output.capacity
           end
         end
     end

From ca459a70aa89362f987e55f37b8088c6a694b939 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Sun, 22 Dec 2019 21:00:32 +0800
Subject: [PATCH 49/77] chore: update api doc

---
 doc/api.raml        | 70 +++++++++++++++++++++++++++++++++++++++++----
 public/api_doc.html | 21 +++++++++-----
 2 files changed, 78 insertions(+), 13 deletions(-)

diff --git a/doc/api.raml b/doc/api.raml
index 153c531d4..56882ecb1 100644
--- a/doc/api.raml
+++ b/doc/api.raml
@@ -617,6 +617,48 @@ types:
     }
   }
 
+  dao_contract_attributes: {
+    type: "object",
+    properties: {
+      "total_deposit": {
+        type: "string"
+      },
+      "deposit_changes": {
+        type: "string"
+      },
+      "depositors_count": {
+        type: "string"
+      },
+      "depositor_changes": {
+        type: "string"
+      },
+      "unclaimed_compensation": {
+        type: "string"
+      },
+      "unclaimed_compensation_changes": {
+        type: "string"
+      },
+      "claimed_compensation": {
+        type: "string"
+      },
+      "claimed_compensation_changes": {
+        type: "string"
+      },
+      "average_deposit_time": {
+        type: "string"
+      },
+      "mining_reward": {
+        type: "string"
+      },
+      "deposit_compensation": {
+        type: "string"
+      },
+      "treasury_amount": {
+        type: "string"
+      }
+    }
+  }
+
   index_statistic_info: {
     type: object,
     properties: {
@@ -707,6 +749,15 @@ types:
     }
   }
 
+  dao_contract_response: {
+    type: object,
+    properties: {
+      "data": {
+        type: dao_contract_attributes
+      }
+    }
+  }
+
   transaction_response: {
     type: object,
     properties: {
@@ -2697,12 +2748,19 @@ types:
                     "id": 1,
                     "type": "dao_contract",
                     "attributes": {
-                      "total_deposit": 1000000,
-                      "interest_granted": 1000,
-                      "deposit_transactions_count": 10,
-                      "withdraw_transactions_count": 1,
-                      "depositors_count": 3,
-                      "total_depositors_count": 10
+                      "average_deposit_time": "22.713185749893621618961218396792431688516315506487629618865127513103881141164341287",
+                      "claimed_compensation": "15011368431049",
+                      "claimed_compensation_changes": "15011368431049.0",
+                      "deposit_changes": "849537119739722.0",
+                      "deposit_compensation": "352265740710938.342252952052525667520572168005043521142110644336070405",
+                      "depositor_changes": "774",
+                      "depositors_count": "774",
+                      "mining_reward": "1814175662411888.0",
+                      "total_deposit": "142247249542395069",
+                      "treasury_amount": "10008525246180062.657747047947474332479427831994956478857889355663929595",
+                      "unclaimed_compensation": "337254372279889.342252952052525667520572168005043521142110644336070405",
+                      "unclaimed_compensation_changes": "337254372279889.342252952052525667520572168005043521142110644336070405",
+                      "estimated_apc": "3.6500101241879927272727272727272723668144763717481197667116764738292367"
                     }
                   }
                 }
diff --git a/public/api_doc.html b/public/api_doc.html
index c25317f74..56fb953c2 100644
--- a/public/api_doc.html
+++ b/public/api_doc.html
@@ -1587,17 +1587,24 @@
     }
   ]
 }
-

/contracts

get

get

Returns a contract by contract name

Returns a contract by contract name

URI Parameters

  • contract_name: required(string)

    The Contract name

HTTP status code 200

Body

Media type: application/vnd.api+json

Type: dao_contract_response

Example:

{
+

/contracts

get

get

Returns a contract by contract name

Returns a contract by contract name

URI Parameters

  • contract_name: required(string)

    The Contract name

HTTP status code 200

Body

Media type: application/vnd.api+json

Type: object

Properties
  • data: required(object)
    • total_deposit: required(string)
    • deposit_changes: required(string)
    • depositors_count: required(string)
    • depositor_changes: required(string)
    • unclaimed_compensation: required(string)
    • unclaimed_compensation_changes: required(string)
    • claimed_compensation: required(string)
    • claimed_compensation_changes: required(string)
    • average_deposit_time: required(string)
    • mining_reward: required(string)
    • deposit_compensation: required(string)
    • treasury_amount: required(string)

Example:

{
   "data": {
     "id": 1,
     "type": "dao_contract",
     "attributes": {
-      "total_deposit": 1000000,
-      "interest_granted": 1000,
-      "deposit_transactions_count": 10,
-      "withdraw_transactions_count": 1,
-      "depositors_count": 3,
-      "total_depositors_count": 10
+      "average_deposit_time": "22.713185749893621618961218396792431688516315506487629618865127513103881141164341287",
+      "claimed_compensation": "15011368431049",
+      "claimed_compensation_changes": "15011368431049.0",
+      "deposit_changes": "849537119739722.0",
+      "deposit_compensation": "352265740710938.342252952052525667520572168005043521142110644336070405",
+      "depositor_changes": "774",
+      "depositors_count": "774",
+      "mining_reward": "1814175662411888.0",
+      "total_deposit": "142247249542395069",
+      "treasury_amount": "10008525246180062.657747047947474332479427831994956478857889355663929595",
+      "unclaimed_compensation": "337254372279889.342252952052525667520572168005043521142110644336070405",
+      "unclaimed_compensation_changes": "337254372279889.342252952052525667520572168005043521142110644336070405",
+      "estimated_apc": "3.6500101241879927272727272727272723668144763717481197667116764738292367"
     }
   }
 }

From cf58ecc208e8abba958cc7e81afe0b528513d66f Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Sun, 22 Dec 2019 22:29:10 +0800
Subject: [PATCH 50/77] chore: adjust tests

---
 app/models/dao_contract.rb                           | 2 +-
 test/controllers/api/v1/contracts_controller_test.rb | 3 +--
 test/models/dao_event_test.rb                        | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/app/models/dao_contract.rb b/app/models/dao_contract.rb
index 82e9721a4..009374dc1 100644
--- a/app/models/dao_contract.rb
+++ b/app/models/dao_contract.rb
@@ -88,7 +88,7 @@ def treasury_amount
   private
 
   def tip_block_fraction_epoch
-    Block.recent.first.fraction_epoch
+    Block.recent.first&.fraction_epoch || OpenStruct.new(number: 0, index: 0, length: 1800)
   end
 
   def latest_daily_statistic
diff --git a/test/controllers/api/v1/contracts_controller_test.rb b/test/controllers/api/v1/contracts_controller_test.rb
index ae10bc8e0..895166758 100644
--- a/test/controllers/api/v1/contracts_controller_test.rb
+++ b/test/controllers/api/v1/contracts_controller_test.rb
@@ -56,8 +56,7 @@ class ContractsControllerTest < ActionDispatch::IntegrationTest
 
         response_contract = json["data"]
         assert_equal %w(
-          dao_type_hash total_deposit interest_granted deposit_transactions_count withdraw_transactions_count
-          depositors_count total_depositors_count
+           average_deposit_time claimed_compensation claimed_compensation_changes deposit_changes deposit_compensation depositor_changes depositors_count estimated_apc mining_reward total_deposit treasury_amount unclaimed_compensation unclaimed_compensation_changes
         ).sort, response_contract["attributes"].keys.sort
       end
 
diff --git a/test/models/dao_event_test.rb b/test/models/dao_event_test.rb
index fbd29cd1c..d2a7f9199 100644
--- a/test/models/dao_event_test.rb
+++ b/test/models/dao_event_test.rb
@@ -7,7 +7,7 @@ class DaoEventTest < ActiveSupport::TestCase
 
   test "should have correct columns" do
     dao_event = create(:dao_event)
-    expected_attributes = %w(address_id block_id contract_id created_at event_type id status ckb_transaction_id updated_at value).sort
+    expected_attributes = %w(address_id block_id block_timestamp contract_id created_at event_type id status ckb_transaction_id updated_at value).sort
     assert_equal expected_attributes, dao_event.attributes.keys.sort
   end
 end

From 1f8ea4c6b5f852ef513bb8cbcc75ddd7fabba5ea Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Tue, 24 Dec 2019 15:21:24 +0800
Subject: [PATCH 51/77] test: deposit_changes

---
 test/factories/dao_contract.rb   | 2 ++
 test/models/dao_contract_test.rb | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/test/factories/dao_contract.rb b/test/factories/dao_contract.rb
index 84d5a807a..3a20895fe 100644
--- a/test/factories/dao_contract.rb
+++ b/test/factories/dao_contract.rb
@@ -1,5 +1,7 @@
 FactoryBot.define do
   factory :dao_contract do
     total_deposit { 10**8 * 1_000_000 }
+    depositors_count { 1000 }
+
   end
 end
diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb
index f5886f069..d84350c17 100644
--- a/test/models/dao_contract_test.rb
+++ b/test/models/dao_contract_test.rb
@@ -50,4 +50,13 @@ class DaoContractTest < ActiveSupport::TestCase
 
     assert_equal expected_estimated_apc, dao_contract.estimated_apc(deposit_epoch, 2190 * 5.5).round(2)
   end
+
+  test "deposit_changes should return difference between yesterday and today" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create(:daily_statistic)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+    expected_deposit_changes = dao_contract.total_deposit - latest_daily_statistic.total_dao_deposit.to_d
+
+    assert_equal expected_deposit_changes, dao_contract.deposit_changes
+  end
 end

From 99e3ce3f215a704637e03247caff2a511a2e4055 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Tue, 24 Dec 2019 16:12:20 +0800
Subject: [PATCH 52/77] test: depositor_changes

---
 test/models/dao_contract_test.rb | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb
index d84350c17..748574bb5 100644
--- a/test/models/dao_contract_test.rb
+++ b/test/models/dao_contract_test.rb
@@ -59,4 +59,13 @@ class DaoContractTest < ActiveSupport::TestCase
 
     assert_equal expected_deposit_changes, dao_contract.deposit_changes
   end
+
+  test "depositor_changes should return difference between yesterday and today" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create(:daily_statistic)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+    expected_depositor_changes = dao_contract.depositors_count - latest_daily_statistic.dao_depositors_count.to_d
+
+    assert_equal expected_depositor_changes, dao_contract.depositor_changes
+  end
 end

From 4a404f406c3d96f285f41d64929f583a9f809f73 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Tue, 24 Dec 2019 17:40:36 +0800
Subject: [PATCH 53/77] test: unclaimed_compensation_changes

---
 test/models/dao_contract_test.rb | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb
index 748574bb5..0cb3aff32 100644
--- a/test/models/dao_contract_test.rb
+++ b/test/models/dao_contract_test.rb
@@ -51,7 +51,7 @@ class DaoContractTest < ActiveSupport::TestCase
     assert_equal expected_estimated_apc, dao_contract.estimated_apc(deposit_epoch, 2190 * 5.5).round(2)
   end
 
-  test "deposit_changes should return difference between yesterday and today" do
+  test "deposit_changes should return difference between beginning of today and current" do
     dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
     create(:daily_statistic)
     latest_daily_statistic = DailyStatistic.order(id: :desc).first
@@ -60,7 +60,7 @@ class DaoContractTest < ActiveSupport::TestCase
     assert_equal expected_deposit_changes, dao_contract.deposit_changes
   end
 
-  test "depositor_changes should return difference between yesterday and today" do
+  test "depositor_changes should return difference between beginning of today and current" do
     dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
     create(:daily_statistic)
     latest_daily_statistic = DailyStatistic.order(id: :desc).first
@@ -68,4 +68,14 @@ class DaoContractTest < ActiveSupport::TestCase
 
     assert_equal expected_depositor_changes, dao_contract.depositor_changes
   end
+
+  test "unclaimed_compensation_changes should return difference within 24 hours" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create_list(:daily_statistic, 2)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+    penultimate_daily_statistic = DailyStatistic.order(id: :desc).second
+    expected_unclaimed_compensation_changes = penultimate_daily_statistic.unclaimed_compensation.to_d - latest_daily_statistic.unclaimed_compensation.to_d
+
+    assert_equal expected_unclaimed_compensation_changes, dao_contract.unclaimed_compensation_changes
+  end
 end

From bb0adf05836c92abf76fae7f1b17539db7800880 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Tue, 24 Dec 2019 17:43:25 +0800
Subject: [PATCH 54/77] test: claimed_compensation_changes

---
 test/models/dao_contract_test.rb | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb
index 0cb3aff32..f56a263a1 100644
--- a/test/models/dao_contract_test.rb
+++ b/test/models/dao_contract_test.rb
@@ -78,4 +78,14 @@ class DaoContractTest < ActiveSupport::TestCase
 
     assert_equal expected_unclaimed_compensation_changes, dao_contract.unclaimed_compensation_changes
   end
+
+  test "claimed_compensation_changes should return difference within 24 hours" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create_list(:daily_statistic, 2)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+    penultimate_daily_statistic = DailyStatistic.order(id: :desc).second
+    expected_claimed_compensation_changes = penultimate_daily_statistic.claimed_compensation.to_d - latest_daily_statistic.claimed_compensation.to_d
+
+    assert_equal expected_claimed_compensation_changes, dao_contract.claimed_compensation_changes
+  end
 end

From 799f38d06505049aef3a34b2876032ce59ce2e41 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Tue, 24 Dec 2019 17:49:22 +0800
Subject: [PATCH 55/77] test: other new fields

unclaimed_compensation, claimed_compensation, average_deposit_time, mining_reward, deposit_compensation, treasury_amount
---
 test/models/dao_contract_test.rb | 48 ++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb
index f56a263a1..485afcc4d 100644
--- a/test/models/dao_contract_test.rb
+++ b/test/models/dao_contract_test.rb
@@ -88,4 +88,52 @@ class DaoContractTest < ActiveSupport::TestCase
 
     assert_equal expected_claimed_compensation_changes, dao_contract.claimed_compensation_changes
   end
+
+  test "unclaimed_compensation should return beginning of today value" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create(:daily_statistic)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+
+    assert_equal latest_daily_statistic.unclaimed_compensation, dao_contract.unclaimed_compensation
+  end
+
+  test "claimed_compensation should return beginning of today value" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create(:daily_statistic)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+
+    assert_equal latest_daily_statistic.claimed_compensation, dao_contract.claimed_compensation
+  end
+
+  test "average_deposit_time should return beginning of today value" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create(:daily_statistic)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+
+    assert_equal latest_daily_statistic.average_deposit_time, dao_contract.average_deposit_time
+  end
+
+  test "mining_reward should return beginning of today value" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create(:daily_statistic)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+
+    assert_equal latest_daily_statistic.mining_reward, dao_contract.mining_reward
+  end
+
+  test "deposit_compensation should return beginning of today value" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create(:daily_statistic)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+
+    assert_equal latest_daily_statistic.deposit_compensation, dao_contract.deposit_compensation
+  end
+
+  test "treasury_amount should return beginning of today value" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+    create(:daily_statistic)
+    latest_daily_statistic = DailyStatistic.order(id: :desc).first
+
+    assert_equal latest_daily_statistic.treasury_amount, dao_contract.treasury_amount
+  end
 end

From 273ab0e16f7b18d5070df024f3bd12e0f619ab3c Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Tue, 24 Dec 2019 17:57:36 +0800
Subject: [PATCH 56/77] test: changes when there is no daily statistic

---
 test/models/dao_contract_test.rb | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/test/models/dao_contract_test.rb b/test/models/dao_contract_test.rb
index 485afcc4d..23aa0cc49 100644
--- a/test/models/dao_contract_test.rb
+++ b/test/models/dao_contract_test.rb
@@ -57,15 +57,17 @@ class DaoContractTest < ActiveSupport::TestCase
     latest_daily_statistic = DailyStatistic.order(id: :desc).first
     expected_deposit_changes = dao_contract.total_deposit - latest_daily_statistic.total_dao_deposit.to_d
 
+    assert_not_equal dao_contract.total_deposit, dao_contract.deposit_changes
     assert_equal expected_deposit_changes, dao_contract.deposit_changes
   end
 
   test "depositor_changes should return difference between beginning of today and current" do
     dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
-    create(:daily_statistic)
+    create(:daily_statistic, dao_depositors_count: 80)
     latest_daily_statistic = DailyStatistic.order(id: :desc).first
     expected_depositor_changes = dao_contract.depositors_count - latest_daily_statistic.dao_depositors_count.to_d
 
+    assert_not_equal dao_contract.depositors_count, dao_contract.depositor_changes
     assert_equal expected_depositor_changes, dao_contract.depositor_changes
   end
 
@@ -136,4 +138,28 @@ class DaoContractTest < ActiveSupport::TestCase
 
     assert_equal latest_daily_statistic.treasury_amount, dao_contract.treasury_amount
   end
+
+  test "deposit_changes should return current value when there is no daily_statistic" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+
+    assert_equal dao_contract.total_deposit, dao_contract.deposit_changes
+  end
+
+  test "depositor_changes should return current value when there is no daily_statistic" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+
+    assert_equal dao_contract.depositors_count, dao_contract.depositor_changes
+  end
+
+  test "unclaimed_compensation_changes should return zero when there is no daily_statistic" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+
+    assert_equal 0, dao_contract.unclaimed_compensation_changes
+  end
+
+  test "claimed_compensation_changes should return zero when there is no daily_statistic" do
+    dao_contract = create(:dao_contract, total_deposit: 10**21 * 100)
+
+    assert_equal 0, dao_contract.claimed_compensation_changes
+  end
 end

From 5e7ccbb0a0cd25573ac9600a617e1b36bb7be93f Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 15:52:18 +0800
Subject: [PATCH 57/77] refactor: use daily statistic generator service

---
 .../charts/daily_statistic_generator.rb       | 96 +++++++++++++++++++
 app/workers/charts/daily_statistic.rb         | 85 +---------------
 2 files changed, 97 insertions(+), 84 deletions(-)
 create mode 100644 app/services/charts/daily_statistic_generator.rb

diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb
new file mode 100644
index 000000000..9a431cd87
--- /dev/null
+++ b/app/services/charts/daily_statistic_generator.rb
@@ -0,0 +1,96 @@
+module Charts
+  class DailyStatisticGenerator
+    MILLISECONDS_IN_DAY = BigDecimal(24 * 60 * 60 * 1000)
+
+    def initialize(datetime)
+      @datetime = datetime
+    end
+
+    def call
+      to_be_counted_date = datetime.presence || DateTime.now - 1.day
+      started_at = to_be_counted_date.beginning_of_day.strftime("%Q")
+      ended_at = to_be_counted_date.end_of_day.strftime("%Q")
+      daily_ckb_transactions_count = CkbTransaction.where("block_timestamp >= ? and block_timestamp <= ?", started_at, ended_at).count
+      addresses_count = Address.where("block_timestamp <= ?", ended_at).count
+      deposit_cells = CellOutput.where(cell_type: "nervos_dao_deposit").where("block_timestamp <= ?", ended_at)
+      total_dao_deposit = datetime.blank? ? deposit_cells.where(status: "live").sum(:capacity) : deposit_cells.sum(:capacity)
+      dao_depositors_count = DaoEvent.processed.where("block_timestamp <= ?", ended_at).count
+      cell_outputs = CellOutput.where("block_timestamp <= ?", ended_at)
+      current_tip_block = Block.where("timestamp <= ?", ended_at).recent.first
+      mining_reward = Block.where("timestamp <= ?", ended_at).sum(:secondary_reward)
+      deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs)
+      estimated_apc = DaoContract.default_contract.estimated_apc(current_tip_block.fraction_epoch)
+      block_timestamp = Block.created_after(started_at).created_before(ended_at).recent.pick(:timestamp)
+      daily_statistic = ::DailyStatistic.find_or_create_by!(block_timestamp: block_timestamp)
+      daily_statistic.update(created_at_unixtimestamp: to_be_counted_date.to_i, transactions_count: daily_ckb_transactions_count,
+                             addresses_count: addresses_count, total_dao_deposit: total_dao_deposit,
+                             dao_depositors_count: dao_depositors_count, unclaimed_compensation: unclaimed_compensation(cell_outputs, current_tip_block),
+                             claimed_compensation: claimed_compensation(cell_outputs), average_deposit_time: average_deposit_time(cell_outputs),
+                             mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount(cell_outputs, current_tip_block),
+                             estimated_apc: estimated_apc)
+    end
+
+    private
+
+    attr_reader :datetime
+
+    def unclaimed_compensation(cell_outputs, current_tip_block)
+      @unclaimed_compensation ||=
+        begin
+          phase1_dao_interests(cell_outputs) + unmade_dao_interests(cell_outputs, current_tip_block)
+        end
+    end
+
+    def claimed_compensation(cell_outputs)
+      @claimed_compensation ||=
+        begin
+          cell_outputs.nervos_dao_withdrawing.dead.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+            memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell)
+          end
+        end
+    end
+
+    def phase1_dao_interests(cell_outputs)
+      cell_outputs.nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+        memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell)
+      end
+    end
+
+    def unmade_dao_interests(cell_outputs, current_tip_block)
+      @unmade_dao_interests ||=
+        begin
+          cell_outputs.nervos_dao_deposit.live.reduce(0) do |memo, cell_output|
+            dao = cell_output.block.dao
+            tip_dao = current_tip_block.dao
+            parse_dao = CkbUtils.parse_dao(dao)
+            tip_parse_dao = CkbUtils.parse_dao(tip_dao)
+            memo + (cell_output.capacity * tip_parse_dao.ar_i / parse_dao.ar_i) - cell_output.capacity
+          end
+        end
+    end
+
+    def average_deposit_time(cell_outputs)
+      interest_bearing_deposits = 0
+      uninterest_bearing_deposits = 0
+      sum_interest_bearing = cell_outputs.nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell|
+        nervos_dao_withdrawing_cell_generated_tx = nervos_dao_withdrawing_cell.generated_by
+        nervos_dao_deposit_cell = nervos_dao_withdrawing_cell_generated_tx.cell_inputs.order(:id)[nervos_dao_withdrawing_cell.cell_index].previous_cell_output
+        interest_bearing_deposits += nervos_dao_deposit_cell.capacity
+        memo + nervos_dao_deposit_cell.capacity * (nervos_dao_withdrawing_cell.block_timestamp - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
+      end
+      sum_uninterest_bearing = cell_outputs.nervos_dao_deposit.live.reduce(0) do |memo, nervos_dao_deposit_cell|
+        current_time = DateTime.now.strftime("%Q").to_i
+        uninterest_bearing_deposits += nervos_dao_deposit_cell.capacity
+
+        memo + nervos_dao_deposit_cell.capacity * (current_time - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
+      end
+
+      (sum_interest_bearing + sum_uninterest_bearing) / (interest_bearing_deposits + uninterest_bearing_deposits)
+    end
+
+    def treasury_amount(cell_outputs, current_tip_block)
+      parse_dao = CkbUtils.parse_dao(current_tip_block.dao)
+      parse_dao.s_i - unmade_dao_interests(cell_outputs, current_tip_block)
+    end
+  end
+end
diff --git a/app/workers/charts/daily_statistic.rb b/app/workers/charts/daily_statistic.rb
index 3b38601cc..6b462b2fc 100644
--- a/app/workers/charts/daily_statistic.rb
+++ b/app/workers/charts/daily_statistic.rb
@@ -5,91 +5,8 @@ class DailyStatistic
     sidekiq_options retry: false
     sidekiq_options queue: "critical"
 
-    MILLISECONDS_IN_DAY = BigDecimal(24 * 60 * 60 * 1000)
-
     def perform(datetime = nil)
-      to_be_counted_date = datetime.presence || DateTime.now - 1.day
-      started_at = to_be_counted_date.beginning_of_day.strftime("%Q")
-      ended_at = to_be_counted_date.end_of_day.strftime("%Q")
-      daily_ckb_transactions_count = CkbTransaction.where("block_timestamp >= ? and block_timestamp <= ?", started_at, ended_at).count
-      addresses_count = Address.where("block_timestamp <= ?", ended_at).count
-      deposit_cells = CellOutput.where(cell_type: "nervos_dao_deposit").where("block_timestamp <= ?", ended_at)
-      total_dao_deposit = datetime.blank? ? deposit_cells.where(status: "live").sum(:capacity) : deposit_cells.sum(:capacity)
-      dao_depositors_count = DaoEvent.processed.where("block_timestamp <= ?", ended_at).count
-      cell_outputs = CellOutput.where("block_timestamp <= ?", ended_at)
-      current_tip_block = Block.where("timestamp <= ?", ended_at).recent.first
-      mining_reward = Block.where("timestamp <= ?", ended_at).sum(:secondary_reward)
-      deposit_compensation = unclaimed_compensation(cell_outputs, current_tip_block) + claimed_compensation(cell_outputs)
-      estimated_apc = DaoContract.default_contract.estimated_apc(current_tip_block.fraction_epoch)
-      block_timestamp = Block.created_after(started_at).created_before(ended_at).recent.pick(:timestamp)
-      daily_statistic = ::DailyStatistic.find_or_create_by!(block_timestamp: block_timestamp)
-      daily_statistic.update(created_at_unixtimestamp: to_be_counted_date.to_i, transactions_count: daily_ckb_transactions_count,
-                             addresses_count: addresses_count, total_dao_deposit: total_dao_deposit,
-                             dao_depositors_count: dao_depositors_count, unclaimed_compensation: unclaimed_compensation(cell_outputs, current_tip_block),
-                             claimed_compensation: claimed_compensation(cell_outputs), average_deposit_time: average_deposit_time(cell_outputs),
-                             mining_reward: mining_reward, deposit_compensation: deposit_compensation, treasury_amount: treasury_amount(cell_outputs, current_tip_block),
-                             estimated_apc: estimated_apc)
-    end
-
-    private
-
-    def unclaimed_compensation(cell_outputs, current_tip_block)
-      @unclaimed_compensation ||=
-        begin
-          phase1_dao_interests(cell_outputs) + unmade_dao_interests(cell_outputs, current_tip_block)
-        end
-    end
-
-    def claimed_compensation(cell_outputs)
-      @claimed_compensation ||=
-        begin
-          cell_outputs.nervos_dao_withdrawing.dead.reduce(0) do |memo, nervos_dao_withdrawing_cell|
-            memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell)
-          end
-        end
-    end
-
-    def phase1_dao_interests(cell_outputs)
-      cell_outputs.nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell|
-        memo + CkbUtils.dao_interest(nervos_dao_withdrawing_cell)
-      end
-    end
-
-    def unmade_dao_interests(cell_outputs, current_tip_block)
-      @unmade_dao_interests ||=
-        begin
-          cell_outputs.nervos_dao_deposit.live.reduce(0) do |memo, cell_output|
-            dao = cell_output.block.dao
-            tip_dao = current_tip_block.dao
-            parse_dao = CkbUtils.parse_dao(dao)
-            tip_parse_dao = CkbUtils.parse_dao(tip_dao)
-            memo + (cell_output.capacity * tip_parse_dao.ar_i / parse_dao.ar_i) - cell_output.capacity
-          end
-        end
-    end
-
-    def average_deposit_time(cell_outputs)
-      interest_bearing_deposits = 0
-      uninterest_bearing_deposits = 0
-      sum_interest_bearing = cell_outputs.nervos_dao_withdrawing.live.reduce(0) do |memo, nervos_dao_withdrawing_cell|
-        nervos_dao_withdrawing_cell_generated_tx = nervos_dao_withdrawing_cell.generated_by
-        nervos_dao_deposit_cell = nervos_dao_withdrawing_cell_generated_tx.cell_inputs.order(:id)[nervos_dao_withdrawing_cell.cell_index].previous_cell_output
-        interest_bearing_deposits += nervos_dao_deposit_cell.capacity
-        memo + nervos_dao_deposit_cell.capacity * (nervos_dao_withdrawing_cell.block_timestamp - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
-      end
-      sum_uninterest_bearing = cell_outputs.nervos_dao_deposit.live.reduce(0) do |memo, nervos_dao_deposit_cell|
-        current_time = DateTime.now.strftime("%Q").to_i
-        uninterest_bearing_deposits += nervos_dao_deposit_cell.capacity
-
-        memo + nervos_dao_deposit_cell.capacity * (current_time - nervos_dao_deposit_cell.block_timestamp) / MILLISECONDS_IN_DAY
-      end
-
-      (sum_interest_bearing + sum_uninterest_bearing) / (interest_bearing_deposits + uninterest_bearing_deposits)
-    end
-
-    def treasury_amount(cell_outputs, current_tip_block)
-      parse_dao = CkbUtils.parse_dao(current_tip_block.dao)
-      parse_dao.s_i - unmade_dao_interests(cell_outputs, current_tip_block)
+      Charts::DailyStatisticGenerator.new(datetime).call
     end
   end
 end

From dd90cc9034f74dcb48d6758019e79a15bf72c0f2 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 15:52:38 +0800
Subject: [PATCH 58/77] test: daily statistic worker

---
 test/workers/charts/daily_statistic_test.rb | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 test/workers/charts/daily_statistic_test.rb

diff --git a/test/workers/charts/daily_statistic_test.rb b/test/workers/charts/daily_statistic_test.rb
new file mode 100644
index 000000000..618edd129
--- /dev/null
+++ b/test/workers/charts/daily_statistic_test.rb
@@ -0,0 +1,12 @@
+require "test_helper"
+
+module Charts
+  class DailyStatisticTest < ActiveSupport::TestCase
+    test "daily statistic job should enqueue critical queue" do
+      assert_difference -> { Charts::DailyStatistic.jobs.size }, 1 do
+        Charts::DailyStatistic.perform_async
+      end
+      assert_equal "critical", Charts::DailyStatistic.queue
+    end
+  end
+end

From 7aa6305627cec4cd89f7ab356afc39d6be4a7550 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Wed, 25 Dec 2019 16:31:21 +0800
Subject: [PATCH 59/77] test: daily statistic generator service should create
 new record

---
 app/services/charts/daily_statistic_generator.rb   |  2 +-
 .../charts/daily_statistic_generator_test.rb       | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 test/services/charts/daily_statistic_generator_test.rb

diff --git a/app/services/charts/daily_statistic_generator.rb b/app/services/charts/daily_statistic_generator.rb
index 9a431cd87..df2a533e4 100644
--- a/app/services/charts/daily_statistic_generator.rb
+++ b/app/services/charts/daily_statistic_generator.rb
@@ -2,7 +2,7 @@ module Charts
   class DailyStatisticGenerator
     MILLISECONDS_IN_DAY = BigDecimal(24 * 60 * 60 * 1000)
 
-    def initialize(datetime)
+    def initialize(datetime = nil)
       @datetime = datetime
     end
 
diff --git a/test/services/charts/daily_statistic_generator_test.rb b/test/services/charts/daily_statistic_generator_test.rb
new file mode 100644
index 000000000..328429f69
--- /dev/null
+++ b/test/services/charts/daily_statistic_generator_test.rb
@@ -0,0 +1,14 @@
+require "test_helper"
+
+module Charts
+  class DailyStatisticGeneratorTest < ActiveSupport::TestCase
+    test "should create daily statistic record" do
+      block = create(:block, dao: "0xaff1568bbe49672f8a02516252ab2300df8c9e15dad428000035a1d671700007")
+      tx = create(:ckb_transaction, block: block)
+      create(:cell_output, cell_type: "nervos_dao_deposit", generated_by: tx, ckb_transaction: tx, block: block, capacity: 10**8 * 1000, block_timestamp: (DateTime.now - 1.day).end_of_day.strftime("%Q"))
+      assert_difference -> { ::DailyStatistic.count }, 1 do
+        Charts::DailyStatisticGenerator.new.call
+      end
+    end
+  end
+end

From 2cf8bffd7355b39c35cf25898344041b1a454db3 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 09:31:07 +0800
Subject: [PATCH 60/77] feat: add forked event model

---
 app/models/forked_event.rb                    | 20 +++++++++++++++++++
 .../20191230012431_create_forked_events.rb    | 12 +++++++++++
 db/schema.rb                                  | 12 ++++++++++-
 test/models/forked_event_test.rb              |  7 +++++++
 4 files changed, 50 insertions(+), 1 deletion(-)
 create mode 100644 app/models/forked_event.rb
 create mode 100644 db/migrate/20191230012431_create_forked_events.rb
 create mode 100644 test/models/forked_event_test.rb

diff --git a/app/models/forked_event.rb b/app/models/forked_event.rb
new file mode 100644
index 000000000..51ad9f79f
--- /dev/null
+++ b/app/models/forked_event.rb
@@ -0,0 +1,20 @@
+class ForkedEvent < ApplicationRecord
+  enum status: { pending: 0, processed: 1 }
+end
+
+# == Schema Information
+#
+# Table name: forked_events
+#
+#  id              :bigint           not null, primary key
+#  block_number    :decimal(30, )
+#  epoch_number    :decimal(30, )
+#  block_timestamp :decimal(30, )
+#  status          :integer          default("pending")
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#
+# Indexes
+#
+#  index_forked_events_on_status  (status)
+#
diff --git a/db/migrate/20191230012431_create_forked_events.rb b/db/migrate/20191230012431_create_forked_events.rb
new file mode 100644
index 000000000..aaabf231f
--- /dev/null
+++ b/db/migrate/20191230012431_create_forked_events.rb
@@ -0,0 +1,12 @@
+class CreateForkedEvents < ActiveRecord::Migration[6.0]
+  def change
+    create_table :forked_events do |t|
+      t.decimal :block_number, precision: 30
+      t.decimal :epoch_number, precision: 30
+      t.decimal :block_timestamp, precision: 30
+      t.integer :status, limit: 1, default: 0, index: true
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1637ea4da..a681a8e1b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_12_26_084920) do
+ActiveRecord::Schema.define(version: 2019_12_30_012431) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -246,6 +246,16 @@
     t.integer "live_cell_changes"
   end
 
+  create_table "forked_events", force: :cascade do |t|
+    t.decimal "block_number", precision: 30
+    t.decimal "epoch_number", precision: 30
+    t.decimal "block_timestamp", precision: 30
+    t.integer "status", limit: 2, default: 0
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["status"], name: "index_forked_events_on_status"
+  end
+
   create_table "lock_scripts", force: :cascade do |t|
     t.string "args"
     t.binary "code_hash"
diff --git a/test/models/forked_event_test.rb b/test/models/forked_event_test.rb
new file mode 100644
index 000000000..8d86d76de
--- /dev/null
+++ b/test/models/forked_event_test.rb
@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class ForkedEventTest < ActiveSupport::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end

From 378ab3e64c6c06ae14cd21a3fafae82489d9fcf8 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 09:36:59 +0800
Subject: [PATCH 61/77] test: should create forked event when forked

---
 test/models/ckb_sync/node_data_processor_test.rb | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb
index a3aff6b96..39314a11c 100644
--- a/test/models/ckb_sync/node_data_processor_test.rb
+++ b/test/models/ckb_sync/node_data_processor_test.rb
@@ -954,6 +954,22 @@ class NodeDataProcessorTest < ActiveSupport::TestCase
       end
     end
 
+    test "should create forked event when block is invalid " do
+      node_block = fake_node_block
+      VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}") do
+        fake_dao_deposit_transaction(node_block)
+        node_data_processor.process_block(node_block)
+      end
+      local_block = Block.find_by(number: DEFAULT_NODE_BLOCK_NUMBER)
+      local_block.update(block_hash: "0x419c632366c8eb9635acbb39ea085f7552ae62e1fdd480893375334a0f37d1bx")
+      VCR.use_cassette("blocks/#{DEFAULT_NODE_BLOCK_NUMBER}", record: :new_episodes) do
+        assert_difference -> { ForkedEvent.count }, 1 do
+          node_data_processor.call
+        end
+        assert_equal "pending", ForkedEvent.first.status
+      end
+    end
+
     test "should revert dao contract total depositors count when block is invalid and there is dao cell" do
       CkbSync::Api.any_instance.stubs(:calculate_dao_maximum_withdraw).returns("0x2faf0be8")
       node_block = fake_node_block

From ce61a1f471856549a97b29d051776e09afbe3210 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 09:37:13 +0800
Subject: [PATCH 62/77] feat: create forked event when forked

---
 app/models/ckb_sync/node_data_processor.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb
index 6de92719c..e13b91c11 100644
--- a/app/models/ckb_sync/node_data_processor.rb
+++ b/app/models/ckb_sync/node_data_processor.rb
@@ -155,6 +155,7 @@ def invalid_block(local_tip_block)
         local_tip_block.invalid!
         local_tip_block.contained_addresses.each(&method(:update_address_balance_and_ckb_transactions_count))
         revert_block_rewards(local_tip_block)
+        ForkedEvent.create!(block_number: local_tip_block.number, epoch_number: local_tip_block.epoch, block_timestamp: local_tip_block.timestamp)
 
         local_tip_block
       end

From a99926c6ef07af0a2f41c179c478d62b397a4ee6 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 11:25:42 +0800
Subject: [PATCH 63/77] feat: add epoch statistic generator service

---
 .../charts/epoch_statistic_generator.rb       | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 app/services/charts/epoch_statistic_generator.rb

diff --git a/app/services/charts/epoch_statistic_generator.rb b/app/services/charts/epoch_statistic_generator.rb
new file mode 100644
index 000000000..0e367c7af
--- /dev/null
+++ b/app/services/charts/epoch_statistic_generator.rb
@@ -0,0 +1,22 @@
+module Charts
+  class EpochStatisticGenerator
+    def initialize(epoch_number)
+      @epoch_number = epoch_number
+    end
+
+    def call
+      return if Block.where(epoch: epoch_number + 1).blank?
+
+      blocks_count = Block.where(epoch: epoch_number).count
+      uncles_count = Block.where(epoch: epoch_number).sum(:uncles_count)
+      uncle_rate = uncles_count / blocks_count.to_d
+      difficulty = Block.where(epoch: epoch_number).first.difficulty
+
+      epoch_statistic = ::EpochStatistic.find_or_create_by(epoch_number: epoch_number)
+      epoch_statistic.update(difficulty: difficulty, uncle_rate: uncle_rate)
+    end
+
+    private
+    attr_reader :epoch_number
+  end
+end

From 8d003f12b9dab039135108f25f891b83557848b5 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 11:26:09 +0800
Subject: [PATCH 64/77] chore: use epoch statistic generator service on epoch
 statistic worker

---
 .../charts/epoch_statistic_generator.rb       | 21 ++++++++++++-------
 app/workers/charts/epoch_statistic.rb         | 12 +----------
 2 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/app/services/charts/epoch_statistic_generator.rb b/app/services/charts/epoch_statistic_generator.rb
index 0e367c7af..767d2eef1 100644
--- a/app/services/charts/epoch_statistic_generator.rb
+++ b/app/services/charts/epoch_statistic_generator.rb
@@ -1,22 +1,27 @@
 module Charts
   class EpochStatisticGenerator
-    def initialize(epoch_number)
-      @epoch_number = epoch_number
+    def initialize(target_epoch_number)
+      @target_epoch_number = target_epoch_number
     end
 
     def call
-      return if Block.where(epoch: epoch_number + 1).blank?
+      return if Block.where(epoch: target_epoch_number).blank?
 
       blocks_count = Block.where(epoch: epoch_number).count
-      uncles_count = Block.where(epoch: epoch_number).sum(:uncles_count)
+      uncles_count = Block.where(epoch: target_epoch_number).sum(:uncles_count)
       uncle_rate = uncles_count / blocks_count.to_d
-      difficulty = Block.where(epoch: epoch_number).first.difficulty
+      difficulty = Block.where(epoch: target_epoch_number).first.difficulty
+      first_block_in_epoch = Block.where(epoch: target_epoch_number).order(:number).first
+      last_lock_in_epoch = Block.where(epoch: target_epoch_number).order(:number).last
+      block_time = last_lock_in_epoch.timestamp - first_block_in_epoch.timestamp
+      epoch_length = Block.where(epoch: target_epoch_number).count
+      hash_rate = difficulty * epoch_length / block_time
 
-      epoch_statistic = ::EpochStatistic.find_or_create_by(epoch_number: epoch_number)
-      epoch_statistic.update(difficulty: difficulty, uncle_rate: uncle_rate)
+      epoch_statistic = ::EpochStatistic.find_or_create_by(epoch_number: target_epoch_number)
+      epoch_statistic.update(difficulty: difficulty, uncle_rate: uncle_rate, hash_rate: hash_rate)
     end
 
     private
-    attr_reader :epoch_number
+    attr_reader :target_epoch_number
   end
 end
diff --git a/app/workers/charts/epoch_statistic.rb b/app/workers/charts/epoch_statistic.rb
index 35929ff4f..58d2e8d35 100644
--- a/app/workers/charts/epoch_statistic.rb
+++ b/app/workers/charts/epoch_statistic.rb
@@ -10,17 +10,7 @@ def perform
       target_epoch_number = latest_epoch_number + 1
       return if Block.where(epoch: target_epoch_number + 1).blank? || ::EpochStatistic.where(epoch_number: target_epoch_number).exists?
 
-      blocks_count = Block.where(epoch: target_epoch_number).count
-      uncles_count = Block.where(epoch: target_epoch_number).sum(:uncles_count)
-      uncle_rate = uncles_count / blocks_count.to_d
-      difficulty = Block.where(epoch: target_epoch_number).first.difficulty
-      first_block_in_epoch = Block.where(epoch: target_epoch_number).order(:number).first
-      last_lock_in_epoch = Block.where(epoch: target_epoch_number).order(:number).last
-      block_time = last_lock_in_epoch.timestamp - first_block_in_epoch.timestamp
-      epoch_length = Block.where(epoch: target_epoch_number).count
-      hash_rate = difficulty * epoch_length / block_time
-
-      ::EpochStatistic.create(epoch_number: target_epoch_number, difficulty: difficulty, uncle_rate: uncle_rate, hash_rate: hash_rate)
+      Charts::EpochStatisticGenerator.new(target_epoch_number).call
     end
   end
 end

From db9161492318dd8a61bbf239e7cc2b0000a86bca Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 11:26:40 +0800
Subject: [PATCH 65/77] feat: add block statistic generator service

---
 .../charts/block_statistic_generator.rb       | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 app/services/charts/block_statistic_generator.rb

diff --git a/app/services/charts/block_statistic_generator.rb b/app/services/charts/block_statistic_generator.rb
new file mode 100644
index 000000000..ef0c1a385
--- /dev/null
+++ b/app/services/charts/block_statistic_generator.rb
@@ -0,0 +1,22 @@
+module Charts
+  class BlockStatisticGenerator
+    def initialize(block_number)
+      @block_number = block_number
+    end
+
+    def call
+      statistic_info = StatisticInfo.new
+      hash_rate = statistic_info.hash_rate(block_number)
+      live_cells_count = CellOutput.live.count
+      dead_cells_count = CellOutput.dead.count
+      target_block = Block.find_by(number: block_number)
+      return if target_block.blank?
+
+      block_statistic = ::BlockStatistic.find_or_create_by(block_number: block_number)
+      block_statistic.update(difficulty: target_block.difficulty, hash_rate: hash_rate, live_cell_count: live_cells_count, dead_cell_count: dead_cells_count)
+    end
+
+    private
+    attr_reader :block_number
+  end
+end

From ee0281166b5cc19a87a3bfb455a223bed972da09 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 11:27:03 +0800
Subject: [PATCH 66/77] chore: user block statistic generator on block
 statistic worker

---
 app/services/charts/block_statistic_generator.rb | 6 +++---
 app/workers/charts/block_statistic.rb            | 7 +------
 2 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/app/services/charts/block_statistic_generator.rb b/app/services/charts/block_statistic_generator.rb
index ef0c1a385..ac847abf3 100644
--- a/app/services/charts/block_statistic_generator.rb
+++ b/app/services/charts/block_statistic_generator.rb
@@ -5,13 +5,13 @@ def initialize(block_number)
     end
 
     def call
+      target_block = Block.find_by(number: block_number)
+      return if target_block.blank?
+
       statistic_info = StatisticInfo.new
       hash_rate = statistic_info.hash_rate(block_number)
       live_cells_count = CellOutput.live.count
       dead_cells_count = CellOutput.dead.count
-      target_block = Block.find_by(number: block_number)
-      return if target_block.blank?
-
       block_statistic = ::BlockStatistic.find_or_create_by(block_number: block_number)
       block_statistic.update(difficulty: target_block.difficulty, hash_rate: hash_rate, live_cell_count: live_cells_count, dead_cell_count: dead_cells_count)
     end
diff --git a/app/workers/charts/block_statistic.rb b/app/workers/charts/block_statistic.rb
index 7edf1e30b..83246f5c6 100644
--- a/app/workers/charts/block_statistic.rb
+++ b/app/workers/charts/block_statistic.rb
@@ -11,12 +11,7 @@ def perform
       target_block = Block.find_by(number: target_block_number)
       return if target_block.blank? || ::BlockStatistic.where(block_number: target_block_number).exists?
 
-      statistic_info = StatisticInfo.new
-      hash_rate = statistic_info.hash_rate(target_block_number)
-      live_cells_count = CellOutput.live.count
-      dead_cells_count = CellOutput.dead.count
-
-      ::BlockStatistic.create(epoch_number: target_block.epoch,block_number: target_block.number, difficulty: target_block.difficulty, hash_rate: hash_rate, live_cells_count: live_cells_count, dead_cells_count: dead_cells_count)
+      Charts::BlockStatisticGenerator.new(target_block_number).call
     end
   end
 end

From c5a444e49f0b7b13d83ab18403affff1b2cfbaf9 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 11:27:38 +0800
Subject: [PATCH 67/77] feat: regenerate block statistic data when block forked

---
 app/models/ckb_sync/node_data_processor.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb
index e13b91c11..154060e78 100644
--- a/app/models/ckb_sync/node_data_processor.rb
+++ b/app/models/ckb_sync/node_data_processor.rb
@@ -156,6 +156,7 @@ def invalid_block(local_tip_block)
         local_tip_block.contained_addresses.each(&method(:update_address_balance_and_ckb_transactions_count))
         revert_block_rewards(local_tip_block)
         ForkedEvent.create!(block_number: local_tip_block.number, epoch_number: local_tip_block.epoch, block_timestamp: local_tip_block.timestamp)
+        Charts::BlockStatisticGenerator.new(local_tip_block.number).call
 
         local_tip_block
       end

From 83927bfea61b7416e3b5ee41e67f32f5ad532fbe Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Mon, 30 Dec 2019 11:28:02 +0800
Subject: [PATCH 68/77] feat: add chart forked event processor

---
 app/workers/charts/forked_event_processor.rb | 20 ++++++++++++++++++++
 config/schedule.yml                          |  5 +++++
 2 files changed, 25 insertions(+)
 create mode 100644 app/workers/charts/forked_event_processor.rb

diff --git a/app/workers/charts/forked_event_processor.rb b/app/workers/charts/forked_event_processor.rb
new file mode 100644
index 000000000..887f10600
--- /dev/null
+++ b/app/workers/charts/forked_event_processor.rb
@@ -0,0 +1,20 @@
+module Charts
+  class ForkedEventProcessor
+    include Sidekiq::Worker
+    sidekiq_options unique: :until_executed
+    sidekiq_options retry: false
+    sidekiq_options queue: "critical"
+
+    def perform
+      forked_events = ForkedEvent.pending
+      epoch_numbers = forked_events.pluck(:epoch_number).uniq
+      epoch_numbers.each { |epoch_number| Charts::EpochStatisticGenerator.new(epoch_number).call }
+      block_timestamps = forked_events.pluck(:block_timestamp).uniq
+      latest_daily_statistic = ::DailyStatistic.last
+      if block_timestamps.any? { |block_timestamp| block_timestamp < latest_daily_statistic.block_timestamp }
+        datetime = DateTime.strptime(latest_daily_statistic.created_at_unixtimestamp.to_s, "%s")
+        Charts::DailyStatisticGenerator.new(datetime).call
+      end
+    end
+  end
+end
\ No newline at end of file
diff --git a/config/schedule.yml b/config/schedule.yml
index 8d1108a67..074dee849 100644
--- a/config/schedule.yml
+++ b/config/schedule.yml
@@ -11,4 +11,9 @@ blockstatistic:
 epoch_statistic:
   cron: "*/30 * * * *"
   class: "Charts::EpochStatistic"
+  queue: critical
+
+chart_forked_event_processor:
+  cron: "30 0 * * *"
+  class: "Charts::ForkedEventProcessor"
   queue: critical
\ No newline at end of file

From a4f1a9e76f278f7e7fe2292805f7ea8487ba3c9d Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 14:37:29 +0800
Subject: [PATCH 69/77] chore: adjust tests

---
 app/models/ckb_sync/node_data_processor.rb       | 12 +++++++-----
 test/models/ckb_sync/node_data_processor_test.rb |  3 +++
 test/workers/charts/daily_statistic_test.rb      |  1 +
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/app/models/ckb_sync/node_data_processor.rb b/app/models/ckb_sync/node_data_processor.rb
index 154060e78..acb2564b5 100644
--- a/app/models/ckb_sync/node_data_processor.rb
+++ b/app/models/ckb_sync/node_data_processor.rb
@@ -52,7 +52,8 @@ def process_block(node_block)
     def build_new_dao_depositor_events(new_dao_depositor_events)
       new_dao_depositor_events.map do |address_id, tx_hash|
         ckb_transaction = CkbTransaction.find_by(tx_hash: tx_hash)
-        ckb_transaction.dao_events.build(block: ckb_transaction.block, address_id: address_id, event_type: "new_dao_depositor", value: 1, contract_id: DaoContract.default_contract.id)
+        ckb_transaction.dao_events.build(block: ckb_transaction.block, address_id: address_id, event_type: "new_dao_depositor",
+                                         value: 1, contract_id: DaoContract.default_contract.id, block_timestamp: ckb_transaction.block_timestamp)
       end
     end
 
@@ -385,7 +386,8 @@ def build_cell_outputs(node_outputs, ckb_transaction, addresses, outputs_data, o
     def build_deposit_dao_events(address, cell_output, ckb_transaction, new_dao_depositor_events)
       if cell_output.nervos_dao_deposit?
         dao_contract = DaoContract.find_or_create_by(id: 1)
-        ckb_transaction.dao_events.build(block: ckb_transaction.block, address_id: address.id, event_type: "deposit_to_dao", value: cell_output.capacity, contract_id: dao_contract.id)
+        ckb_transaction.dao_events.build(block: ckb_transaction.block, address_id: address.id, event_type: "deposit_to_dao",
+                                         value: cell_output.capacity, contract_id: dao_contract.id, block_timestamp: ckb_transaction.block_timestamp)
         if address.dao_deposit.zero? && !new_dao_depositor_events.key?(address.id)
           new_dao_depositor_events[address.id] = ckb_transaction.tx_hash
         end
@@ -396,12 +398,12 @@ def build_withdraw_dao_events(address_id, ckb_transaction_id, local_block, previ
       if previous_cell_output.nervos_dao_withdrawing?
         withdraw_amount = previous_cell_output.capacity
         ckb_transaction = CkbTransaction.find(ckb_transaction_id)
-        ckb_transaction.dao_events.create!(block: local_block, address_id: address_id, event_type: "withdraw_from_dao", value: withdraw_amount, contract_id: DaoContract.default_contract.id)
+        ckb_transaction.dao_events.create!(block: local_block, block_timestamp: local_block.timestamp, address_id: address_id, event_type: "withdraw_from_dao", value: withdraw_amount, contract_id: DaoContract.default_contract.id)
         interest = CkbUtils.dao_interest(previous_cell_output)
-        ckb_transaction.dao_events.create!(block: local_block, address_id: address_id, event_type: "issue_interest", value: interest, contract_id: DaoContract.default_contract.id)
+        ckb_transaction.dao_events.create!(block: local_block, block_timestamp: local_block.timestamp, address_id: address_id, event_type: "issue_interest", value: interest, contract_id: DaoContract.default_contract.id)
         address = Address.find(address_id)
         if (address.dao_deposit - withdraw_amount).zero?
-          ckb_transaction.dao_events.create!(block: local_block, address_id: address_id, event_type: "take_away_all_deposit", value: 1, contract_id: DaoContract.default_contract.id)
+          ckb_transaction.dao_events.create!(block: local_block, block_timestamp: local_block.timestamp, address_id: address_id, event_type: "take_away_all_deposit", value: 1, contract_id: DaoContract.default_contract.id)
         end
       end
     end
diff --git a/test/models/ckb_sync/node_data_processor_test.rb b/test/models/ckb_sync/node_data_processor_test.rb
index 39314a11c..6870a31f7 100644
--- a/test/models/ckb_sync/node_data_processor_test.rb
+++ b/test/models/ckb_sync/node_data_processor_test.rb
@@ -626,6 +626,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase
 
         deposit_to_dao_events = Block.find_by(number: node_block.header.number).dao_events.where(event_type: "new_dao_depositor")
         assert_equal ["processed"], deposit_to_dao_events.pluck(:status).uniq
+        assert_equal %w(block_id ckb_transaction_id address_id contract_id event_type value status block_timestamp), deposit_to_dao_events.first.attribute_names.reject { |attribute| attribute.in?(%w(created_at updated_at id)) }
       end
     end
 
@@ -683,6 +684,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase
 
         deposit_to_dao_events = Block.find_by(number: node_block.header.number).dao_events.where(event_type: "withdraw_from_dao")
         assert_equal ["processed"], deposit_to_dao_events.pluck(:status).uniq
+        assert_equal %w(block_id ckb_transaction_id address_id contract_id event_type value status block_timestamp), deposit_to_dao_events.first.attribute_names.reject { |attribute| attribute.in?(%w(created_at updated_at id)) }
       end
     end
 
@@ -731,6 +733,7 @@ class NodeDataProcessorTest < ActiveSupport::TestCase
 
         deposit_to_dao_events = Block.find_by(number: node_block.header.number).dao_events.where(event_type: "withdraw_from_dao")
         assert_equal ["processed"], deposit_to_dao_events.pluck(:status).uniq
+        assert_equal %w(block_id ckb_transaction_id address_id contract_id event_type value status block_timestamp), deposit_to_dao_events.first.attribute_names.reject { |attribute| attribute.in?(%w(created_at updated_at id)) }
       end
     end
 
diff --git a/test/workers/charts/daily_statistic_test.rb b/test/workers/charts/daily_statistic_test.rb
index 618edd129..487666279 100644
--- a/test/workers/charts/daily_statistic_test.rb
+++ b/test/workers/charts/daily_statistic_test.rb
@@ -4,6 +4,7 @@ module Charts
   class DailyStatisticTest < ActiveSupport::TestCase
     test "daily statistic job should enqueue critical queue" do
       assert_difference -> { Charts::DailyStatistic.jobs.size }, 1 do
+        DaoContract.any_instance.stubs(:estimated_apc).returns(nil)
         Charts::DailyStatistic.perform_async
       end
       assert_equal "critical", Charts::DailyStatistic.queue

From 05ab165b539923117652ab4fd7af42be3b850499 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 14:43:58 +0800
Subject: [PATCH 70/77] test: chart workers

---
 app/services/charts/block_statistic_generator.rb |  2 +-
 app/services/charts/epoch_statistic_generator.rb |  2 +-
 app/workers/charts/block_statistic.rb            |  2 +-
 app/workers/charts/epoch_statistic.rb            |  4 ++--
 test/workers/charts/block_statistic_test.rb      | 13 +++++++++++++
 test/workers/charts/daily_statistic_test.rb      |  1 +
 test/workers/charts/epoch_statistic_test.rb      | 13 +++++++++++++
 7 files changed, 32 insertions(+), 5 deletions(-)
 create mode 100644 test/workers/charts/block_statistic_test.rb
 create mode 100644 test/workers/charts/epoch_statistic_test.rb

diff --git a/app/services/charts/block_statistic_generator.rb b/app/services/charts/block_statistic_generator.rb
index ac847abf3..6783c827a 100644
--- a/app/services/charts/block_statistic_generator.rb
+++ b/app/services/charts/block_statistic_generator.rb
@@ -13,7 +13,7 @@ def call
       live_cells_count = CellOutput.live.count
       dead_cells_count = CellOutput.dead.count
       block_statistic = ::BlockStatistic.find_or_create_by(block_number: block_number)
-      block_statistic.update(difficulty: target_block.difficulty, hash_rate: hash_rate, live_cell_count: live_cells_count, dead_cell_count: dead_cells_count)
+      block_statistic.update(difficulty: target_block.difficulty, hash_rate: hash_rate, live_cells_count: live_cells_count, dead_cells_count: dead_cells_count)
     end
 
     private
diff --git a/app/services/charts/epoch_statistic_generator.rb b/app/services/charts/epoch_statistic_generator.rb
index 767d2eef1..d5b7be70a 100644
--- a/app/services/charts/epoch_statistic_generator.rb
+++ b/app/services/charts/epoch_statistic_generator.rb
@@ -7,7 +7,7 @@ def initialize(target_epoch_number)
     def call
       return if Block.where(epoch: target_epoch_number).blank?
 
-      blocks_count = Block.where(epoch: epoch_number).count
+      blocks_count = Block.where(epoch: target_epoch_number).count
       uncles_count = Block.where(epoch: target_epoch_number).sum(:uncles_count)
       uncle_rate = uncles_count / blocks_count.to_d
       difficulty = Block.where(epoch: target_epoch_number).first.difficulty
diff --git a/app/workers/charts/block_statistic.rb b/app/workers/charts/block_statistic.rb
index 83246f5c6..febe3292b 100644
--- a/app/workers/charts/block_statistic.rb
+++ b/app/workers/charts/block_statistic.rb
@@ -6,7 +6,7 @@ class BlockStatistic
     sidekiq_options queue: "critical"
 
     def perform
-      latest_block_number = ::BlockStatistic.order(block_number: :desc).pick(:block_number)
+      latest_block_number = ::BlockStatistic.order(block_number: :desc).pick(:block_number) || 0
       target_block_number = latest_block_number + 100
       target_block = Block.find_by(number: target_block_number)
       return if target_block.blank? || ::BlockStatistic.where(block_number: target_block_number).exists?
diff --git a/app/workers/charts/epoch_statistic.rb b/app/workers/charts/epoch_statistic.rb
index 58d2e8d35..4b7854f97 100644
--- a/app/workers/charts/epoch_statistic.rb
+++ b/app/workers/charts/epoch_statistic.rb
@@ -6,9 +6,9 @@ class EpochStatistic
     sidekiq_options queue: "critical"
 
     def perform
-      latest_epoch_number = ::EpochStatistic.order(epoch_number: :desc).pick(:epoch_number)
+      latest_epoch_number = ::EpochStatistic.order(epoch_number: :desc).pick(:epoch_number) || 0
       target_epoch_number = latest_epoch_number + 1
-      return if Block.where(epoch: target_epoch_number + 1).blank? || ::EpochStatistic.where(epoch_number: target_epoch_number).exists?
+      return if Block.where(epoch: target_epoch_number).blank? || ::EpochStatistic.where(epoch_number: target_epoch_number).exists?
 
       Charts::EpochStatisticGenerator.new(target_epoch_number).call
     end
diff --git a/test/workers/charts/block_statistic_test.rb b/test/workers/charts/block_statistic_test.rb
new file mode 100644
index 000000000..ec41788a7
--- /dev/null
+++ b/test/workers/charts/block_statistic_test.rb
@@ -0,0 +1,13 @@
+require "test_helper"
+
+module Charts
+  class BlockStatisticTest < ActiveSupport::TestCase
+    test "block statistic job should enqueue critical queue" do
+      Sidekiq::Testing.fake!
+      assert_difference -> { Charts::BlockStatistic.jobs.size }, 1 do
+        Charts::BlockStatistic.perform_async
+      end
+      assert_equal "critical", Charts::BlockStatistic.queue
+    end
+  end
+end
diff --git a/test/workers/charts/daily_statistic_test.rb b/test/workers/charts/daily_statistic_test.rb
index 487666279..bc0ffad38 100644
--- a/test/workers/charts/daily_statistic_test.rb
+++ b/test/workers/charts/daily_statistic_test.rb
@@ -5,6 +5,7 @@ class DailyStatisticTest < ActiveSupport::TestCase
     test "daily statistic job should enqueue critical queue" do
       assert_difference -> { Charts::DailyStatistic.jobs.size }, 1 do
         DaoContract.any_instance.stubs(:estimated_apc).returns(nil)
+        Charts::DailyStatisticGenerator.any_instance.stubs(:call).returns(true)
         Charts::DailyStatistic.perform_async
       end
       assert_equal "critical", Charts::DailyStatistic.queue
diff --git a/test/workers/charts/epoch_statistic_test.rb b/test/workers/charts/epoch_statistic_test.rb
new file mode 100644
index 000000000..67b78ba00
--- /dev/null
+++ b/test/workers/charts/epoch_statistic_test.rb
@@ -0,0 +1,13 @@
+require "test_helper"
+
+module Charts
+  class EpochStatisticTest < ActiveSupport::TestCase
+    test "block statistic job should enqueue critical queue" do
+      Sidekiq::Testing.fake!
+      assert_difference -> { Charts::EpochStatistic.jobs.size }, 1 do
+        Charts::EpochStatistic.perform_async
+      end
+      assert_equal "critical", Charts::EpochStatistic.queue
+    end
+  end
+end

From 86f2fa61d3ad37b27dc76cc98e442501d9a6959a Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 15:29:41 +0800
Subject: [PATCH 71/77] chore: handle sidekiq web ui forbidden

---
 config/routes.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/routes.rb b/config/routes.rb
index 70f5744c9..effc37f9c 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -4,6 +4,7 @@
   require "sidekiq_unique_jobs/web"
   require "sidekiq/cron/web"
   mount Sidekiq::Web => "/sidekiq"
+  Sidekiq::Web.set :session_secret, Rails.application.credentials[:secret_key_base]
 
   root "application#homepage"
 

From 12d25ef957b9d5f0ed2bbe8ff8e603a3f5131f36 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 15:50:16 +0800
Subject: [PATCH 72/77] chore: fill block timestamp for all dao events

---
 lib/tasks/migration/fill_block_timestamp_to_dao_event.rake | 6 +++---
 test/workers/charts/daily_statistic_test.rb                | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/tasks/migration/fill_block_timestamp_to_dao_event.rake b/lib/tasks/migration/fill_block_timestamp_to_dao_event.rake
index 8e468b2d1..52c6d4f5e 100644
--- a/lib/tasks/migration/fill_block_timestamp_to_dao_event.rake
+++ b/lib/tasks/migration/fill_block_timestamp_to_dao_event.rake
@@ -1,6 +1,6 @@
 namespace :migration do
   task fill_block_timestamp_to_dao_event: :environment do
-    dao_events = DaoEvent.processed
+    dao_events = DaoEvent.all
     progress_bar = ProgressBar.create({
       total: dao_events.count,
       format: "%e %B %p%% %c/%C"
@@ -9,10 +9,10 @@ namespace :migration do
     values =
       dao_events.map do |dao_event|
         progress_bar.increment
-        [dao_event.id, dao_event.block.timestamp]
+        [dao_event.id, dao_event.block_id, dao_event.ckb_transaction_id, dao_event.address_id, dao_event.block&.timestamp]
       end
 
-    columns = [:id, :block_timestamp]
+    columns = [:id, :block_id, :ckb_transaction_id, :address_id, :block_timestamp]
     DaoEvent.import columns, values, on_duplicate_key_update: [:block_timestamp]
 
     puts "done"
diff --git a/test/workers/charts/daily_statistic_test.rb b/test/workers/charts/daily_statistic_test.rb
index bc0ffad38..adc2a36b9 100644
--- a/test/workers/charts/daily_statistic_test.rb
+++ b/test/workers/charts/daily_statistic_test.rb
@@ -4,6 +4,7 @@ module Charts
   class DailyStatisticTest < ActiveSupport::TestCase
     test "daily statistic job should enqueue critical queue" do
       assert_difference -> { Charts::DailyStatistic.jobs.size }, 1 do
+        Sidekiq::Testing.fake!
         DaoContract.any_instance.stubs(:estimated_apc).returns(nil)
         Charts::DailyStatisticGenerator.any_instance.stubs(:call).returns(true)
         Charts::DailyStatistic.perform_async

From c16ca67aea1cdc938083bce34986845a7425c4a4 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 16:55:25 +0800
Subject: [PATCH 73/77] Bump version to v0.9.0

---
 CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 154b1ecc1..e5874be76 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,45 @@
+# [0.9.0](https://github.com/shaojunda/ckb-explorer/compare/v0.8.3...v0.9.0) (2020-01-02)
+
+
+### Bug Fixes
+
+* fix BlockStatistic worker bug ([550308a](https://github.com/shaojunda/ckb-explorer/commit/550308a))
+* fix cell status not changed ([7522100](https://github.com/shaojunda/ckb-explorer/commit/7522100))
+
+
+### Features
+
+* add block list serializer ([8d729d7](https://github.com/shaojunda/ckb-explorer/commit/8d729d7))
+* add block statistic generator service ([db91614](https://github.com/shaojunda/ckb-explorer/commit/db91614))
+* add block timestamp to dao event ([a40453b](https://github.com/shaojunda/ckb-explorer/commit/a40453b))
+* add capacity_involved column to ckb transaction ([7bcd7a5](https://github.com/shaojunda/ckb-explorer/commit/7bcd7a5))
+* add chart forked event processor ([83927bf](https://github.com/shaojunda/ckb-explorer/commit/83927bf))
+* add epoch statistic generator service ([a99926c](https://github.com/shaojunda/ckb-explorer/commit/a99926c))
+* add estimated_apc to dao contract ([cb31d1c](https://github.com/shaojunda/ckb-explorer/commit/cb31d1c))
+* add external stats api ([d9a3fdb](https://github.com/shaojunda/ckb-explorer/commit/d9a3fdb))
+* add forked event model ([2cf8bff](https://github.com/shaojunda/ckb-explorer/commit/2cf8bff))
+* add hash rate to epoch statistic ([e0746df](https://github.com/shaojunda/ckb-explorer/commit/e0746df))
+* add index action to ckb transactions controller ([e71fef7](https://github.com/shaojunda/ckb-explorer/commit/e71fef7))
+* add live_cell_changes to block and ckb_transaction ([2f04c62](https://github.com/shaojunda/ckb-explorer/commit/2f04c62))
+* add live_cell_changes to forked blocks ([a2d42b2](https://github.com/shaojunda/ckb-explorer/commit/a2d42b2))
+* add more field to daily statistics ([3912139](https://github.com/shaojunda/ckb-explorer/commit/3912139))
+* add pagination to ckb transactions controller ([2251ba2](https://github.com/shaojunda/ckb-explorer/commit/2251ba2))
+* add ratio scale ([3b68742](https://github.com/shaojunda/ckb-explorer/commit/3b68742))
+* calculate estimated apc ([4a4f04f](https://github.com/shaojunda/ckb-explorer/commit/4a4f04f))
+* create forked event when forked ([ce61a1f](https://github.com/shaojunda/ckb-explorer/commit/ce61a1f))
+* implement ckb transactions index action ([26e5a36](https://github.com/shaojunda/ckb-explorer/commit/26e5a36))
+* regenerate block statistic data when block forked ([c5a444e](https://github.com/shaojunda/ckb-explorer/commit/c5a444e))
+* return hash_rate ([354ebf1](https://github.com/shaojunda/ckb-explorer/commit/354ebf1))
+* save capacity involved to ckb_transaction ([61fc5ba](https://github.com/shaojunda/ckb-explorer/commit/61fc5ba))
+* save hash rate on epoch statistic worker ([48be8e4](https://github.com/shaojunda/ckb-explorer/commit/48be8e4))
+* save live_cell_changes to block ([76f3efc](https://github.com/shaojunda/ckb-explorer/commit/76f3efc))
+* save live_cell_changes to ckb_transaction ([0c5589f](https://github.com/shaojunda/ckb-explorer/commit/0c5589f))
+* show more attributes on dao contract ([6369a86](https://github.com/shaojunda/ckb-explorer/commit/6369a86))
+* show nervos dao phase1 cells ([09ebc1b](https://github.com/shaojunda/ckb-explorer/commit/09ebc1b))
+* use ckb transaction list serializer ([7d12caa](https://github.com/shaojunda/ckb-explorer/commit/7d12caa))
+
+
+
 # [0.8.4](https://github.com/shaojunda/ckb-explorer/compare/v0.8.3...v0.8.4) (2019-12-21)
 
 

From efe97fa172c1210667daaa23e55e3c6dd46d66fa Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 15:49:56 +0800
Subject: [PATCH 74/77] feat: add miner reward to block

---
 app/serializers/block_serializer.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/serializers/block_serializer.rb b/app/serializers/block_serializer.rb
index 3b6af71ae..f2a78a5f1 100644
--- a/app/serializers/block_serializer.rb
+++ b/app/serializers/block_serializer.rb
@@ -55,4 +55,7 @@ class BlockSerializer
   attribute :difficulty do |object|
     object.difficulty.to_s
   end
+  attribute :miner_reward do |object|
+    (object.received_tx_fee + object.reward).to_s
+  end
 end

From 40d51e6dc506c39b492d316c747b7a66cb8dda61 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 26 Dec 2019 15:50:11 +0800
Subject: [PATCH 75/77] chore: adjust tests

---
 test/controllers/api/v1/blocks_controller_test.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/controllers/api/v1/blocks_controller_test.rb b/test/controllers/api/v1/blocks_controller_test.rb
index 1257abb79..086c468d3 100644
--- a/test/controllers/api/v1/blocks_controller_test.rb
+++ b/test/controllers/api/v1/blocks_controller_test.rb
@@ -72,7 +72,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest
         valid_get api_v1_blocks_url
 
         response_block = json["data"].first
-        assert_equal %w(number transactions_count reward miner_hash timestamp live_cell_changes).sort, response_block["attributes"].keys.sort
+        assert_equal %w(number transactions_count reward miner_hash timestamp live_cell_changes miner_reward).sort, response_block["attributes"].keys.sort
       end
 
       test "should return error object when page param is invalid" do
@@ -301,7 +301,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest
         valid_get api_v1_block_url(block.block_hash)
 
         response_block = json["data"]
-        assert_equal %w(block_hash number transactions_count proposals_count uncles_count uncle_block_hashes reward total_transaction_fee cell_consumed total_cell_capacity miner_hash timestamp difficulty version nonce epoch start_number length transactions_root reward_status received_tx_fee received_tx_fee_status block_index_in_epoch).sort, response_block["attributes"].keys.sort
+        assert_equal %w(block_hash number transactions_count proposals_count uncles_count uncle_block_hashes reward total_transaction_fee cell_consumed total_cell_capacity miner_hash timestamp difficulty version nonce epoch start_number length transactions_root reward_status received_tx_fee received_tx_fee_status block_index_in_epoch miner_reward).sort, response_block["attributes"].keys.sort
       end
 
       test "should return error object when no records found by id" do

From 2aa9501ddd54afc93931fe4f9f0d8cc924af5207 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 18:16:22 +0800
Subject: [PATCH 76/77] chore: adjust test

---
 test/controllers/api/v1/blocks_controller_test.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/controllers/api/v1/blocks_controller_test.rb b/test/controllers/api/v1/blocks_controller_test.rb
index 086c468d3..09fa0e633 100644
--- a/test/controllers/api/v1/blocks_controller_test.rb
+++ b/test/controllers/api/v1/blocks_controller_test.rb
@@ -72,7 +72,7 @@ class BlocksControllerTest < ActionDispatch::IntegrationTest
         valid_get api_v1_blocks_url
 
         response_block = json["data"].first
-        assert_equal %w(number transactions_count reward miner_hash timestamp live_cell_changes miner_reward).sort, response_block["attributes"].keys.sort
+        assert_equal %w(number transactions_count reward miner_hash timestamp live_cell_changes).sort, response_block["attributes"].keys.sort
       end
 
       test "should return error object when page param is invalid" do

From 5b3bc50ab17ba22507c8f8b848f0d08afee2bb04 Mon Sep 17 00:00:00 2001
From: shaojunda 
Date: Thu, 2 Jan 2020 17:12:46 +0800
Subject: [PATCH 77/77] chore: setup redis on github actions

---
 .github/workflows/ruby.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
index 88465284a..0ab113d48 100644
--- a/.github/workflows/ruby.yml
+++ b/.github/workflows/ruby.yml
@@ -8,6 +8,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - uses: zhulik/redis-action@v1.1.0
+      - name: Set up Redis
+        with:
+          redis version: "5"
+          number of databases: 2
       - uses: actions/checkout@v1
       - name: Set up Ruby 2.6.4
         run: |