From e4dad7c724be4fce53990fa5fb1642a602608955 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BB=A3=E7=A0=81=E4=B9=8B=E5=8A=9B?= <9770+ShiningRay@users.noreply.github.com> Date: Sun, 11 Jun 2023 23:13:24 +0800 Subject: [PATCH] Optimize block query performance (#1307) --- app/controllers/api/v1/blocks_controller.rb | 80 +++++++++++++-------- app/models/block.rb | 2 +- lib/scheduler.rb | 8 ++- 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/app/controllers/api/v1/blocks_controller.rb b/app/controllers/api/v1/blocks_controller.rb index 0a42d0c4d..983d97736 100644 --- a/app/controllers/api/v1/blocks_controller.rb +++ b/app/controllers/api/v1/blocks_controller.rb @@ -1,4 +1,4 @@ -require 'csv' +require "csv" module Api module V1 class BlocksController < ApplicationController @@ -7,29 +7,37 @@ class BlocksController < ApplicationController def index if from_home_page? - blocks = Block.recent.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, :live_cell_changes, :updated_at).limit(ENV["HOMEPAGE_BLOCK_RECORDS_COUNT"].to_i) + blocks = Block.recent.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, + :live_cell_changes, :updated_at).limit(ENV["HOMEPAGE_BLOCK_RECORDS_COUNT"].to_i) json = Rails.cache.realize(blocks.cache_key, version: blocks.cache_version, race_condition_ttl: 3.seconds) do BlockListSerializer.new(blocks).serialized_json end else - blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, :live_cell_changes, :updated_at) - params[:sort] ||= "timestamp.desc" - - order_by, asc_or_desc = params[:sort].split('.', 2) - order_by = case order_by - when 'height' then 'number' - when 'transactions' then 'ckb_transactions_count' - else order_by - end + blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, + :live_cell_changes, :updated_at) + params[:sort] ||= "number.desc" + + order_by, asc_or_desc = params[:sort].split(".", 2) + order_by = + case order_by + when "height" + "number" + when "transactions" + "ckb_transactions_count" + else + order_by + end head :not_found and return unless order_by.in? %w[number reward timestamp ckb_transactions_count] - blocks = blocks.order(order_by => asc_or_desc).page(@page).per(@page_size).fast_page + + blocks = blocks.order(order_by => asc_or_desc).page(@page).per(@page_size) json = Rails.cache.realize(blocks.cache_key, version: blocks.cache_version, race_condition_ttl: 3.seconds) do records_counter = RecordCounters::Blocks.new - options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: @page, page_size: @page_size, records_counter: records_counter).call + options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: @page, + page_size: @page_size, records_counter: records_counter).call BlockListSerializer.new(blocks, options).serialized_json end end @@ -44,25 +52,39 @@ def show end def download_csv - blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, :live_cell_changes, :updated_at) - - blocks = blocks.where('timestamp >= ?', DateTime.strptime(params[:start_date], '%Y-%m-%d').to_time.to_i * 1000 ) if params[:start_date].present? - blocks = blocks.where('timestamp <= ?', DateTime.strptime(params[:end_date], '%Y-%m-%d').to_time.to_i * 1000 ) if params[:end_date].present? - blocks = blocks.where('number >= ?', params[:start_number]) if params[:start_number].present? - blocks = blocks.where('number <= ?', params[:end_number]) if params[:end_number].present? - - blocks = blocks.order('number desc').limit(5000) + blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, + :live_cell_changes, :updated_at) - file = CSV.generate do |csv| - csv << ["Blockno", "Transactions", "UnixTimestamp", "Reward(CKB)", "Miner", "date(UTC)"] - blocks.find_each.with_index do |block, index| - row = [block.number, block.ckb_transactions_count, (block.timestamp / 1000), block.reward, block.miner_hash, - Time.at((block.timestamp / 1000).to_i).in_time_zone('UTC').strftime('%Y-%m-%d %H:%M:%S') ] - csv << row - end + if params[:start_date].present? + blocks = blocks.where("timestamp >= ?", + DateTime.strptime(params[:start_date], + "%Y-%m-%d").to_time.to_i * 1000) + end + if params[:end_date].present? + blocks = blocks.where("timestamp <= ?", + DateTime.strptime(params[:end_date], + "%Y-%m-%d").to_time.to_i * 1000) end - send_data file, :type => 'text/csv; charset=utf-8; header=present', :disposition => "attachment;filename=blocks.csv" + blocks = blocks.where("number >= ?", params[:start_number]) if params[:start_number].present? + blocks = blocks.where("number <= ?", params[:end_number]) if params[:end_number].present? + + blocks = blocks.order("number desc").limit(5000) + + file = + CSV.generate do |csv| + csv << ["Blockno", "Transactions", "UnixTimestamp", "Reward(CKB)", "Miner", "date(UTC)"] + blocks.find_each.with_index do |block, _index| + row = [ + block.number, block.ckb_transactions_count, (block.timestamp / 1000), block.reward, block.miner_hash, + Time.at((block.timestamp / 1000).to_i).in_time_zone("UTC").strftime("%Y-%m-%d %H:%M:%S") + ] + csv << row + end + end + send_data file, type: "text/csv; charset=utf-8; header=present", + disposition: "attachment;filename=blocks.csv" end + private def from_home_page? diff --git a/app/models/block.rb b/app/models/block.rb index 079bcbc22..0beefadc4 100644 --- a/app/models/block.rb +++ b/app/models/block.rb @@ -51,7 +51,7 @@ class Block < ApplicationRecord attribute :uncle_block_hashes, :ckb_array_hash, hash_length: Settings.default_hash_length attribute :proposals, :ckb_array_hash, hash_length: Settings.default_short_hash_length - scope :recent, -> { order("timestamp desc nulls last") } + scope :recent, -> { order("number" => "desc") } scope :created_after, ->(timestamp) { where("timestamp >= ?", timestamp) } scope :created_before, ->(timestamp) { where("timestamp <= ?", timestamp) } scope :created_between, ->(from, to) { where(timestamp: from..to) } diff --git a/lib/scheduler.rb b/lib/scheduler.rb index 22bb52562..b9f765911 100644 --- a/lib/scheduler.rb +++ b/lib/scheduler.rb @@ -11,10 +11,14 @@ require "rufus-scheduler" s = Rufus::Scheduler.singleton -def s.around_trigger(job) +def s.around_trigger(job, &block) t = Time.now puts "Starting job #{job.id} at #{Time.now}" - yield + Rails.application.executor.wrap do + ActiveRecord::Base.connection_pool.with_connection do + ActiveRecord::Base.cache(&block) + end + end puts "job #{job.id} finished in #{Time.now - t} seconds." end