Optimize block query performance (#1307)

nervosnetwork · Jun 11, 2023 · e4dad7c · e4dad7c
1 parent 8b76cc7
commit e4dad7c
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 32 deletions.
diff --git a/app/controllers/api/v1/blocks_controller.rb b/app/controllers/api/v1/blocks_controller.rb
@@ -1,4 +1,4 @@
-require 'csv'
+require "csv"
 module Api
   module V1
     class BlocksController < ApplicationController
@@ -7,29 +7,37 @@ class BlocksController < ApplicationController
 
       def index
         if from_home_page?
-          blocks = Block.recent.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, :live_cell_changes, :updated_at).limit(ENV["HOMEPAGE_BLOCK_RECORDS_COUNT"].to_i)
+          blocks = Block.recent.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count,
+                                       :live_cell_changes, :updated_at).limit(ENV["HOMEPAGE_BLOCK_RECORDS_COUNT"].to_i)
           json =
             Rails.cache.realize(blocks.cache_key, version: blocks.cache_version, race_condition_ttl: 3.seconds) do
               BlockListSerializer.new(blocks).serialized_json
             end
         else
-          blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, :live_cell_changes, :updated_at)
-          params[:sort] ||= "timestamp.desc"
-
-          order_by, asc_or_desc = params[:sort].split('.', 2)
-          order_by = case order_by
-          when 'height' then 'number'
-          when 'transactions' then 'ckb_transactions_count'
-          else order_by
-          end
+          blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count,
+                                :live_cell_changes, :updated_at)
+          params[:sort] ||= "number.desc"
+
+          order_by, asc_or_desc = params[:sort].split(".", 2)
+          order_by =
+            case order_by
+                     when "height"
+                       "number"
+                     when "transactions"
+                       "ckb_transactions_count"
+                     else
+                       order_by
+            end
 
           head :not_found and return unless order_by.in? %w[number reward timestamp ckb_transactions_count]
-          blocks = blocks.order(order_by => asc_or_desc).page(@page).per(@page_size).fast_page
+
+          blocks = blocks.order(order_by => asc_or_desc).page(@page).per(@page_size)
 
           json =
             Rails.cache.realize(blocks.cache_key, version: blocks.cache_version, race_condition_ttl: 3.seconds) do
               records_counter = RecordCounters::Blocks.new
-              options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: @page, page_size: @page_size, records_counter: records_counter).call
+              options = FastJsonapi::PaginationMetaGenerator.new(request: request, records: blocks, page: @page,
+                                                                 page_size: @page_size, records_counter: records_counter).call
               BlockListSerializer.new(blocks, options).serialized_json
             end
         end
@@ -44,25 +52,39 @@ def show
       end
 
       def download_csv
-        blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count, :live_cell_changes, :updated_at)
-
-        blocks = blocks.where('timestamp >= ?', DateTime.strptime(params[:start_date], '%Y-%m-%d').to_time.to_i * 1000 ) if params[:start_date].present?
-        blocks = blocks.where('timestamp <= ?', DateTime.strptime(params[:end_date], '%Y-%m-%d').to_time.to_i * 1000 ) if params[:end_date].present?
-        blocks = blocks.where('number >= ?', params[:start_number]) if params[:start_number].present?
-        blocks = blocks.where('number <= ?', params[:end_number]) if params[:end_number].present?
-
-        blocks = blocks.order('number desc').limit(5000)
+        blocks = Block.select(:id, :miner_hash, :number, :timestamp, :reward, :ckb_transactions_count,
+                              :live_cell_changes, :updated_at)
 
-        file = CSV.generate do |csv|
-          csv << ["Blockno", "Transactions", "UnixTimestamp", "Reward(CKB)", "Miner", "date(UTC)"]
-          blocks.find_each.with_index do |block, index|
-            row = [block.number, block.ckb_transactions_count, (block.timestamp / 1000), block.reward, block.miner_hash,
-                   Time.at((block.timestamp / 1000).to_i).in_time_zone('UTC').strftime('%Y-%m-%d %H:%M:%S') ]
-            csv << row
-          end
+        if params[:start_date].present?
+          blocks = blocks.where("timestamp >= ?",
+                                DateTime.strptime(params[:start_date],
+                                                  "%Y-%m-%d").to_time.to_i * 1000)
+        end
+        if params[:end_date].present?
+          blocks = blocks.where("timestamp <= ?",
+                                DateTime.strptime(params[:end_date],
+                                                  "%Y-%m-%d").to_time.to_i * 1000)
         end
-        send_data file, :type => 'text/csv; charset=utf-8; header=present', :disposition => "attachment;filename=blocks.csv"
+        blocks = blocks.where("number >= ?", params[:start_number]) if params[:start_number].present?
+        blocks = blocks.where("number <= ?", params[:end_number]) if params[:end_number].present?
+
+        blocks = blocks.order("number desc").limit(5000)
+
+        file =
+          CSV.generate do |csv|
+            csv << ["Blockno", "Transactions", "UnixTimestamp", "Reward(CKB)", "Miner", "date(UTC)"]
+            blocks.find_each.with_index do |block, _index|
+              row = [
+                block.number, block.ckb_transactions_count, (block.timestamp / 1000), block.reward, block.miner_hash,
+                Time.at((block.timestamp / 1000).to_i).in_time_zone("UTC").strftime("%Y-%m-%d %H:%M:%S")
+              ]
+              csv << row
+            end
+          end
+        send_data file, type: "text/csv; charset=utf-8; header=present",
+                        disposition: "attachment;filename=blocks.csv"
       end
+
       private
 
       def from_home_page?

diff --git a/app/models/block.rb b/app/models/block.rb
@@ -51,7 +51,7 @@ class Block < ApplicationRecord
   attribute :uncle_block_hashes, :ckb_array_hash, hash_length: Settings.default_hash_length
   attribute :proposals, :ckb_array_hash, hash_length: Settings.default_short_hash_length
 
-  scope :recent, -> { order("timestamp desc nulls last") }
+  scope :recent, -> { order("number" => "desc") }
   scope :created_after, ->(timestamp) { where("timestamp >= ?", timestamp) }
   scope :created_before, ->(timestamp) { where("timestamp <= ?", timestamp) }
   scope :created_between, ->(from, to) { where(timestamp: from..to) }

diff --git a/lib/scheduler.rb b/lib/scheduler.rb
@@ -11,10 +11,14 @@
 require "rufus-scheduler"
 s = Rufus::Scheduler.singleton
 
-def s.around_trigger(job)
+def s.around_trigger(job, &block)
   t = Time.now
   puts "Starting job #{job.id} at #{Time.now}"
-  yield
+  Rails.application.executor.wrap do
+    ActiveRecord::Base.connection_pool.with_connection do
+      ActiveRecord::Base.cache(&block)
+    end
+  end
   puts "job #{job.id} finished in #{Time.now - t} seconds."
 end